singularity-forge/scripts/upgrade-vega-source-server.mjs

#!/usr/bin/env node
/**
 * upgrade-vega-source-server.mjs — blue/green upgrade the shared vega SF
 * webserver.
 *
 * Purpose: prove a candidate source-mounted server on a side port before
 * replacing the shared production container on port 4000.
 *
 * Consumer: `npm run docker:vega:upgrade` locally and Forgejo/host-side deploy
 * automation when vega is the target.
 */
import { spawnSync } from "node:child_process";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const root = resolve(fileURLToPath(new URL("..", import.meta.url)));
const bind = process.env.SF_VEGA_BIND || "127.0.0.1";
const prodName = process.env.SF_VEGA_CONTAINER || "sf-server-vega";
const candidateName =
	process.env.SF_VEGA_CANDIDATE_CONTAINER || "sf-server-vega-candidate";
const prodPort = process.env.SF_VEGA_PORT || "4000";
const candidatePort = process.env.SF_VEGA_CANDIDATE_PORT || "4001";
const workspacesRoot = process.env.SF_WORKSPACES_DIR || dirname(root);
const skipBuild = process.env.SF_VEGA_UPGRADE_SKIP_BUILD === "1";

if (!skipBuild) {
	run("npm", ["run", "build:web-host"]);
	run(process.execPath, [
		"scripts/generate-release-manifest.mjs",
		"--out",
		"dist/sf-release-manifest.json",
	]);
}
run("docker", [
	"build",
	"-f",
	"docker/Dockerfile.source-server",
	"-t",
	process.env.SF_VEGA_IMAGE || "sf-source-server:vega",
	".",
]);

startServer(candidateName, candidatePort);
await probeServer(candidatePort, "candidate");

drainContainer(prodName);
startServer(prodName, prodPort);
await probeServer(prodPort, "prod");

drainContainer(candidateName);
process.stdout.write(
	`sf server upgraded: ${prodName} is healthy on ${bind}:${prodPort}\n`,
);

function startServer(name, port) {
	run("node", ["scripts/run-vega-source-server.mjs", "up"], {
		env: {
			...process.env,
			SF_VEGA_CONTAINER: name,
			SF_VEGA_PORT: port,
			SF_VEGA_SKIP_IMAGE_BUILD: "1",
		},
	});
}

async function probeServer(port, label) {
	const baseUrl = `http://${bind}:${port}`;
	const checks = [
		["healthz", `${baseUrl}/api/healthz`],
		["ready", `${baseUrl}/api/ready`],
		["version", `${baseUrl}/api/version`],
		[
			"projects",
			`${baseUrl}/api/projects?root=${encodeURIComponent(workspacesRoot)}&detail=true`,
		],
	];
	const deadline = Date.now() + 60_000;
	let lastError = "";
	while (Date.now() < deadline) {
		try {
			for (const [name, url] of checks) {
				const response = await fetch(url, { cache: "no-store" });
				if (!response.ok) {
					throw new Error(`${name} returned ${response.status}`);
				}
				if (name === "projects") {
					const projects = await response.json();
					if (!Array.isArray(projects) || projects.length === 0) {
						throw new Error("projects returned no configured repos");
					}
				} else {
					await response.arrayBuffer();
				}
			}
			process.stdout.write(`${label} probes passed on ${baseUrl}\n`);
			return;
		} catch (error) {
			lastError = error instanceof Error ? error.message : String(error);
			await delay(1000);
		}
	}
	showLogs(label === "candidate" ? candidateName : prodName);
	throw new Error(`${label} probes failed: ${lastError}`);
}

function showLogs(name) {
	spawnSync("docker", ["logs", "--tail=120", name], {
		cwd: root,
		stdio: "inherit",
		env: process.env,
	});
}

function drainContainer(name) {
	// 610s: matches SF_RPC_SHUTDOWN_GRACE_MS=600000 in rpc-mode's
	// graceful-shutdown handler with a 10s safety margin for Node exit.
	// Normal drains finish in <1s; the long ceiling is for pathological
	// lock contention so queued self-feedback writes are never lost
	// across an upgrade. Override per-deployment via env if needed.
	const stopTime = process.env.SF_VEGA_DRAIN_STOP_TIME || "610";
	run("docker", ["stop", "--timeout", stopTime, name], { allowFailure: true });
	run("docker", ["rm", "-f", name], { allowFailure: true });
}

function delay(ms) {
	return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
}

function run(command, args, options = {}) {
	const result = spawnSync(command, args, {
		cwd: root,
		stdio: "inherit",
		env: options.env ?? process.env,
	});
	if (result.status !== 0 && !options.allowFailure) {
		process.exit(result.status ?? 1);
	}
}