feat: add durable release controller and admin deploy modes
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
# Gateway release controller
|
||||
|
||||
`release-controller`는 Gateway API·frontend·orchestrator와 분리된 PM2
|
||||
프로세스입니다. 관리자 GUI가 `GatewayReleaseOperation`을 만들면 controller가
|
||||
선택 commit을 고정하고 다음 순서로 전환합니다.
|
||||
|
||||
1. commit 전용 worktree를 준비하고 frozen lockfile로 의존성을 설치합니다.
|
||||
2. `release-manifest.json`의 protocol, component와 실제 migration head를
|
||||
확인합니다.
|
||||
3. Gateway API와 frontend를 빌드하고 gateway migration을 적용합니다.
|
||||
4. 기존 `sammo:gateway-api`, `sammo:gateway-frontend`,
|
||||
`sammo:gateway-orchestrator`를 중지하고 새 worktree에서 시작합니다.
|
||||
5. 두 HTTP endpoint와 세 PM2 process가 모두 준비된 경우에만 현재·이전
|
||||
릴리스 상태를 게시합니다. 실패하면 이전 세 프로세스를 복구합니다.
|
||||
|
||||
## 환경 변수
|
||||
|
||||
- `GATEWAY_DATABASE_URL`: Gateway PostgreSQL URL입니다. 필수입니다.
|
||||
- `GATEWAY_DB_SCHEMA`: Gateway schema이며 기본값은 `public`입니다.
|
||||
- `RELEASE_CONTROLLER_WORKSPACE_ROOT`: Git checkout입니다.
|
||||
- `RELEASE_CONTROLLER_WORKTREE_ROOT`: commit worktree 상위 경로입니다.
|
||||
- `GATEWAY_API_PORT`, `GATEWAY_FRONTEND_PORT`, `GATEWAY_BASE_PATH`: readiness와
|
||||
frontend build 계약입니다.
|
||||
- `RELEASE_CONTROLLER_POLL_MS`, `RELEASE_CONTROLLER_READINESS_TIMEOUT_MS`: queue
|
||||
poll과 준비 제한 시간입니다.
|
||||
|
||||
비밀값은 Git에서 제외된 환경 파일 또는 process 환경으로 전달해 주세요.
|
||||
`VITE_*`에는 공개 URL만 넣어 주세요.
|
||||
|
||||
## 설치와 실행
|
||||
|
||||
먼저 controller가 읽을 Gateway schema를 migration하고 의존 package를 함께
|
||||
빌드합니다.
|
||||
|
||||
```sh
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm --filter @sammo-ts/infra prisma:generate
|
||||
pnpm --filter @sammo-ts/common build
|
||||
pnpm --filter @sammo-ts/infra build
|
||||
pnpm --filter @sammo-ts/logic build
|
||||
pnpm --filter @sammo-ts/game-engine build
|
||||
pnpm --filter @sammo-ts/gateway-api build
|
||||
pnpm --filter @sammo-ts/release-controller build
|
||||
pnpm --filter @sammo-ts/infra prisma:migrate:deploy:gateway
|
||||
pnpm --filter @sammo-ts/release-controller start
|
||||
```
|
||||
|
||||
운영에서는 마지막 명령 대신 `sammo:release-controller`라는 PM2 process로
|
||||
`app/release-controller/dist/index.js daemon`을 실행해 주세요. 상태와 queue
|
||||
한 건 처리는 다음 CLI로 확인할 수 있습니다.
|
||||
|
||||
```sh
|
||||
pnpm --filter @sammo-ts/release-controller status
|
||||
pnpm --filter @sammo-ts/release-controller run-once
|
||||
```
|
||||
|
||||
## Controller self-upgrade
|
||||
|
||||
이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를
|
||||
빌드하고 gateway migration을 적용한 뒤 `sammo:release-controller`만 새
|
||||
worktree로 전환합니다. 새 daemon 시작에 실패하면 이전 definition을
|
||||
복구합니다.
|
||||
|
||||
```sh
|
||||
pnpm --filter @sammo-ts/release-controller build
|
||||
pnpm --filter @sammo-ts/release-controller self-upgrade -- BRANCH main
|
||||
# 또는
|
||||
pnpm --filter @sammo-ts/release-controller self-upgrade -- COMMIT <full-sha>
|
||||
```
|
||||
|
||||
Database migration은 일반적으로 되돌리지 않습니다. 이전 애플리케이션으로
|
||||
rollback하려면 새 schema와의 하위 호환성을 릴리스 전에 확인해 주세요.
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "@sammo-ts/release-controller",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/release-controller",
|
||||
"start": "node dist/index.js daemon",
|
||||
"run-once": "node dist/index.js run-once",
|
||||
"status": "node dist/index.js status",
|
||||
"self-upgrade": "node dist/index.js self-upgrade",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint . --fix",
|
||||
"test": "vitest run --config vitest.config.ts",
|
||||
"typecheck": "tsc -b"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sammo-ts/gateway-api": "workspace:*",
|
||||
"@sammo-ts/infra": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tsdown": "^0.22.14",
|
||||
"vite-tsconfig-paths": "^6.0.3",
|
||||
"vitest": "^4.0.16"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import path from 'node:path';
|
||||
|
||||
const parsePositiveInt = (value: string | undefined, fallback: number, name: string): number => {
|
||||
if (!value) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer.`);
|
||||
return parsed;
|
||||
};
|
||||
|
||||
const applySchema = (databaseUrl: string, schema: string): string => {
|
||||
const parsed = new URL(databaseUrl);
|
||||
parsed.searchParams.set('schema', schema);
|
||||
return parsed.toString();
|
||||
};
|
||||
|
||||
export interface ReleaseControllerConfig {
|
||||
workspaceRoot: string;
|
||||
worktreeRoot: string;
|
||||
gatewayDatabaseUrl: string;
|
||||
gatewayDbSchema: string;
|
||||
gatewayApiPort: number;
|
||||
gatewayFrontendPort: number;
|
||||
gatewayBasePath: string;
|
||||
pollIntervalMs: number;
|
||||
readinessTimeoutMs: number;
|
||||
baseEnv: Record<string, string>;
|
||||
}
|
||||
|
||||
export const resolveReleaseControllerConfig = (env: NodeJS.ProcessEnv = process.env): ReleaseControllerConfig => {
|
||||
const rawGatewayDatabaseUrl = env.GATEWAY_DATABASE_URL ?? env.DATABASE_URL ?? '';
|
||||
if (!rawGatewayDatabaseUrl) {
|
||||
throw new Error('GATEWAY_DATABASE_URL or DATABASE_URL is required.');
|
||||
}
|
||||
const workspaceRoot = path.resolve(env.RELEASE_CONTROLLER_WORKSPACE_ROOT ?? process.cwd());
|
||||
const gatewayDbSchema = env.GATEWAY_DB_SCHEMA?.trim() || 'public';
|
||||
return {
|
||||
workspaceRoot,
|
||||
worktreeRoot: path.resolve(
|
||||
env.RELEASE_CONTROLLER_WORKTREE_ROOT ?? path.join(workspaceRoot, '.release-worktrees')
|
||||
),
|
||||
gatewayDatabaseUrl: applySchema(rawGatewayDatabaseUrl, gatewayDbSchema),
|
||||
gatewayDbSchema,
|
||||
gatewayApiPort: parsePositiveInt(env.GATEWAY_API_PORT, 15001, 'GATEWAY_API_PORT'),
|
||||
gatewayFrontendPort: parsePositiveInt(env.GATEWAY_FRONTEND_PORT, 15000, 'GATEWAY_FRONTEND_PORT'),
|
||||
gatewayBasePath: env.GATEWAY_BASE_PATH?.trim() || '/gateway',
|
||||
pollIntervalMs: parsePositiveInt(env.RELEASE_CONTROLLER_POLL_MS, 5000, 'RELEASE_CONTROLLER_POLL_MS'),
|
||||
readinessTimeoutMs: parsePositiveInt(
|
||||
env.RELEASE_CONTROLLER_READINESS_TIMEOUT_MS,
|
||||
60000,
|
||||
'RELEASE_CONTROLLER_READINESS_TIMEOUT_MS'
|
||||
),
|
||||
baseEnv: Object.fromEntries(
|
||||
Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === 'string')
|
||||
),
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,83 @@
|
||||
import { createGatewayPostgresConnector, type GatewayPrismaClient } from '@sammo-ts/infra';
|
||||
import {
|
||||
createGatewayReleaseRepository,
|
||||
GitWorkspaceManager,
|
||||
Pm2ProcessManager,
|
||||
PnpmBuildRunner,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
|
||||
import { resolveReleaseControllerConfig } from './config.js';
|
||||
import { GatewayReleaseController } from './releaseController.js';
|
||||
import { upgradeReleaseController } from './selfUpgrade.js';
|
||||
|
||||
export * from './config.js';
|
||||
export * from './releaseController.js';
|
||||
export * from './selfUpgrade.js';
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const config = resolveReleaseControllerConfig();
|
||||
const postgres = createGatewayPostgresConnector({ url: config.gatewayDatabaseUrl });
|
||||
await postgres.connect();
|
||||
const repository = createGatewayReleaseRepository(postgres.prisma as GatewayPrismaClient);
|
||||
const workspaceManager = new GitWorkspaceManager({
|
||||
repoRoot: config.workspaceRoot,
|
||||
worktreeRoot: config.worktreeRoot,
|
||||
baseEnv: config.baseEnv,
|
||||
});
|
||||
const buildRunner = new PnpmBuildRunner();
|
||||
const processManager = new Pm2ProcessManager();
|
||||
const controller = new GatewayReleaseController(repository, workspaceManager, buildRunner, processManager, config);
|
||||
const command = process.argv[2] ?? 'daemon';
|
||||
if (command === 'status') {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{ state: await repository.getState(), operations: await repository.listOperations(20) },
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
await postgres.disconnect();
|
||||
return;
|
||||
}
|
||||
if (command === 'run-once') {
|
||||
console.log(JSON.stringify(await controller.runOnce(), null, 2));
|
||||
await postgres.disconnect();
|
||||
return;
|
||||
}
|
||||
if (command === 'self-upgrade') {
|
||||
const sourceMode = process.argv[3];
|
||||
const sourceRef = process.argv[4];
|
||||
if ((sourceMode !== 'BRANCH' && sourceMode !== 'COMMIT') || !sourceRef) {
|
||||
throw new Error('usage: release-controller self-upgrade <BRANCH|COMMIT> <ref>');
|
||||
}
|
||||
const result = await upgradeReleaseController({
|
||||
sourceMode,
|
||||
sourceRef,
|
||||
workspaceManager,
|
||||
buildRunner,
|
||||
processManager,
|
||||
config,
|
||||
});
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
await postgres.disconnect();
|
||||
return;
|
||||
}
|
||||
if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`);
|
||||
let stopping = false;
|
||||
const stop = async (): Promise<void> => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
await postgres.disconnect();
|
||||
};
|
||||
process.once('SIGINT', () => void stop());
|
||||
process.once('SIGTERM', () => void stop());
|
||||
while (!stopping) {
|
||||
await controller.runOnce();
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, config.pollIntervalMs));
|
||||
}
|
||||
};
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('[release-controller] failed', error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,247 @@
|
||||
import path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import {
|
||||
assertReleaseComponents,
|
||||
type BuildCommand,
|
||||
type BuildRunner,
|
||||
type GatewayReleaseOperationRecord,
|
||||
type GatewayReleaseRepository,
|
||||
type GatewayReleaseStateRecord,
|
||||
type GitWorkspaceManager,
|
||||
type ProcessDefinition,
|
||||
type ProcessManager,
|
||||
readReleaseManifest,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
|
||||
import type { ReleaseControllerConfig } from './config.js';
|
||||
|
||||
const LEASE_DURATION_MS = 10 * 60_000;
|
||||
const HEARTBEAT_INTERVAL_MS = 60_000;
|
||||
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
|
||||
|
||||
export const buildGatewayReleaseCommands = (
|
||||
workspaceRoot: string,
|
||||
needsInstall: boolean,
|
||||
config: ReleaseControllerConfig
|
||||
): BuildCommand[] => {
|
||||
const env = {
|
||||
...config.baseEnv,
|
||||
VITE_APP_BASE_PATH: config.gatewayBasePath,
|
||||
VITE_GATEWAY_API_URL: `${config.gatewayBasePath}/api/trpc`,
|
||||
VITE_GAME_API_URL_TEMPLATE: '/{profile}/api/trpc',
|
||||
VITE_GAME_WEB_URL_TEMPLATE: '/{profile}/',
|
||||
};
|
||||
return [
|
||||
...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []),
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'prisma:generate'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/logic', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/game-engine', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/gateway-api', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/gateway-frontend', 'build'], cwd: workspaceRoot, env },
|
||||
];
|
||||
};
|
||||
|
||||
export const buildGatewayMigrationCommand = (workspaceRoot: string, config: ReleaseControllerConfig): BuildCommand => ({
|
||||
command: 'pnpm',
|
||||
args: ['--filter', '@sammo-ts/infra', 'prisma:migrate:deploy:gateway'],
|
||||
cwd: workspaceRoot,
|
||||
env: {
|
||||
...config.baseEnv,
|
||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||
},
|
||||
});
|
||||
|
||||
export const buildGatewayProcessDefinitions = (
|
||||
workspaceRoot: string,
|
||||
config: ReleaseControllerConfig
|
||||
): ProcessDefinition[] => {
|
||||
const apiCwd = path.join(workspaceRoot, 'app', 'gateway-api');
|
||||
const frontendCwd = path.join(workspaceRoot, 'app', 'gateway-frontend');
|
||||
const apiScript = path.join(apiCwd, 'dist', 'index.js');
|
||||
const frontendScript = path.join(workspaceRoot, 'node_modules', 'vite', 'bin', 'vite.js');
|
||||
const env = {
|
||||
...config.baseEnv,
|
||||
GATEWAY_API_HOST: '0.0.0.0',
|
||||
GATEWAY_API_PORT: String(config.gatewayApiPort),
|
||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||
};
|
||||
return [
|
||||
{ name: 'sammo:gateway-api', script: apiScript, cwd: apiCwd, env: { ...env, GATEWAY_ROLE: 'api' } },
|
||||
{
|
||||
name: 'sammo:gateway-frontend',
|
||||
script: frontendScript,
|
||||
cwd: frontendCwd,
|
||||
args: ['preview', '--host', '0.0.0.0', '--port', String(config.gatewayFrontendPort)],
|
||||
env,
|
||||
},
|
||||
{
|
||||
name: 'sammo:gateway-orchestrator',
|
||||
script: apiScript,
|
||||
cwd: apiCwd,
|
||||
env: { ...env, GATEWAY_ROLE: 'orchestrator' },
|
||||
},
|
||||
];
|
||||
};
|
||||
|
||||
const isMissingProcessError = (error: unknown): boolean =>
|
||||
error instanceof Error && /process or namespace not found/i.test(error.message);
|
||||
|
||||
export class GatewayReleaseController {
|
||||
private readonly ownerId = randomUUID();
|
||||
|
||||
constructor(
|
||||
private readonly repository: GatewayReleaseRepository,
|
||||
private readonly workspaceManager: GitWorkspaceManager,
|
||||
private readonly buildRunner: BuildRunner,
|
||||
private readonly processManager: ProcessManager,
|
||||
private readonly config: ReleaseControllerConfig,
|
||||
private readonly now: () => Date = () => new Date(),
|
||||
private readonly fetchImpl: typeof fetch = fetch
|
||||
) {}
|
||||
|
||||
async runOnce(): Promise<GatewayReleaseOperationRecord | null> {
|
||||
const operation = await this.repository.claimNextOperation(this.now(), {
|
||||
ownerId: this.ownerId,
|
||||
durationMs: LEASE_DURATION_MS,
|
||||
});
|
||||
if (!operation) return null;
|
||||
const heartbeat = setInterval(() => {
|
||||
void this.repository.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS);
|
||||
}, HEARTBEAT_INTERVAL_MS);
|
||||
let resolvedCommitSha: string | undefined;
|
||||
try {
|
||||
const state = await this.repository.getState();
|
||||
const deploymentState: GatewayReleaseStateRecord = {
|
||||
...state,
|
||||
activeCommitSha: state.activeCommitSha ?? (await this.workspaceManager.resolveCommit('COMMIT', 'HEAD')),
|
||||
activeWorkspace: state.activeWorkspace ?? this.config.workspaceRoot,
|
||||
};
|
||||
const sourceMode = operation.sourceMode ?? 'COMMIT';
|
||||
const sourceRef = operation.sourceRef ?? state.previousCommitSha;
|
||||
if (!sourceRef) throw new Error('Release source is missing.');
|
||||
resolvedCommitSha = await this.workspaceManager.resolveCommit(sourceMode, sourceRef);
|
||||
if (!(await this.repository.pinOperationResolvedCommit(operation.id, this.ownerId, resolvedCommitSha))) {
|
||||
throw new Error('Gateway release lease was lost while pinning the commit.');
|
||||
}
|
||||
await this.deploy(operation, deploymentState, resolvedCommitSha);
|
||||
return await this.repository.completeOperation(
|
||||
operation.id,
|
||||
'SUCCEEDED',
|
||||
{ resolvedCommitSha, error: null },
|
||||
this.ownerId
|
||||
);
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
await this.repository.recordStateError(detail);
|
||||
return await this.repository.completeOperation(
|
||||
operation.id,
|
||||
'FAILED',
|
||||
{ resolvedCommitSha, error: detail },
|
||||
this.ownerId
|
||||
);
|
||||
} finally {
|
||||
clearInterval(heartbeat);
|
||||
}
|
||||
}
|
||||
|
||||
private async deploy(
|
||||
operation: GatewayReleaseOperationRecord,
|
||||
state: GatewayReleaseStateRecord,
|
||||
commitSha: string
|
||||
): Promise<void> {
|
||||
const workspace = await this.workspaceManager.prepare(commitSha);
|
||||
const manifest = await readReleaseManifest(workspace.root);
|
||||
assertReleaseComponents(manifest, ['gateway-api', 'gateway-frontend']);
|
||||
const build = await this.buildRunner.run(
|
||||
buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config)
|
||||
);
|
||||
if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`);
|
||||
const migration = await this.buildRunner.run([buildGatewayMigrationCommand(workspace.root, this.config)]);
|
||||
if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`);
|
||||
|
||||
const previousDefinitions = state.activeWorkspace
|
||||
? buildGatewayProcessDefinitions(state.activeWorkspace, this.config)
|
||||
: [];
|
||||
await this.stopManagedProcesses();
|
||||
try {
|
||||
await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config));
|
||||
await this.waitForReadiness();
|
||||
} catch (error) {
|
||||
await this.stopManagedProcesses();
|
||||
if (previousDefinitions.length) {
|
||||
await this.startDefinitions(previousDefinitions);
|
||||
await this.waitForReadiness();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await this.repository.publishRelease(operation.id, this.ownerId, {
|
||||
commitSha,
|
||||
workspace: workspace.root,
|
||||
previousCommitSha: state.activeCommitSha,
|
||||
previousWorkspace: state.activeWorkspace,
|
||||
});
|
||||
}
|
||||
|
||||
private async startDefinitions(definitions: ProcessDefinition[]): Promise<void> {
|
||||
const started: string[] = [];
|
||||
try {
|
||||
for (const definition of definitions) {
|
||||
await this.processManager.start(definition);
|
||||
started.push(definition.name);
|
||||
}
|
||||
} catch (error) {
|
||||
for (const name of started.reverse()) {
|
||||
try {
|
||||
await this.processManager.delete(name);
|
||||
} catch {
|
||||
// Preserve the start failure.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async stopManagedProcesses(): Promise<void> {
|
||||
const existing = new Set((await this.processManager.list()).map((process) => process.name));
|
||||
const failures: string[] = [];
|
||||
for (const name of [...PROCESS_NAMES].reverse()) {
|
||||
if (!existing.has(name)) continue;
|
||||
try {
|
||||
await this.processManager.stop(name);
|
||||
} catch {
|
||||
// Delete below is authoritative.
|
||||
}
|
||||
try {
|
||||
await this.processManager.delete(name);
|
||||
} catch (error) {
|
||||
if (!isMissingProcessError(error)) failures.push(`${name}: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
if (failures.length) throw new Error(`Failed to stop gateway processes: ${failures.join('; ')}`);
|
||||
}
|
||||
|
||||
private async waitForReadiness(): Promise<void> {
|
||||
const deadline = Date.now() + this.config.readinessTimeoutMs;
|
||||
const apiUrl = `http://127.0.0.1:${this.config.gatewayApiPort}/healthz`;
|
||||
const frontendUrl = `http://127.0.0.1:${this.config.gatewayFrontendPort}${this.config.gatewayBasePath}/`;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const [api, frontend] = await Promise.all([this.fetchImpl(apiUrl), this.fetchImpl(frontendUrl)]);
|
||||
const processes = await this.processManager.list();
|
||||
const online = new Set(
|
||||
processes
|
||||
.filter((process) => process.status.toLowerCase() === 'online')
|
||||
.map((process) => process.name)
|
||||
);
|
||||
if (api.ok && frontend.ok && PROCESS_NAMES.every((name) => online.has(name))) return;
|
||||
} catch {
|
||||
// Retry until the bounded deadline.
|
||||
}
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
throw new Error('Gateway release did not become ready before the timeout.');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
assertReleaseComponents,
|
||||
type BuildCommand,
|
||||
type BuildRunner,
|
||||
type GitWorkspaceManager,
|
||||
type ProcessDefinition,
|
||||
type ProcessManager,
|
||||
readReleaseManifest,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
|
||||
import type { ReleaseControllerConfig } from './config.js';
|
||||
import { buildGatewayMigrationCommand } from './releaseController.js';
|
||||
|
||||
const CONTROLLER_PROCESS_NAME = 'sammo:release-controller';
|
||||
|
||||
export const buildReleaseControllerCommands = (
|
||||
workspaceRoot: string,
|
||||
needsInstall: boolean,
|
||||
config: ReleaseControllerConfig
|
||||
): BuildCommand[] => {
|
||||
const env = config.baseEnv;
|
||||
return [
|
||||
...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []),
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'prisma:generate'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/logic', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/game-engine', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/gateway-api', 'build'], cwd: workspaceRoot, env },
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/release-controller', 'build'], cwd: workspaceRoot, env },
|
||||
];
|
||||
};
|
||||
|
||||
export const buildReleaseControllerDefinition = (
|
||||
workspaceRoot: string,
|
||||
config: ReleaseControllerConfig
|
||||
): ProcessDefinition => ({
|
||||
name: CONTROLLER_PROCESS_NAME,
|
||||
script: path.join(workspaceRoot, 'app', 'release-controller', 'dist', 'index.js'),
|
||||
cwd: path.join(workspaceRoot, 'app', 'release-controller'),
|
||||
args: ['daemon'],
|
||||
env: {
|
||||
...config.baseEnv,
|
||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||
GATEWAY_DB_SCHEMA: config.gatewayDbSchema,
|
||||
RELEASE_CONTROLLER_WORKSPACE_ROOT: workspaceRoot,
|
||||
RELEASE_CONTROLLER_WORKTREE_ROOT: config.worktreeRoot,
|
||||
},
|
||||
});
|
||||
|
||||
const workspaceFromControllerCwd = (cwd: string | undefined, fallback: string): string =>
|
||||
cwd ? path.resolve(cwd, '..', '..') : fallback;
|
||||
|
||||
export const upgradeReleaseController = async (options: {
|
||||
sourceMode: 'BRANCH' | 'COMMIT';
|
||||
sourceRef: string;
|
||||
workspaceManager: GitWorkspaceManager;
|
||||
buildRunner: BuildRunner;
|
||||
processManager: ProcessManager;
|
||||
config: ReleaseControllerConfig;
|
||||
readinessTimeoutMs?: number;
|
||||
}): Promise<{ commitSha: string; workspace: string }> => {
|
||||
const commitSha = await options.workspaceManager.resolveCommit(options.sourceMode, options.sourceRef);
|
||||
const workspace = await options.workspaceManager.prepare(commitSha);
|
||||
const manifest = await readReleaseManifest(workspace.root);
|
||||
assertReleaseComponents(manifest, ['release-controller']);
|
||||
const build = await options.buildRunner.run(
|
||||
buildReleaseControllerCommands(workspace.root, workspace.needsInstall, options.config)
|
||||
);
|
||||
if (!build.ok) throw new Error(`Release controller build failed: ${build.output.slice(-4000)}`);
|
||||
const migration = await options.buildRunner.run([buildGatewayMigrationCommand(workspace.root, options.config)]);
|
||||
if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`);
|
||||
|
||||
const existing = (await options.processManager.list()).find((process) => process.name === CONTROLLER_PROCESS_NAME);
|
||||
const previousDefinition = buildReleaseControllerDefinition(
|
||||
workspaceFromControllerCwd(existing?.cwd, options.config.workspaceRoot),
|
||||
options.config
|
||||
);
|
||||
if (existing) {
|
||||
try {
|
||||
await options.processManager.stop(CONTROLLER_PROCESS_NAME);
|
||||
} finally {
|
||||
await options.processManager.delete(CONTROLLER_PROCESS_NAME);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await options.processManager.start(buildReleaseControllerDefinition(workspace.root, options.config));
|
||||
const deadline = Date.now() + (options.readinessTimeoutMs ?? options.config.readinessTimeoutMs);
|
||||
while (Date.now() < deadline) {
|
||||
const active = (await options.processManager.list()).find(
|
||||
(process) => process.name === CONTROLLER_PROCESS_NAME && process.status.toLowerCase() === 'online'
|
||||
);
|
||||
if (active) return { commitSha, workspace: workspace.root };
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
throw new Error('Release controller did not become online before the timeout.');
|
||||
} catch (error) {
|
||||
try {
|
||||
await options.processManager.delete(CONTROLLER_PROCESS_NAME);
|
||||
} catch {
|
||||
// The failed new process may already be absent.
|
||||
}
|
||||
if (existing) await options.processManager.start(previousDefinition);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,304 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import type {
|
||||
BuildRunner,
|
||||
GatewayReleaseOperationRecord,
|
||||
GatewayReleaseRepository,
|
||||
GatewayReleaseStateRecord,
|
||||
GitWorkspaceManager,
|
||||
ProcessDefinition,
|
||||
ProcessManager,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { resolveReleaseControllerConfig, type ReleaseControllerConfig } from '../src/config.js';
|
||||
import { GatewayReleaseController } from '../src/releaseController.js';
|
||||
import { upgradeReleaseController } from '../src/selfUpgrade.js';
|
||||
|
||||
const SHA = '1111111111111111111111111111111111111111';
|
||||
const OLD_SHA = '2222222222222222222222222222222222222222';
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
const createReleaseWorkspace = async (): Promise<string> => {
|
||||
const workspace = await fs.mkdtemp(path.join(os.tmpdir(), 'sammo-release-controller-'));
|
||||
temporaryDirectories.push(workspace);
|
||||
const gatewayHead = '20260801000000_gateway';
|
||||
const gameHead = '20260801000000_game';
|
||||
await fs.mkdir(path.join(workspace, 'packages/infra/prisma/gateway-migrations', gatewayHead), {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.mkdir(path.join(workspace, 'packages/infra/prisma/migrations', gameHead), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(workspace, 'release-manifest.json'),
|
||||
JSON.stringify({
|
||||
formatVersion: 1,
|
||||
controllerProtocol: 1,
|
||||
gatewaySchemaHead: gatewayHead,
|
||||
gameSchemaHead: gameHead,
|
||||
components: ['gateway-api', 'gateway-frontend', 'release-controller'],
|
||||
})
|
||||
);
|
||||
return workspace;
|
||||
};
|
||||
|
||||
const operation: GatewayReleaseOperationRecord = {
|
||||
id: '11111111-1111-4111-8111-111111111111',
|
||||
type: 'DEPLOY',
|
||||
status: 'RUNNING',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: SHA,
|
||||
payload: {},
|
||||
requestedBy: 'admin',
|
||||
attempts: 1,
|
||||
createdAt: '2026-08-01T00:00:00.000Z',
|
||||
updatedAt: '2026-08-01T00:00:00.000Z',
|
||||
};
|
||||
|
||||
const state: GatewayReleaseStateRecord = {
|
||||
id: 'gateway',
|
||||
activeCommitSha: OLD_SHA,
|
||||
activeWorkspace: '/srv/sammo/old',
|
||||
updatedAt: '2026-08-01T00:00:00.000Z',
|
||||
};
|
||||
|
||||
const config: ReleaseControllerConfig = {
|
||||
workspaceRoot: '/srv/sammo/controller',
|
||||
worktreeRoot: '/srv/sammo/releases',
|
||||
gatewayDatabaseUrl: 'postgresql://integration.invalid/sammo?schema=gateway',
|
||||
gatewayDbSchema: 'gateway',
|
||||
gatewayApiPort: 15001,
|
||||
gatewayFrontendPort: 15000,
|
||||
gatewayBasePath: '/gateway',
|
||||
pollIntervalMs: 5,
|
||||
readinessTimeoutMs: 10,
|
||||
baseEnv: {},
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true })));
|
||||
});
|
||||
|
||||
const createRepository = () => {
|
||||
let next: GatewayReleaseOperationRecord | null = operation;
|
||||
const completions: string[] = [];
|
||||
const published: Array<{ commitSha: string; workspace: string; previousCommitSha?: string }> = [];
|
||||
const errors: string[] = [];
|
||||
const repository: GatewayReleaseRepository = {
|
||||
getState: async () => state,
|
||||
listOperations: async () => [],
|
||||
getOperation: async () => operation,
|
||||
createOperation: async () => operation,
|
||||
claimNextOperation: async () => {
|
||||
const claimed = next;
|
||||
next = null;
|
||||
return claimed;
|
||||
},
|
||||
renewOperationLease: async () => true,
|
||||
pinOperationResolvedCommit: async () => true,
|
||||
completeOperation: async (_id, statusValue) => {
|
||||
completions.push(statusValue);
|
||||
return { ...operation, status: statusValue };
|
||||
},
|
||||
publishRelease: async (_id, _owner, release) => {
|
||||
published.push(release);
|
||||
return { ...state, activeCommitSha: release.commitSha, activeWorkspace: release.workspace };
|
||||
},
|
||||
recordStateError: async (detail) => {
|
||||
errors.push(detail);
|
||||
},
|
||||
cancelOperation: async () => false,
|
||||
retryOperation: async () => null,
|
||||
};
|
||||
return { repository, completions, published, errors };
|
||||
};
|
||||
|
||||
const gatewayNames = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'];
|
||||
|
||||
describe('GatewayReleaseController', () => {
|
||||
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
const harness = createRepository();
|
||||
const commandGroups: string[][] = [];
|
||||
const running = new Map(gatewayNames.map((name) => [name, '/srv/sammo/old']));
|
||||
const processManager: ProcessManager = {
|
||||
list: async () =>
|
||||
[...running].map(([name, cwd]) => ({ name, cwd, status: 'online', script: path.join(cwd, 'dist.js') })),
|
||||
start: async (definition) => {
|
||||
running.set(definition.name, definition.cwd);
|
||||
},
|
||||
stop: async () => {},
|
||||
delete: async (name) => {
|
||||
running.delete(name);
|
||||
},
|
||||
};
|
||||
const buildRunner: BuildRunner = {
|
||||
run: async (commands) => {
|
||||
commandGroups.push(commands.map((command) => command.args.join(' ')));
|
||||
return { ok: true, exitCode: 0, output: '' };
|
||||
},
|
||||
};
|
||||
const workspaceManager = {
|
||||
resolveCommit: async () => SHA,
|
||||
prepare: async () => ({ root: workspace, created: true, needsInstall: true }),
|
||||
} as unknown as GitWorkspaceManager;
|
||||
const controller = new GatewayReleaseController(
|
||||
harness.repository,
|
||||
workspaceManager,
|
||||
buildRunner,
|
||||
processManager,
|
||||
config,
|
||||
() => new Date('2026-08-01T00:00:00.000Z'),
|
||||
async () => new Response('', { status: 200 })
|
||||
);
|
||||
|
||||
await controller.runOnce();
|
||||
|
||||
expect(commandGroups).toHaveLength(2);
|
||||
expect(commandGroups[0]?.[0]).toBe('install --frozen-lockfile');
|
||||
expect(commandGroups[1]).toEqual(['--filter @sammo-ts/infra prisma:migrate:deploy:gateway']);
|
||||
expect([...running.keys()].sort()).toEqual([...gatewayNames].sort());
|
||||
expect(harness.published).toEqual([
|
||||
{ commitSha: SHA, workspace, previousCommitSha: OLD_SHA, previousWorkspace: '/srv/sammo/old' },
|
||||
]);
|
||||
expect(harness.completions).toEqual(['SUCCEEDED']);
|
||||
});
|
||||
|
||||
it('restores the previous gateway processes when the new process set cannot start', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
const harness = createRepository();
|
||||
const started: ProcessDefinition[] = [];
|
||||
const running = new Map(gatewayNames.map((name) => [name, '/srv/sammo/old']));
|
||||
const processManager: ProcessManager = {
|
||||
list: async () => [...running].map(([name, cwd]) => ({ name, cwd, status: 'online' })),
|
||||
start: async (definition) => {
|
||||
if (definition.cwd.startsWith(workspace) && definition.name === 'sammo:gateway-api') {
|
||||
throw new Error('new gateway failed');
|
||||
}
|
||||
started.push(definition);
|
||||
running.set(definition.name, definition.cwd);
|
||||
},
|
||||
stop: async () => {},
|
||||
delete: async (name) => {
|
||||
running.delete(name);
|
||||
},
|
||||
};
|
||||
const workspaceManager = {
|
||||
resolveCommit: async () => SHA,
|
||||
prepare: async () => ({ root: workspace, created: true, needsInstall: false }),
|
||||
} as unknown as GitWorkspaceManager;
|
||||
const controller = new GatewayReleaseController(
|
||||
harness.repository,
|
||||
workspaceManager,
|
||||
{ run: async () => ({ ok: true, exitCode: 0, output: '' }) },
|
||||
processManager,
|
||||
config,
|
||||
() => new Date('2026-08-01T00:00:00.000Z'),
|
||||
async () => new Response('', { status: 200 })
|
||||
);
|
||||
|
||||
await controller.runOnce();
|
||||
|
||||
expect(started.filter((definition) => definition.cwd.startsWith('/srv/sammo/old'))).toHaveLength(3);
|
||||
expect(harness.published).toEqual([]);
|
||||
expect(harness.completions).toEqual(['FAILED']);
|
||||
expect(harness.errors.at(-1)).toContain('new gateway failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveReleaseControllerConfig', () => {
|
||||
it('applies the configured gateway schema to the controller database URL', () => {
|
||||
const resolved = resolveReleaseControllerConfig({
|
||||
GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo?schema=wrong',
|
||||
GATEWAY_DB_SCHEMA: 'gateway_release',
|
||||
RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller',
|
||||
});
|
||||
|
||||
expect(new URL(resolved.gatewayDatabaseUrl).searchParams.get('schema')).toBe('gateway_release');
|
||||
});
|
||||
});
|
||||
|
||||
describe('upgradeReleaseController', () => {
|
||||
it('switches the controller daemon from a separately invoked CLI process', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]);
|
||||
const starts: ProcessDefinition[] = [];
|
||||
const processManager: ProcessManager = {
|
||||
list: async () => [...running].map(([name, cwd]) => ({ name, cwd, status: 'online' })),
|
||||
start: async (definition) => {
|
||||
starts.push(definition);
|
||||
running.set(definition.name, definition.cwd);
|
||||
},
|
||||
stop: async () => {},
|
||||
delete: async (name) => {
|
||||
running.delete(name);
|
||||
},
|
||||
};
|
||||
const workspaceManager = {
|
||||
resolveCommit: async () => SHA,
|
||||
prepare: async () => ({ root: workspace, created: true, needsInstall: true }),
|
||||
} as unknown as GitWorkspaceManager;
|
||||
const commandGroups: string[][] = [];
|
||||
|
||||
await expect(
|
||||
upgradeReleaseController({
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: SHA,
|
||||
workspaceManager,
|
||||
buildRunner: {
|
||||
run: async (commands) => {
|
||||
commandGroups.push(commands.map((command) => command.args.join(' ')));
|
||||
return { ok: true, exitCode: 0, output: '' };
|
||||
},
|
||||
},
|
||||
processManager,
|
||||
config,
|
||||
readinessTimeoutMs: 10,
|
||||
})
|
||||
).resolves.toEqual({ commitSha: SHA, workspace });
|
||||
|
||||
expect(commandGroups).toHaveLength(2);
|
||||
expect(commandGroups[0]?.at(-1)).toBe('--filter @sammo-ts/release-controller build');
|
||||
expect(starts.at(-1)).toMatchObject({
|
||||
name: 'sammo:release-controller',
|
||||
cwd: path.join(workspace, 'app', 'release-controller'),
|
||||
args: ['daemon'],
|
||||
});
|
||||
});
|
||||
|
||||
it('restores the old controller definition when the new daemon cannot start', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
const starts: ProcessDefinition[] = [];
|
||||
const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]);
|
||||
const processManager: ProcessManager = {
|
||||
list: async () => [...running].map(([name, cwd]) => ({ name, cwd, status: 'online' })),
|
||||
start: async (definition) => {
|
||||
if (definition.cwd.startsWith(workspace)) throw new Error('new controller failed');
|
||||
starts.push(definition);
|
||||
running.set(definition.name, definition.cwd);
|
||||
},
|
||||
stop: async () => {},
|
||||
delete: async (name) => {
|
||||
running.delete(name);
|
||||
},
|
||||
};
|
||||
const workspaceManager = {
|
||||
resolveCommit: async () => SHA,
|
||||
prepare: async () => ({ root: workspace, created: true, needsInstall: false }),
|
||||
} as unknown as GitWorkspaceManager;
|
||||
|
||||
await expect(
|
||||
upgradeReleaseController({
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: SHA,
|
||||
workspaceManager,
|
||||
buildRunner: { run: async () => ({ ok: true, exitCode: 0, output: '' }) },
|
||||
processManager,
|
||||
config,
|
||||
readinessTimeoutMs: 10,
|
||||
})
|
||||
).rejects.toThrow('new controller failed');
|
||||
expect(starts.at(-1)?.cwd).toBe('/srv/sammo/old/app/release-controller');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"extends": "../../tsconfig.paths.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"composite": true
|
||||
},
|
||||
"include": ["src", "test", "*.ts"],
|
||||
"references": [
|
||||
{ "path": "../../packages/common" },
|
||||
{ "path": "../../packages/infra" },
|
||||
{ "path": "../gateway-api" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import path from 'node:path';
|
||||
|
||||
import { defineConfig } from 'vitest/config';
|
||||
import tsconfigPaths from 'vite-tsconfig-paths';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [tsconfigPaths({ projects: [path.resolve(__dirname, '../../tsconfig.paths.json')] })],
|
||||
test: {
|
||||
environment: 'node',
|
||||
include: ['test/**/*.test.ts'],
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user