From 7b466de0a6c8528b9448581e5b9d3805c21d8931 Mon Sep 17 00:00:00 2001 From: hided62 Date: Sat, 1 Aug 2026 03:27:38 +0000 Subject: [PATCH] feat: add durable release controller and admin deploy modes --- README.md | 11 + app/gateway-api/src/adminRouter.ts | 159 +++++++++ app/gateway-api/src/context.ts | 7 + app/gateway-api/src/index.ts | 6 + .../src/orchestrator/gatewayOrchestrator.ts | 289 +++++++++++++++- .../orchestrator/gatewayReleaseRepository.ts | 323 ++++++++++++++++++ .../src/orchestrator/profileRepository.ts | 2 +- .../src/orchestrator/releaseManifest.ts | 65 ++++ app/gateway-api/src/server.ts | 3 + app/gateway-api/test/adminOperations.test.ts | 131 ++++++- ...tewayReleaseRepository.integration.test.ts | 107 ++++++ .../test/orchestratorOperations.test.ts | 9 +- app/gateway-api/test/orchestratorPlan.test.ts | 22 +- .../test/profileDeployOperation.test.ts | 178 ++++++++++ app/gateway-api/test/releaseManifest.test.ts | 51 +++ .../e2e/admin-runtime-actions.spec.ts | 7 + .../e2e/lobby-admin-navigation.spec.ts | 6 +- .../e2e/server-operations.spec.ts | 103 +++++- .../src/views/ServerOperationsView.vue | 255 +++++++++++++- app/release-controller/README.md | 72 ++++ app/release-controller/package.json | 28 ++ app/release-controller/src/config.ts | 56 +++ app/release-controller/src/index.ts | 83 +++++ .../src/releaseController.ts | 247 ++++++++++++++ app/release-controller/src/selfUpgrade.ts | 108 ++++++ .../test/releaseController.test.ts | 304 +++++++++++++++++ app/release-controller/tsconfig.json | 13 + app/release-controller/vitest.config.ts | 12 + docs/architecture/runtime.md | 21 ++ docs/e2e-orchestrator-tests.md | 4 + .../migration.sql | 47 +++ packages/infra/prisma/gateway.prisma | 46 +++ pnpm-lock.yaml | 19 ++ release-manifest.json | 7 + tools/conditional-integration-registry.tsv | 4 + tools/run-conditional-integration.sh | 7 +- tsconfig.json | 1 + 37 files changed, 2778 insertions(+), 35 deletions(-) create mode 100644 app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts create mode 100644 app/gateway-api/src/orchestrator/releaseManifest.ts create mode 100644 app/gateway-api/test/gatewayReleaseRepository.integration.test.ts create mode 100644 app/gateway-api/test/profileDeployOperation.test.ts create mode 100644 app/gateway-api/test/releaseManifest.test.ts create mode 100644 app/release-controller/README.md create mode 100644 app/release-controller/package.json create mode 100644 app/release-controller/src/config.ts create mode 100644 app/release-controller/src/index.ts create mode 100644 app/release-controller/src/releaseController.ts create mode 100644 app/release-controller/src/selfUpgrade.ts create mode 100644 app/release-controller/test/releaseController.test.ts create mode 100644 app/release-controller/tsconfig.json create mode 100644 app/release-controller/vitest.config.ts create mode 100644 packages/infra/prisma/gateway-migrations/20260801000000_add_release_operations/migration.sql create mode 100644 release-manifest.json diff --git a/README.md b/README.md index def4c8b..f9de523 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ | ------------------------------ | -------------------------------------------------------------- | | `app/gateway-frontend` | 가입, 로그인, 로비, 계정, 관리자 UI | | `app/gateway-api` | 계정·세션, profile 정책, operation queue, PM2 orchestration | +| `app/release-controller` | Gateway 전체 릴리스와 controller CLI self-upgrade | | `app/game-frontend` | profile별 게임 SPA와 ref 호환 화면 | | `app/game-api` | tRPC, SSE, 인증, 조회·입력 API, 비동기 worker | | `app/game-engine` | turn daemon, AI, 월간 lifecycle, in-memory world와 DB flush | @@ -34,6 +35,8 @@ Gateway는 계정과 profile 운영을 소유합니다. `gateway-api`가 gateway PostgreSQL과 Redis session을 사용하며, game session token을 발급합니다. 관리 operation은 `GatewayProfile`과 `GatewayOperation`에 저장되고 orchestrator가 commit별 worktree와 PM2 process를 조정합니다. +Gateway 자체 릴리스는 Gateway 프로세스 밖의 `release-controller`가 별도 +`GatewayReleaseOperation` queue를 처리합니다. 각 game profile은 별도 PostgreSQL schema를 사용합니다. `game-api`는 인증된 요청을 검증하고 직접 처리할 mutation 또는 daemon 입력을 @@ -163,3 +166,11 @@ direct-navigation URL을 사용합니다. `/image/*`는 외부 Caddy가 소유 `build:server`는 profile resource를 `dist/`에 복사하는 도구입니다. 완전한 API·daemon·frontend 배포 bundle은 gateway orchestrator의 commit-worktree build 경로에서 구성합니다. + +관리자 화면의 `DB 유지 배포`는 profile의 game migration만 적용하고 현재 +게임 DB를 seed하지 않습니다. `DB 초기화 배포`는 현재 시즌 테이블을 새 +시나리오로 교체하지만 `hall`, `ng_games`, 연감, 과거 장수·국가와 상속 자료는 +보존합니다. Gateway API·frontend·orchestrator는 외부 release-controller가 +함께 전환합니다. 설치와 CLI self-upgrade 절차는 +[`app/release-controller/README.md`](app/release-controller/README.md)를 확인해 +주세요. diff --git a/app/gateway-api/src/adminRouter.ts b/app/gateway-api/src/adminRouter.ts index 5baad8a..07761b6 100644 --- a/app/gateway-api/src/adminRouter.ts +++ b/app/gateway-api/src/adminRouter.ts @@ -39,6 +39,7 @@ const ROLE_SUPERUSER = 'superuser'; const ROLE_ADMIN_USERS = 'admin.users.manage'; const ROLE_ADMIN_USERS_CREATE = 'admin.users.create'; const ROLE_ADMIN_PROFILES = 'admin.profiles.manage'; +const ROLE_ADMIN_RELEASES = 'admin.releases.manage'; const ROLE_ADMIN_NOTICE = 'admin.notice.manage'; const ROLE_RESET_SCHEDULE = 'admin.reset.schedule'; const ROLE_RESUME_WHEN_STOPPED = 'admin.resume.when-stopped'; @@ -242,6 +243,12 @@ const profileAdminProcedure = adminProcedure.use(({ ctx, next }) => { return next(); }); +const releaseAdminProcedure = adminProcedure.use(({ ctx, next }) => { + const adminAuth = requireAdminAuth(ctx); + assertPermission(adminAuth, ROLE_ADMIN_RELEASES); + return next(); +}); + const zUserLookupInput = z .object({ id: z.string().min(1).optional(), @@ -766,6 +773,60 @@ export const adminRouter = router({ }); } }), + requestDeploy: adminProcedure + .input( + z.object({ + profileName: z.string().min(1), + sourceMode: zSourceMode, + sourceRef: z.string().min(1).max(128), + reason: z.string().max(200).optional(), + }) + ) + .mutation(async ({ ctx, input }) => { + const adminAuth = requireAdminAuth(ctx); + assertPermission(adminAuth, ROLE_ADMIN_PROFILES, input.profileName); + const profile = await ctx.profiles.getProfile(input.profileName); + if (!profile) { + throw new TRPCError({ code: 'NOT_FOUND', message: 'Profile not found.' }); + } + let sourceRef = input.sourceRef.trim(); + try { + const resolved = + input.sourceMode === 'BRANCH' + ? await resolveGitBranchCommitSha(sourceRef) + : await resolveGitCommitSha(sourceRef); + if (input.sourceMode === 'COMMIT') { + sourceRef = resolved; + } + const scenarios = await listScenarioPreviews({ gitRef: resolved }); + if (!scenarios.some((scenario) => String(scenario.id) === profile.scenario)) { + throw new Error('Current scenario is not available at source.'); + } + } catch { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: 'Source is invalid or does not contain the current scenario.', + }); + } + try { + return await ctx.profiles.createOperation({ + profileName: input.profileName, + type: 'DEPLOY', + sourceMode: input.sourceMode, + sourceRef, + reason: input.reason, + requestedBy: adminAuth.user.id, + }); + } catch (error) { + if (!isUniqueConstraintError(error)) { + throw error; + } + throw new TRPCError({ + code: 'CONFLICT', + message: 'This profile already has a queued or running operation.', + }); + } + }), requestRuntime: adminProcedure .input( z.object({ @@ -845,6 +906,103 @@ export const adminRouter = router({ } }), }), + releases: router({ + gatewayState: releaseAdminProcedure.query(({ ctx }) => ctx.releases.getState()), + list: releaseAdminProcedure + .input(z.object({ limit: z.number().int().min(1).max(200).optional() }).optional()) + .query(({ ctx, input }) => ctx.releases.listOperations(input?.limit)), + requestGatewayDeploy: releaseAdminProcedure + .input( + z.object({ + sourceMode: zSourceMode, + sourceRef: z.string().min(1).max(128), + reason: z.string().max(200).optional(), + }) + ) + .mutation(async ({ ctx, input }) => { + const adminAuth = requireAdminAuth(ctx); + let sourceRef = input.sourceRef.trim(); + try { + const resolved = + input.sourceMode === 'BRANCH' + ? await resolveGitBranchCommitSha(sourceRef) + : await resolveGitCommitSha(sourceRef); + if (input.sourceMode === 'COMMIT') { + sourceRef = resolved; + } + } catch { + throw new TRPCError({ code: 'BAD_REQUEST', message: 'Gateway release source is invalid.' }); + } + try { + return await ctx.releases.createOperation({ + type: 'DEPLOY', + sourceMode: input.sourceMode, + sourceRef, + reason: input.reason, + requestedBy: adminAuth.user.id, + }); + } catch (error) { + if (!isUniqueConstraintError(error)) { + throw error; + } + throw new TRPCError({ code: 'CONFLICT', message: 'A gateway release is already active.' }); + } + }), + requestGatewayRollback: releaseAdminProcedure + .input(z.object({ reason: z.string().max(200).optional() }).optional()) + .mutation(async ({ ctx, input }) => { + const adminAuth = requireAdminAuth(ctx); + const state = await ctx.releases.getState(); + if (!state.previousCommitSha || !state.previousWorkspace) { + throw new TRPCError({ code: 'BAD_REQUEST', message: 'No previous gateway release is available.' }); + } + try { + return await ctx.releases.createOperation({ + type: 'ROLLBACK', + sourceMode: 'COMMIT', + sourceRef: state.previousCommitSha, + payload: { + expectedWorkspace: state.previousWorkspace, + replacedCommitSha: state.activeCommitSha ?? null, + }, + reason: input?.reason, + requestedBy: adminAuth.user.id, + }); + } catch (error) { + if (!isUniqueConstraintError(error)) { + throw error; + } + throw new TRPCError({ code: 'CONFLICT', message: 'A gateway release is already active.' }); + } + }), + cancel: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => { + if (!(await ctx.releases.cancelOperation(input.id))) { + throw new TRPCError({ code: 'CONFLICT', message: 'Only queued releases can be cancelled.' }); + } + return { ok: true }; + }), + retry: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => { + const adminAuth = requireAdminAuth(ctx); + try { + const operation = await ctx.releases.retryOperation(input.id, adminAuth.user.id); + if (!operation) { + throw new TRPCError({ + code: 'CONFLICT', + message: 'Only failed or cancelled releases can be retried.', + }); + } + return operation; + } catch (error) { + if (error instanceof TRPCError) { + throw error; + } + if (!isUniqueConstraintError(error)) { + throw error; + } + throw new TRPCError({ code: 'CONFLICT', message: 'A gateway release is already active.' }); + } + }), + }), profiles: router({ list: adminProcedure.query(async ({ ctx }) => { const profiles = await ctx.profiles.listProfiles(); @@ -883,6 +1041,7 @@ export const adminRouter = router({ activeOperation: activeOperationByProfile.get(profile.profileName) ?? null, runtime: runtimeMap.get(profile.profileName) ?? { profileName: profile.profileName, + frontendRunning: false, apiRunning: false, daemonRunning: false, auctionRunning: false, diff --git a/app/gateway-api/src/context.ts b/app/gateway-api/src/context.ts index b10827f..21bde45 100644 --- a/app/gateway-api/src/context.ts +++ b/app/gateway-api/src/context.ts @@ -4,6 +4,10 @@ import type { UserRepository } from './auth/userRepository.js'; import type { KakaoOAuthClient } from './auth/kakaoClient.js'; import type { OAuthSessionStore } from './auth/oauthSessionStore.js'; import type { GatewayProfileRepository } from './orchestrator/profileRepository.js'; +import { + createGatewayReleaseRepository, + type GatewayReleaseRepository, +} from './orchestrator/gatewayReleaseRepository.js'; import type { GatewayOrchestratorHandle } from './orchestrator/gatewayOrchestrator.js'; import type { GatewayProfileStatusService } from './lobby/profileStatusService.js'; import type { GatewayPrismaClient } from '@sammo-ts/infra'; @@ -26,6 +30,7 @@ export interface GatewayApiContext { localAccountGraceDays: number; passwordEnvelope: PasswordEnvelopeService; profiles: GatewayProfileRepository; + releases: GatewayReleaseRepository; orchestrator: GatewayOrchestratorHandle; profileStatus: GatewayProfileStatusService; requestHeaders: Record; @@ -49,6 +54,7 @@ export const createGatewayApiContext = (options: { localAccountGraceDays: number; passwordEnvelope: PasswordEnvelopeService; profiles: GatewayProfileRepository; + releases?: GatewayReleaseRepository; orchestrator: GatewayOrchestratorHandle; profileStatus: GatewayProfileStatusService; requestHeaders?: Record; @@ -69,6 +75,7 @@ export const createGatewayApiContext = (options: { localAccountGraceDays: options.localAccountGraceDays, passwordEnvelope: options.passwordEnvelope, profiles: options.profiles, + releases: options.releases ?? createGatewayReleaseRepository(options.prisma), orchestrator: options.orchestrator, profileStatus: options.profileStatus, requestHeaders: options.requestHeaders ?? {}, diff --git a/app/gateway-api/src/index.ts b/app/gateway-api/src/index.ts index 4abc427..3464f56 100644 --- a/app/gateway-api/src/index.ts +++ b/app/gateway-api/src/index.ts @@ -14,7 +14,13 @@ export { GatewayPrisma }; export type JsonObject = GatewayPrisma.JsonObject; export type JsonArray = GatewayPrisma.JsonArray; export * from './orchestrator/profileRepository.js'; +export * from './orchestrator/gatewayReleaseRepository.js'; export * from './orchestrator/gatewayOrchestrator.js'; +export * from './orchestrator/workspaceManager.js'; +export * from './orchestrator/buildRunner.js'; +export * from './orchestrator/processManager.js'; +export * from './orchestrator/pm2ProcessManager.js'; +export * from './orchestrator/releaseManifest.js'; export * from './auth/userRepository.js'; export * from './auth/passwordHasher.js'; export * from './auth/inMemoryUserRepository.js'; diff --git a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts index c559349..1bbf29c 100644 --- a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts +++ b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts @@ -18,6 +18,7 @@ import type { } from './profileRepository.js'; import type { GitWorkspaceManager } from './workspaceManager.js'; import type { AdminSeedUser } from './seedProfileDatabase.js'; +import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js'; export interface GatewayProcessConfig { workspaceRoot: string; @@ -37,10 +38,13 @@ export interface GatewayOrchestratorOptions { scheduleIntervalMs: number; buildIntervalMs: number; adminActionIntervalMs: number; + profileReadinessTimeoutMs?: number; now?: () => Date; + fetchImpl?: typeof fetch; } export interface ProfileRuntimeState { + frontendRunning: boolean; apiRunning: boolean; daemonRunning: boolean; auctionRunning: boolean; @@ -73,6 +77,7 @@ export const planProfileReconcile = ( if (status === 'RUNNING' || status === 'PREOPEN' || status === 'PAUSED' || status === 'COMPLETED') { return { shouldStart: !( + runtime.frontendRunning && runtime.apiRunning && runtime.daemonRunning && runtime.auctionRunning && @@ -85,6 +90,7 @@ export const planProfileReconcile = ( return { shouldStart: false, shouldStop: + runtime.frontendRunning || runtime.apiRunning || runtime.daemonRunning || runtime.auctionRunning || @@ -301,18 +307,20 @@ const parseInstallOptions = ( const buildProcessName = ( profileName: string, - role: 'api' | 'daemon' | 'auction' | 'battle-sim' | 'tournament' + role: 'frontend' | 'api' | 'daemon' | 'auction' | 'battle-sim' | 'tournament' ): string => `sammo:${profileName}:${ - role === 'api' - ? 'game-api' - : role === 'daemon' - ? 'turn-daemon' - : role === 'auction' - ? 'auction-worker' - : role === 'battle-sim' - ? 'battle-sim-worker' - : 'tournament-worker' + role === 'frontend' + ? 'game-frontend' + : role === 'api' + ? 'game-api' + : role === 'daemon' + ? 'turn-daemon' + : role === 'auction' + ? 'auction-worker' + : role === 'battle-sim' + ? 'battle-sim-worker' + : 'tournament-worker' }`; const isMissingProcessError = (error: unknown): boolean => @@ -335,6 +343,7 @@ export const buildProcessDefinitions = ( profile: GatewayProfileRecord, config: GatewayProcessConfig ): { + frontend: { name: string; script: string; cwd: string; args: string[]; env: Record }; api: { name: string; script: string; cwd: string; env: Record }; daemon: { name: string; script: string; cwd: string; env: Record }; auction: { name: string; script: string; cwd: string; env: Record }; @@ -342,12 +351,16 @@ export const buildProcessDefinitions = ( tournament: { name: string; script: string; cwd: string; env: Record }; } => { const baseEnv = { ...(config.baseEnv ?? {}) }; + const frontendName = buildProcessName(profile.profileName, 'frontend'); const apiName = buildProcessName(profile.profileName, 'api'); const daemonName = buildProcessName(profile.profileName, 'daemon'); const auctionName = buildProcessName(profile.profileName, 'auction'); const battleSimName = buildProcessName(profile.profileName, 'battle-sim'); const tournamentName = buildProcessName(profile.profileName, 'tournament'); const runtimeWorkspace = profile.buildWorkspace ?? config.workspaceRoot; + const frontendCwd = path.join(runtimeWorkspace, 'app', 'game-frontend'); + const frontendOutDir = buildProfileFrontendOutDir(runtimeWorkspace, profile.profileName); + const frontendScript = path.join(runtimeWorkspace, 'node_modules', 'vite', 'bin', 'vite.js'); const apiCwd = path.join(runtimeWorkspace, 'app', 'game-api'); const daemonCwd = path.join(runtimeWorkspace, 'app', 'game-engine'); const apiScript = path.join(apiCwd, 'dist', 'index.js'); @@ -373,6 +386,13 @@ export const buildProcessDefinitions = ( TURN_PROFILE_NAME: profile.profileName, }; return { + frontend: { + name: frontendName, + script: frontendScript, + cwd: frontendCwd, + args: ['preview', '--host', '0.0.0.0', '--port', String(profile.apiPort - 1), '--outDir', frontendOutDir], + env: baseEnv, + }, api: { name: apiName, script: apiScript, @@ -415,6 +435,39 @@ export const buildProcessDefinitions = ( }; }; +const sanitizeArtifactName = (value: string): string => value.replace(/[^0-9A-Za-z._-]+/g, '_'); + +export const buildProfileFrontendOutDir = (workspaceRoot: string, profileName: string): string => + path.join(workspaceRoot, '.release-dist', sanitizeArtifactName(profileName), 'game-frontend'); + +export const buildProfileFrontendCommands = ( + workspaceRoot: string, + profile: Pick, + env?: Record +): BuildCommand[] => { + const buildEnv = { + ...(env ?? {}), + VITE_APP_BASE_PATH: `/${profile.profile}`, + VITE_GAME_API_URL: `/${profile.profile}/api/trpc`, + VITE_GAME_SSE_URL: `/${profile.profile}/api/events`, + }; + const outDir = buildProfileFrontendOutDir(workspaceRoot, profile.profileName); + return [ + { + command: 'pnpm', + args: ['--filter', '@sammo-ts/game-frontend', 'exec', 'vue-tsc', '--noEmit'], + cwd: workspaceRoot, + env: buildEnv, + }, + { + command: 'pnpm', + args: ['--filter', '@sammo-ts/game-frontend', 'exec', 'vite', 'build', '--outDir', outDir], + cwd: workspaceRoot, + env: buildEnv, + }, + ]; +}; + export const buildWorkspaceCommands = ( workspaceRoot: string, needsInstall: boolean, @@ -424,7 +477,7 @@ export const buildWorkspaceCommands = ( if (needsInstall) { commands.push({ command: 'pnpm', - args: ['install'], + args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env, }); @@ -462,6 +515,7 @@ export const buildProfileMigrationCommand = ( const mapRuntimeStates = (profileNames: string[], processNames: Map): ProfileRuntimeSnapshot[] => profileNames.map((profileName) => { + const frontendName = buildProcessName(profileName, 'frontend'); const apiName = buildProcessName(profileName, 'api'); const daemonName = buildProcessName(profileName, 'daemon'); const auctionName = buildProcessName(profileName, 'auction'); @@ -469,6 +523,7 @@ const mapRuntimeStates = (profileNames: string[], processNames: Map Date; + private readonly fetchImpl: typeof fetch; private reconcileTimer?: NodeJS.Timeout; private scheduleTimer?: NodeJS.Timeout; private buildTimer?: NodeJS.Timeout; @@ -513,7 +570,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { this.scheduleIntervalMs = options.scheduleIntervalMs; this.buildIntervalMs = options.buildIntervalMs; this.adminActionIntervalMs = options.adminActionIntervalMs; + this.profileReadinessTimeoutMs = options.profileReadinessTimeoutMs ?? 30_000; this.now = options.now ?? (() => new Date()); + this.fetchImpl = options.fetchImpl ?? fetch; } start(): void { @@ -668,7 +727,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { startedAt, error: null, }); - const { result, workspace } = await this.runBuildCommands(queued.buildCommitSha); + const { result, workspace } = await this.runBuildCommands(queued.buildCommitSha, queued); const completedAt = this.now().toISOString(); if (result.ok) { await this.repository.updateWorkspaceUsage( @@ -863,6 +922,19 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } } await assertLease(); + if (operation.type === 'DEPLOY') { + const result = await this.handleProfileDeploy(profile, commitSha, assertLease, operation.id); + if (!result.ok) { + throw new Error(result.detail); + } + await this.repository.completeOperation( + operation.id, + 'SUCCEEDED', + { resolvedCommitSha: commitSha, error: null }, + this.operationLeaseOwner + ); + return; + } const payload = normalizeMeta(operation.payload); const install = isRecord(payload.install) ? payload.install : {}; const installOperationId = @@ -923,6 +995,149 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } } + private async handleProfileDeploy( + profile: GatewayProfileRecord, + commitSha: string, + assertLease: () => Promise, + operationId: string + ): Promise<{ ok: true } | { ok: false; detail: string }> { + if (this.buildInFlight) { + return { ok: false, detail: 'build already in progress' }; + } + this.buildInFlight = true; + const shouldRun = ['RUNNING', 'PREOPEN', 'PAUSED', 'COMPLETED'].includes(profile.status); + const updateClaimedProfile = async (patch: GatewayClaimedProfileUpdate): Promise => { + if (!this.repository.updateProfileForOperation) { + throw new Error('Profile deploy requires lease-fenced profile updates.'); + } + const updated = await this.repository.updateProfileForOperation( + operationId, + this.operationLeaseOwner, + profile.profileName, + patch + ); + if (!updated) { + throw new OperationLeaseLostError(`Operation lease lost while deploying profile: ${operationId}`); + } + return updated; + }; + let oldRuntimeStopped = false; + try { + const startedAt = this.now().toISOString(); + await updateClaimedProfile({ + buildStatus: 'RUNNING', + buildRequestedAt: startedAt, + buildStartedAt: startedAt, + buildError: null, + }); + const workspace = await this.workspaceManager.prepare(commitSha); + const manifest = await readReleaseManifest(workspace.root); + assertReleaseComponents(manifest, ['game-api', 'game-engine', 'game-frontend']); + const commands = [ + ...buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv), + ...buildProfileFrontendCommands(workspace.root, profile, this.processConfig.baseEnv), + ]; + const result = await this.buildRunner.run(commands); + await assertLease(); + if (!result.ok) { + const detail = result.output.slice(-4000) || 'selected workspace build failed'; + await updateClaimedProfile({ + buildStatus: 'FAILED', + buildCompletedAt: this.now().toISOString(), + buildError: detail, + }); + return { ok: false, detail }; + } + + await this.stopProfile(profile, assertLease); + oldRuntimeStopped = true; + const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile); + const migration = await this.runProfileMigration(workspace.root, profileDatabaseUrl); + await assertLease(); + if (!migration.ok) { + const detail = migration.output.slice(-4000) || 'profile database migration failed'; + if (shouldRun) { + await this.startProfile(profile, assertLease); + oldRuntimeStopped = false; + } + await updateClaimedProfile({ + buildStatus: 'FAILED', + buildCompletedAt: this.now().toISOString(), + buildError: detail, + }); + return { ok: false, detail }; + } + + const completedAt = this.now().toISOString(); + const candidate: GatewayProfileRecord = { + ...profile, + buildStatus: 'SUCCEEDED', + buildCommitSha: commitSha, + buildWorkspace: workspace.root, + buildLastUsedAt: completedAt, + buildCompletedAt: completedAt, + buildError: undefined, + }; + if (shouldRun) { + const started = await this.startProfile(candidate, assertLease); + const ready = started && (await this.waitForProfileReadiness(candidate, assertLease)); + if (!ready) { + if (started) { + await this.stopProfile(candidate, assertLease); + } + const rollbackStarted = + (await this.startProfile(profile, assertLease)) && + (await this.waitForProfileReadiness(profile, assertLease)); + oldRuntimeStopped = !rollbackStarted; + const detail = rollbackStarted + ? 'new profile release failed readiness; previous runtime restored' + : 'new profile release failed and previous runtime could not be restored'; + await updateClaimedProfile({ + buildStatus: 'FAILED', + buildCompletedAt: completedAt, + buildError: detail, + lastError: detail, + status: rollbackStarted ? profile.status : 'STOPPED', + }); + return { ok: false, detail }; + } + } + await assertLease(); + await updateClaimedProfile({ + buildStatus: 'SUCCEEDED', + buildCommitSha: commitSha, + buildWorkspace: workspace.root, + buildLastUsedAt: completedAt, + buildCompletedAt: completedAt, + buildError: null, + lastError: null, + }); + oldRuntimeStopped = false; + return { ok: true }; + } catch (error) { + if (error instanceof OperationLeaseLostError) { + throw error; + } + const detail = error instanceof Error ? error.message : String(error); + if (oldRuntimeStopped && shouldRun) { + try { + await this.startProfile(profile, assertLease); + } catch { + // The original error remains authoritative; reconciliation records the stopped runtime. + } + } + await updateClaimedProfile({ + buildStatus: 'FAILED', + buildCompletedAt: this.now().toISOString(), + buildError: detail, + lastError: detail, + }); + return { ok: false, detail }; + } finally { + this.buildInFlight = false; + } + } + private async runAdminActionsNow(): Promise { if (this.stopping || this.adminActionInFlight) { return; @@ -1095,7 +1310,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { commitSha, }) ); - const { result, workspace } = await this.runBuildCommands(commitSha); + const { result, workspace } = await this.runBuildCommands(commitSha, profile); await assertLease?.(); if (!result.ok) { const completedAt = this.now().toISOString(); @@ -1289,12 +1504,18 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { return { databaseUrl, scenarioId, tickSeconds, meta }; } - private async runBuildCommands(commitSha: string): Promise<{ + private async runBuildCommands( + commitSha: string, + profile?: GatewayProfileRecord + ): Promise<{ result: Awaited>; workspace: Awaited>; }> { const workspace = await this.workspaceManager.prepare(commitSha); - const commands = buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv); + const commands = [ + ...buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv), + ...(profile ? buildProfileFrontendCommands(workspace.root, profile, this.processConfig.baseEnv) : []), + ]; return { result: await this.buildRunner.run(commands), workspace }; } @@ -1403,6 +1624,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { for (const [workspace, entry] of workspaceMap.entries()) { const profileProcessNames = new Set( entry.profileNames.flatMap((profileName) => [ + buildProcessName(profileName, 'frontend'), buildProcessName(profileName, 'api'), buildProcessName(profileName, 'daemon'), buildProcessName(profileName, 'auction'), @@ -1451,6 +1673,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { private async startProfile(profile: GatewayProfileRecord, assertLease?: () => Promise): Promise { const definitions = buildProcessDefinitions(profile, this.processConfig); const orderedDefinitions = [ + definitions.frontend, definitions.api, definitions.daemon, definitions.auction, @@ -1493,7 +1716,41 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } } + private async waitForProfileReadiness( + profile: GatewayProfileRecord, + assertLease?: () => Promise + ): Promise { + const deadline = Date.now() + this.profileReadinessTimeoutMs; + const definitions = buildProcessDefinitions(profile, this.processConfig); + const expectedNames = Object.values(definitions).map((definition) => definition.name); + const apiUrl = `http://127.0.0.1:${profile.apiPort}/healthz`; + const frontendUrl = `http://127.0.0.1:${profile.apiPort - 1}/${profile.profile}/`; + while (Date.now() < deadline) { + await assertLease?.(); + try { + const [api, frontend, processes] = await Promise.all([ + this.fetchImpl(apiUrl), + this.fetchImpl(frontendUrl), + this.processManager.list(), + ]); + const online = new Set( + processes + .filter((process) => process.status.toLowerCase() === 'online') + .map((process) => process.name) + ); + if (api.ok && frontend.ok && expectedNames.every((name) => online.has(name))) { + return true; + } + } catch { + // Retry until the bounded deadline. + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + return false; + } + private async stopProfile(profile: GatewayProfileRecord, assertLease?: () => Promise): Promise { + const frontendName = buildProcessName(profile.profileName, 'frontend'); const apiName = buildProcessName(profile.profileName, 'api'); const daemonName = buildProcessName(profile.profileName, 'daemon'); const auctionName = buildProcessName(profile.profileName, 'auction'); @@ -1503,7 +1760,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { const existingNames = new Set((await this.processManager.list()).map((process) => process.name)); await assertLease?.(); const failures: string[] = []; - for (const name of [apiName, daemonName, auctionName, battleSimName, tournamentName]) { + for (const name of [frontendName, apiName, daemonName, auctionName, battleSimName, tournamentName]) { if (!existingNames.has(name)) { continue; } diff --git a/app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts b/app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts new file mode 100644 index 0000000..b02b9fb --- /dev/null +++ b/app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts @@ -0,0 +1,323 @@ +import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra'; + +import type { GatewayOperationStatus, GatewaySourceMode } from './profileRepository.js'; + +export const GATEWAY_RELEASE_OPERATION_TYPES = ['DEPLOY', 'ROLLBACK'] as const; +export type GatewayReleaseOperationType = (typeof GATEWAY_RELEASE_OPERATION_TYPES)[number]; + +export interface GatewayReleaseStateRecord { + id: string; + activeCommitSha?: string; + activeWorkspace?: string; + previousCommitSha?: string; + previousWorkspace?: string; + lastSuccessfulAt?: string; + lastError?: string; + updatedAt: string; +} + +export interface GatewayReleaseOperationRecord { + id: string; + type: GatewayReleaseOperationType; + status: GatewayOperationStatus; + sourceMode?: GatewaySourceMode; + sourceRef?: string; + resolvedCommitSha?: string; + payload: GatewayPrisma.JsonObject; + reason?: string; + requestedBy: string; + startedAt?: string; + completedAt?: string; + error?: string; + leaseOwner?: string; + leaseUntil?: string; + heartbeatAt?: string; + attempts: number; + createdAt: string; + updatedAt: string; +} + +export interface GatewayReleaseOperationCreateInput { + type: GatewayReleaseOperationType; + sourceMode?: GatewaySourceMode; + sourceRef?: string; + payload?: GatewayPrisma.JsonObject; + reason?: string; + requestedBy: string; +} + +export interface GatewayReleaseRepository { + getState(): Promise; + listOperations(limit?: number): Promise; + getOperation(id: string): Promise; + createOperation(input: GatewayReleaseOperationCreateInput): Promise; + claimNextOperation( + now: Date, + lease: { ownerId: string; durationMs: number } + ): Promise; + renewOperationLease(id: string, ownerId: string, now: Date, durationMs: number): Promise; + pinOperationResolvedCommit(id: string, ownerId: string, resolvedCommitSha: string): Promise; + completeOperation( + id: string, + status: Extract, + fields: { resolvedCommitSha?: string | null; error?: string | null }, + leaseOwner: string + ): Promise; + publishRelease( + operationId: string, + leaseOwner: string, + release: { commitSha: string; workspace: string; previousCommitSha?: string; previousWorkspace?: string } + ): Promise; + recordStateError(detail: string): Promise; + cancelOperation(id: string): Promise; + retryOperation(id: string, requestedBy: string): Promise; +} + +const toIso = (value: Date | null): string | undefined => (value ? value.toISOString() : undefined); + +const mapState = (row: { + id: string; + activeCommitSha: string | null; + activeWorkspace: string | null; + previousCommitSha: string | null; + previousWorkspace: string | null; + lastSuccessfulAt: Date | null; + lastError: string | null; + updatedAt: Date; +}): GatewayReleaseStateRecord => ({ + id: row.id, + activeCommitSha: row.activeCommitSha ?? undefined, + activeWorkspace: row.activeWorkspace ?? undefined, + previousCommitSha: row.previousCommitSha ?? undefined, + previousWorkspace: row.previousWorkspace ?? undefined, + lastSuccessfulAt: toIso(row.lastSuccessfulAt), + lastError: row.lastError ?? undefined, + updatedAt: row.updatedAt.toISOString(), +}); + +const mapOperation = (row: { + id: string; + type: GatewayReleaseOperationType; + status: GatewayOperationStatus; + sourceMode: GatewaySourceMode | null; + sourceRef: string | null; + resolvedCommitSha: string | null; + payload: GatewayPrisma.JsonValue; + reason: string | null; + requestedBy: string; + startedAt: Date | null; + completedAt: Date | null; + error: string | null; + leaseOwner: string | null; + leaseUntil: Date | null; + heartbeatAt: Date | null; + attempts: number; + createdAt: Date; + updatedAt: Date; +}): GatewayReleaseOperationRecord => ({ + id: row.id, + type: row.type, + status: row.status, + sourceMode: row.sourceMode ?? undefined, + sourceRef: row.sourceRef ?? undefined, + resolvedCommitSha: row.resolvedCommitSha ?? undefined, + payload: (row.payload ?? {}) as GatewayPrisma.JsonObject, + reason: row.reason ?? undefined, + requestedBy: row.requestedBy, + startedAt: toIso(row.startedAt), + completedAt: toIso(row.completedAt), + error: row.error ?? undefined, + leaseOwner: row.leaseOwner ?? undefined, + leaseUntil: toIso(row.leaseUntil), + heartbeatAt: toIso(row.heartbeatAt), + attempts: row.attempts, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), +}); + +export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): GatewayReleaseRepository => ({ + async getState() { + const row = await prisma.gatewayReleaseState.upsert({ + where: { id: 'gateway' }, + create: { id: 'gateway' }, + update: {}, + }); + return mapState(row); + }, + async listOperations(limit = 50) { + const rows = await prisma.gatewayReleaseOperation.findMany({ + orderBy: { createdAt: 'desc' }, + take: Math.min(Math.max(limit, 1), 200), + }); + return rows.map(mapOperation); + }, + async getOperation(id) { + const row = await prisma.gatewayReleaseOperation.findUnique({ where: { id } }); + return row ? mapOperation(row) : null; + }, + async createOperation(input) { + const row = await prisma.gatewayReleaseOperation.create({ + data: { + type: input.type, + sourceMode: input.sourceMode, + sourceRef: input.sourceRef, + payload: input.payload ?? {}, + reason: input.reason, + requestedBy: input.requestedBy, + }, + }); + return mapOperation(row); + }, + async claimNextOperation(now, lease) { + const row = await prisma.$transaction(async (tx) => { + await tx.$queryRaw>` + SELECT pg_advisory_xact_lock(hashtextextended('gateway_release_operation_claim', 0))::text AS lock_result + `; + const staleBefore = new Date(now.getTime() - lease.durationMs); + const running = await tx.gatewayReleaseOperation.findFirst({ + where: { status: 'RUNNING' }, + orderBy: { createdAt: 'asc' }, + }); + const runningIsStale = Boolean( + running && + ((running.leaseUntil && running.leaseUntil < now) || + (!running.leaseUntil && running.startedAt && running.startedAt <= staleBefore)) + ); + if (running && !runningIsStale) { + return null; + } + const candidate = + running ?? + (await tx.gatewayReleaseOperation.findFirst({ + where: { status: 'QUEUED' }, + orderBy: { createdAt: 'asc' }, + })); + if (!candidate) { + return null; + } + const claimed = await tx.gatewayReleaseOperation.updateMany({ + where: + candidate.status === 'QUEUED' + ? { id: candidate.id, status: 'QUEUED' } + : { id: candidate.id, status: 'RUNNING', leaseUntil: candidate.leaseUntil }, + data: { + status: 'RUNNING', + startedAt: candidate.startedAt ?? now, + completedAt: null, + error: null, + leaseOwner: lease.ownerId, + leaseUntil: new Date(now.getTime() + lease.durationMs), + heartbeatAt: now, + attempts: { increment: 1 }, + }, + }); + return claimed.count === 1 ? tx.gatewayReleaseOperation.findUnique({ where: { id: candidate.id } }) : null; + }); + return row ? mapOperation(row) : null; + }, + async renewOperationLease(id, ownerId, now, durationMs) { + const updated = await prisma.gatewayReleaseOperation.updateMany({ + where: { id, status: 'RUNNING', leaseOwner: ownerId }, + data: { + leaseUntil: new Date(now.getTime() + durationMs), + heartbeatAt: now, + }, + }); + return updated.count === 1; + }, + async pinOperationResolvedCommit(id, ownerId, resolvedCommitSha) { + const updated = await prisma.gatewayReleaseOperation.updateMany({ + where: { + id, + status: 'RUNNING', + leaseOwner: ownerId, + OR: [{ resolvedCommitSha: null }, { resolvedCommitSha }], + }, + data: { resolvedCommitSha }, + }); + return updated.count === 1; + }, + async completeOperation(id, status, fields, leaseOwner) { + const updated = await prisma.gatewayReleaseOperation.updateMany({ + where: { id, status: 'RUNNING', leaseOwner }, + data: { + status, + completedAt: new Date(), + resolvedCommitSha: fields.resolvedCommitSha, + error: fields.error, + leaseOwner: null, + leaseUntil: null, + heartbeatAt: null, + }, + }); + if (updated.count !== 1) { + throw new Error(`Gateway release lease lost before completion: ${id}`); + } + return mapOperation(await prisma.gatewayReleaseOperation.findUniqueOrThrow({ where: { id } })); + }, + async publishRelease(operationId, leaseOwner, release) { + const row = await prisma.$transaction(async (tx) => { + const owned = await tx.gatewayReleaseOperation.findFirst({ + where: { id: operationId, status: 'RUNNING', leaseOwner }, + select: { id: true }, + }); + if (!owned) { + throw new Error(`Gateway release lease lost before publish: ${operationId}`); + } + return tx.gatewayReleaseState.upsert({ + where: { id: 'gateway' }, + create: { + id: 'gateway', + activeCommitSha: release.commitSha, + activeWorkspace: release.workspace, + previousCommitSha: release.previousCommitSha, + previousWorkspace: release.previousWorkspace, + lastSuccessfulAt: new Date(), + lastError: null, + }, + update: { + activeCommitSha: release.commitSha, + activeWorkspace: release.workspace, + previousCommitSha: release.previousCommitSha, + previousWorkspace: release.previousWorkspace, + lastSuccessfulAt: new Date(), + lastError: null, + }, + }); + }); + return mapState(row); + }, + async recordStateError(detail) { + await prisma.gatewayReleaseState.upsert({ + where: { id: 'gateway' }, + create: { id: 'gateway', lastError: detail }, + update: { lastError: detail }, + }); + }, + async cancelOperation(id) { + const updated = await prisma.gatewayReleaseOperation.updateMany({ + where: { id, status: 'QUEUED' }, + data: { status: 'CANCELLED', completedAt: new Date() }, + }); + return updated.count === 1; + }, + async retryOperation(id, requestedBy) { + const row = await prisma.$transaction(async (tx) => { + const previous = await tx.gatewayReleaseOperation.findUnique({ where: { id } }); + if (!previous || (previous.status !== 'FAILED' && previous.status !== 'CANCELLED')) { + return null; + } + return tx.gatewayReleaseOperation.create({ + data: { + type: previous.type, + sourceMode: previous.resolvedCommitSha ? 'COMMIT' : previous.sourceMode, + sourceRef: previous.resolvedCommitSha ?? previous.sourceRef, + payload: previous.payload as GatewayPrisma.JsonObject, + reason: previous.reason, + requestedBy, + }, + }); + }); + return row ? mapOperation(row) : null; + }, +}); diff --git a/app/gateway-api/src/orchestrator/profileRepository.ts b/app/gateway-api/src/orchestrator/profileRepository.ts index c648999..45d9819 100644 --- a/app/gateway-api/src/orchestrator/profileRepository.ts +++ b/app/gateway-api/src/orchestrator/profileRepository.ts @@ -14,7 +14,7 @@ export type GatewayProfileStatus = (typeof GATEWAY_PROFILE_STATUSES)[number]; export const GATEWAY_BUILD_STATUSES = ['IDLE', 'QUEUED', 'RUNNING', 'FAILED', 'SUCCEEDED'] as const; export type GatewayBuildStatus = (typeof GATEWAY_BUILD_STATUSES)[number]; -export const GATEWAY_OPERATION_TYPES = ['RESET', 'START', 'STOP'] as const; +export const GATEWAY_OPERATION_TYPES = ['RESET', 'DEPLOY', 'START', 'STOP'] as const; export type GatewayOperationType = (typeof GATEWAY_OPERATION_TYPES)[number]; export const GATEWAY_OPERATION_STATUSES = ['QUEUED', 'RUNNING', 'SUCCEEDED', 'FAILED', 'CANCELLED'] as const; diff --git a/app/gateway-api/src/orchestrator/releaseManifest.ts b/app/gateway-api/src/orchestrator/releaseManifest.ts new file mode 100644 index 0000000..5564ffb --- /dev/null +++ b/app/gateway-api/src/orchestrator/releaseManifest.ts @@ -0,0 +1,65 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { isRecord } from '@sammo-ts/common'; + +export const RELEASE_CONTROLLER_PROTOCOL = 1; + +export interface ReleaseManifest { + formatVersion: 1; + controllerProtocol: number; + gatewaySchemaHead: string; + gameSchemaHead: string; + components: string[]; +} + +const assertMigrationHead = async (workspaceRoot: string, directory: string, expected: string): Promise => { + const migrationRoot = path.join(workspaceRoot, 'packages', 'infra', 'prisma', directory); + const entries = await fs.readdir(migrationRoot, { withFileTypes: true }); + const latest = entries + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() + .at(-1); + if (!latest || latest !== expected) { + throw new Error( + `Release manifest ${directory} head ${expected} does not match workspace head ${latest ?? 'none'}.` + ); + } +}; + +export const readReleaseManifest = async (workspaceRoot: string): Promise => { + const manifestPath = path.join(workspaceRoot, 'release-manifest.json'); + const parsed = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as unknown; + if ( + !isRecord(parsed) || + parsed.formatVersion !== 1 || + typeof parsed.controllerProtocol !== 'number' || + !Number.isInteger(parsed.controllerProtocol) || + typeof parsed.gatewaySchemaHead !== 'string' || + typeof parsed.gameSchemaHead !== 'string' || + !Array.isArray(parsed.components) || + !parsed.components.every((component) => typeof component === 'string') + ) { + throw new Error(`Invalid release manifest: ${manifestPath}`); + } + if (parsed.controllerProtocol > RELEASE_CONTROLLER_PROTOCOL) { + throw new Error( + `Release requires controller protocol ${parsed.controllerProtocol}; this controller supports ${RELEASE_CONTROLLER_PROTOCOL}.` + ); + } + const manifest = parsed as unknown as ReleaseManifest; + await Promise.all([ + assertMigrationHead(workspaceRoot, 'gateway-migrations', manifest.gatewaySchemaHead), + assertMigrationHead(workspaceRoot, 'migrations', manifest.gameSchemaHead), + ]); + return manifest; +}; + +export const assertReleaseComponents = (manifest: ReleaseManifest, required: string[]): void => { + const available = new Set(manifest.components); + const missing = required.filter((component) => !available.has(component)); + if (missing.length) { + throw new Error(`Release manifest is missing components: ${missing.join(', ')}`); + } +}; diff --git a/app/gateway-api/src/server.ts b/app/gateway-api/src/server.ts index 3e25649..5ac6944 100644 --- a/app/gateway-api/src/server.ts +++ b/app/gateway-api/src/server.ts @@ -22,6 +22,7 @@ import { createPasswordHasher } from './auth/passwordHasher.js'; import { createPasswordEnvelopeService } from './auth/passwordEnvelope.js'; import { RedisGatewaySessionService } from './auth/redisSessionService.js'; import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js'; +import { createGatewayReleaseRepository } from './orchestrator/gatewayReleaseRepository.js'; import { appRouter } from './router.js'; import { RepositoryProfileStatusService } from './lobby/profileStatusService.js'; import { registerAccountIconInternalRoute } from './auth/accountIconInternalRoute.js'; @@ -64,6 +65,7 @@ export const createGatewayApiServer = async () => { config, process.env ); + const releases = createGatewayReleaseRepository(postgres.prisma as GatewayPrismaClient); const profileStatus = new RepositoryProfileStatusService(profiles, orchestrator); const app = fastify({ @@ -106,6 +108,7 @@ export const createGatewayApiServer = async () => { localAccountGraceDays: config.localAccountGraceDays, passwordEnvelope, profiles, + releases, orchestrator, profileStatus, requestHeaders: req.headers, diff --git a/app/gateway-api/test/adminOperations.test.ts b/app/gateway-api/test/adminOperations.test.ts index 25ca96d..84bd929 100644 --- a/app/gateway-api/test/adminOperations.test.ts +++ b/app/gateway-api/test/adminOperations.test.ts @@ -9,6 +9,11 @@ import type { GatewayProfileRecord, GatewayProfileRepository, } from '../src/orchestrator/profileRepository.js'; +import type { + GatewayReleaseOperationCreateInput, + GatewayReleaseOperationRecord, + GatewayReleaseRepository, +} from '../src/orchestrator/gatewayReleaseRepository.js'; import { createGatewayApiContext } from '../src/context.js'; import { InMemoryProfileStatusService } from '../src/lobby/profileStatusService.js'; import { appRouter } from '../src/router.js'; @@ -22,6 +27,7 @@ const buildCaller = async ( runtimeActionCreateError?: unknown; initialNotice?: string; initialProfileStatus?: GatewayProfileRecord['status']; + profileScenario?: string; } = {} ) => { const users = createInMemoryUserRepository(); @@ -38,6 +44,7 @@ const buildCaller = async ( }); const session = await sessions.createSession({ ...admin, roles: adminRoles }); const createdInputs: GatewayOperationCreateInput[] = []; + const createdReleaseInputs: GatewayReleaseOperationCreateInput[] = []; const operationRecords = new Map>>(); const createdRuntimeActions: Array> = []; const flushes: Array<{ userId: string; reason?: string; iconRevision?: string }> = []; @@ -48,7 +55,7 @@ const buildCaller = async ( const profile = { profileName: 'che:2', profile: 'che', - scenario: '2', + scenario: options.profileScenario ?? '2', apiPort: 15003, status: options.initialProfileStatus ?? ('STOPPED' as const), buildStatus: 'SUCCEEDED' as const, @@ -93,6 +100,46 @@ const buildCaller = async ( cancelOperation: async () => false, retryOperation: async () => null, }; + const releases: GatewayReleaseRepository = { + getState: async () => ({ + id: 'gateway', + activeCommitSha: '1111111111111111111111111111111111111111', + activeWorkspace: '/srv/sammo/current', + previousCommitSha: '2222222222222222222222222222222222222222', + previousWorkspace: '/srv/sammo/previous', + updatedAt: '2026-08-01T00:00:00.000Z', + }), + listOperations: async () => [], + getOperation: async () => null, + createOperation: async (input) => { + createdReleaseInputs.push(input); + return { + id: '44444444-4444-4444-8444-444444444444', + type: input.type, + status: 'QUEUED', + sourceMode: input.sourceMode, + sourceRef: input.sourceRef, + payload: input.payload ?? {}, + reason: input.reason, + requestedBy: input.requestedBy, + attempts: 0, + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T00:00:00.000Z', + } satisfies GatewayReleaseOperationRecord; + }, + claimNextOperation: async () => null, + renewOperationLease: async () => false, + pinOperationResolvedCommit: async () => false, + completeOperation: async () => { + throw new Error('not used'); + }, + publishRelease: async () => { + throw new Error('not used'); + }, + recordStateError: async () => {}, + cancelOperation: async () => false, + retryOperation: async () => null, + }; const caller = appRouter.createCaller( createGatewayApiContext({ users, @@ -116,6 +163,7 @@ const buildCaller = async ( localAccountGraceDays: 7, passwordEnvelope: createPasswordEnvelopeService(), profiles, + releases, orchestrator: { start: () => {}, stop: async () => {}, @@ -170,6 +218,7 @@ const buildCaller = async ( return { caller, createdInputs, + createdReleaseInputs, createdRuntimeActions, users, admin, @@ -270,6 +319,86 @@ describe('admin operation API', () => { }) ).rejects.toMatchObject({ code: 'CONFLICT' }); }); + + it('queues a DB-preserving profile deployment without reset payload', async () => { + const harness = await buildCaller( + async (input) => ({ + id: '33333333-3333-4333-8333-333333333333', + profileName: input.profileName, + type: 'DEPLOY', + status: 'QUEUED', + sourceMode: input.sourceMode, + sourceRef: input.sourceRef, + payload: {}, + requestedBy: input.requestedBy, + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T00:00:00.000Z', + }), + { profileScenario: '1010' } + ); + + await harness.caller.admin.operations.requestDeploy({ + profileName: 'che:2', + sourceMode: 'COMMIT', + sourceRef: 'HEAD', + reason: 'preserve live season', + }); + + expect(harness.createdInputs[0]).toMatchObject({ + profileName: 'che:2', + type: 'DEPLOY', + sourceMode: 'COMMIT', + reason: 'preserve live season', + }); + expect(harness.createdInputs[0]).not.toHaveProperty('payload'); + }); +}); + +describe('gateway release API', () => { + it('queues a gateway deployment for the external release controller', async () => { + const harness = await buildCaller(async () => { + throw new Error('not used'); + }); + + await harness.caller.admin.releases.requestGatewayDeploy({ + sourceMode: 'COMMIT', + sourceRef: 'HEAD', + reason: 'gateway rollout', + }); + + expect(harness.createdReleaseInputs[0]).toMatchObject({ + type: 'DEPLOY', + sourceMode: 'COMMIT', + reason: 'gateway rollout', + requestedBy: harness.admin.id, + }); + expect(harness.createdReleaseInputs[0]?.sourceRef).toMatch(/^[0-9a-f]{40}$/u); + }); + + it('queues rollback to the previously published gateway commit', async () => { + const harness = await buildCaller(async () => { + throw new Error('not used'); + }); + + await harness.caller.admin.releases.requestGatewayRollback({ reason: 'readiness regression' }); + + expect(harness.createdReleaseInputs[0]).toMatchObject({ + type: 'ROLLBACK', + sourceMode: 'COMMIT', + sourceRef: '2222222222222222222222222222222222222222', + }); + }); + + it('requires the global release permission even for profile-scoped administrators', async () => { + const harness = await buildCaller( + async () => { + throw new Error('not used'); + }, + { adminRoles: ['admin.profiles.manage:che:2'], firstUserIsAdmin: false } + ); + + await expect(harness.caller.admin.releases.gatewayState()).rejects.toMatchObject({ code: 'FORBIDDEN' }); + }); }); describe('legacy profile install API', () => { diff --git a/app/gateway-api/test/gatewayReleaseRepository.integration.test.ts b/app/gateway-api/test/gatewayReleaseRepository.integration.test.ts new file mode 100644 index 0000000..392423a --- /dev/null +++ b/app/gateway-api/test/gatewayReleaseRepository.integration.test.ts @@ -0,0 +1,107 @@ +import { createGatewayPostgresConnector } from '@sammo-ts/infra'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { createGatewayReleaseRepository } from '../src/orchestrator/gatewayReleaseRepository.js'; + +const databaseUrl = process.env.GATEWAY_RELEASE_DATABASE_URL; +const describeDatabase = describe.runIf(Boolean(databaseUrl)); + +describeDatabase('gateway release operation persistence', () => { + const connector = createGatewayPostgresConnector({ url: databaseUrl ?? '' }); + const repository = createGatewayReleaseRepository(connector.prisma); + + beforeAll(async () => { + await connector.connect(); + }); + + afterEach(async () => { + await connector.prisma.gatewayReleaseOperation.deleteMany(); + await connector.prisma.gatewayReleaseState.deleteMany(); + }); + + afterAll(async () => { + await connector.disconnect(); + }); + + it('serializes releases, fences publication by lease owner, and records rollback state', async () => { + const operation = await repository.createOperation({ + type: 'DEPLOY', + sourceMode: 'BRANCH', + sourceRef: 'main', + requestedBy: 'admin-a', + }); + await expect( + repository.createOperation({ + type: 'ROLLBACK', + sourceMode: 'COMMIT', + sourceRef: '2222222222222222222222222222222222222222', + requestedBy: 'admin-b', + }) + ).rejects.toMatchObject({ code: 'P2002' }); + + const now = new Date('2030-01-01T00:00:00.000Z'); + await expect( + repository.claimNextOperation(now, { ownerId: 'controller-a', durationMs: 1_000 }) + ).resolves.toMatchObject({ + id: operation.id, + attempts: 1, + leaseOwner: 'controller-a', + }); + await expect(repository.pinOperationResolvedCommit(operation.id, 'controller-a', 'a'.repeat(40))).resolves.toBe( + true + ); + await expect( + repository.publishRelease(operation.id, 'stale-controller', { + commitSha: 'a'.repeat(40), + workspace: '/srv/sammo/new', + }) + ).rejects.toThrow('lease lost before publish'); + + await expect( + repository.publishRelease(operation.id, 'controller-a', { + commitSha: 'a'.repeat(40), + workspace: '/srv/sammo/new', + previousCommitSha: 'b'.repeat(40), + previousWorkspace: '/srv/sammo/old', + }) + ).resolves.toMatchObject({ + activeCommitSha: 'a'.repeat(40), + activeWorkspace: '/srv/sammo/new', + previousCommitSha: 'b'.repeat(40), + previousWorkspace: '/srv/sammo/old', + }); + await expect( + repository.completeOperation( + operation.id, + 'SUCCEEDED', + { resolvedCommitSha: 'a'.repeat(40), error: null }, + 'controller-a' + ) + ).resolves.toMatchObject({ status: 'SUCCEEDED' }); + }); + + it('reclaims an expired release while preserving its pinned commit', async () => { + const operation = await repository.createOperation({ + type: 'DEPLOY', + sourceMode: 'BRANCH', + sourceRef: 'main', + requestedBy: 'admin', + }); + const now = new Date('2030-01-01T00:00:00.000Z'); + await repository.claimNextOperation(now, { ownerId: 'controller-a', durationMs: 1_000 }); + await repository.pinOperationResolvedCommit(operation.id, 'controller-a', 'c'.repeat(40)); + + await expect( + repository.claimNextOperation(new Date(now.getTime() + 1_001), { + ownerId: 'controller-b', + durationMs: 1_000, + }) + ).resolves.toMatchObject({ + id: operation.id, + attempts: 2, + leaseOwner: 'controller-b', + resolvedCommitSha: 'c'.repeat(40), + }); + await expect(repository.renewOperationLease(operation.id, 'controller-a', now, 1_000)).resolves.toBe(false); + }); +}); diff --git a/app/gateway-api/test/orchestratorOperations.test.ts b/app/gateway-api/test/orchestratorOperations.test.ts index bdec9a9..ecfe3a3 100644 --- a/app/gateway-api/test/orchestratorOperations.test.ts +++ b/app/gateway-api/test/orchestratorOperations.test.ts @@ -91,6 +91,7 @@ const createHarness = ( list: async () => processesPresent ? [ + { name: 'sammo:che:2:game-frontend', status: 'online' }, { name: 'sammo:che:2:game-api', status: 'online' }, { name: 'sammo:che:2:turn-daemon', status: 'online' }, { name: 'sammo:che:2:auction-worker', status: 'online' }, @@ -166,6 +167,7 @@ describe('GatewayOrchestrator first-class operations', () => { expect(harness.statuses).toEqual(['RUNNING']); expect(harness.started.map((definition) => definition.name)).toEqual([ + 'sammo:che:2:game-frontend', 'sammo:che:2:game-api', 'sammo:che:2:turn-daemon', 'sammo:che:2:auction-worker', @@ -182,6 +184,7 @@ describe('GatewayOrchestrator first-class operations', () => { expect(harness.statuses).toEqual(['STOPPED']); expect(harness.stopped).toEqual([ + 'sammo:che:2:game-frontend', 'sammo:che:2:game-api', 'sammo:che:2:turn-daemon', 'sammo:che:2:auction-worker', @@ -189,6 +192,7 @@ describe('GatewayOrchestrator first-class operations', () => { 'sammo:che:2:tournament-worker', ]); expect(harness.deleted).toEqual([ + 'sammo:che:2:game-frontend', 'sammo:che:2:game-api', 'sammo:che:2:turn-daemon', 'sammo:che:2:auction-worker', @@ -215,6 +219,7 @@ describe('GatewayOrchestrator first-class operations', () => { await harness.orchestrator.runOperationsNow(); expect(harness.deleted).toEqual([ + 'sammo:che:2:game-frontend', 'sammo:che:2:game-api', 'sammo:che:2:turn-daemon', 'sammo:che:2:auction-worker', @@ -231,9 +236,9 @@ describe('GatewayOrchestrator first-class operations', () => { expect(harness.completions).toEqual(['FAILED']); expect(harness.deleted).toEqual([ - 'sammo:che:2:auction-worker', 'sammo:che:2:turn-daemon', 'sammo:che:2:game-api', + 'sammo:che:2:game-frontend', ]); }); @@ -243,6 +248,7 @@ describe('GatewayOrchestrator first-class operations', () => { await harness.orchestrator.runOperationsNow(); expect(harness.stopped).toEqual([ + 'sammo:che:2:game-frontend', 'sammo:che:2:game-api', 'sammo:che:2:turn-daemon', 'sammo:che:2:auction-worker', @@ -250,6 +256,7 @@ describe('GatewayOrchestrator first-class operations', () => { 'sammo:che:2:tournament-worker', ]); expect(harness.deleted).toEqual([ + 'sammo:che:2:game-frontend', 'sammo:che:2:game-api', 'sammo:che:2:turn-daemon', 'sammo:che:2:auction-worker', diff --git a/app/gateway-api/test/orchestratorPlan.test.ts b/app/gateway-api/test/orchestratorPlan.test.ts index d96a84a..2dbe774 100644 --- a/app/gateway-api/test/orchestratorPlan.test.ts +++ b/app/gateway-api/test/orchestratorPlan.test.ts @@ -28,6 +28,7 @@ describe('planProfileReconcile', () => { it('starts missing processes for running profiles', () => { expect( planProfileReconcile('RUNNING', { + frontendRunning: true, apiRunning: true, daemonRunning: false, auctionRunning: true, @@ -40,6 +41,7 @@ describe('planProfileReconcile', () => { it('starts processes for preopen profiles', () => { expect( planProfileReconcile('PREOPEN', { + frontendRunning: false, apiRunning: false, daemonRunning: false, auctionRunning: false, @@ -52,6 +54,7 @@ describe('planProfileReconcile', () => { it('does nothing when running profile is healthy', () => { expect( planProfileReconcile('RUNNING', { + frontendRunning: true, apiRunning: true, daemonRunning: true, auctionRunning: true, @@ -64,6 +67,7 @@ describe('planProfileReconcile', () => { it('restarts a running profile when only the auction worker is missing', () => { expect( planProfileReconcile('RUNNING', { + frontendRunning: true, apiRunning: true, daemonRunning: true, auctionRunning: false, @@ -76,6 +80,7 @@ describe('planProfileReconcile', () => { it('stops processes for non-running profiles', () => { expect( planProfileReconcile('STOPPED', { + frontendRunning: false, apiRunning: false, daemonRunning: true, auctionRunning: false, @@ -88,6 +93,7 @@ describe('planProfileReconcile', () => { it('keeps reserved profiles off', () => { expect( planProfileReconcile('RESERVED', { + frontendRunning: false, apiRunning: false, daemonRunning: false, auctionRunning: false, @@ -110,6 +116,19 @@ describe('buildProcessDefinitions', () => { const buildWorkspace = '/srv/sammo/worktrees/0123456789abcdef'; const definitions = buildProcessDefinitions(buildProfile(buildWorkspace), processConfig); + expect(definitions.frontend).toMatchObject({ + cwd: path.join(buildWorkspace, 'app', 'game-frontend'), + script: path.join(buildWorkspace, 'node_modules', 'vite', 'bin', 'vite.js'), + args: [ + 'preview', + '--host', + '0.0.0.0', + '--port', + '15002', + '--outDir', + path.join(buildWorkspace, '.release-dist', 'che_2', 'game-frontend'), + ], + }); expect(definitions.api.cwd).toBe(path.join(buildWorkspace, 'app', 'game-api')); expect(definitions.api.script).toBe(path.join(buildWorkspace, 'app', 'game-api', 'dist', 'index.js')); expect(definitions.api.env).toMatchObject({ @@ -141,6 +160,7 @@ describe('buildProcessDefinitions', () => { it('keeps main as the runtime for profiles without a commit worktree', () => { const definitions = buildProcessDefinitions(buildProfile(), processConfig); + expect(definitions.frontend.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-frontend')); expect(definitions.api.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-api')); expect(definitions.daemon.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-engine')); expect(definitions.auction.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-api')); @@ -155,7 +175,7 @@ describe('buildWorkspaceCommands', () => { const commands = buildWorkspaceCommands(workspaceRoot, true); expect(commands.map(({ args }) => args)).toEqual([ - ['install'], + ['install', '--frozen-lockfile'], ['--filter', '@sammo-ts/common', 'build'], ['--filter', '@sammo-ts/infra', 'prisma:generate'], ['--filter', '@sammo-ts/infra', 'build'], diff --git a/app/gateway-api/test/profileDeployOperation.test.ts b/app/gateway-api/test/profileDeployOperation.test.ts new file mode 100644 index 0000000..f2f86b6 --- /dev/null +++ b/app/gateway-api/test/profileDeployOperation.test.ts @@ -0,0 +1,178 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js'; +import type { BuildCommand } from '../src/orchestrator/buildRunner.js'; +import type { ProcessManager } from '../src/orchestrator/processManager.js'; +import type { + GatewayClaimedProfileUpdate, + GatewayOperationRecord, + GatewayProfileRecord, + GatewayProfileRepository, +} from '../src/orchestrator/profileRepository.js'; +import type { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js'; + +const SHA = '1111111111111111111111111111111111111111'; +const temporaryDirectories: string[] = []; + +const createReleaseWorkspace = async (): Promise => { + const workspace = await fs.mkdtemp(path.join(os.tmpdir(), 'sammo-profile-deploy-')); + temporaryDirectories.push(workspace); + await fs.mkdir(path.join(workspace, 'packages/infra/prisma/gateway-migrations/20260801000000_gateway'), { + recursive: true, + }); + await fs.mkdir(path.join(workspace, 'packages/infra/prisma/migrations/20260801000000_game'), { + recursive: true, + }); + await fs.writeFile( + path.join(workspace, 'release-manifest.json'), + JSON.stringify({ + formatVersion: 1, + controllerProtocol: 1, + gatewaySchemaHead: '20260801000000_gateway', + gameSchemaHead: '20260801000000_game', + components: ['game-api', 'game-engine', 'game-frontend'], + }) + ); + return workspace; +}; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true }))); +}); + +describe('profile DEPLOY operation', () => { + it('migrates and switches the selected release without executing the reset seed path', async () => { + const workspace = await createReleaseWorkspace(); + const profile: GatewayProfileRecord = { + profileName: 'che:1010', + profile: 'che', + scenario: '1010', + apiPort: 15003, + status: 'RUNNING', + buildStatus: 'SUCCEEDED', + buildCommitSha: '2222222222222222222222222222222222222222', + buildWorkspace: '/srv/sammo/old', + meta: {}, + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T00:00:00.000Z', + }; + const operation: GatewayOperationRecord = { + id: '33333333-3333-4333-8333-333333333333', + profileName: profile.profileName, + type: 'DEPLOY', + status: 'RUNNING', + sourceMode: 'COMMIT', + sourceRef: SHA, + payload: {}, + requestedBy: 'admin', + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T00:00:00.000Z', + }; + let nextOperation: GatewayOperationRecord | null = operation; + const patches: GatewayClaimedProfileUpdate[] = []; + const completions: string[] = []; + const repository: GatewayProfileRepository = { + listProfiles: async () => [profile], + getProfile: async () => profile, + upsertProfile: async () => profile, + updateScenario: async () => profile, + updateStatus: async () => profile, + updateBuildStatus: async () => profile, + updateMeta: async () => profile, + listReservedToStart: async () => [], + findQueuedBuild: async () => null, + updateLastError: async () => {}, + updateWorkspaceUsage: async () => {}, + clearWorkspaceUsage: async () => {}, + listOperations: async () => [], + getOperation: async () => operation, + createOperation: async () => operation, + claimNextOperation: async () => { + const value = nextOperation; + nextOperation = null; + return value; + }, + renewOperationLease: async () => true, + pinOperationResolvedCommit: async () => true, + updateProfileForOperation: async (_id, _owner, _profileName, patch) => { + patches.push(patch); + return { ...profile, ...patch } as GatewayProfileRecord; + }, + completeOperation: async (_id, status) => { + completions.push(status); + return { ...operation, status }; + }, + requeueOperation: async () => operation, + cancelOperation: async () => false, + retryOperation: async () => null, + }; + const processNames = [ + 'sammo:che:1010:game-frontend', + 'sammo:che:1010:game-api', + 'sammo:che:1010:turn-daemon', + 'sammo:che:1010:auction-worker', + 'sammo:che:1010:battle-sim-worker', + 'sammo:che:1010:tournament-worker', + ]; + const running = new Set(processNames); + const processManager: ProcessManager = { + list: async () => [...running].map((name) => ({ name, status: 'online' })), + start: async (definition) => { + running.add(definition.name); + }, + stop: async () => {}, + delete: async (name) => { + running.delete(name); + }, + }; + const commandGroups: BuildCommand[][] = []; + const workspaceManager = { + resolveCommit: async () => SHA, + prepare: async () => ({ root: workspace, created: true, needsInstall: true }), + } as unknown as GitWorkspaceManager; + const orchestrator = new GatewayOrchestrator({ + repository, + processManager, + buildRunner: { + run: async (commands) => { + commandGroups.push(commands); + return { ok: true, exitCode: 0, output: '' }; + }, + }, + workspaceManager, + processConfig: { + workspaceRoot: '/srv/sammo/controller', + redisKeyPrefix: 'sammo:test', + gameTokenSecret: 'test-secret', + gatewayInternalApiUrl: 'http://127.0.0.1:15001', + baseEnv: { DATABASE_URL: 'postgresql://integration.invalid/sammo' }, + }, + reconcileIntervalMs: 60_000, + scheduleIntervalMs: 60_000, + buildIntervalMs: 60_000, + adminActionIntervalMs: 60_000, + profileReadinessTimeoutMs: 10, + fetchImpl: async () => new Response('', { status: 200 }), + }); + + await orchestrator.runOperationsNow(); + + expect(commandGroups).toHaveLength(2); + expect(commandGroups[0]?.[0]?.args).toEqual(['install', '--frozen-lockfile']); + expect(commandGroups[1]?.map((command) => command.args)).toEqual([ + ['--filter', '@sammo-ts/infra', 'prisma:migrate:deploy:game'], + ]); + expect(commandGroups.flat().some((command) => command.env?.GATEWAY_ROLE === 'profile-seed')).toBe(false); + expect(patches.at(-1)).toMatchObject({ + buildStatus: 'SUCCEEDED', + buildCommitSha: SHA, + buildWorkspace: workspace, + }); + expect(completions).toEqual(['SUCCEEDED']); + expect([...running].sort()).toEqual([...processNames].sort()); + }); +}); diff --git a/app/gateway-api/test/releaseManifest.test.ts b/app/gateway-api/test/releaseManifest.test.ts new file mode 100644 index 0000000..61cc83f --- /dev/null +++ b/app/gateway-api/test/releaseManifest.test.ts @@ -0,0 +1,51 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { readReleaseManifest } from '../src/orchestrator/releaseManifest.js'; + +const temporaryDirectories: string[] = []; + +const createWorkspace = async (gatewayHead: string, gameHead: string): Promise => { + const workspace = await fs.mkdtemp(path.join(os.tmpdir(), 'sammo-release-manifest-')); + temporaryDirectories.push(workspace); + await fs.mkdir(path.join(workspace, 'packages/infra/prisma/gateway-migrations', gatewayHead), { + recursive: true, + }); + await fs.mkdir(path.join(workspace, 'packages/infra/prisma/migrations', gameHead), { recursive: true }); + await fs.writeFile( + path.join(workspace, 'release-manifest.json'), + JSON.stringify({ + formatVersion: 1, + controllerProtocol: 1, + gatewaySchemaHead: gatewayHead, + gameSchemaHead: gameHead, + components: ['gateway-api', 'gateway-frontend', 'game-api', 'game-engine', 'game-frontend'], + }) + ); + return workspace; +}; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true }))); +}); + +describe('readReleaseManifest', () => { + it('accepts a manifest whose schema heads match the selected workspace', async () => { + const workspace = await createWorkspace('20260801000000_gateway', '20260801000000_game'); + + await expect(readReleaseManifest(workspace)).resolves.toMatchObject({ + gatewaySchemaHead: '20260801000000_gateway', + gameSchemaHead: '20260801000000_game', + }); + }); + + it('rejects a stale schema head before any deployment command runs', async () => { + const workspace = await createWorkspace('20260801000000_gateway', '20260801000000_game'); + await fs.mkdir(path.join(workspace, 'packages/infra/prisma/migrations/20260802000000_newer')); + + await expect(readReleaseManifest(workspace)).rejects.toThrow('does not match workspace head'); + }); +}); diff --git a/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts b/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts index ded240f..c3afddb 100644 --- a/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts +++ b/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts @@ -121,6 +121,7 @@ const installFixture = async ( : null, runtime: { profileName: 'hwe:default', + frontendRunning: true, apiRunning: true, daemonRunning: true, auctionRunning: true, @@ -180,6 +181,12 @@ const installFixture = async ( : [] ); } + if (operation === 'admin.releases.gatewayState') { + return response({ id: 'gateway', updatedAt: '2026-08-01T00:00:00.000Z' }); + } + if (operation === 'admin.releases.list') { + return response([]); + } throw new Error(`Unhandled tRPC operation: ${operation}`); }); await route.fulfill({ diff --git a/app/gateway-frontend/e2e/lobby-admin-navigation.spec.ts b/app/gateway-frontend/e2e/lobby-admin-navigation.spec.ts index 8ffd3d8..f6b5a56 100644 --- a/app/gateway-frontend/e2e/lobby-admin-navigation.spec.ts +++ b/app/gateway-frontend/e2e/lobby-admin-navigation.spec.ts @@ -29,10 +29,14 @@ const installGatewayFixture = async (page: Page, roles: string[]) => { operation === 'lobby.profiles' || operation === 'admin.profiles.list' || operation === 'admin.profiles.listScenarios' || - operation === 'admin.operations.list' + operation === 'admin.operations.list' || + operation === 'admin.releases.list' ) { return response([]); } + if (operation === 'admin.releases.gatewayState') { + return response({ id: 'gateway', updatedAt: '2026-08-01T00:00:00.000Z' }); + } if (operation === 'admin.users.getLocalAccountStatus') { return response({ enabled: true }); } diff --git a/app/gateway-frontend/e2e/server-operations.spec.ts b/app/gateway-frontend/e2e/server-operations.spec.ts index a06ff7d..3056705 100644 --- a/app/gateway-frontend/e2e/server-operations.spec.ts +++ b/app/gateway-frontend/e2e/server-operations.spec.ts @@ -5,7 +5,7 @@ type OperationStatus = 'QUEUED' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'CANCELLE type Operation = { id: string; profileName: string; - type: 'RESET' | 'START' | 'STOP'; + type: 'RESET' | 'DEPLOY' | 'START' | 'STOP'; status: OperationStatus; sourceMode?: 'BRANCH' | 'COMMIT'; sourceRef?: string; @@ -20,6 +20,17 @@ type Operation = { type FixtureState = { operations: Operation[]; + gatewayOperations: Array<{ + id: string; + type: 'DEPLOY' | 'ROLLBACK'; + status: OperationStatus; + sourceMode?: 'BRANCH' | 'COMMIT'; + sourceRef?: string; + payload: Record; + requestedBy: string; + createdAt: string; + updatedAt: string; + }>; runtimeRunning: boolean; requestBodies: Array<{ operation: string; body: unknown }>; }; @@ -38,6 +49,7 @@ const profile = (runtimeRunning: boolean) => ({ updatedAt: '2026-07-25T00:00:00.000Z', runtime: { profileName: 'che:2', + frontendRunning: runtimeRunning, apiRunning: runtimeRunning, daemonRunning: runtimeRunning, auctionRunning: runtimeRunning, @@ -90,6 +102,19 @@ const installFixture = async (page: Page, state: FixtureState) => { if (name === 'admin.operations.list') { return response(state.operations); } + if (name === 'admin.releases.gatewayState') { + return response({ + id: 'gateway', + activeCommitSha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + activeWorkspace: '/srv/sammo/current', + previousCommitSha: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + previousWorkspace: '/srv/sammo/previous', + updatedAt: '2026-08-01T00:00:00.000Z', + }); + } + if (name === 'admin.releases.list') { + return response(state.gatewayOperations); + } if (name === 'admin.profiles.listScenarios') { return response(scenarios); } @@ -109,6 +134,37 @@ const installFixture = async (page: Page, state: FixtureState) => { state.operations = [operation]; return response(operation); } + if (name === 'admin.operations.requestDeploy') { + const operation: Operation = { + id: '66666666-6666-4666-8666-666666666666', + profileName: 'che:2', + type: 'DEPLOY', + status: 'QUEUED', + sourceMode: 'BRANCH', + sourceRef: 'main', + payload: {}, + requestedBy: 'admin', + createdAt: '2026-08-01T01:00:00.000Z', + updatedAt: '2026-08-01T01:00:00.000Z', + }; + state.operations = [operation]; + return response(operation); + } + if (name === 'admin.releases.requestGatewayDeploy' || name === 'admin.releases.requestGatewayRollback') { + const releaseOperation = { + id: '77777777-7777-4777-8777-777777777777', + type: name.endsWith('Rollback') ? ('ROLLBACK' as const) : ('DEPLOY' as const), + status: 'QUEUED' as const, + sourceMode: 'BRANCH' as const, + sourceRef: 'main', + payload: {}, + requestedBy: 'admin', + createdAt: '2026-08-01T02:00:00.000Z', + updatedAt: '2026-08-01T02:00:00.000Z', + }; + state.gatewayOperations = [releaseOperation]; + return response(releaseOperation); + } if (name === 'admin.operations.requestRuntime') { const serialized = JSON.stringify(body); const type = serialized.includes('"STOP"') ? 'STOP' : 'START'; @@ -158,7 +214,7 @@ const installFixture = async (page: Page, state: FixtureState) => { test('separates branch and commit semantics and submits a reset from the dedicated page', async ({ page, }, testInfo) => { - const state: FixtureState = { operations: [], runtimeRunning: false, requestBodies: [] }; + const state: FixtureState = { operations: [], gatewayOperations: [], runtimeRunning: false, requestBodies: [] }; await installFixture(page, state); page.on('dialog', (dialog) => dialog.accept()); @@ -233,7 +289,7 @@ test('separates branch and commit semantics and submits a reset from the dedicat }); test('starts and stops all runtime roles through the operation controls', async ({ page }) => { - const state: FixtureState = { operations: [], runtimeRunning: false, requestBodies: [] }; + const state: FixtureState = { operations: [], gatewayOperations: [], runtimeRunning: false, requestBodies: [] }; await installFixture(page, state); page.on('dialog', (dialog) => dialog.accept()); @@ -251,6 +307,46 @@ test('starts and stops all runtime roles through the operation controls', async expect(serializedRequests).toContain('"action":"STOP"'); }); +test('separates DB-preserving profile deployment from DB reset', async ({ page }) => { + const state: FixtureState = { operations: [], gatewayOperations: [], runtimeRunning: true, requestBodies: [] }; + await installFixture(page, state); + page.on('dialog', (dialog) => dialog.accept()); + + await page.goto('admin/server-operations'); + await expect(page.getByText('Game frontend')).toBeVisible(); + await page.getByTestId('request-deploy').click(); + + await expect(page.getByText('DB 보존 배포 작업을 등록했습니다.')).toBeVisible(); + await expect(page.getByTestId('operations-table')).toContainText('DEPLOY'); + expect(state.requestBodies.some((entry) => entry.operation === 'admin.operations.requestDeploy')).toBe(true); + expect(state.requestBodies.some((entry) => entry.operation === 'admin.operations.requestReset')).toBe(false); +}); + +test('controls gateway deployment and rollback through the external controller queue', async ({ page }, testInfo) => { + const state: FixtureState = { operations: [], gatewayOperations: [], runtimeRunning: true, requestBodies: [] }; + await installFixture(page, state); + page.on('dialog', (dialog) => dialog.accept()); + + await page.goto('admin/server-operations'); + const panel = page.getByTestId('gateway-release-panel'); + await expect(panel).toBeVisible(); + await expect(panel).toContainText('aaaaaaaaaaaa'); + await expect(panel).toContainText('bbbbbbbbbbbb'); + await page.getByTestId('gateway-source-ref').fill('release/2026-08'); + await page.getByTestId('request-gateway-deploy').click(); + + await expect(page.getByText(/Gateway 배포 작업을 등록했습니다/)).toBeVisible(); + await expect(page.getByTestId('gateway-release-table')).toContainText('DEPLOY'); + expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.requestGatewayDeploy')).toBe(true); + await page.screenshot({ path: testInfo.outputPath('gateway-release-desktop.png'), fullPage: true }); + + state.gatewayOperations = []; + await page.getByTestId('refresh-operations').click(); + await page.getByTestId('request-gateway-rollback').click(); + await expect(page.getByText('Gateway rollback 작업을 등록했습니다.')).toBeVisible(); + expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.requestGatewayRollback')).toBe(true); +}); + test('renders a failed reset, retries it as a new operation, and reaches success', async ({ page }, testInfo) => { const longError = '선택한 커밋의 프로필 프로세스를 시작하지 못했습니다. 실패 원인을 확인한 뒤 동일 generation으로 재시도해 주세요.'; @@ -272,6 +368,7 @@ test('renders a failed reset, retries it as a new operation, and reaches success updatedAt: '2026-07-25T03:30:00.000Z', }, ], + gatewayOperations: [], runtimeRunning: false, requestBodies: [], }; diff --git a/app/gateway-frontend/src/views/ServerOperationsView.vue b/app/gateway-frontend/src/views/ServerOperationsView.vue index 14ec4a8..ea98e7a 100644 --- a/app/gateway-frontend/src/views/ServerOperationsView.vue +++ b/app/gateway-frontend/src/views/ServerOperationsView.vue @@ -17,6 +17,7 @@ type Profile = { buildError?: string; lastError?: string; runtime: { + frontendRunning: boolean; apiRunning: boolean; daemonRunning: boolean; auctionRunning: boolean; @@ -35,7 +36,7 @@ type Scenario = { type Operation = { id: string; profileName: string; - type: 'RESET' | 'START' | 'STOP'; + type: 'RESET' | 'DEPLOY' | 'START' | 'STOP'; status: 'QUEUED' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'CANCELLED'; sourceMode?: 'BRANCH' | 'COMMIT'; sourceRef?: string; @@ -49,9 +50,35 @@ type Operation = { createdAt: string; }; +type GatewayReleaseState = { + activeCommitSha?: string; + activeWorkspace?: string; + previousCommitSha?: string; + previousWorkspace?: string; + lastSuccessfulAt?: string; + lastError?: string; +}; + +type GatewayReleaseOperation = { + id: string; + type: 'DEPLOY' | 'ROLLBACK'; + status: 'QUEUED' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'CANCELLED'; + sourceMode?: 'BRANCH' | 'COMMIT'; + sourceRef?: string; + resolvedCommitSha?: string; + requestedBy: string; + reason?: string; + error?: string; + createdAt: string; + completedAt?: string; +}; + const profiles = ref([]); const scenarios = ref([]); const operations = ref([]); +const gatewayReleaseState = ref(null); +const gatewayReleaseOperations = ref([]); +const gatewayReleaseAvailable = ref(false); const selectedProfileName = ref(''); const loading = ref(false); const catalogLoading = ref(false); @@ -86,6 +113,11 @@ const form = reactive({ scheduledAt: '', reason: '', }); +const gatewayForm = reactive({ + sourceMode: 'BRANCH' as 'BRANCH' | 'COMMIT', + sourceRef: 'main', + reason: '', +}); const selectedProfile = computed( () => profiles.value.find((profile) => profile.profileName === selectedProfileName.value) ?? null @@ -135,6 +167,17 @@ const loadState = async (quiet = false) => { const operationResult = await adminClient.operations.list.query({ limit: 100 }); profiles.value = profileResult as Profile[]; operations.value = operationResult as Operation[]; + try { + const [state, releaseOperations] = await Promise.all([ + adminClient.releases.gatewayState.query(), + adminClient.releases.list.query({ limit: 30 }), + ]); + gatewayReleaseState.value = state as GatewayReleaseState; + gatewayReleaseOperations.value = releaseOperations as GatewayReleaseOperation[]; + gatewayReleaseAvailable.value = true; + } catch { + gatewayReleaseAvailable.value = false; + } if (!selectedProfileName.value && profiles.value.length > 0) { selectedProfileName.value = profiles.value[0].profileName; } @@ -146,6 +189,78 @@ const loadState = async (quiet = false) => { } }; +const requestDeploy = async () => { + clearStatus(); + if (!selectedProfile.value || activeOperation.value || !form.sourceRef.trim()) { + return; + } + if ( + !window.confirm( + `${selectedProfile.value.profileName}의 인게임 DB를 유지하고 ${form.sourceRef.trim()} 버전으로 배포하시겠습니까?` + ) + ) { + return; + } + submitting.value = true; + try { + await adminClient.operations.requestDeploy.mutate({ + profileName: selectedProfile.value.profileName, + sourceMode: form.sourceMode, + sourceRef: form.sourceRef.trim(), + reason: form.reason.trim() || undefined, + }); + message.value = 'DB 보존 배포 작업을 등록했습니다.'; + await loadState(true); + } catch (error) { + errorMessage.value = error instanceof Error ? error.message : 'DB 보존 배포 요청에 실패했습니다.'; + } finally { + submitting.value = false; + } +}; + +const requestGatewayDeploy = async () => { + clearStatus(); + if (!gatewayForm.sourceRef.trim()) return; + if (!window.confirm(`Gateway 전체를 ${gatewayForm.sourceRef.trim()} 버전으로 전환하시겠습니까?`)) return; + submitting.value = true; + try { + await adminClient.releases.requestGatewayDeploy.mutate({ + sourceMode: gatewayForm.sourceMode, + sourceRef: gatewayForm.sourceRef.trim(), + reason: gatewayForm.reason.trim() || undefined, + }); + message.value = 'Gateway 배포 작업을 등록했습니다. 외부 release-controller가 처리합니다.'; + await loadState(true); + } catch (error) { + errorMessage.value = error instanceof Error ? error.message : 'Gateway 배포 요청에 실패했습니다.'; + } finally { + submitting.value = false; + } +}; + +const requestGatewayRollback = async () => { + clearStatus(); + if (!gatewayReleaseState.value?.previousCommitSha) return; + if ( + !window.confirm( + `Gateway를 이전 버전 ${shortSha(gatewayReleaseState.value.previousCommitSha)}로 되돌리시겠습니까?` + ) + ) + return; + submitting.value = true; + try { + await adminClient.releases.requestGatewayRollback.mutate({ + reason: gatewayForm.reason.trim() || undefined, + }); + message.value = 'Gateway rollback 작업을 등록했습니다.'; + await loadState(true); + } catch (error) { + errorMessage.value = error instanceof Error ? error.message : 'Gateway rollback 요청에 실패했습니다.'; + } finally { + submitting.value = false; + } +}; + const loadScenarios = async () => { clearStatus(); if (!form.sourceRef.trim()) { @@ -371,6 +486,14 @@ onBeforeUnmount(() => {
빌드
{{ selectedProfile.buildStatus }}
+
+
Game frontend
+
+ {{ selectedProfile.runtime.frontendRunning ? 'RUNNING' : 'STOPPED' }} +
+
Game API
@@ -446,7 +569,7 @@ onBeforeUnmount(() => { @submit.prevent="requestReset" >
-

시나리오 초기화

+

프로필 배포 · 시나리오 초기화

{ class="w-full rounded border border-zinc-700 bg-zinc-950 px-3 py-2 text-sm text-white" placeholder="작업 사유 또는 운영 메모" /> - +
+ + +
+
+
+
+

Gateway 릴리스

+

+ 외부 release-controller가 Gateway API·frontend·orchestrator를 전환합니다. +

+
+
+ 현재 + {{ + shortSha(gatewayReleaseState?.activeCommitSha) + }} + · 이전 + {{ + shortSha(gatewayReleaseState?.previousCommitSha) + }} +
+
+
+ + + +
+
+ + +
+
+ {{ gatewayReleaseState.lastError }} +
+
+ + + + + + + + + + + + + + + + + + + + + +
시각작업상태소스해석 커밋오류
{{ formatTime(operation.createdAt) }}{{ operation.type }}{{ operation.status }}{{ operation.sourceRef }}{{ shortSha(operation.resolvedCommitSha) }}{{ operation.error }}
+
+
+

작업 이력

diff --git a/app/release-controller/README.md b/app/release-controller/README.md new file mode 100644 index 0000000..d01039d --- /dev/null +++ b/app/release-controller/README.md @@ -0,0 +1,72 @@ +# Gateway release controller + +`release-controller`는 Gateway API·frontend·orchestrator와 분리된 PM2 +프로세스입니다. 관리자 GUI가 `GatewayReleaseOperation`을 만들면 controller가 +선택 commit을 고정하고 다음 순서로 전환합니다. + +1. commit 전용 worktree를 준비하고 frozen lockfile로 의존성을 설치합니다. +2. `release-manifest.json`의 protocol, component와 실제 migration head를 + 확인합니다. +3. Gateway API와 frontend를 빌드하고 gateway migration을 적용합니다. +4. 기존 `sammo:gateway-api`, `sammo:gateway-frontend`, + `sammo:gateway-orchestrator`를 중지하고 새 worktree에서 시작합니다. +5. 두 HTTP endpoint와 세 PM2 process가 모두 준비된 경우에만 현재·이전 + 릴리스 상태를 게시합니다. 실패하면 이전 세 프로세스를 복구합니다. + +## 환경 변수 + +- `GATEWAY_DATABASE_URL`: Gateway PostgreSQL URL입니다. 필수입니다. +- `GATEWAY_DB_SCHEMA`: Gateway schema이며 기본값은 `public`입니다. +- `RELEASE_CONTROLLER_WORKSPACE_ROOT`: Git checkout입니다. +- `RELEASE_CONTROLLER_WORKTREE_ROOT`: commit worktree 상위 경로입니다. +- `GATEWAY_API_PORT`, `GATEWAY_FRONTEND_PORT`, `GATEWAY_BASE_PATH`: readiness와 + frontend build 계약입니다. +- `RELEASE_CONTROLLER_POLL_MS`, `RELEASE_CONTROLLER_READINESS_TIMEOUT_MS`: queue + poll과 준비 제한 시간입니다. + +비밀값은 Git에서 제외된 환경 파일 또는 process 환경으로 전달해 주세요. +`VITE_*`에는 공개 URL만 넣어 주세요. + +## 설치와 실행 + +먼저 controller가 읽을 Gateway schema를 migration하고 의존 package를 함께 +빌드합니다. + +```sh +pnpm install --frozen-lockfile +pnpm --filter @sammo-ts/infra prisma:generate +pnpm --filter @sammo-ts/common build +pnpm --filter @sammo-ts/infra build +pnpm --filter @sammo-ts/logic build +pnpm --filter @sammo-ts/game-engine build +pnpm --filter @sammo-ts/gateway-api build +pnpm --filter @sammo-ts/release-controller build +pnpm --filter @sammo-ts/infra prisma:migrate:deploy:gateway +pnpm --filter @sammo-ts/release-controller start +``` + +운영에서는 마지막 명령 대신 `sammo:release-controller`라는 PM2 process로 +`app/release-controller/dist/index.js daemon`을 실행해 주세요. 상태와 queue +한 건 처리는 다음 CLI로 확인할 수 있습니다. + +```sh +pnpm --filter @sammo-ts/release-controller status +pnpm --filter @sammo-ts/release-controller run-once +``` + +## Controller self-upgrade + +이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를 +빌드하고 gateway migration을 적용한 뒤 `sammo:release-controller`만 새 +worktree로 전환합니다. 새 daemon 시작에 실패하면 이전 definition을 +복구합니다. + +```sh +pnpm --filter @sammo-ts/release-controller build +pnpm --filter @sammo-ts/release-controller self-upgrade -- BRANCH main +# 또는 +pnpm --filter @sammo-ts/release-controller self-upgrade -- COMMIT +``` + +Database migration은 일반적으로 되돌리지 않습니다. 이전 애플리케이션으로 +rollback하려면 새 schema와의 하위 호환성을 릴리스 전에 확인해 주세요. diff --git a/app/release-controller/package.json b/app/release-controller/package.json new file mode 100644 index 0000000..fc219e4 --- /dev/null +++ b/app/release-controller/package.json @@ -0,0 +1,28 @@ +{ + "name": "@sammo-ts/release-controller", + "private": true, + "version": "0.0.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/release-controller", + "start": "node dist/index.js daemon", + "run-once": "node dist/index.js run-once", + "status": "node dist/index.js status", + "self-upgrade": "node dist/index.js self-upgrade", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test": "vitest run --config vitest.config.ts", + "typecheck": "tsc -b" + }, + "dependencies": { + "@sammo-ts/gateway-api": "workspace:*", + "@sammo-ts/infra": "workspace:*" + }, + "devDependencies": { + "tsdown": "^0.22.14", + "vite-tsconfig-paths": "^6.0.3", + "vitest": "^4.0.16" + } +} diff --git a/app/release-controller/src/config.ts b/app/release-controller/src/config.ts new file mode 100644 index 0000000..c94b6a0 --- /dev/null +++ b/app/release-controller/src/config.ts @@ -0,0 +1,56 @@ +import path from 'node:path'; + +const parsePositiveInt = (value: string | undefined, fallback: number, name: string): number => { + if (!value) return fallback; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer.`); + return parsed; +}; + +const applySchema = (databaseUrl: string, schema: string): string => { + const parsed = new URL(databaseUrl); + parsed.searchParams.set('schema', schema); + return parsed.toString(); +}; + +export interface ReleaseControllerConfig { + workspaceRoot: string; + worktreeRoot: string; + gatewayDatabaseUrl: string; + gatewayDbSchema: string; + gatewayApiPort: number; + gatewayFrontendPort: number; + gatewayBasePath: string; + pollIntervalMs: number; + readinessTimeoutMs: number; + baseEnv: Record; +} + +export const resolveReleaseControllerConfig = (env: NodeJS.ProcessEnv = process.env): ReleaseControllerConfig => { + const rawGatewayDatabaseUrl = env.GATEWAY_DATABASE_URL ?? env.DATABASE_URL ?? ''; + if (!rawGatewayDatabaseUrl) { + throw new Error('GATEWAY_DATABASE_URL or DATABASE_URL is required.'); + } + const workspaceRoot = path.resolve(env.RELEASE_CONTROLLER_WORKSPACE_ROOT ?? process.cwd()); + const gatewayDbSchema = env.GATEWAY_DB_SCHEMA?.trim() || 'public'; + return { + workspaceRoot, + worktreeRoot: path.resolve( + env.RELEASE_CONTROLLER_WORKTREE_ROOT ?? path.join(workspaceRoot, '.release-worktrees') + ), + gatewayDatabaseUrl: applySchema(rawGatewayDatabaseUrl, gatewayDbSchema), + gatewayDbSchema, + gatewayApiPort: parsePositiveInt(env.GATEWAY_API_PORT, 15001, 'GATEWAY_API_PORT'), + gatewayFrontendPort: parsePositiveInt(env.GATEWAY_FRONTEND_PORT, 15000, 'GATEWAY_FRONTEND_PORT'), + gatewayBasePath: env.GATEWAY_BASE_PATH?.trim() || '/gateway', + pollIntervalMs: parsePositiveInt(env.RELEASE_CONTROLLER_POLL_MS, 5000, 'RELEASE_CONTROLLER_POLL_MS'), + readinessTimeoutMs: parsePositiveInt( + env.RELEASE_CONTROLLER_READINESS_TIMEOUT_MS, + 60000, + 'RELEASE_CONTROLLER_READINESS_TIMEOUT_MS' + ), + baseEnv: Object.fromEntries( + Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === 'string') + ), + }; +}; diff --git a/app/release-controller/src/index.ts b/app/release-controller/src/index.ts new file mode 100644 index 0000000..5883d95 --- /dev/null +++ b/app/release-controller/src/index.ts @@ -0,0 +1,83 @@ +import { createGatewayPostgresConnector, type GatewayPrismaClient } from '@sammo-ts/infra'; +import { + createGatewayReleaseRepository, + GitWorkspaceManager, + Pm2ProcessManager, + PnpmBuildRunner, +} from '@sammo-ts/gateway-api'; + +import { resolveReleaseControllerConfig } from './config.js'; +import { GatewayReleaseController } from './releaseController.js'; +import { upgradeReleaseController } from './selfUpgrade.js'; + +export * from './config.js'; +export * from './releaseController.js'; +export * from './selfUpgrade.js'; + +const main = async (): Promise => { + const config = resolveReleaseControllerConfig(); + const postgres = createGatewayPostgresConnector({ url: config.gatewayDatabaseUrl }); + await postgres.connect(); + const repository = createGatewayReleaseRepository(postgres.prisma as GatewayPrismaClient); + const workspaceManager = new GitWorkspaceManager({ + repoRoot: config.workspaceRoot, + worktreeRoot: config.worktreeRoot, + baseEnv: config.baseEnv, + }); + const buildRunner = new PnpmBuildRunner(); + const processManager = new Pm2ProcessManager(); + const controller = new GatewayReleaseController(repository, workspaceManager, buildRunner, processManager, config); + const command = process.argv[2] ?? 'daemon'; + if (command === 'status') { + console.log( + JSON.stringify( + { state: await repository.getState(), operations: await repository.listOperations(20) }, + null, + 2 + ) + ); + await postgres.disconnect(); + return; + } + if (command === 'run-once') { + console.log(JSON.stringify(await controller.runOnce(), null, 2)); + await postgres.disconnect(); + return; + } + if (command === 'self-upgrade') { + const sourceMode = process.argv[3]; + const sourceRef = process.argv[4]; + if ((sourceMode !== 'BRANCH' && sourceMode !== 'COMMIT') || !sourceRef) { + throw new Error('usage: release-controller self-upgrade '); + } + const result = await upgradeReleaseController({ + sourceMode, + sourceRef, + workspaceManager, + buildRunner, + processManager, + config, + }); + console.log(JSON.stringify(result, null, 2)); + await postgres.disconnect(); + return; + } + if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`); + let stopping = false; + const stop = async (): Promise => { + if (stopping) return; + stopping = true; + await postgres.disconnect(); + }; + process.once('SIGINT', () => void stop()); + process.once('SIGTERM', () => void stop()); + while (!stopping) { + await controller.runOnce(); + await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); + } +}; + +main().catch((error) => { + console.error('[release-controller] failed', error); + process.exitCode = 1; +}); diff --git a/app/release-controller/src/releaseController.ts b/app/release-controller/src/releaseController.ts new file mode 100644 index 0000000..6f6f11f --- /dev/null +++ b/app/release-controller/src/releaseController.ts @@ -0,0 +1,247 @@ +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; + +import { + assertReleaseComponents, + type BuildCommand, + type BuildRunner, + type GatewayReleaseOperationRecord, + type GatewayReleaseRepository, + type GatewayReleaseStateRecord, + type GitWorkspaceManager, + type ProcessDefinition, + type ProcessManager, + readReleaseManifest, +} from '@sammo-ts/gateway-api'; + +import type { ReleaseControllerConfig } from './config.js'; + +const LEASE_DURATION_MS = 10 * 60_000; +const HEARTBEAT_INTERVAL_MS = 60_000; +const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const; + +export const buildGatewayReleaseCommands = ( + workspaceRoot: string, + needsInstall: boolean, + config: ReleaseControllerConfig +): BuildCommand[] => { + const env = { + ...config.baseEnv, + VITE_APP_BASE_PATH: config.gatewayBasePath, + VITE_GATEWAY_API_URL: `${config.gatewayBasePath}/api/trpc`, + VITE_GAME_API_URL_TEMPLATE: '/{profile}/api/trpc', + VITE_GAME_WEB_URL_TEMPLATE: '/{profile}/', + }; + return [ + ...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []), + { command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'prisma:generate'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/logic', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/game-engine', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/gateway-api', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/gateway-frontend', 'build'], cwd: workspaceRoot, env }, + ]; +}; + +export const buildGatewayMigrationCommand = (workspaceRoot: string, config: ReleaseControllerConfig): BuildCommand => ({ + command: 'pnpm', + args: ['--filter', '@sammo-ts/infra', 'prisma:migrate:deploy:gateway'], + cwd: workspaceRoot, + env: { + ...config.baseEnv, + GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl, + }, +}); + +export const buildGatewayProcessDefinitions = ( + workspaceRoot: string, + config: ReleaseControllerConfig +): ProcessDefinition[] => { + const apiCwd = path.join(workspaceRoot, 'app', 'gateway-api'); + const frontendCwd = path.join(workspaceRoot, 'app', 'gateway-frontend'); + const apiScript = path.join(apiCwd, 'dist', 'index.js'); + const frontendScript = path.join(workspaceRoot, 'node_modules', 'vite', 'bin', 'vite.js'); + const env = { + ...config.baseEnv, + GATEWAY_API_HOST: '0.0.0.0', + GATEWAY_API_PORT: String(config.gatewayApiPort), + GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl, + }; + return [ + { name: 'sammo:gateway-api', script: apiScript, cwd: apiCwd, env: { ...env, GATEWAY_ROLE: 'api' } }, + { + name: 'sammo:gateway-frontend', + script: frontendScript, + cwd: frontendCwd, + args: ['preview', '--host', '0.0.0.0', '--port', String(config.gatewayFrontendPort)], + env, + }, + { + name: 'sammo:gateway-orchestrator', + script: apiScript, + cwd: apiCwd, + env: { ...env, GATEWAY_ROLE: 'orchestrator' }, + }, + ]; +}; + +const isMissingProcessError = (error: unknown): boolean => + error instanceof Error && /process or namespace not found/i.test(error.message); + +export class GatewayReleaseController { + private readonly ownerId = randomUUID(); + + constructor( + private readonly repository: GatewayReleaseRepository, + private readonly workspaceManager: GitWorkspaceManager, + private readonly buildRunner: BuildRunner, + private readonly processManager: ProcessManager, + private readonly config: ReleaseControllerConfig, + private readonly now: () => Date = () => new Date(), + private readonly fetchImpl: typeof fetch = fetch + ) {} + + async runOnce(): Promise { + const operation = await this.repository.claimNextOperation(this.now(), { + ownerId: this.ownerId, + durationMs: LEASE_DURATION_MS, + }); + if (!operation) return null; + const heartbeat = setInterval(() => { + void this.repository.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS); + }, HEARTBEAT_INTERVAL_MS); + let resolvedCommitSha: string | undefined; + try { + const state = await this.repository.getState(); + const deploymentState: GatewayReleaseStateRecord = { + ...state, + activeCommitSha: state.activeCommitSha ?? (await this.workspaceManager.resolveCommit('COMMIT', 'HEAD')), + activeWorkspace: state.activeWorkspace ?? this.config.workspaceRoot, + }; + const sourceMode = operation.sourceMode ?? 'COMMIT'; + const sourceRef = operation.sourceRef ?? state.previousCommitSha; + if (!sourceRef) throw new Error('Release source is missing.'); + resolvedCommitSha = await this.workspaceManager.resolveCommit(sourceMode, sourceRef); + if (!(await this.repository.pinOperationResolvedCommit(operation.id, this.ownerId, resolvedCommitSha))) { + throw new Error('Gateway release lease was lost while pinning the commit.'); + } + await this.deploy(operation, deploymentState, resolvedCommitSha); + return await this.repository.completeOperation( + operation.id, + 'SUCCEEDED', + { resolvedCommitSha, error: null }, + this.ownerId + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + await this.repository.recordStateError(detail); + return await this.repository.completeOperation( + operation.id, + 'FAILED', + { resolvedCommitSha, error: detail }, + this.ownerId + ); + } finally { + clearInterval(heartbeat); + } + } + + private async deploy( + operation: GatewayReleaseOperationRecord, + state: GatewayReleaseStateRecord, + commitSha: string + ): Promise { + const workspace = await this.workspaceManager.prepare(commitSha); + const manifest = await readReleaseManifest(workspace.root); + assertReleaseComponents(manifest, ['gateway-api', 'gateway-frontend']); + const build = await this.buildRunner.run( + buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config) + ); + if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`); + const migration = await this.buildRunner.run([buildGatewayMigrationCommand(workspace.root, this.config)]); + if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`); + + const previousDefinitions = state.activeWorkspace + ? buildGatewayProcessDefinitions(state.activeWorkspace, this.config) + : []; + await this.stopManagedProcesses(); + try { + await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config)); + await this.waitForReadiness(); + } catch (error) { + await this.stopManagedProcesses(); + if (previousDefinitions.length) { + await this.startDefinitions(previousDefinitions); + await this.waitForReadiness(); + } + throw error; + } + await this.repository.publishRelease(operation.id, this.ownerId, { + commitSha, + workspace: workspace.root, + previousCommitSha: state.activeCommitSha, + previousWorkspace: state.activeWorkspace, + }); + } + + private async startDefinitions(definitions: ProcessDefinition[]): Promise { + const started: string[] = []; + try { + for (const definition of definitions) { + await this.processManager.start(definition); + started.push(definition.name); + } + } catch (error) { + for (const name of started.reverse()) { + try { + await this.processManager.delete(name); + } catch { + // Preserve the start failure. + } + } + throw error; + } + } + + private async stopManagedProcesses(): Promise { + const existing = new Set((await this.processManager.list()).map((process) => process.name)); + const failures: string[] = []; + for (const name of [...PROCESS_NAMES].reverse()) { + if (!existing.has(name)) continue; + try { + await this.processManager.stop(name); + } catch { + // Delete below is authoritative. + } + try { + await this.processManager.delete(name); + } catch (error) { + if (!isMissingProcessError(error)) failures.push(`${name}: ${String(error)}`); + } + } + if (failures.length) throw new Error(`Failed to stop gateway processes: ${failures.join('; ')}`); + } + + private async waitForReadiness(): Promise { + const deadline = Date.now() + this.config.readinessTimeoutMs; + const apiUrl = `http://127.0.0.1:${this.config.gatewayApiPort}/healthz`; + const frontendUrl = `http://127.0.0.1:${this.config.gatewayFrontendPort}${this.config.gatewayBasePath}/`; + while (Date.now() < deadline) { + try { + const [api, frontend] = await Promise.all([this.fetchImpl(apiUrl), this.fetchImpl(frontendUrl)]); + const processes = await this.processManager.list(); + const online = new Set( + processes + .filter((process) => process.status.toLowerCase() === 'online') + .map((process) => process.name) + ); + if (api.ok && frontend.ok && PROCESS_NAMES.every((name) => online.has(name))) return; + } catch { + // Retry until the bounded deadline. + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error('Gateway release did not become ready before the timeout.'); + } +} diff --git a/app/release-controller/src/selfUpgrade.ts b/app/release-controller/src/selfUpgrade.ts new file mode 100644 index 0000000..8f8d1a9 --- /dev/null +++ b/app/release-controller/src/selfUpgrade.ts @@ -0,0 +1,108 @@ +import path from 'node:path'; + +import { + assertReleaseComponents, + type BuildCommand, + type BuildRunner, + type GitWorkspaceManager, + type ProcessDefinition, + type ProcessManager, + readReleaseManifest, +} from '@sammo-ts/gateway-api'; + +import type { ReleaseControllerConfig } from './config.js'; +import { buildGatewayMigrationCommand } from './releaseController.js'; + +const CONTROLLER_PROCESS_NAME = 'sammo:release-controller'; + +export const buildReleaseControllerCommands = ( + workspaceRoot: string, + needsInstall: boolean, + config: ReleaseControllerConfig +): BuildCommand[] => { + const env = config.baseEnv; + return [ + ...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []), + { command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'prisma:generate'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/infra', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/logic', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/game-engine', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/gateway-api', 'build'], cwd: workspaceRoot, env }, + { command: 'pnpm', args: ['--filter', '@sammo-ts/release-controller', 'build'], cwd: workspaceRoot, env }, + ]; +}; + +export const buildReleaseControllerDefinition = ( + workspaceRoot: string, + config: ReleaseControllerConfig +): ProcessDefinition => ({ + name: CONTROLLER_PROCESS_NAME, + script: path.join(workspaceRoot, 'app', 'release-controller', 'dist', 'index.js'), + cwd: path.join(workspaceRoot, 'app', 'release-controller'), + args: ['daemon'], + env: { + ...config.baseEnv, + GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl, + GATEWAY_DB_SCHEMA: config.gatewayDbSchema, + RELEASE_CONTROLLER_WORKSPACE_ROOT: workspaceRoot, + RELEASE_CONTROLLER_WORKTREE_ROOT: config.worktreeRoot, + }, +}); + +const workspaceFromControllerCwd = (cwd: string | undefined, fallback: string): string => + cwd ? path.resolve(cwd, '..', '..') : fallback; + +export const upgradeReleaseController = async (options: { + sourceMode: 'BRANCH' | 'COMMIT'; + sourceRef: string; + workspaceManager: GitWorkspaceManager; + buildRunner: BuildRunner; + processManager: ProcessManager; + config: ReleaseControllerConfig; + readinessTimeoutMs?: number; +}): Promise<{ commitSha: string; workspace: string }> => { + const commitSha = await options.workspaceManager.resolveCommit(options.sourceMode, options.sourceRef); + const workspace = await options.workspaceManager.prepare(commitSha); + const manifest = await readReleaseManifest(workspace.root); + assertReleaseComponents(manifest, ['release-controller']); + const build = await options.buildRunner.run( + buildReleaseControllerCommands(workspace.root, workspace.needsInstall, options.config) + ); + if (!build.ok) throw new Error(`Release controller build failed: ${build.output.slice(-4000)}`); + const migration = await options.buildRunner.run([buildGatewayMigrationCommand(workspace.root, options.config)]); + if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`); + + const existing = (await options.processManager.list()).find((process) => process.name === CONTROLLER_PROCESS_NAME); + const previousDefinition = buildReleaseControllerDefinition( + workspaceFromControllerCwd(existing?.cwd, options.config.workspaceRoot), + options.config + ); + if (existing) { + try { + await options.processManager.stop(CONTROLLER_PROCESS_NAME); + } finally { + await options.processManager.delete(CONTROLLER_PROCESS_NAME); + } + } + try { + await options.processManager.start(buildReleaseControllerDefinition(workspace.root, options.config)); + const deadline = Date.now() + (options.readinessTimeoutMs ?? options.config.readinessTimeoutMs); + while (Date.now() < deadline) { + const active = (await options.processManager.list()).find( + (process) => process.name === CONTROLLER_PROCESS_NAME && process.status.toLowerCase() === 'online' + ); + if (active) return { commitSha, workspace: workspace.root }; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error('Release controller did not become online before the timeout.'); + } catch (error) { + try { + await options.processManager.delete(CONTROLLER_PROCESS_NAME); + } catch { + // The failed new process may already be absent. + } + if (existing) await options.processManager.start(previousDefinition); + throw error; + } +}; diff --git a/app/release-controller/test/releaseController.test.ts b/app/release-controller/test/releaseController.test.ts new file mode 100644 index 0000000..3b0163a --- /dev/null +++ b/app/release-controller/test/releaseController.test.ts @@ -0,0 +1,304 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import type { + BuildRunner, + GatewayReleaseOperationRecord, + GatewayReleaseRepository, + GatewayReleaseStateRecord, + GitWorkspaceManager, + ProcessDefinition, + ProcessManager, +} from '@sammo-ts/gateway-api'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { resolveReleaseControllerConfig, type ReleaseControllerConfig } from '../src/config.js'; +import { GatewayReleaseController } from '../src/releaseController.js'; +import { upgradeReleaseController } from '../src/selfUpgrade.js'; + +const SHA = '1111111111111111111111111111111111111111'; +const OLD_SHA = '2222222222222222222222222222222222222222'; +const temporaryDirectories: string[] = []; + +const createReleaseWorkspace = async (): Promise => { + const workspace = await fs.mkdtemp(path.join(os.tmpdir(), 'sammo-release-controller-')); + temporaryDirectories.push(workspace); + const gatewayHead = '20260801000000_gateway'; + const gameHead = '20260801000000_game'; + await fs.mkdir(path.join(workspace, 'packages/infra/prisma/gateway-migrations', gatewayHead), { + recursive: true, + }); + await fs.mkdir(path.join(workspace, 'packages/infra/prisma/migrations', gameHead), { recursive: true }); + await fs.writeFile( + path.join(workspace, 'release-manifest.json'), + JSON.stringify({ + formatVersion: 1, + controllerProtocol: 1, + gatewaySchemaHead: gatewayHead, + gameSchemaHead: gameHead, + components: ['gateway-api', 'gateway-frontend', 'release-controller'], + }) + ); + return workspace; +}; + +const operation: GatewayReleaseOperationRecord = { + id: '11111111-1111-4111-8111-111111111111', + type: 'DEPLOY', + status: 'RUNNING', + sourceMode: 'COMMIT', + sourceRef: SHA, + payload: {}, + requestedBy: 'admin', + attempts: 1, + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T00:00:00.000Z', +}; + +const state: GatewayReleaseStateRecord = { + id: 'gateway', + activeCommitSha: OLD_SHA, + activeWorkspace: '/srv/sammo/old', + updatedAt: '2026-08-01T00:00:00.000Z', +}; + +const config: ReleaseControllerConfig = { + workspaceRoot: '/srv/sammo/controller', + worktreeRoot: '/srv/sammo/releases', + gatewayDatabaseUrl: 'postgresql://integration.invalid/sammo?schema=gateway', + gatewayDbSchema: 'gateway', + gatewayApiPort: 15001, + gatewayFrontendPort: 15000, + gatewayBasePath: '/gateway', + pollIntervalMs: 5, + readinessTimeoutMs: 10, + baseEnv: {}, +}; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true }))); +}); + +const createRepository = () => { + let next: GatewayReleaseOperationRecord | null = operation; + const completions: string[] = []; + const published: Array<{ commitSha: string; workspace: string; previousCommitSha?: string }> = []; + const errors: string[] = []; + const repository: GatewayReleaseRepository = { + getState: async () => state, + listOperations: async () => [], + getOperation: async () => operation, + createOperation: async () => operation, + claimNextOperation: async () => { + const claimed = next; + next = null; + return claimed; + }, + renewOperationLease: async () => true, + pinOperationResolvedCommit: async () => true, + completeOperation: async (_id, statusValue) => { + completions.push(statusValue); + return { ...operation, status: statusValue }; + }, + publishRelease: async (_id, _owner, release) => { + published.push(release); + return { ...state, activeCommitSha: release.commitSha, activeWorkspace: release.workspace }; + }, + recordStateError: async (detail) => { + errors.push(detail); + }, + cancelOperation: async () => false, + retryOperation: async () => null, + }; + return { repository, completions, published, errors }; +}; + +const gatewayNames = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator']; + +describe('GatewayReleaseController', () => { + it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => { + const workspace = await createReleaseWorkspace(); + const harness = createRepository(); + const commandGroups: string[][] = []; + const running = new Map(gatewayNames.map((name) => [name, '/srv/sammo/old'])); + const processManager: ProcessManager = { + list: async () => + [...running].map(([name, cwd]) => ({ name, cwd, status: 'online', script: path.join(cwd, 'dist.js') })), + start: async (definition) => { + running.set(definition.name, definition.cwd); + }, + stop: async () => {}, + delete: async (name) => { + running.delete(name); + }, + }; + const buildRunner: BuildRunner = { + run: async (commands) => { + commandGroups.push(commands.map((command) => command.args.join(' '))); + return { ok: true, exitCode: 0, output: '' }; + }, + }; + const workspaceManager = { + resolveCommit: async () => SHA, + prepare: async () => ({ root: workspace, created: true, needsInstall: true }), + } as unknown as GitWorkspaceManager; + const controller = new GatewayReleaseController( + harness.repository, + workspaceManager, + buildRunner, + processManager, + config, + () => new Date('2026-08-01T00:00:00.000Z'), + async () => new Response('', { status: 200 }) + ); + + await controller.runOnce(); + + expect(commandGroups).toHaveLength(2); + expect(commandGroups[0]?.[0]).toBe('install --frozen-lockfile'); + expect(commandGroups[1]).toEqual(['--filter @sammo-ts/infra prisma:migrate:deploy:gateway']); + expect([...running.keys()].sort()).toEqual([...gatewayNames].sort()); + expect(harness.published).toEqual([ + { commitSha: SHA, workspace, previousCommitSha: OLD_SHA, previousWorkspace: '/srv/sammo/old' }, + ]); + expect(harness.completions).toEqual(['SUCCEEDED']); + }); + + it('restores the previous gateway processes when the new process set cannot start', async () => { + const workspace = await createReleaseWorkspace(); + const harness = createRepository(); + const started: ProcessDefinition[] = []; + const running = new Map(gatewayNames.map((name) => [name, '/srv/sammo/old'])); + const processManager: ProcessManager = { + list: async () => [...running].map(([name, cwd]) => ({ name, cwd, status: 'online' })), + start: async (definition) => { + if (definition.cwd.startsWith(workspace) && definition.name === 'sammo:gateway-api') { + throw new Error('new gateway failed'); + } + started.push(definition); + running.set(definition.name, definition.cwd); + }, + stop: async () => {}, + delete: async (name) => { + running.delete(name); + }, + }; + const workspaceManager = { + resolveCommit: async () => SHA, + prepare: async () => ({ root: workspace, created: true, needsInstall: false }), + } as unknown as GitWorkspaceManager; + const controller = new GatewayReleaseController( + harness.repository, + workspaceManager, + { run: async () => ({ ok: true, exitCode: 0, output: '' }) }, + processManager, + config, + () => new Date('2026-08-01T00:00:00.000Z'), + async () => new Response('', { status: 200 }) + ); + + await controller.runOnce(); + + expect(started.filter((definition) => definition.cwd.startsWith('/srv/sammo/old'))).toHaveLength(3); + expect(harness.published).toEqual([]); + expect(harness.completions).toEqual(['FAILED']); + expect(harness.errors.at(-1)).toContain('new gateway failed'); + }); +}); + +describe('resolveReleaseControllerConfig', () => { + it('applies the configured gateway schema to the controller database URL', () => { + const resolved = resolveReleaseControllerConfig({ + GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo?schema=wrong', + GATEWAY_DB_SCHEMA: 'gateway_release', + RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller', + }); + + expect(new URL(resolved.gatewayDatabaseUrl).searchParams.get('schema')).toBe('gateway_release'); + }); +}); + +describe('upgradeReleaseController', () => { + it('switches the controller daemon from a separately invoked CLI process', async () => { + const workspace = await createReleaseWorkspace(); + const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]); + const starts: ProcessDefinition[] = []; + const processManager: ProcessManager = { + list: async () => [...running].map(([name, cwd]) => ({ name, cwd, status: 'online' })), + start: async (definition) => { + starts.push(definition); + running.set(definition.name, definition.cwd); + }, + stop: async () => {}, + delete: async (name) => { + running.delete(name); + }, + }; + const workspaceManager = { + resolveCommit: async () => SHA, + prepare: async () => ({ root: workspace, created: true, needsInstall: true }), + } as unknown as GitWorkspaceManager; + const commandGroups: string[][] = []; + + await expect( + upgradeReleaseController({ + sourceMode: 'COMMIT', + sourceRef: SHA, + workspaceManager, + buildRunner: { + run: async (commands) => { + commandGroups.push(commands.map((command) => command.args.join(' '))); + return { ok: true, exitCode: 0, output: '' }; + }, + }, + processManager, + config, + readinessTimeoutMs: 10, + }) + ).resolves.toEqual({ commitSha: SHA, workspace }); + + expect(commandGroups).toHaveLength(2); + expect(commandGroups[0]?.at(-1)).toBe('--filter @sammo-ts/release-controller build'); + expect(starts.at(-1)).toMatchObject({ + name: 'sammo:release-controller', + cwd: path.join(workspace, 'app', 'release-controller'), + args: ['daemon'], + }); + }); + + it('restores the old controller definition when the new daemon cannot start', async () => { + const workspace = await createReleaseWorkspace(); + const starts: ProcessDefinition[] = []; + const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]); + const processManager: ProcessManager = { + list: async () => [...running].map(([name, cwd]) => ({ name, cwd, status: 'online' })), + start: async (definition) => { + if (definition.cwd.startsWith(workspace)) throw new Error('new controller failed'); + starts.push(definition); + running.set(definition.name, definition.cwd); + }, + stop: async () => {}, + delete: async (name) => { + running.delete(name); + }, + }; + const workspaceManager = { + resolveCommit: async () => SHA, + prepare: async () => ({ root: workspace, created: true, needsInstall: false }), + } as unknown as GitWorkspaceManager; + + await expect( + upgradeReleaseController({ + sourceMode: 'COMMIT', + sourceRef: SHA, + workspaceManager, + buildRunner: { run: async () => ({ ok: true, exitCode: 0, output: '' }) }, + processManager, + config, + readinessTimeoutMs: 10, + }) + ).rejects.toThrow('new controller failed'); + expect(starts.at(-1)?.cwd).toBe('/srv/sammo/old/app/release-controller'); + }); +}); diff --git a/app/release-controller/tsconfig.json b/app/release-controller/tsconfig.json new file mode 100644 index 0000000..fdf3380 --- /dev/null +++ b/app/release-controller/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.paths.json", + "compilerOptions": { + "outDir": "dist", + "composite": true + }, + "include": ["src", "test", "*.ts"], + "references": [ + { "path": "../../packages/common" }, + { "path": "../../packages/infra" }, + { "path": "../gateway-api" } + ] +} diff --git a/app/release-controller/vitest.config.ts b/app/release-controller/vitest.config.ts new file mode 100644 index 0000000..c324e94 --- /dev/null +++ b/app/release-controller/vitest.config.ts @@ -0,0 +1,12 @@ +import path from 'node:path'; + +import { defineConfig } from 'vitest/config'; +import tsconfigPaths from 'vite-tsconfig-paths'; + +export default defineConfig({ + plugins: [tsconfigPaths({ projects: [path.resolve(__dirname, '../../tsconfig.paths.json')] })], + test: { + environment: 'node', + include: ['test/**/*.test.ts'], + }, +}); diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 266722f..7f9834b 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -6,6 +6,8 @@ | -------------------- | -------------------------------------------------------- | ------------------------------------------- | | gateway API | `app/gateway-api/src/server.ts` | 계정, session, profile, admin operation | | gateway orchestrator | `app/gateway-api/src/orchestrator/orchestratorServer.ts` | DB queue, build, PM2 reconciliation | +| release controller | `app/release-controller/src/index.ts` | Gateway 전체 릴리스와 controller CLI 전환 | +| game frontend | Vite preview | profile별 commit frontend artifact | | game API | `app/game-api/src/server.ts` | profile tRPC, SSE, worker transport | | turn daemon | `app/game-engine/src/turn/cli.ts` | schedule, command, 월간 lifecycle, DB flush | | battle worker | `app/game-api/src/battleSim/worker.ts` | 격리된 전투 시뮬레이션 | @@ -26,6 +28,7 @@ Gateway API는 다음 저장 경계를 사용합니다. - `AppUser`, `SystemSetting`: 계정과 정책 - `GatewayProfile`: profile, scenario, port, 상태와 build 결과 - `GatewayOperation`: build/reset/open/close 등 실행 요청과 결과 +- `GatewayReleaseOperation`, `GatewayReleaseState`: Gateway 전체 릴리스 queue와 현재·이전 commit - `GatewayRuntimeAction`: profile별 시간 가속·연기 요청, 부분 적용과 최종 결과 - Redis: gateway session, OAuth 임시 상태, flush channel @@ -34,6 +37,15 @@ Orchestrator는 `GatewayOperation`을 claim하고 source ref를 commit으로 artifact를 만들며 `Pm2ProcessManager`가 profile process를 조정합니다. 재시작 시 DB 상태와 process 상태를 reconciliation합니다. +Profile `DEPLOY` operation은 현재 game schema와 시즌 데이터를 유지한 채 선택 +commit의 game API, engine과 profile 전용 frontend artifact를 빌드합니다. 기존 +프로세스를 멈춘 뒤 `prisma migrate deploy`만 실행하고 seed는 호출하지 않습니다. +새 API·frontend와 모든 worker가 PM2 `online`이고 HTTP readiness가 성공해야 +build commit을 게시합니다. 실패하면 이전 worktree 프로세스를 다시 시작합니다. +`RESET` operation은 같은 build 경계를 사용한 뒤 현재 시즌 테이블을 seed로 +교체합니다. Seeder의 reset 목록에는 `hall`, `ng_games`, `yearbook_history`, +과거 장수·국가와 상속·진단 자료가 포함되지 않습니다. + 시간 가속·연기는 일반 profile meta log가 아니라 UUID가 있는 `GatewayRuntimeAction`으로 접수합니다. Profile별 `REQUESTED`/`PARTIAL`은 DB partial unique index로 한 건만 허용합니다. Turn daemon은 자신의 lease를 @@ -137,6 +149,15 @@ Gateway operation은 source commit, worktree, build artifact와 process를 연결합니다. `tools/build-scripts/build-server.mjs`는 profile resource 복사만 담당합니다. +Gateway API·frontend·orchestrator 자신을 Gateway orchestrator가 교체하면 +작업 중인 실행자가 사라질 수 있습니다. 따라서 `app/release-controller`가 +별도 PM2 process로 `GatewayReleaseOperation`을 claim합니다. 선택 worktree의 +`release-manifest.json`에서 controller protocol, component와 gateway/game +migration head를 확인하고 build와 gateway migration을 마친 뒤 Gateway 세 +process를 전환합니다. HTTP와 PM2 readiness 실패 시 이전 Gateway worktree를 +복구하고 성공한 경우에만 `GatewayReleaseState`의 현재·이전 commit을 바꿉니다. +Controller 자체 갱신은 별도 CLI process의 `self-upgrade` 명령이 수행합니다. + 외부 공개 경로는 `/gateway/`, `/che/`, `/hwe/`입니다. frontend base, tRPC, SSE, upload와 direct navigation은 해당 prefix를 유지합니다. `/image/*`는 Caddy의 별도 파일 시스템 경로입니다. diff --git a/docs/e2e-orchestrator-tests.md b/docs/e2e-orchestrator-tests.md index e464cdc..ca23a31 100644 --- a/docs/e2e-orchestrator-tests.md +++ b/docs/e2e-orchestrator-tests.md @@ -10,6 +10,9 @@ commit worktree, build와 PM2 process를 조정합니다. 검증은 다음 경 process plan 계산 - `orchestratorOperations.test.ts`: operation claim, 상태 전이와 오류 - `adminOperations.test.ts`: 관리자 API와 operation 생성 +- `profileDeployOperation.test.ts`: DB 보존 배포에서 migration은 실행하고 seed는 실행하지 않는 계약 +- `gatewayReleaseRepository.integration.test.ts`: release lease, 단일 active queue와 publish fencing +- `app/release-controller/test/releaseController.test.ts`: Gateway 세 역할 전환·복구와 controller self-upgrade - `workspaceManager.test.ts`: source commit과 worktree - `app/gateway-frontend/e2e/server-operations.spec.ts`: 관리자 화면 - `app/gateway-frontend/e2e/hwe-lifecycle.spec.ts`: reset/build/seed/PM2와 @@ -80,6 +83,7 @@ HTTPS·host·firewall 검증을 구분합니다. - operation의 requested/running/succeeded 또는 failed 상태 - resolved commit과 build workspace - `sammo::game-api`, `sammo::turn-daemon` process +- `sammo::game-frontend`와 API·daemon·worker 전체 process - game API와 daemon의 profile name 일치 - 관리자 capability와 일반 사용자 거부 - 두 사용자의 서로 분리된 session·장수 소유권 diff --git a/packages/infra/prisma/gateway-migrations/20260801000000_add_release_operations/migration.sql b/packages/infra/prisma/gateway-migrations/20260801000000_add_release_operations/migration.sql new file mode 100644 index 0000000..f328fe8 --- /dev/null +++ b/packages/infra/prisma/gateway-migrations/20260801000000_add_release_operations/migration.sql @@ -0,0 +1,47 @@ +ALTER TYPE "GatewayOperationType" ADD VALUE IF NOT EXISTS 'DEPLOY'; + +CREATE TYPE "GatewayReleaseOperationType" AS ENUM ('DEPLOY', 'ROLLBACK'); + +CREATE TABLE "gateway_release_state" ( + "id" TEXT NOT NULL DEFAULT 'gateway', + "active_commit_sha" TEXT, + "active_workspace" TEXT, + "previous_commit_sha" TEXT, + "previous_workspace" TEXT, + "last_successful_at" TIMESTAMP(3), + "last_error" TEXT, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "gateway_release_state_pkey" PRIMARY KEY ("id") +); + +CREATE TABLE "gateway_release_operation" ( + "id" TEXT NOT NULL, + "type" "GatewayReleaseOperationType" NOT NULL, + "status" "GatewayOperationStatus" NOT NULL DEFAULT 'QUEUED', + "source_mode" "GatewaySourceMode", + "source_ref" TEXT, + "resolved_commit_sha" TEXT, + "payload" JSONB NOT NULL DEFAULT '{}'::jsonb, + "reason" TEXT, + "requested_by" TEXT NOT NULL, + "started_at" TIMESTAMP(3), + "completed_at" TIMESTAMP(3), + "error" TEXT, + "lease_owner" TEXT, + "lease_until" TIMESTAMPTZ, + "heartbeat_at" TIMESTAMPTZ, + "attempts" INTEGER NOT NULL DEFAULT 0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "gateway_release_operation_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX "gateway_release_operation_status_lease_until_created_at_idx" + ON "gateway_release_operation" ("status", "lease_until", "created_at"); +CREATE INDEX "gateway_release_operation_created_at_idx" + ON "gateway_release_operation" ("created_at"); +CREATE UNIQUE INDEX "gateway_release_operation_one_active_idx" + ON "gateway_release_operation" ((1)) + WHERE "status" IN ('QUEUED', 'RUNNING'); diff --git a/packages/infra/prisma/gateway.prisma b/packages/infra/prisma/gateway.prisma index 7d65690..76ff15b 100644 --- a/packages/infra/prisma/gateway.prisma +++ b/packages/infra/prisma/gateway.prisma @@ -33,10 +33,16 @@ enum GatewayBuildStatus { enum GatewayOperationType { RESET + DEPLOY START STOP } +enum GatewayReleaseOperationType { + DEPLOY + ROLLBACK +} + enum GatewayOperationStatus { QUEUED RUNNING @@ -204,6 +210,46 @@ model GatewayOperation { @@map("gateway_operation") } +model GatewayReleaseState { + id String @id @default("gateway") + activeCommitSha String? @map("active_commit_sha") + activeWorkspace String? @map("active_workspace") + previousCommitSha String? @map("previous_commit_sha") + previousWorkspace String? @map("previous_workspace") + lastSuccessfulAt DateTime? @map("last_successful_at") + lastError String? @map("last_error") + updatedAt DateTime @updatedAt @map("updated_at") + + @@map("gateway_release_state") +} + +model GatewayReleaseOperation { + /// A partial unique index in the gateway migration chain permits only one + /// QUEUED or RUNNING control-plane release at a time. + id String @id @default(uuid()) + type GatewayReleaseOperationType + status GatewayOperationStatus @default(QUEUED) + sourceMode GatewaySourceMode? @map("source_mode") + sourceRef String? @map("source_ref") + resolvedCommitSha String? @map("resolved_commit_sha") + payload Json @default(dbgenerated("'{}'::jsonb")) + reason String? + requestedBy String @map("requested_by") + startedAt DateTime? @map("started_at") + completedAt DateTime? @map("completed_at") + error String? + leaseOwner String? @map("lease_owner") + leaseUntil DateTime? @map("lease_until") + heartbeatAt DateTime? @map("heartbeat_at") + attempts Int @default(0) + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + @@index([status, leaseUntil, createdAt]) + @@index([createdAt]) + @@map("gateway_release_operation") +} + model SystemSetting { id Int @id @default(1) @map("no") registrationEnabled Boolean @default(false) @map("registration_enabled") diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d59a58f..be22db2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -375,6 +375,25 @@ importers: specifier: ^3.2.1 version: 3.2.1(typescript@6.0.2) + app/release-controller: + dependencies: + '@sammo-ts/gateway-api': + specifier: workspace:* + version: link:../gateway-api + '@sammo-ts/infra': + specifier: workspace:* + version: link:../../packages/infra + devDependencies: + tsdown: + specifier: ^0.22.14 + version: 0.22.14(@volar/typescript@2.4.27)(tsx@4.21.0)(typescript@6.0.2)(unrun@0.2.22(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)(synckit@0.11.13))(vue-tsc@3.2.2(typescript@6.0.2)) + vite-tsconfig-paths: + specifier: ^6.0.3 + version: 6.0.3(supports-color@7.2.0)(typescript@6.0.2)(vite@7.3.1(@types/node@26.1.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)) + vitest: + specifier: ^4.0.16 + version: 4.0.16(@types/node@26.1.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0) + packages/common: dependencies: '@noble/hashes': diff --git a/release-manifest.json b/release-manifest.json new file mode 100644 index 0000000..cbd6018 --- /dev/null +++ b/release-manifest.json @@ -0,0 +1,7 @@ +{ + "formatVersion": 1, + "controllerProtocol": 1, + "gatewaySchemaHead": "20260801000000_add_release_operations", + "gameSchemaHead": "20260731001000_add_unification_finalization", + "components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"] +} diff --git a/tools/conditional-integration-registry.tsv b/tools/conditional-integration-registry.tsv index 579247e..f687012 100644 --- a/tools/conditional-integration-registry.tsv +++ b/tools/conditional-integration-registry.tsv @@ -1,12 +1,16 @@ # Environment variable Execution mode CREATE_GENERAL_DATABASE_URL create_general GATEWAY_RUNTIME_ACTION_DATABASE_URL gateway_runtime +GATEWAY_OPERATION_DATABASE_URL gateway_runtime +GATEWAY_RELEASE_DATABASE_URL gateway_runtime GENERAL_LIFECYCLE_DATABASE_URL core IMMEDIATE_ACTION_DATABASE_URL immediate_action INPUT_EVENT_DATABASE_URL core LIVE_SORTIE_PERSISTENCE_DATABASE_URL reference_live_sortie NPC_POSSESSION_DATABASE_URL npc_possession NPC_POSSESSION_DIFFERENTIAL_DATABASE_URL reference_npc_possession +PROFILE_SEED_CLI_DATABASE_URL core +PROFILE_SEED_DATABASE_URL core RESERVED_TURN_DATABASE_URL core SELECT_POOL_DATABASE_URL select_pool TURN_DAEMON_LEASE_DATABASE_URL core diff --git a/tools/run-conditional-integration.sh b/tools/run-conditional-integration.sh index 69d86dd..9a2122e 100755 --- a/tools/run-conditional-integration.sh +++ b/tools/run-conditional-integration.sh @@ -470,6 +470,8 @@ export GENERAL_LIFECYCLE_DATABASE_URL=$database_url export TURN_DAEMON_LEASE_DATABASE_URL=$database_url export TURN_DIFFERENTIAL_DATABASE_URL=$database_url export RESERVED_TURN_DATABASE_URL=$database_url +export PROFILE_SEED_CLI_DATABASE_URL=$database_url +export PROFILE_SEED_DATABASE_URL=$database_url pnpm --filter @sammo-ts/infra prisma:db:push:game @@ -526,9 +528,12 @@ gateway_runtime_database_url=$(build_database_url "$gateway_runtime_schema") export POSTGRES_SCHEMA=$gateway_runtime_schema export DATABASE_URL=$gateway_runtime_database_url export GATEWAY_DATABASE_URL=$gateway_runtime_database_url - pnpm --filter @sammo-ts/infra prisma:db:push:gateway + pnpm --filter @sammo-ts/infra prisma:migrate:deploy:gateway + pnpm --filter @sammo-ts/infra prisma:migrate:deploy:gateway ) export GATEWAY_RUNTIME_ACTION_DATABASE_URL=$gateway_runtime_database_url +export GATEWAY_OPERATION_DATABASE_URL=$gateway_runtime_database_url +export GATEWAY_RELEASE_DATABASE_URL=$gateway_runtime_database_url export GATEWAY_RUNTIME_INTEGRATION_SCHEMA=$gateway_runtime_schema run_marked_tests app/gateway-api \ "$(markers_for_mode gateway_runtime)" \ diff --git a/tsconfig.json b/tsconfig.json index 7c6f81d..c89e04c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,6 +9,7 @@ { "path": "./app/game-engine" }, { "path": "./app/game-frontend" }, { "path": "./app/gateway-api" }, + { "path": "./app/release-controller" }, { "path": "./app/gateway-frontend" } ] }