diff --git a/app/game-api/src/daemon/types.ts b/app/game-api/src/daemon/types.ts index 389fc9e..f0e44dc 100644 --- a/app/game-api/src/daemon/types.ts +++ b/app/game-api/src/daemon/types.ts @@ -28,6 +28,7 @@ export interface TurnDaemonStatus { state: TurnDaemonState; running: boolean; paused: boolean; + lastError?: string; lastRunAt?: string; lastDurationMs?: number; lastTurnTime?: string; diff --git a/app/game-engine/package.json b/app/game-engine/package.json index e309009..79457e6 100644 --- a/app/game-engine/package.json +++ b/app/game-engine/package.json @@ -15,6 +15,7 @@ "@sammo-ts/common": "workspace:*", "@sammo-ts/infra": "workspace:*", "@sammo-ts/logic": "workspace:*", + "@prisma/client": "^7.2.0", "zod": "^4.2.1" }, "devDependencies": { diff --git a/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts b/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts index caa8301..1e47b42 100644 --- a/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts +++ b/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts @@ -31,6 +31,7 @@ export interface TurnDaemonLifecycleDeps { stateStore: TurnStateStore; processor: TurnProcessor; hooks?: TurnDaemonHooks; + pauseGate?: () => Promise; } export class TurnDaemonLifecycle { @@ -41,12 +42,15 @@ export class TurnDaemonLifecycle { private readonly stateStore: TurnStateStore; private readonly processor: TurnProcessor; private readonly hooks?: TurnDaemonHooks; + private readonly pauseGate?: () => Promise; private readonly options: TurnDaemonLifecycleOptions; private status: TurnDaemonStatus; private pendingRun: PendingRun | null = null; private stopping = false; private loopPromise: Promise | null = null; + private manualPaused = false; + private errorPaused = false; constructor(deps: TurnDaemonLifecycleDeps, options: TurnDaemonLifecycleOptions) { this.clock = deps.clock; @@ -55,6 +59,7 @@ export class TurnDaemonLifecycle { this.stateStore = deps.stateStore; this.processor = deps.processor; this.hooks = deps.hooks; + this.pauseGate = deps.pauseGate; this.options = options; this.status = { state: 'idle', @@ -109,10 +114,24 @@ export class TurnDaemonLifecycle { if (this.stopping) { break; } + const gatePaused = (await this.pauseGate?.()) ?? false; + if (this.errorPaused && !gatePaused) { + this.errorPaused = false; + this.status.lastError = undefined; + } + this.status.paused = this.manualPaused || gatePaused || this.errorPaused; if (this.status.paused) { - await this.waitForResume(); + this.status.state = 'paused'; + if (this.manualPaused) { + await this.waitForResume(); + } else { + await this.clock.sleepMs(500); + } continue; } + if (this.status.state === 'paused') { + this.status.state = 'idle'; + } if (this.pendingRun) { await this.runOnce(this.pendingRun); @@ -186,11 +205,13 @@ export class TurnDaemonLifecycle { private async handleCommand(command: TurnDaemonCommand): Promise { switch (command.type) { case 'pause': + this.manualPaused = true; this.status.paused = true; this.status.state = 'paused'; return; case 'resume': - this.status.paused = false; + this.manualPaused = false; + this.status.paused = this.errorPaused; this.status.state = 'idle'; return; case 'shutdown': @@ -221,6 +242,15 @@ export class TurnDaemonLifecycle { try { result = await this.processor.run(targetTime, budget, checkpoint); + } catch (error) { + this.status.running = false; + this.status.state = 'paused'; + this.status.paused = true; + this.errorPaused = true; + this.status.lastError = + error instanceof Error ? error.message : 'Unknown turn daemon error.'; + await this.hooks?.onRunError?.(error); + return; } finally { this.status.running = false; } diff --git a/app/game-engine/src/lifecycle/types.ts b/app/game-engine/src/lifecycle/types.ts index 58fd90b..51ea5aa 100644 --- a/app/game-engine/src/lifecycle/types.ts +++ b/app/game-engine/src/lifecycle/types.ts @@ -28,6 +28,7 @@ export interface TurnDaemonStatus { state: TurnDaemonState; running: boolean; paused: boolean; + lastError?: string; lastRunAt?: string; lastDurationMs?: number; lastTurnTime?: string; @@ -70,4 +71,5 @@ export interface TurnDaemonControlQueue { export interface TurnDaemonHooks { flushChanges?(result: TurnRunResult): Promise; publishEvents?(result: TurnRunResult): Promise; + onRunError?(error: unknown): Promise; } diff --git a/app/game-engine/src/turn/cli.ts b/app/game-engine/src/turn/cli.ts index d0da412..ce3e2b1 100644 --- a/app/game-engine/src/turn/cli.ts +++ b/app/game-engine/src/turn/cli.ts @@ -6,6 +6,8 @@ import { createTurnDaemonRuntime } from './turnDaemon.js'; export interface TurnDaemonCliOptions { profile?: string; + profileName?: string; + scenario?: string; databaseUrl?: string; tickMinutes?: number; schedule?: TurnSchedule; @@ -71,6 +73,11 @@ export const runTurnDaemonCli = async ( const env = options.env ?? process.env; const profile = options.profile ?? env.TURN_PROFILE ?? env.PROFILE ?? 'che'; + const scenario = options.scenario ?? env.TURN_SCENARIO ?? env.SCENARIO; + const profileName = + options.profileName ?? + env.TURN_PROFILE_NAME ?? + (scenario ? `${profile}:${scenario}` : profile); const databaseUrl = options.databaseUrl ?? (await resolveDatabaseUrl({ env })); const budget = buildBudgetOverride(env, options.budget); @@ -80,14 +87,17 @@ export const runTurnDaemonCli = async ( options.enableDatabaseFlush ?? parseBoolean(env.TURN_FLUSH_DB) ?? true; + const pauseGateIntervalMs = parseNumber(env.TURN_PAUSE_GATE_MS); const runtime = await createTurnDaemonRuntime({ profile, + profileName, databaseUrl, defaultBudget: budget, tickMinutes, schedule: options.schedule, enableDatabaseFlush, + pauseGateIntervalMs, }); let closed = false; diff --git a/app/game-engine/src/turn/gatewayProfileGate.ts b/app/game-engine/src/turn/gatewayProfileGate.ts new file mode 100644 index 0000000..7c6c242 --- /dev/null +++ b/app/game-engine/src/turn/gatewayProfileGate.ts @@ -0,0 +1,78 @@ +import { createPostgresConnector } from '@sammo-ts/infra'; +import type { PrismaClient } from '@prisma/client'; + +export interface GatewayProfileGateOptions { + databaseUrl: string; + profileName: string; + cacheMs?: number; +} + +export interface GatewayProfileGate { + shouldPause(): Promise; + markPaused(error?: unknown): Promise; + close(): Promise; +} + +const DEFAULT_CACHE_MS = 2000; + +const isRunningStatus = (status: string | null | undefined): boolean => + status === 'RUNNING'; + +export const createGatewayProfileGate = async ( + options: GatewayProfileGateOptions +): Promise => { + const connector = createPostgresConnector({ url: options.databaseUrl }); + await connector.connect(); + const prisma = connector.prisma as PrismaClient; + let lastCheckedAt = 0; + let cachedPause = false; + + const loadStatus = async (): Promise => { + try { + const profile = await prisma.gatewayProfile.findUnique({ + where: { profileName: options.profileName }, + }); + if (!profile) { + return false; + } + return !isRunningStatus(profile.status); + } catch { + return false; + } + }; + + return { + // 게이트웨이 프로필 상태를 읽어 턴 실행을 멈춰야 하는지 판단한다. + async shouldPause(): Promise { + const now = Date.now(); + if (now - lastCheckedAt < (options.cacheMs ?? DEFAULT_CACHE_MS)) { + return cachedPause; + } + cachedPause = await loadStatus(); + lastCheckedAt = now; + return cachedPause; + }, + async markPaused(error?: unknown): Promise { + const message = + error instanceof Error + ? error.message + : error + ? String(error) + : null; + try { + await prisma.gatewayProfile.update({ + where: { profileName: options.profileName }, + data: { + status: 'PAUSED', + lastError: message, + }, + }); + } catch { + return; + } + }, + async close(): Promise { + await connector.disconnect(); + }, + }; +}; diff --git a/app/game-engine/src/turn/turnDaemon.ts b/app/game-engine/src/turn/turnDaemon.ts index 93e67c9..b63c73e 100644 --- a/app/game-engine/src/turn/turnDaemon.ts +++ b/app/game-engine/src/turn/turnDaemon.ts @@ -20,12 +20,14 @@ import type { import { InMemoryTurnWorld } from './inMemoryWorld.js'; import { InMemoryTurnProcessor } from './inMemoryTurnProcessor.js'; import { InMemoryTurnStateStore } from './inMemoryStateStore.js'; +import { createGatewayProfileGate } from './gatewayProfileGate.js'; import { createReservedTurnHandler } from './reservedTurnHandler.js'; import { createReservedTurnStore } from './reservedTurnStore.js'; import { loadTurnWorldFromDatabase } from './worldLoader.js'; export interface TurnDaemonRuntimeOptions { profile: string; + profileName?: string; databaseUrl: string; defaultBudget?: TurnRunBudget; clock?: Clock; @@ -36,6 +38,7 @@ export interface TurnDaemonRuntimeOptions { generalTurnHandler?: GeneralTurnHandler; calendarHandler?: TurnCalendarHandler; enableDatabaseFlush?: boolean; + pauseGateIntervalMs?: number; } export interface TurnDaemonRuntime { @@ -119,19 +122,55 @@ export const createTurnDaemonRuntime = async ( let hooks: TurnDaemonHooks | undefined; let close = async () => {}; + let pauseGate: (() => Promise) | undefined; + const gatewayGate = + options.profileName + ? await createGatewayProfileGate({ + databaseUrl: options.databaseUrl, + profileName: options.profileName, + cacheMs: options.pauseGateIntervalMs, + }) + : null; + if (gatewayGate) { + pauseGate = gatewayGate.shouldPause; + } if (options.enableDatabaseFlush ?? true) { const dbHooks = await createDatabaseTurnHooks(options.databaseUrl, world, { reservedTurns: reservedTurnStoreHandle?.store, }); - hooks = dbHooks.hooks; + hooks = { + ...dbHooks.hooks, + onRunError: async (error) => { + await dbHooks.hooks.onRunError?.(error); + await gatewayGate?.markPaused(error); + }, + }; close = async () => { await dbHooks.close(); if (reservedTurnStoreHandle) { await reservedTurnStoreHandle.close(); } + await gatewayGate?.close(); }; } else if (reservedTurnStoreHandle) { - close = async () => reservedTurnStoreHandle.close(); + hooks = { + onRunError: async (error) => { + await gatewayGate?.markPaused(error); + }, + }; + close = async () => { + await reservedTurnStoreHandle.close(); + await gatewayGate?.close(); + }; + } else if (gatewayGate) { + hooks = { + onRunError: async (error) => { + await gatewayGate?.markPaused(error); + }, + }; + close = async () => { + await gatewayGate.close(); + }; } const defaultBudget: TurnRunBudget = options.defaultBudget ?? { @@ -149,6 +188,7 @@ export const createTurnDaemonRuntime = async ( stateStore, processor, hooks, + pauseGate, }, { profile: options.profile, defaultBudget } ); diff --git a/app/gateway-api/package.json b/app/gateway-api/package.json index 07a2791..f764812 100644 --- a/app/gateway-api/package.json +++ b/app/gateway-api/package.json @@ -6,6 +6,7 @@ "scripts": { "build": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api", "dev": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api --watch", + "orchestrator": "GATEWAY_ROLE=orchestrator node dist/index.js", "lint": "node -e \"console.log('lint not configured')\"", "test": "vitest run --config vitest.config.ts", "typecheck": "tsc --noEmit" diff --git a/app/gateway-api/src/adminRouter.ts b/app/gateway-api/src/adminRouter.ts index 733fab5..13e7d51 100644 --- a/app/gateway-api/src/adminRouter.ts +++ b/app/gateway-api/src/adminRouter.ts @@ -58,7 +58,10 @@ export const adminRouter = router({ scenario: z.string().min(1).max(64), apiPort: z.number().int().min(1).max(65535), status: zProfileStatus.optional(), + preopenAt: z.string().datetime().optional(), + openAt: z.string().datetime().optional(), scheduledStartAt: z.string().datetime().optional(), + buildCommitSha: z.string().min(7).max(64).optional(), }) ) .mutation(async ({ ctx, input }) => { @@ -68,7 +71,10 @@ export const adminRouter = router({ scenario: input.scenario, apiPort: input.apiPort, status, + preopenAt: input.preopenAt, + openAt: input.openAt, scheduledStartAt: input.scheduledStartAt, + buildCommitSha: input.buildCommitSha, }); }), setStatus: adminProcedure @@ -76,21 +82,34 @@ export const adminRouter = router({ z.object({ profileName: z.string().min(1), status: zProfileStatus, + preopenAt: z.string().datetime().optional(), + openAt: z.string().datetime().optional(), scheduledStartAt: z.string().datetime().optional(), + buildCommitSha: z.string().min(7).max(64).optional(), }) ) .mutation(async ({ ctx, input }) => { - if (input.status === 'RESERVED' && !input.scheduledStartAt) { + if (input.status === 'RESERVED' && (!input.preopenAt || !input.openAt)) { throw new TRPCError({ code: 'BAD_REQUEST', - message: 'scheduledStartAt is required for RESERVED status.', + message: 'preopenAt and openAt are required for RESERVED status.', }); } const result = await ctx.profiles.updateStatus( input.profileName, input.status, - input.status === 'RESERVED' ? input.scheduledStartAt : null + { + preopenAt: input.preopenAt, + openAt: input.openAt, + scheduledStartAt: + input.status === 'RESERVED' ? input.scheduledStartAt : null, + } ); + if (input.buildCommitSha) { + await ctx.profiles.updateBuildStatus(input.profileName, 'IDLE', { + commitSha: input.buildCommitSha, + }); + } await ctx.orchestrator.reconcileNow(); return result; }), diff --git a/app/gateway-api/src/config.ts b/app/gateway-api/src/config.ts index 53e4fe6..63972f9 100644 --- a/app/gateway-api/src/config.ts +++ b/app/gateway-api/src/config.ts @@ -23,6 +23,16 @@ export interface GatewayApiConfig { worktreeRoot: string; } +export interface GatewayOrchestratorConfig { + redisKeyPrefix: string; + gameTokenSecret: string; + orchestratorReconcileIntervalMs: number; + orchestratorScheduleIntervalMs: number; + orchestratorBuildIntervalMs: number; + workspaceRootHint: string; + worktreeRoot: string; +} + const parseNumber = (value: string | undefined, fallback: number, label: string): number => { if (!value) { return fallback; @@ -85,7 +95,40 @@ export const resolveGatewayApiConfigFromEnv = ( kakaoRedirectUri, publicBaseUrl, adminToken: env.GATEWAY_ADMIN_TOKEN, - orchestratorEnabled: parseBoolean(env.GATEWAY_ORCHESTRATOR_ENABLED, true), + orchestratorEnabled: parseBoolean(env.GATEWAY_ORCHESTRATOR_ENABLED, false), + orchestratorReconcileIntervalMs: parseNumber( + env.GATEWAY_ORCHESTRATOR_RECONCILE_MS, + 15000, + 'GATEWAY_ORCHESTRATOR_RECONCILE_MS' + ), + orchestratorScheduleIntervalMs: parseNumber( + env.GATEWAY_ORCHESTRATOR_SCHEDULE_MS, + 5000, + 'GATEWAY_ORCHESTRATOR_SCHEDULE_MS' + ), + orchestratorBuildIntervalMs: parseNumber( + env.GATEWAY_ORCHESTRATOR_BUILD_MS, + 10000, + 'GATEWAY_ORCHESTRATOR_BUILD_MS' + ), + workspaceRootHint: env.GATEWAY_WORKSPACE_ROOT ?? process.cwd(), + worktreeRoot: + env.GATEWAY_WORKTREE_ROOT ?? + path.resolve(env.GATEWAY_WORKSPACE_ROOT ?? process.cwd(), '.worktrees'), + }; +}; + +export const resolveGatewayOrchestratorConfigFromEnv = ( + env: NodeJS.ProcessEnv = process.env +): GatewayOrchestratorConfig => { + const secret = env.GAME_TOKEN_SECRET ?? env.GATEWAY_TOKEN_SECRET ?? ''; + if (!secret) { + throw new Error('GAME_TOKEN_SECRET is required for game server processes.'); + } + const redisKeyPrefix = env.GATEWAY_REDIS_PREFIX ?? 'sammo:gateway'; + return { + redisKeyPrefix, + gameTokenSecret: secret, orchestratorReconcileIntervalMs: parseNumber( env.GATEWAY_ORCHESTRATOR_RECONCILE_MS, 15000, diff --git a/app/gateway-api/src/index.ts b/app/gateway-api/src/index.ts index c9ca0f5..40ab679 100644 --- a/app/gateway-api/src/index.ts +++ b/app/gateway-api/src/index.ts @@ -2,6 +2,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { runGatewayApiServer } from './server.js'; +import { runGatewayOrchestrator } from './orchestrator/orchestratorServer.js'; export * from './config.js'; export * from './context.js'; @@ -27,8 +28,11 @@ const isMain = (): boolean => { }; if (isMain()) { - runGatewayApiServer().catch((error) => { - console.error('[gateway-api] failed to start', error); + const role = process.env.GATEWAY_ROLE ?? 'api'; + const run = role === 'orchestrator' ? runGatewayOrchestrator : runGatewayApiServer; + run().catch((error) => { + const prefix = role === 'orchestrator' ? 'gateway-orchestrator' : 'gateway-api'; + console.error(`[${prefix}] failed to start`, error); process.exitCode = 1; }); } diff --git a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts index 56df369..e646e7a 100644 --- a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts +++ b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts @@ -54,7 +54,12 @@ export const planProfileReconcile = ( status: GatewayProfileStatus, runtime: ProfileRuntimeState ): { shouldStart: boolean; shouldStop: boolean } => { - if (status === 'RUNNING') { + if ( + status === 'RUNNING' || + status === 'PREOPEN' || + status === 'PAUSED' || + status === 'COMPLETED' + ) { return { shouldStart: !(runtime.apiRunning && runtime.daemonRunning), shouldStop: false, @@ -94,6 +99,7 @@ const buildProcessDefinitions = ( TURN_PROFILE: profile.profile, PROFILE: profile.profile, SCENARIO: profile.scenario, + TURN_PROFILE_NAME: profile.profileName, }; return { api: { @@ -221,18 +227,41 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { const now = this.now(); const due = await this.repository.listReservedToStart(now); for (const profile of due) { - try { - await this.repository.updateStatus( - profile.profileName, - 'RUNNING', - null - ); - await this.startProfile(profile); - } catch (error) { + if (!profile.preopenAt || !profile.openAt) { await this.repository.updateLastError( profile.profileName, - error instanceof Error ? error.message : 'Failed to start scheduled profile.' + 'Reserved profile is missing preopen/open schedule.' ); + continue; + } + if (!profile.buildCommitSha) { + await this.repository.updateLastError( + profile.profileName, + 'Reserved profile is missing build commit SHA.' + ); + continue; + } + const queued = + profile.buildStatus === 'QUEUED' || profile.buildStatus === 'RUNNING'; + if (!queued) { + await this.repository.updateBuildStatus(profile.profileName, 'QUEUED', { + requestedAt: now.toISOString(), + error: null, + commitSha: profile.buildCommitSha, + }); + } + } + const profiles = await this.repository.listProfiles(); + for (const profile of profiles) { + if ( + profile.status === 'PREOPEN' && + profile.openAt && + new Date(profile.openAt) <= now + ) { + await this.repository.updateStatus(profile.profileName, 'RUNNING', { + preopenAt: profile.preopenAt ?? null, + openAt: profile.openAt ?? null, + }); } } } finally { @@ -304,12 +333,24 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { completedAt, error: null, }); - if (queued.status !== 'RUNNING' && queued.status !== 'DISABLED') { + if (queued.status === 'RESERVED') { await this.repository.updateStatus( queued.profileName, - 'COMPLETED', - queued.scheduledStartAt ? queued.scheduledStartAt : null + queued.openAt && new Date(queued.openAt) <= this.now() + ? 'RUNNING' + : 'PREOPEN', + { + preopenAt: queued.preopenAt ?? null, + openAt: queued.openAt ?? null, + } ); + } else if (queued.status === 'PREOPEN' && queued.openAt) { + if (new Date(queued.openAt) <= this.now()) { + await this.repository.updateStatus(queued.profileName, 'RUNNING', { + preopenAt: queued.preopenAt ?? null, + openAt: queued.openAt ?? null, + }); + } } } else { await this.repository.updateBuildStatus(queued.profileName, 'FAILED', { diff --git a/app/gateway-api/src/orchestrator/orchestratorFactory.ts b/app/gateway-api/src/orchestrator/orchestratorFactory.ts new file mode 100644 index 0000000..b34438a --- /dev/null +++ b/app/gateway-api/src/orchestrator/orchestratorFactory.ts @@ -0,0 +1,52 @@ +import type { PrismaClient } from '@prisma/client'; + +import type { GatewayOrchestratorConfig } from '../config.js'; +import { createGatewayProfileRepository } from './profileRepository.js'; +import { GatewayOrchestrator } from './gatewayOrchestrator.js'; +import { Pm2ProcessManager } from './pm2ProcessManager.js'; +import { PnpmBuildRunner } from './buildRunner.js'; +import { resolveWorkspaceRoot } from './workspaceRoot.js'; +import { GitWorkspaceManager } from './workspaceManager.js'; + +export const buildEnvMap = (env: NodeJS.ProcessEnv): Record => { + const entries = Object.entries(env).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string' + ); + return Object.fromEntries(entries); +}; + +export const createGatewayOrchestrator = ( + prisma: PrismaClient, + config: GatewayOrchestratorConfig, + env: NodeJS.ProcessEnv = process.env +): { + orchestrator: GatewayOrchestrator; + profiles: ReturnType; +} => { + const profiles = createGatewayProfileRepository(prisma); + const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint); + const processManager = new Pm2ProcessManager(); + const buildRunner = new PnpmBuildRunner(); + const baseEnv = buildEnvMap(env); + const workspaceManager = new GitWorkspaceManager({ + repoRoot: workspaceRoot, + worktreeRoot: config.worktreeRoot, + baseEnv, + }); + const orchestrator = new GatewayOrchestrator({ + repository: profiles, + processManager, + buildRunner, + workspaceManager, + processConfig: { + workspaceRoot, + redisKeyPrefix: config.redisKeyPrefix, + gameTokenSecret: config.gameTokenSecret, + baseEnv, + }, + reconcileIntervalMs: config.orchestratorReconcileIntervalMs, + scheduleIntervalMs: config.orchestratorScheduleIntervalMs, + buildIntervalMs: config.orchestratorBuildIntervalMs, + }); + return { orchestrator, profiles }; +}; diff --git a/app/gateway-api/src/orchestrator/orchestratorServer.ts b/app/gateway-api/src/orchestrator/orchestratorServer.ts new file mode 100644 index 0000000..a20c33e --- /dev/null +++ b/app/gateway-api/src/orchestrator/orchestratorServer.ts @@ -0,0 +1,32 @@ +import type { PrismaClient } from '@prisma/client'; +import { + createPostgresConnector, + resolvePostgresConfigFromEnv, +} from '@sammo-ts/infra'; + +import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js'; +import { createGatewayOrchestrator } from './orchestratorFactory.js'; + +export const runGatewayOrchestrator = async (): Promise => { + const config = resolveGatewayOrchestratorConfigFromEnv(); + const postgres = createPostgresConnector(resolvePostgresConfigFromEnv()); + await postgres.connect(); + + const { orchestrator } = createGatewayOrchestrator( + postgres.prisma as PrismaClient, + config, + process.env + ); + + const stop = async (reason: string): Promise => { + console.info(`[gateway-orchestrator] stopping: ${reason}`); + await orchestrator.stop(); + await postgres.disconnect(); + }; + + process.on('SIGINT', () => void stop('SIGINT')); + process.on('SIGTERM', () => void stop('SIGTERM')); + + orchestrator.start(); + console.info('[gateway-orchestrator] started'); +}; diff --git a/app/gateway-api/src/orchestrator/profileRepository.ts b/app/gateway-api/src/orchestrator/profileRepository.ts index 2a728a2..5a8ff5e 100644 --- a/app/gateway-api/src/orchestrator/profileRepository.ts +++ b/app/gateway-api/src/orchestrator/profileRepository.ts @@ -1,9 +1,11 @@ import { Prisma, type PrismaClient } from '@prisma/client'; export const GATEWAY_PROFILE_STATUSES = [ - 'COMPLETED', 'RESERVED', + 'PREOPEN', 'RUNNING', + 'PAUSED', + 'COMPLETED', 'STOPPED', 'DISABLED', ] as const; @@ -28,6 +30,8 @@ export interface GatewayProfileRecord { buildCommitSha?: string; buildWorkspace?: string; buildLastUsedAt?: string; + preopenAt?: string; + openAt?: string; scheduledStartAt?: string; buildRequestedAt?: string; buildStartedAt?: string; @@ -44,7 +48,10 @@ export interface GatewayProfileUpsertInput { scenario: string; apiPort: number; status?: GatewayProfileStatus; + preopenAt?: string; + openAt?: string; scheduledStartAt?: string; + buildCommitSha?: string; meta?: Prisma.JsonObject; } @@ -55,7 +62,11 @@ export interface GatewayProfileRepository { updateStatus( profileName: string, status: GatewayProfileStatus, - scheduledStartAt?: string | null + schedule?: { + preopenAt?: string | null; + openAt?: string | null; + scheduledStartAt?: string | null; + } ): Promise; updateBuildStatus( profileName: string, @@ -94,6 +105,8 @@ const mapProfile = (row: { buildCommitSha: string | null; buildWorkspace: string | null; buildLastUsedAt: Date | null; + preopenAt: Date | null; + openAt: Date | null; scheduledStartAt: Date | null; buildRequestedAt: Date | null; buildStartedAt: Date | null; @@ -113,6 +126,8 @@ const mapProfile = (row: { buildCommitSha: row.buildCommitSha ?? undefined, buildWorkspace: row.buildWorkspace ?? undefined, buildLastUsedAt: toIso(row.buildLastUsedAt), + preopenAt: toIso(row.preopenAt), + openAt: toIso(row.openAt), scheduledStartAt: toIso(row.scheduledStartAt), buildRequestedAt: toIso(row.buildRequestedAt), buildStartedAt: toIso(row.buildStartedAt), @@ -152,19 +167,36 @@ export const createGatewayProfileRepository = ( scenario: input.scenario, apiPort: input.apiPort, status: input.status ?? 'STOPPED', + preopenAt: input.preopenAt ? new Date(input.preopenAt) : null, + openAt: input.openAt ? new Date(input.openAt) : null, scheduledStartAt: input.scheduledStartAt ? new Date(input.scheduledStartAt) : null, + buildCommitSha: input.buildCommitSha ?? null, meta: (input.meta ?? {}) as Prisma.JsonObject, }, update: { apiPort: input.apiPort, status: input.status, + preopenAt: input.preopenAt + ? new Date(input.preopenAt) + : input.preopenAt === null + ? null + : undefined, + openAt: input.openAt + ? new Date(input.openAt) + : input.openAt === null + ? null + : undefined, scheduledStartAt: input.scheduledStartAt ? new Date(input.scheduledStartAt) : input.scheduledStartAt === null ? null : undefined, + buildCommitSha: + input.buildCommitSha === undefined + ? undefined + : input.buildCommitSha, meta: input.meta ? (input.meta as Prisma.JsonObject) : undefined, }, }); @@ -173,17 +205,33 @@ export const createGatewayProfileRepository = ( async updateStatus( profileName: string, status: GatewayProfileStatus, - scheduledStartAt?: string | null + schedule?: { + preopenAt?: string | null; + openAt?: string | null; + scheduledStartAt?: string | null; + } ): Promise { const row = await prisma.gatewayProfile.update({ where: { profileName }, data: { status, - scheduledStartAt: - scheduledStartAt === undefined + preopenAt: + schedule?.preopenAt === undefined ? undefined - : scheduledStartAt - ? new Date(scheduledStartAt) + : schedule?.preopenAt + ? new Date(schedule.preopenAt) + : null, + openAt: + schedule?.openAt === undefined + ? undefined + : schedule?.openAt + ? new Date(schedule.openAt) + : null, + scheduledStartAt: + schedule?.scheduledStartAt === undefined + ? undefined + : schedule?.scheduledStartAt + ? new Date(schedule.scheduledStartAt) : null, }, }); @@ -240,7 +288,7 @@ export const createGatewayProfileRepository = ( const rows = await prisma.gatewayProfile.findMany({ where: { status: 'RESERVED', - scheduledStartAt: { + preopenAt: { lte: now, }, }, diff --git a/app/gateway-api/src/server.ts b/app/gateway-api/src/server.ts index 0eacec3..0634a44 100644 --- a/app/gateway-api/src/server.ts +++ b/app/gateway-api/src/server.ts @@ -16,21 +16,9 @@ import { KakaoOAuthClient } from './auth/kakaoClient.js'; import { RedisOAuthSessionStore } from './auth/oauthSessionStore.js'; import { createPostgresUserRepository } from './auth/postgresUserRepository.js'; import { RedisGatewaySessionService } from './auth/redisSessionService.js'; -import { createGatewayProfileRepository } from './orchestrator/profileRepository.js'; -import { GatewayOrchestrator } from './orchestrator/gatewayOrchestrator.js'; -import { Pm2ProcessManager } from './orchestrator/pm2ProcessManager.js'; -import { PnpmBuildRunner } from './orchestrator/buildRunner.js'; -import { resolveWorkspaceRoot } from './orchestrator/workspaceRoot.js'; -import { GitWorkspaceManager } from './orchestrator/workspaceManager.js'; +import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js'; import { appRouter } from './router.js'; -const buildEnvMap = (env: NodeJS.ProcessEnv): Record => { - const entries = Object.entries(env).filter( - (entry): entry is [string, string] => typeof entry[1] === 'string' - ); - return Object.fromEntries(entries); -}; - export const createGatewayApiServer = async () => { const config = resolveGatewayApiConfigFromEnv(); const postgres = createPostgresConnector(resolvePostgresConfigFromEnv()); @@ -58,33 +46,11 @@ export const createGatewayApiServer = async () => { config.oauthSessionTtlSeconds ); - const profiles = createGatewayProfileRepository( - postgres.prisma as PrismaClient + const { orchestrator, profiles } = createGatewayOrchestrator( + postgres.prisma as PrismaClient, + config, + process.env ); - const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint); - const processManager = new Pm2ProcessManager(); - const buildRunner = new PnpmBuildRunner(); - const baseEnv = buildEnvMap(process.env); - const workspaceManager = new GitWorkspaceManager({ - repoRoot: workspaceRoot, - worktreeRoot: config.worktreeRoot, - baseEnv, - }); - const orchestrator = new GatewayOrchestrator({ - repository: profiles, - processManager, - buildRunner, - workspaceManager, - processConfig: { - workspaceRoot, - redisKeyPrefix: config.redisKeyPrefix, - gameTokenSecret: config.gameTokenSecret, - baseEnv, - }, - reconcileIntervalMs: config.orchestratorReconcileIntervalMs, - scheduleIntervalMs: config.orchestratorScheduleIntervalMs, - buildIntervalMs: config.orchestratorBuildIntervalMs, - }); const app = fastify({ logger: true, @@ -121,10 +87,6 @@ export const createGatewayApiServer = async () => { ok: true, })); - if (config.orchestratorEnabled) { - orchestrator.start(); - } - app.addHook('onClose', async () => { await orchestrator.stop(); await redis.disconnect(); diff --git a/app/gateway-api/test/orchestratorPlan.test.ts b/app/gateway-api/test/orchestratorPlan.test.ts index c0eb76a..3e7e299 100644 --- a/app/gateway-api/test/orchestratorPlan.test.ts +++ b/app/gateway-api/test/orchestratorPlan.test.ts @@ -12,6 +12,15 @@ describe('planProfileReconcile', () => { ).toEqual({ shouldStart: true, shouldStop: false }); }); + it('starts processes for preopen profiles', () => { + expect( + planProfileReconcile('PREOPEN', { + apiRunning: false, + daemonRunning: false, + }) + ).toEqual({ shouldStart: true, shouldStop: false }); + }); + it('does nothing when running profile is healthy', () => { expect( planProfileReconcile('RUNNING', { diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 1804e2a..bf3da22 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -20,7 +20,8 @@ selection is required because it drives unit sets and DB settings. ## Gateway Orchestration (Single Host Draft) Gateway API is the single source of truth for profile state and reconciles -PM2-managed processes on boot and on a short interval. The DB owns the desired +PM2-managed processes on boot and on a short interval. The orchestrator runs +as a separate process (`GATEWAY_ROLE=orchestrator`). The DB owns the desired state; PM2 is treated as the actuator. The runtime state is grouped so that `game-api` + `turn-daemon` are either on together or off together. @@ -29,10 +30,12 @@ state; PM2 is treated as the actuator. The runtime state is grouped so that Profiles are tracked by `profileName` (= `${profile}:${scenario}`). Gateway loads the profile table on boot, then reconciles PM2 to match. -- `완료됨` (COMPLETED): build output exists; processes should be off. -- `예약됨` (RESERVED): start is scheduled for a timestamp; processes off. +- `예약됨` (RESERVED): preopen/open timestamps are set; processes off. +- `가오픈` (PREOPEN): build done; API+daemon on, daemon paused. - `가동중` (RUNNING): both `game-api` and `turn-daemon` should be on. - `정지됨` (STOPPED): processes off; may be resumed later. +- `정지(오류)` (PAUSED): fatal error; daemon paused but process remains on. +- `천하통일` (COMPLETED): game finished; API on, daemon paused. - `비활성화` (DISABLED): excluded from orchestration; start forbidden. ### Boot Reconciliation @@ -40,29 +43,30 @@ Gateway loads the profile table on boot, then reconciles PM2 to match. 1) Load profile rows from DB. 2) List PM2 processes and map `profileName -> running state`. 3) For each profile: - - If desired `RUNNING` and any process is missing, start missing processes. - - If desired not `RUNNING` and any process is running, stop both processes. + - If desired `RUNNING/PREOPEN/PAUSED/COMPLETED` and any process is missing, start. + - If desired `RESERVED/STOPPED/DISABLED` and any process is running, stop both. 4) Persist errors to DB for audit. ### Internal Scheduler (Gateway Cron) Gateway runs a lightweight cron loop (setInterval) that: -- Promotes `RESERVED` profiles whose `scheduledStartAt <= now` to `RUNNING`. -- Triggers a reconcile immediately after promotion. +- When `RESERVED` and `preopenAt <= now`, queue a build for the reserved commit. +- When build succeeds, status becomes `PREOPEN` (daemon paused). +- When `openAt <= now`, status becomes `RUNNING` and daemon resumes. - Optionally drains a build queue (see build workflow). ### Build Workflow (Admin) - Admin triggers a build request for a profile. - Gateway queues a build job with `(profileName, commitSha)` and prepares a - per-commit workspace (`/var/sammo/workspaces/{commitSha}` recommended). + per-commit workspace (`/.worktrees/{commitSha}` by default). - Workspace is backed by `git worktree` and is reused across builds for the same commit. - Each workspace stores `lastUsedAt` in DB so cleanup can remove stale worktrees. - Cleanup is invoked manually by admin API and removes worktrees unused for 6+ months. - Build runs `pnpm install` when workspace is created, then executes `pnpm --filter @sammo-ts/game-api build` and `pnpm --filter @sammo-ts/game-engine build`, then marks build success/failure. -- On success, profile status remains `COMPLETED` (or stays `RUNNING` if already on). +- On success, status moves to `PREOPEN` for reserved builds or stays unchanged for manual builds. ## Current Implementation Status diff --git a/packages/infra/prisma/migrations/20250105000000_add_gateway_profile/migration.sql b/packages/infra/prisma/migrations/20250105000000_add_gateway_profile/migration.sql index 16676e1..a793f5b 100644 --- a/packages/infra/prisma/migrations/20250105000000_add_gateway_profile/migration.sql +++ b/packages/infra/prisma/migrations/20250105000000_add_gateway_profile/migration.sql @@ -1,8 +1,10 @@ -- Create enums for gateway profile tracking CREATE TYPE "GatewayProfileStatus" AS ENUM ( - 'COMPLETED', 'RESERVED', + 'PREOPEN', 'RUNNING', + 'PAUSED', + 'COMPLETED', 'STOPPED', 'DISABLED' ); @@ -26,6 +28,8 @@ CREATE TABLE "gateway_profile" ( "build_commit_sha" TEXT, "build_workspace" TEXT, "build_last_used_at" TIMESTAMP(3), + "preopen_at" TIMESTAMP(3), + "open_at" TIMESTAMP(3), "scheduled_start_at" TIMESTAMP(3), "build_requested_at" TIMESTAMP(3), "build_started_at" TIMESTAMP(3), @@ -39,4 +43,3 @@ CREATE TABLE "gateway_profile" ( CONSTRAINT "gateway_profile_pkey" PRIMARY KEY ("profile_name"), CONSTRAINT "gateway_profile_profile_scenario_key" UNIQUE ("profile", "scenario") ); - diff --git a/packages/infra/prisma/schema.prisma b/packages/infra/prisma/schema.prisma index aaca480..85cd7f9 100644 --- a/packages/infra/prisma/schema.prisma +++ b/packages/infra/prisma/schema.prisma @@ -28,9 +28,11 @@ enum OAuthType { } enum GatewayProfileStatus { - COMPLETED RESERVED + PREOPEN RUNNING + PAUSED + COMPLETED STOPPED DISABLED } @@ -72,6 +74,8 @@ model GatewayProfile { buildCommitSha String? @map("build_commit_sha") buildWorkspace String? @map("build_workspace") buildLastUsedAt DateTime? @map("build_last_used_at") + preopenAt DateTime? @map("preopen_at") + openAt DateTime? @map("open_at") scheduledStartAt DateTime? @map("scheduled_start_at") buildRequestedAt DateTime? @map("build_requested_at") buildStartedAt DateTime? @map("build_started_at")