feat: Enhance Gateway Orchestrator with preopen and open scheduling, add error handling for paused states, and implement new profile management features

This commit is contained in:
2026-01-01 11:27:10 +00:00
parent e64962c3fd
commit 8c9f61700e
20 changed files with 470 additions and 86 deletions
+1
View File
@@ -28,6 +28,7 @@ export interface TurnDaemonStatus {
state: TurnDaemonState; state: TurnDaemonState;
running: boolean; running: boolean;
paused: boolean; paused: boolean;
lastError?: string;
lastRunAt?: string; lastRunAt?: string;
lastDurationMs?: number; lastDurationMs?: number;
lastTurnTime?: string; lastTurnTime?: string;
+1
View File
@@ -15,6 +15,7 @@
"@sammo-ts/common": "workspace:*", "@sammo-ts/common": "workspace:*",
"@sammo-ts/infra": "workspace:*", "@sammo-ts/infra": "workspace:*",
"@sammo-ts/logic": "workspace:*", "@sammo-ts/logic": "workspace:*",
"@prisma/client": "^7.2.0",
"zod": "^4.2.1" "zod": "^4.2.1"
}, },
"devDependencies": { "devDependencies": {
@@ -31,6 +31,7 @@ export interface TurnDaemonLifecycleDeps {
stateStore: TurnStateStore; stateStore: TurnStateStore;
processor: TurnProcessor; processor: TurnProcessor;
hooks?: TurnDaemonHooks; hooks?: TurnDaemonHooks;
pauseGate?: () => Promise<boolean>;
} }
export class TurnDaemonLifecycle { export class TurnDaemonLifecycle {
@@ -41,12 +42,15 @@ export class TurnDaemonLifecycle {
private readonly stateStore: TurnStateStore; private readonly stateStore: TurnStateStore;
private readonly processor: TurnProcessor; private readonly processor: TurnProcessor;
private readonly hooks?: TurnDaemonHooks; private readonly hooks?: TurnDaemonHooks;
private readonly pauseGate?: () => Promise<boolean>;
private readonly options: TurnDaemonLifecycleOptions; private readonly options: TurnDaemonLifecycleOptions;
private status: TurnDaemonStatus; private status: TurnDaemonStatus;
private pendingRun: PendingRun | null = null; private pendingRun: PendingRun | null = null;
private stopping = false; private stopping = false;
private loopPromise: Promise<void> | null = null; private loopPromise: Promise<void> | null = null;
private manualPaused = false;
private errorPaused = false;
constructor(deps: TurnDaemonLifecycleDeps, options: TurnDaemonLifecycleOptions) { constructor(deps: TurnDaemonLifecycleDeps, options: TurnDaemonLifecycleOptions) {
this.clock = deps.clock; this.clock = deps.clock;
@@ -55,6 +59,7 @@ export class TurnDaemonLifecycle {
this.stateStore = deps.stateStore; this.stateStore = deps.stateStore;
this.processor = deps.processor; this.processor = deps.processor;
this.hooks = deps.hooks; this.hooks = deps.hooks;
this.pauseGate = deps.pauseGate;
this.options = options; this.options = options;
this.status = { this.status = {
state: 'idle', state: 'idle',
@@ -109,10 +114,24 @@ export class TurnDaemonLifecycle {
if (this.stopping) { if (this.stopping) {
break; break;
} }
const gatePaused = (await this.pauseGate?.()) ?? false;
if (this.errorPaused && !gatePaused) {
this.errorPaused = false;
this.status.lastError = undefined;
}
this.status.paused = this.manualPaused || gatePaused || this.errorPaused;
if (this.status.paused) { if (this.status.paused) {
await this.waitForResume(); this.status.state = 'paused';
if (this.manualPaused) {
await this.waitForResume();
} else {
await this.clock.sleepMs(500);
}
continue; continue;
} }
if (this.status.state === 'paused') {
this.status.state = 'idle';
}
if (this.pendingRun) { if (this.pendingRun) {
await this.runOnce(this.pendingRun); await this.runOnce(this.pendingRun);
@@ -186,11 +205,13 @@ export class TurnDaemonLifecycle {
private async handleCommand(command: TurnDaemonCommand): Promise<void> { private async handleCommand(command: TurnDaemonCommand): Promise<void> {
switch (command.type) { switch (command.type) {
case 'pause': case 'pause':
this.manualPaused = true;
this.status.paused = true; this.status.paused = true;
this.status.state = 'paused'; this.status.state = 'paused';
return; return;
case 'resume': case 'resume':
this.status.paused = false; this.manualPaused = false;
this.status.paused = this.errorPaused;
this.status.state = 'idle'; this.status.state = 'idle';
return; return;
case 'shutdown': case 'shutdown':
@@ -221,6 +242,15 @@ export class TurnDaemonLifecycle {
try { try {
result = await this.processor.run(targetTime, budget, checkpoint); result = await this.processor.run(targetTime, budget, checkpoint);
} catch (error) {
this.status.running = false;
this.status.state = 'paused';
this.status.paused = true;
this.errorPaused = true;
this.status.lastError =
error instanceof Error ? error.message : 'Unknown turn daemon error.';
await this.hooks?.onRunError?.(error);
return;
} finally { } finally {
this.status.running = false; this.status.running = false;
} }
+2
View File
@@ -28,6 +28,7 @@ export interface TurnDaemonStatus {
state: TurnDaemonState; state: TurnDaemonState;
running: boolean; running: boolean;
paused: boolean; paused: boolean;
lastError?: string;
lastRunAt?: string; lastRunAt?: string;
lastDurationMs?: number; lastDurationMs?: number;
lastTurnTime?: string; lastTurnTime?: string;
@@ -70,4 +71,5 @@ export interface TurnDaemonControlQueue {
export interface TurnDaemonHooks { export interface TurnDaemonHooks {
flushChanges?(result: TurnRunResult): Promise<void>; flushChanges?(result: TurnRunResult): Promise<void>;
publishEvents?(result: TurnRunResult): Promise<void>; publishEvents?(result: TurnRunResult): Promise<void>;
onRunError?(error: unknown): Promise<void>;
} }
+10
View File
@@ -6,6 +6,8 @@ import { createTurnDaemonRuntime } from './turnDaemon.js';
export interface TurnDaemonCliOptions { export interface TurnDaemonCliOptions {
profile?: string; profile?: string;
profileName?: string;
scenario?: string;
databaseUrl?: string; databaseUrl?: string;
tickMinutes?: number; tickMinutes?: number;
schedule?: TurnSchedule; schedule?: TurnSchedule;
@@ -71,6 +73,11 @@ export const runTurnDaemonCli = async (
const env = options.env ?? process.env; const env = options.env ?? process.env;
const profile = const profile =
options.profile ?? env.TURN_PROFILE ?? env.PROFILE ?? 'che'; options.profile ?? env.TURN_PROFILE ?? env.PROFILE ?? 'che';
const scenario = options.scenario ?? env.TURN_SCENARIO ?? env.SCENARIO;
const profileName =
options.profileName ??
env.TURN_PROFILE_NAME ??
(scenario ? `${profile}:${scenario}` : profile);
const databaseUrl = const databaseUrl =
options.databaseUrl ?? (await resolveDatabaseUrl({ env })); options.databaseUrl ?? (await resolveDatabaseUrl({ env }));
const budget = buildBudgetOverride(env, options.budget); const budget = buildBudgetOverride(env, options.budget);
@@ -80,14 +87,17 @@ export const runTurnDaemonCli = async (
options.enableDatabaseFlush ?? options.enableDatabaseFlush ??
parseBoolean(env.TURN_FLUSH_DB) ?? parseBoolean(env.TURN_FLUSH_DB) ??
true; true;
const pauseGateIntervalMs = parseNumber(env.TURN_PAUSE_GATE_MS);
const runtime = await createTurnDaemonRuntime({ const runtime = await createTurnDaemonRuntime({
profile, profile,
profileName,
databaseUrl, databaseUrl,
defaultBudget: budget, defaultBudget: budget,
tickMinutes, tickMinutes,
schedule: options.schedule, schedule: options.schedule,
enableDatabaseFlush, enableDatabaseFlush,
pauseGateIntervalMs,
}); });
let closed = false; let closed = false;
@@ -0,0 +1,78 @@
import { createPostgresConnector } from '@sammo-ts/infra';
import type { PrismaClient } from '@prisma/client';
export interface GatewayProfileGateOptions {
databaseUrl: string;
profileName: string;
cacheMs?: number;
}
export interface GatewayProfileGate {
shouldPause(): Promise<boolean>;
markPaused(error?: unknown): Promise<void>;
close(): Promise<void>;
}
const DEFAULT_CACHE_MS = 2000;
const isRunningStatus = (status: string | null | undefined): boolean =>
status === 'RUNNING';
export const createGatewayProfileGate = async (
options: GatewayProfileGateOptions
): Promise<GatewayProfileGate> => {
const connector = createPostgresConnector({ url: options.databaseUrl });
await connector.connect();
const prisma = connector.prisma as PrismaClient;
let lastCheckedAt = 0;
let cachedPause = false;
const loadStatus = async (): Promise<boolean> => {
try {
const profile = await prisma.gatewayProfile.findUnique({
where: { profileName: options.profileName },
});
if (!profile) {
return false;
}
return !isRunningStatus(profile.status);
} catch {
return false;
}
};
return {
// 게이트웨이 프로필 상태를 읽어 턴 실행을 멈춰야 하는지 판단한다.
async shouldPause(): Promise<boolean> {
const now = Date.now();
if (now - lastCheckedAt < (options.cacheMs ?? DEFAULT_CACHE_MS)) {
return cachedPause;
}
cachedPause = await loadStatus();
lastCheckedAt = now;
return cachedPause;
},
async markPaused(error?: unknown): Promise<void> {
const message =
error instanceof Error
? error.message
: error
? String(error)
: null;
try {
await prisma.gatewayProfile.update({
where: { profileName: options.profileName },
data: {
status: 'PAUSED',
lastError: message,
},
});
} catch {
return;
}
},
async close(): Promise<void> {
await connector.disconnect();
},
};
};
+42 -2
View File
@@ -20,12 +20,14 @@ import type {
import { InMemoryTurnWorld } from './inMemoryWorld.js'; import { InMemoryTurnWorld } from './inMemoryWorld.js';
import { InMemoryTurnProcessor } from './inMemoryTurnProcessor.js'; import { InMemoryTurnProcessor } from './inMemoryTurnProcessor.js';
import { InMemoryTurnStateStore } from './inMemoryStateStore.js'; import { InMemoryTurnStateStore } from './inMemoryStateStore.js';
import { createGatewayProfileGate } from './gatewayProfileGate.js';
import { createReservedTurnHandler } from './reservedTurnHandler.js'; import { createReservedTurnHandler } from './reservedTurnHandler.js';
import { createReservedTurnStore } from './reservedTurnStore.js'; import { createReservedTurnStore } from './reservedTurnStore.js';
import { loadTurnWorldFromDatabase } from './worldLoader.js'; import { loadTurnWorldFromDatabase } from './worldLoader.js';
export interface TurnDaemonRuntimeOptions { export interface TurnDaemonRuntimeOptions {
profile: string; profile: string;
profileName?: string;
databaseUrl: string; databaseUrl: string;
defaultBudget?: TurnRunBudget; defaultBudget?: TurnRunBudget;
clock?: Clock; clock?: Clock;
@@ -36,6 +38,7 @@ export interface TurnDaemonRuntimeOptions {
generalTurnHandler?: GeneralTurnHandler; generalTurnHandler?: GeneralTurnHandler;
calendarHandler?: TurnCalendarHandler; calendarHandler?: TurnCalendarHandler;
enableDatabaseFlush?: boolean; enableDatabaseFlush?: boolean;
pauseGateIntervalMs?: number;
} }
export interface TurnDaemonRuntime { export interface TurnDaemonRuntime {
@@ -119,19 +122,55 @@ export const createTurnDaemonRuntime = async (
let hooks: TurnDaemonHooks | undefined; let hooks: TurnDaemonHooks | undefined;
let close = async () => {}; let close = async () => {};
let pauseGate: (() => Promise<boolean>) | undefined;
const gatewayGate =
options.profileName
? await createGatewayProfileGate({
databaseUrl: options.databaseUrl,
profileName: options.profileName,
cacheMs: options.pauseGateIntervalMs,
})
: null;
if (gatewayGate) {
pauseGate = gatewayGate.shouldPause;
}
if (options.enableDatabaseFlush ?? true) { if (options.enableDatabaseFlush ?? true) {
const dbHooks = await createDatabaseTurnHooks(options.databaseUrl, world, { const dbHooks = await createDatabaseTurnHooks(options.databaseUrl, world, {
reservedTurns: reservedTurnStoreHandle?.store, reservedTurns: reservedTurnStoreHandle?.store,
}); });
hooks = dbHooks.hooks; hooks = {
...dbHooks.hooks,
onRunError: async (error) => {
await dbHooks.hooks.onRunError?.(error);
await gatewayGate?.markPaused(error);
},
};
close = async () => { close = async () => {
await dbHooks.close(); await dbHooks.close();
if (reservedTurnStoreHandle) { if (reservedTurnStoreHandle) {
await reservedTurnStoreHandle.close(); await reservedTurnStoreHandle.close();
} }
await gatewayGate?.close();
}; };
} else if (reservedTurnStoreHandle) { } else if (reservedTurnStoreHandle) {
close = async () => reservedTurnStoreHandle.close(); hooks = {
onRunError: async (error) => {
await gatewayGate?.markPaused(error);
},
};
close = async () => {
await reservedTurnStoreHandle.close();
await gatewayGate?.close();
};
} else if (gatewayGate) {
hooks = {
onRunError: async (error) => {
await gatewayGate?.markPaused(error);
},
};
close = async () => {
await gatewayGate.close();
};
} }
const defaultBudget: TurnRunBudget = options.defaultBudget ?? { const defaultBudget: TurnRunBudget = options.defaultBudget ?? {
@@ -149,6 +188,7 @@ export const createTurnDaemonRuntime = async (
stateStore, stateStore,
processor, processor,
hooks, hooks,
pauseGate,
}, },
{ profile: options.profile, defaultBudget } { profile: options.profile, defaultBudget }
); );
+1
View File
@@ -6,6 +6,7 @@
"scripts": { "scripts": {
"build": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api", "build": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api",
"dev": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api --watch", "dev": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api --watch",
"orchestrator": "GATEWAY_ROLE=orchestrator node dist/index.js",
"lint": "node -e \"console.log('lint not configured')\"", "lint": "node -e \"console.log('lint not configured')\"",
"test": "vitest run --config vitest.config.ts", "test": "vitest run --config vitest.config.ts",
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
+22 -3
View File
@@ -58,7 +58,10 @@ export const adminRouter = router({
scenario: z.string().min(1).max(64), scenario: z.string().min(1).max(64),
apiPort: z.number().int().min(1).max(65535), apiPort: z.number().int().min(1).max(65535),
status: zProfileStatus.optional(), status: zProfileStatus.optional(),
preopenAt: z.string().datetime().optional(),
openAt: z.string().datetime().optional(),
scheduledStartAt: z.string().datetime().optional(), scheduledStartAt: z.string().datetime().optional(),
buildCommitSha: z.string().min(7).max(64).optional(),
}) })
) )
.mutation(async ({ ctx, input }) => { .mutation(async ({ ctx, input }) => {
@@ -68,7 +71,10 @@ export const adminRouter = router({
scenario: input.scenario, scenario: input.scenario,
apiPort: input.apiPort, apiPort: input.apiPort,
status, status,
preopenAt: input.preopenAt,
openAt: input.openAt,
scheduledStartAt: input.scheduledStartAt, scheduledStartAt: input.scheduledStartAt,
buildCommitSha: input.buildCommitSha,
}); });
}), }),
setStatus: adminProcedure setStatus: adminProcedure
@@ -76,21 +82,34 @@ export const adminRouter = router({
z.object({ z.object({
profileName: z.string().min(1), profileName: z.string().min(1),
status: zProfileStatus, status: zProfileStatus,
preopenAt: z.string().datetime().optional(),
openAt: z.string().datetime().optional(),
scheduledStartAt: z.string().datetime().optional(), scheduledStartAt: z.string().datetime().optional(),
buildCommitSha: z.string().min(7).max(64).optional(),
}) })
) )
.mutation(async ({ ctx, input }) => { .mutation(async ({ ctx, input }) => {
if (input.status === 'RESERVED' && !input.scheduledStartAt) { if (input.status === 'RESERVED' && (!input.preopenAt || !input.openAt)) {
throw new TRPCError({ throw new TRPCError({
code: 'BAD_REQUEST', code: 'BAD_REQUEST',
message: 'scheduledStartAt is required for RESERVED status.', message: 'preopenAt and openAt are required for RESERVED status.',
}); });
} }
const result = await ctx.profiles.updateStatus( const result = await ctx.profiles.updateStatus(
input.profileName, input.profileName,
input.status, input.status,
input.status === 'RESERVED' ? input.scheduledStartAt : null {
preopenAt: input.preopenAt,
openAt: input.openAt,
scheduledStartAt:
input.status === 'RESERVED' ? input.scheduledStartAt : null,
}
); );
if (input.buildCommitSha) {
await ctx.profiles.updateBuildStatus(input.profileName, 'IDLE', {
commitSha: input.buildCommitSha,
});
}
await ctx.orchestrator.reconcileNow(); await ctx.orchestrator.reconcileNow();
return result; return result;
}), }),
+44 -1
View File
@@ -23,6 +23,16 @@ export interface GatewayApiConfig {
worktreeRoot: string; worktreeRoot: string;
} }
export interface GatewayOrchestratorConfig {
redisKeyPrefix: string;
gameTokenSecret: string;
orchestratorReconcileIntervalMs: number;
orchestratorScheduleIntervalMs: number;
orchestratorBuildIntervalMs: number;
workspaceRootHint: string;
worktreeRoot: string;
}
const parseNumber = (value: string | undefined, fallback: number, label: string): number => { const parseNumber = (value: string | undefined, fallback: number, label: string): number => {
if (!value) { if (!value) {
return fallback; return fallback;
@@ -85,7 +95,40 @@ export const resolveGatewayApiConfigFromEnv = (
kakaoRedirectUri, kakaoRedirectUri,
publicBaseUrl, publicBaseUrl,
adminToken: env.GATEWAY_ADMIN_TOKEN, adminToken: env.GATEWAY_ADMIN_TOKEN,
orchestratorEnabled: parseBoolean(env.GATEWAY_ORCHESTRATOR_ENABLED, true), orchestratorEnabled: parseBoolean(env.GATEWAY_ORCHESTRATOR_ENABLED, false),
orchestratorReconcileIntervalMs: parseNumber(
env.GATEWAY_ORCHESTRATOR_RECONCILE_MS,
15000,
'GATEWAY_ORCHESTRATOR_RECONCILE_MS'
),
orchestratorScheduleIntervalMs: parseNumber(
env.GATEWAY_ORCHESTRATOR_SCHEDULE_MS,
5000,
'GATEWAY_ORCHESTRATOR_SCHEDULE_MS'
),
orchestratorBuildIntervalMs: parseNumber(
env.GATEWAY_ORCHESTRATOR_BUILD_MS,
10000,
'GATEWAY_ORCHESTRATOR_BUILD_MS'
),
workspaceRootHint: env.GATEWAY_WORKSPACE_ROOT ?? process.cwd(),
worktreeRoot:
env.GATEWAY_WORKTREE_ROOT ??
path.resolve(env.GATEWAY_WORKSPACE_ROOT ?? process.cwd(), '.worktrees'),
};
};
export const resolveGatewayOrchestratorConfigFromEnv = (
env: NodeJS.ProcessEnv = process.env
): GatewayOrchestratorConfig => {
const secret = env.GAME_TOKEN_SECRET ?? env.GATEWAY_TOKEN_SECRET ?? '';
if (!secret) {
throw new Error('GAME_TOKEN_SECRET is required for game server processes.');
}
const redisKeyPrefix = env.GATEWAY_REDIS_PREFIX ?? 'sammo:gateway';
return {
redisKeyPrefix,
gameTokenSecret: secret,
orchestratorReconcileIntervalMs: parseNumber( orchestratorReconcileIntervalMs: parseNumber(
env.GATEWAY_ORCHESTRATOR_RECONCILE_MS, env.GATEWAY_ORCHESTRATOR_RECONCILE_MS,
15000, 15000,
+6 -2
View File
@@ -2,6 +2,7 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url'; import { fileURLToPath } from 'node:url';
import { runGatewayApiServer } from './server.js'; import { runGatewayApiServer } from './server.js';
import { runGatewayOrchestrator } from './orchestrator/orchestratorServer.js';
export * from './config.js'; export * from './config.js';
export * from './context.js'; export * from './context.js';
@@ -27,8 +28,11 @@ const isMain = (): boolean => {
}; };
if (isMain()) { if (isMain()) {
runGatewayApiServer().catch((error) => { const role = process.env.GATEWAY_ROLE ?? 'api';
console.error('[gateway-api] failed to start', error); const run = role === 'orchestrator' ? runGatewayOrchestrator : runGatewayApiServer;
run().catch((error) => {
const prefix = role === 'orchestrator' ? 'gateway-orchestrator' : 'gateway-api';
console.error(`[${prefix}] failed to start`, error);
process.exitCode = 1; process.exitCode = 1;
}); });
} }
@@ -54,7 +54,12 @@ export const planProfileReconcile = (
status: GatewayProfileStatus, status: GatewayProfileStatus,
runtime: ProfileRuntimeState runtime: ProfileRuntimeState
): { shouldStart: boolean; shouldStop: boolean } => { ): { shouldStart: boolean; shouldStop: boolean } => {
if (status === 'RUNNING') { if (
status === 'RUNNING' ||
status === 'PREOPEN' ||
status === 'PAUSED' ||
status === 'COMPLETED'
) {
return { return {
shouldStart: !(runtime.apiRunning && runtime.daemonRunning), shouldStart: !(runtime.apiRunning && runtime.daemonRunning),
shouldStop: false, shouldStop: false,
@@ -94,6 +99,7 @@ const buildProcessDefinitions = (
TURN_PROFILE: profile.profile, TURN_PROFILE: profile.profile,
PROFILE: profile.profile, PROFILE: profile.profile,
SCENARIO: profile.scenario, SCENARIO: profile.scenario,
TURN_PROFILE_NAME: profile.profileName,
}; };
return { return {
api: { api: {
@@ -221,18 +227,41 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
const now = this.now(); const now = this.now();
const due = await this.repository.listReservedToStart(now); const due = await this.repository.listReservedToStart(now);
for (const profile of due) { for (const profile of due) {
try { if (!profile.preopenAt || !profile.openAt) {
await this.repository.updateStatus(
profile.profileName,
'RUNNING',
null
);
await this.startProfile(profile);
} catch (error) {
await this.repository.updateLastError( await this.repository.updateLastError(
profile.profileName, profile.profileName,
error instanceof Error ? error.message : 'Failed to start scheduled profile.' 'Reserved profile is missing preopen/open schedule.'
); );
continue;
}
if (!profile.buildCommitSha) {
await this.repository.updateLastError(
profile.profileName,
'Reserved profile is missing build commit SHA.'
);
continue;
}
const queued =
profile.buildStatus === 'QUEUED' || profile.buildStatus === 'RUNNING';
if (!queued) {
await this.repository.updateBuildStatus(profile.profileName, 'QUEUED', {
requestedAt: now.toISOString(),
error: null,
commitSha: profile.buildCommitSha,
});
}
}
const profiles = await this.repository.listProfiles();
for (const profile of profiles) {
if (
profile.status === 'PREOPEN' &&
profile.openAt &&
new Date(profile.openAt) <= now
) {
await this.repository.updateStatus(profile.profileName, 'RUNNING', {
preopenAt: profile.preopenAt ?? null,
openAt: profile.openAt ?? null,
});
} }
} }
} finally { } finally {
@@ -304,12 +333,24 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
completedAt, completedAt,
error: null, error: null,
}); });
if (queued.status !== 'RUNNING' && queued.status !== 'DISABLED') { if (queued.status === 'RESERVED') {
await this.repository.updateStatus( await this.repository.updateStatus(
queued.profileName, queued.profileName,
'COMPLETED', queued.openAt && new Date(queued.openAt) <= this.now()
queued.scheduledStartAt ? queued.scheduledStartAt : null ? 'RUNNING'
: 'PREOPEN',
{
preopenAt: queued.preopenAt ?? null,
openAt: queued.openAt ?? null,
}
); );
} else if (queued.status === 'PREOPEN' && queued.openAt) {
if (new Date(queued.openAt) <= this.now()) {
await this.repository.updateStatus(queued.profileName, 'RUNNING', {
preopenAt: queued.preopenAt ?? null,
openAt: queued.openAt ?? null,
});
}
} }
} else { } else {
await this.repository.updateBuildStatus(queued.profileName, 'FAILED', { await this.repository.updateBuildStatus(queued.profileName, 'FAILED', {
@@ -0,0 +1,52 @@
import type { PrismaClient } from '@prisma/client';
import type { GatewayOrchestratorConfig } from '../config.js';
import { createGatewayProfileRepository } from './profileRepository.js';
import { GatewayOrchestrator } from './gatewayOrchestrator.js';
import { Pm2ProcessManager } from './pm2ProcessManager.js';
import { PnpmBuildRunner } from './buildRunner.js';
import { resolveWorkspaceRoot } from './workspaceRoot.js';
import { GitWorkspaceManager } from './workspaceManager.js';
export const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
const entries = Object.entries(env).filter(
(entry): entry is [string, string] => typeof entry[1] === 'string'
);
return Object.fromEntries(entries);
};
export const createGatewayOrchestrator = (
prisma: PrismaClient,
config: GatewayOrchestratorConfig,
env: NodeJS.ProcessEnv = process.env
): {
orchestrator: GatewayOrchestrator;
profiles: ReturnType<typeof createGatewayProfileRepository>;
} => {
const profiles = createGatewayProfileRepository(prisma);
const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint);
const processManager = new Pm2ProcessManager();
const buildRunner = new PnpmBuildRunner();
const baseEnv = buildEnvMap(env);
const workspaceManager = new GitWorkspaceManager({
repoRoot: workspaceRoot,
worktreeRoot: config.worktreeRoot,
baseEnv,
});
const orchestrator = new GatewayOrchestrator({
repository: profiles,
processManager,
buildRunner,
workspaceManager,
processConfig: {
workspaceRoot,
redisKeyPrefix: config.redisKeyPrefix,
gameTokenSecret: config.gameTokenSecret,
baseEnv,
},
reconcileIntervalMs: config.orchestratorReconcileIntervalMs,
scheduleIntervalMs: config.orchestratorScheduleIntervalMs,
buildIntervalMs: config.orchestratorBuildIntervalMs,
});
return { orchestrator, profiles };
};
@@ -0,0 +1,32 @@
import type { PrismaClient } from '@prisma/client';
import {
createPostgresConnector,
resolvePostgresConfigFromEnv,
} from '@sammo-ts/infra';
import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js';
import { createGatewayOrchestrator } from './orchestratorFactory.js';
export const runGatewayOrchestrator = async (): Promise<void> => {
const config = resolveGatewayOrchestratorConfigFromEnv();
const postgres = createPostgresConnector(resolvePostgresConfigFromEnv());
await postgres.connect();
const { orchestrator } = createGatewayOrchestrator(
postgres.prisma as PrismaClient,
config,
process.env
);
const stop = async (reason: string): Promise<void> => {
console.info(`[gateway-orchestrator] stopping: ${reason}`);
await orchestrator.stop();
await postgres.disconnect();
};
process.on('SIGINT', () => void stop('SIGINT'));
process.on('SIGTERM', () => void stop('SIGTERM'));
orchestrator.start();
console.info('[gateway-orchestrator] started');
};
@@ -1,9 +1,11 @@
import { Prisma, type PrismaClient } from '@prisma/client'; import { Prisma, type PrismaClient } from '@prisma/client';
export const GATEWAY_PROFILE_STATUSES = [ export const GATEWAY_PROFILE_STATUSES = [
'COMPLETED',
'RESERVED', 'RESERVED',
'PREOPEN',
'RUNNING', 'RUNNING',
'PAUSED',
'COMPLETED',
'STOPPED', 'STOPPED',
'DISABLED', 'DISABLED',
] as const; ] as const;
@@ -28,6 +30,8 @@ export interface GatewayProfileRecord {
buildCommitSha?: string; buildCommitSha?: string;
buildWorkspace?: string; buildWorkspace?: string;
buildLastUsedAt?: string; buildLastUsedAt?: string;
preopenAt?: string;
openAt?: string;
scheduledStartAt?: string; scheduledStartAt?: string;
buildRequestedAt?: string; buildRequestedAt?: string;
buildStartedAt?: string; buildStartedAt?: string;
@@ -44,7 +48,10 @@ export interface GatewayProfileUpsertInput {
scenario: string; scenario: string;
apiPort: number; apiPort: number;
status?: GatewayProfileStatus; status?: GatewayProfileStatus;
preopenAt?: string;
openAt?: string;
scheduledStartAt?: string; scheduledStartAt?: string;
buildCommitSha?: string;
meta?: Prisma.JsonObject; meta?: Prisma.JsonObject;
} }
@@ -55,7 +62,11 @@ export interface GatewayProfileRepository {
updateStatus( updateStatus(
profileName: string, profileName: string,
status: GatewayProfileStatus, status: GatewayProfileStatus,
scheduledStartAt?: string | null schedule?: {
preopenAt?: string | null;
openAt?: string | null;
scheduledStartAt?: string | null;
}
): Promise<GatewayProfileRecord | null>; ): Promise<GatewayProfileRecord | null>;
updateBuildStatus( updateBuildStatus(
profileName: string, profileName: string,
@@ -94,6 +105,8 @@ const mapProfile = (row: {
buildCommitSha: string | null; buildCommitSha: string | null;
buildWorkspace: string | null; buildWorkspace: string | null;
buildLastUsedAt: Date | null; buildLastUsedAt: Date | null;
preopenAt: Date | null;
openAt: Date | null;
scheduledStartAt: Date | null; scheduledStartAt: Date | null;
buildRequestedAt: Date | null; buildRequestedAt: Date | null;
buildStartedAt: Date | null; buildStartedAt: Date | null;
@@ -113,6 +126,8 @@ const mapProfile = (row: {
buildCommitSha: row.buildCommitSha ?? undefined, buildCommitSha: row.buildCommitSha ?? undefined,
buildWorkspace: row.buildWorkspace ?? undefined, buildWorkspace: row.buildWorkspace ?? undefined,
buildLastUsedAt: toIso(row.buildLastUsedAt), buildLastUsedAt: toIso(row.buildLastUsedAt),
preopenAt: toIso(row.preopenAt),
openAt: toIso(row.openAt),
scheduledStartAt: toIso(row.scheduledStartAt), scheduledStartAt: toIso(row.scheduledStartAt),
buildRequestedAt: toIso(row.buildRequestedAt), buildRequestedAt: toIso(row.buildRequestedAt),
buildStartedAt: toIso(row.buildStartedAt), buildStartedAt: toIso(row.buildStartedAt),
@@ -152,19 +167,36 @@ export const createGatewayProfileRepository = (
scenario: input.scenario, scenario: input.scenario,
apiPort: input.apiPort, apiPort: input.apiPort,
status: input.status ?? 'STOPPED', status: input.status ?? 'STOPPED',
preopenAt: input.preopenAt ? new Date(input.preopenAt) : null,
openAt: input.openAt ? new Date(input.openAt) : null,
scheduledStartAt: input.scheduledStartAt scheduledStartAt: input.scheduledStartAt
? new Date(input.scheduledStartAt) ? new Date(input.scheduledStartAt)
: null, : null,
buildCommitSha: input.buildCommitSha ?? null,
meta: (input.meta ?? {}) as Prisma.JsonObject, meta: (input.meta ?? {}) as Prisma.JsonObject,
}, },
update: { update: {
apiPort: input.apiPort, apiPort: input.apiPort,
status: input.status, status: input.status,
preopenAt: input.preopenAt
? new Date(input.preopenAt)
: input.preopenAt === null
? null
: undefined,
openAt: input.openAt
? new Date(input.openAt)
: input.openAt === null
? null
: undefined,
scheduledStartAt: input.scheduledStartAt scheduledStartAt: input.scheduledStartAt
? new Date(input.scheduledStartAt) ? new Date(input.scheduledStartAt)
: input.scheduledStartAt === null : input.scheduledStartAt === null
? null ? null
: undefined, : undefined,
buildCommitSha:
input.buildCommitSha === undefined
? undefined
: input.buildCommitSha,
meta: input.meta ? (input.meta as Prisma.JsonObject) : undefined, meta: input.meta ? (input.meta as Prisma.JsonObject) : undefined,
}, },
}); });
@@ -173,17 +205,33 @@ export const createGatewayProfileRepository = (
async updateStatus( async updateStatus(
profileName: string, profileName: string,
status: GatewayProfileStatus, status: GatewayProfileStatus,
scheduledStartAt?: string | null schedule?: {
preopenAt?: string | null;
openAt?: string | null;
scheduledStartAt?: string | null;
}
): Promise<GatewayProfileRecord | null> { ): Promise<GatewayProfileRecord | null> {
const row = await prisma.gatewayProfile.update({ const row = await prisma.gatewayProfile.update({
where: { profileName }, where: { profileName },
data: { data: {
status, status,
scheduledStartAt: preopenAt:
scheduledStartAt === undefined schedule?.preopenAt === undefined
? undefined ? undefined
: scheduledStartAt : schedule?.preopenAt
? new Date(scheduledStartAt) ? new Date(schedule.preopenAt)
: null,
openAt:
schedule?.openAt === undefined
? undefined
: schedule?.openAt
? new Date(schedule.openAt)
: null,
scheduledStartAt:
schedule?.scheduledStartAt === undefined
? undefined
: schedule?.scheduledStartAt
? new Date(schedule.scheduledStartAt)
: null, : null,
}, },
}); });
@@ -240,7 +288,7 @@ export const createGatewayProfileRepository = (
const rows = await prisma.gatewayProfile.findMany({ const rows = await prisma.gatewayProfile.findMany({
where: { where: {
status: 'RESERVED', status: 'RESERVED',
scheduledStartAt: { preopenAt: {
lte: now, lte: now,
}, },
}, },
+5 -43
View File
@@ -16,21 +16,9 @@ import { KakaoOAuthClient } from './auth/kakaoClient.js';
import { RedisOAuthSessionStore } from './auth/oauthSessionStore.js'; import { RedisOAuthSessionStore } from './auth/oauthSessionStore.js';
import { createPostgresUserRepository } from './auth/postgresUserRepository.js'; import { createPostgresUserRepository } from './auth/postgresUserRepository.js';
import { RedisGatewaySessionService } from './auth/redisSessionService.js'; import { RedisGatewaySessionService } from './auth/redisSessionService.js';
import { createGatewayProfileRepository } from './orchestrator/profileRepository.js'; import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js';
import { GatewayOrchestrator } from './orchestrator/gatewayOrchestrator.js';
import { Pm2ProcessManager } from './orchestrator/pm2ProcessManager.js';
import { PnpmBuildRunner } from './orchestrator/buildRunner.js';
import { resolveWorkspaceRoot } from './orchestrator/workspaceRoot.js';
import { GitWorkspaceManager } from './orchestrator/workspaceManager.js';
import { appRouter } from './router.js'; import { appRouter } from './router.js';
const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
const entries = Object.entries(env).filter(
(entry): entry is [string, string] => typeof entry[1] === 'string'
);
return Object.fromEntries(entries);
};
export const createGatewayApiServer = async () => { export const createGatewayApiServer = async () => {
const config = resolveGatewayApiConfigFromEnv(); const config = resolveGatewayApiConfigFromEnv();
const postgres = createPostgresConnector(resolvePostgresConfigFromEnv()); const postgres = createPostgresConnector(resolvePostgresConfigFromEnv());
@@ -58,33 +46,11 @@ export const createGatewayApiServer = async () => {
config.oauthSessionTtlSeconds config.oauthSessionTtlSeconds
); );
const profiles = createGatewayProfileRepository( const { orchestrator, profiles } = createGatewayOrchestrator(
postgres.prisma as PrismaClient postgres.prisma as PrismaClient,
config,
process.env
); );
const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint);
const processManager = new Pm2ProcessManager();
const buildRunner = new PnpmBuildRunner();
const baseEnv = buildEnvMap(process.env);
const workspaceManager = new GitWorkspaceManager({
repoRoot: workspaceRoot,
worktreeRoot: config.worktreeRoot,
baseEnv,
});
const orchestrator = new GatewayOrchestrator({
repository: profiles,
processManager,
buildRunner,
workspaceManager,
processConfig: {
workspaceRoot,
redisKeyPrefix: config.redisKeyPrefix,
gameTokenSecret: config.gameTokenSecret,
baseEnv,
},
reconcileIntervalMs: config.orchestratorReconcileIntervalMs,
scheduleIntervalMs: config.orchestratorScheduleIntervalMs,
buildIntervalMs: config.orchestratorBuildIntervalMs,
});
const app = fastify({ const app = fastify({
logger: true, logger: true,
@@ -121,10 +87,6 @@ export const createGatewayApiServer = async () => {
ok: true, ok: true,
})); }));
if (config.orchestratorEnabled) {
orchestrator.start();
}
app.addHook('onClose', async () => { app.addHook('onClose', async () => {
await orchestrator.stop(); await orchestrator.stop();
await redis.disconnect(); await redis.disconnect();
@@ -12,6 +12,15 @@ describe('planProfileReconcile', () => {
).toEqual({ shouldStart: true, shouldStop: false }); ).toEqual({ shouldStart: true, shouldStop: false });
}); });
it('starts processes for preopen profiles', () => {
expect(
planProfileReconcile('PREOPEN', {
apiRunning: false,
daemonRunning: false,
})
).toEqual({ shouldStart: true, shouldStop: false });
});
it('does nothing when running profile is healthy', () => { it('does nothing when running profile is healthy', () => {
expect( expect(
planProfileReconcile('RUNNING', { planProfileReconcile('RUNNING', {
+13 -9
View File
@@ -20,7 +20,8 @@ selection is required because it drives unit sets and DB settings.
## Gateway Orchestration (Single Host Draft) ## Gateway Orchestration (Single Host Draft)
Gateway API is the single source of truth for profile state and reconciles Gateway API is the single source of truth for profile state and reconciles
PM2-managed processes on boot and on a short interval. The DB owns the desired PM2-managed processes on boot and on a short interval. The orchestrator runs
as a separate process (`GATEWAY_ROLE=orchestrator`). The DB owns the desired
state; PM2 is treated as the actuator. The runtime state is grouped so that state; PM2 is treated as the actuator. The runtime state is grouped so that
`game-api` + `turn-daemon` are either on together or off together. `game-api` + `turn-daemon` are either on together or off together.
@@ -29,10 +30,12 @@ state; PM2 is treated as the actuator. The runtime state is grouped so that
Profiles are tracked by `profileName` (= `${profile}:${scenario}`). Profiles are tracked by `profileName` (= `${profile}:${scenario}`).
Gateway loads the profile table on boot, then reconciles PM2 to match. Gateway loads the profile table on boot, then reconciles PM2 to match.
- `완료됨` (COMPLETED): build output exists; processes should be off. - `예약됨` (RESERVED): preopen/open timestamps are set; processes off.
- `예약됨` (RESERVED): start is scheduled for a timestamp; processes off. - `가오픈` (PREOPEN): build done; API+daemon on, daemon paused.
- `가동중` (RUNNING): both `game-api` and `turn-daemon` should be on. - `가동중` (RUNNING): both `game-api` and `turn-daemon` should be on.
- `정지됨` (STOPPED): processes off; may be resumed later. - `정지됨` (STOPPED): processes off; may be resumed later.
- `정지(오류)` (PAUSED): fatal error; daemon paused but process remains on.
- `천하통일` (COMPLETED): game finished; API on, daemon paused.
- `비활성화` (DISABLED): excluded from orchestration; start forbidden. - `비활성화` (DISABLED): excluded from orchestration; start forbidden.
### Boot Reconciliation ### Boot Reconciliation
@@ -40,29 +43,30 @@ Gateway loads the profile table on boot, then reconciles PM2 to match.
1) Load profile rows from DB. 1) Load profile rows from DB.
2) List PM2 processes and map `profileName -> running state`. 2) List PM2 processes and map `profileName -> running state`.
3) For each profile: 3) For each profile:
- If desired `RUNNING` and any process is missing, start missing processes. - If desired `RUNNING/PREOPEN/PAUSED/COMPLETED` and any process is missing, start.
- If desired not `RUNNING` and any process is running, stop both processes. - If desired `RESERVED/STOPPED/DISABLED` and any process is running, stop both.
4) Persist errors to DB for audit. 4) Persist errors to DB for audit.
### Internal Scheduler (Gateway Cron) ### Internal Scheduler (Gateway Cron)
Gateway runs a lightweight cron loop (setInterval) that: Gateway runs a lightweight cron loop (setInterval) that:
- Promotes `RESERVED` profiles whose `scheduledStartAt <= now` to `RUNNING`. - When `RESERVED` and `preopenAt <= now`, queue a build for the reserved commit.
- Triggers a reconcile immediately after promotion. - When build succeeds, status becomes `PREOPEN` (daemon paused).
- When `openAt <= now`, status becomes `RUNNING` and daemon resumes.
- Optionally drains a build queue (see build workflow). - Optionally drains a build queue (see build workflow).
### Build Workflow (Admin) ### Build Workflow (Admin)
- Admin triggers a build request for a profile. - Admin triggers a build request for a profile.
- Gateway queues a build job with `(profileName, commitSha)` and prepares a - Gateway queues a build job with `(profileName, commitSha)` and prepares a
per-commit workspace (`/var/sammo/workspaces/{commitSha}` recommended). per-commit workspace (`/.worktrees/{commitSha}` by default).
- Workspace is backed by `git worktree` and is reused across builds for the same commit. - Workspace is backed by `git worktree` and is reused across builds for the same commit.
- Each workspace stores `lastUsedAt` in DB so cleanup can remove stale worktrees. - Each workspace stores `lastUsedAt` in DB so cleanup can remove stale worktrees.
- Cleanup is invoked manually by admin API and removes worktrees unused for 6+ months. - Cleanup is invoked manually by admin API and removes worktrees unused for 6+ months.
- Build runs `pnpm install` when workspace is created, then executes - Build runs `pnpm install` when workspace is created, then executes
`pnpm --filter @sammo-ts/game-api build` and `pnpm --filter @sammo-ts/game-api build` and
`pnpm --filter @sammo-ts/game-engine build`, then marks build success/failure. `pnpm --filter @sammo-ts/game-engine build`, then marks build success/failure.
- On success, profile status remains `COMPLETED` (or stays `RUNNING` if already on). - On success, status moves to `PREOPEN` for reserved builds or stays unchanged for manual builds.
## Current Implementation Status ## Current Implementation Status
@@ -1,8 +1,10 @@
-- Create enums for gateway profile tracking -- Create enums for gateway profile tracking
CREATE TYPE "GatewayProfileStatus" AS ENUM ( CREATE TYPE "GatewayProfileStatus" AS ENUM (
'COMPLETED',
'RESERVED', 'RESERVED',
'PREOPEN',
'RUNNING', 'RUNNING',
'PAUSED',
'COMPLETED',
'STOPPED', 'STOPPED',
'DISABLED' 'DISABLED'
); );
@@ -26,6 +28,8 @@ CREATE TABLE "gateway_profile" (
"build_commit_sha" TEXT, "build_commit_sha" TEXT,
"build_workspace" TEXT, "build_workspace" TEXT,
"build_last_used_at" TIMESTAMP(3), "build_last_used_at" TIMESTAMP(3),
"preopen_at" TIMESTAMP(3),
"open_at" TIMESTAMP(3),
"scheduled_start_at" TIMESTAMP(3), "scheduled_start_at" TIMESTAMP(3),
"build_requested_at" TIMESTAMP(3), "build_requested_at" TIMESTAMP(3),
"build_started_at" TIMESTAMP(3), "build_started_at" TIMESTAMP(3),
@@ -39,4 +43,3 @@ CREATE TABLE "gateway_profile" (
CONSTRAINT "gateway_profile_pkey" PRIMARY KEY ("profile_name"), CONSTRAINT "gateway_profile_pkey" PRIMARY KEY ("profile_name"),
CONSTRAINT "gateway_profile_profile_scenario_key" UNIQUE ("profile", "scenario") CONSTRAINT "gateway_profile_profile_scenario_key" UNIQUE ("profile", "scenario")
); );
+5 -1
View File
@@ -28,9 +28,11 @@ enum OAuthType {
} }
enum GatewayProfileStatus { enum GatewayProfileStatus {
COMPLETED
RESERVED RESERVED
PREOPEN
RUNNING RUNNING
PAUSED
COMPLETED
STOPPED STOPPED
DISABLED DISABLED
} }
@@ -72,6 +74,8 @@ model GatewayProfile {
buildCommitSha String? @map("build_commit_sha") buildCommitSha String? @map("build_commit_sha")
buildWorkspace String? @map("build_workspace") buildWorkspace String? @map("build_workspace")
buildLastUsedAt DateTime? @map("build_last_used_at") buildLastUsedAt DateTime? @map("build_last_used_at")
preopenAt DateTime? @map("preopen_at")
openAt DateTime? @map("open_at")
scheduledStartAt DateTime? @map("scheduled_start_at") scheduledStartAt DateTime? @map("scheduled_start_at")
buildRequestedAt DateTime? @map("build_requested_at") buildRequestedAt DateTime? @map("build_requested_at")
buildStartedAt DateTime? @map("build_started_at") buildStartedAt DateTime? @map("build_started_at")