feat: Enhance Gateway Orchestrator with preopen and open scheduling, add error handling for paused states, and implement new profile management features
This commit is contained in:
@@ -28,6 +28,7 @@ export interface TurnDaemonStatus {
|
|||||||
state: TurnDaemonState;
|
state: TurnDaemonState;
|
||||||
running: boolean;
|
running: boolean;
|
||||||
paused: boolean;
|
paused: boolean;
|
||||||
|
lastError?: string;
|
||||||
lastRunAt?: string;
|
lastRunAt?: string;
|
||||||
lastDurationMs?: number;
|
lastDurationMs?: number;
|
||||||
lastTurnTime?: string;
|
lastTurnTime?: string;
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
"@sammo-ts/common": "workspace:*",
|
"@sammo-ts/common": "workspace:*",
|
||||||
"@sammo-ts/infra": "workspace:*",
|
"@sammo-ts/infra": "workspace:*",
|
||||||
"@sammo-ts/logic": "workspace:*",
|
"@sammo-ts/logic": "workspace:*",
|
||||||
|
"@prisma/client": "^7.2.0",
|
||||||
"zod": "^4.2.1"
|
"zod": "^4.2.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ export interface TurnDaemonLifecycleDeps {
|
|||||||
stateStore: TurnStateStore;
|
stateStore: TurnStateStore;
|
||||||
processor: TurnProcessor;
|
processor: TurnProcessor;
|
||||||
hooks?: TurnDaemonHooks;
|
hooks?: TurnDaemonHooks;
|
||||||
|
pauseGate?: () => Promise<boolean>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class TurnDaemonLifecycle {
|
export class TurnDaemonLifecycle {
|
||||||
@@ -41,12 +42,15 @@ export class TurnDaemonLifecycle {
|
|||||||
private readonly stateStore: TurnStateStore;
|
private readonly stateStore: TurnStateStore;
|
||||||
private readonly processor: TurnProcessor;
|
private readonly processor: TurnProcessor;
|
||||||
private readonly hooks?: TurnDaemonHooks;
|
private readonly hooks?: TurnDaemonHooks;
|
||||||
|
private readonly pauseGate?: () => Promise<boolean>;
|
||||||
private readonly options: TurnDaemonLifecycleOptions;
|
private readonly options: TurnDaemonLifecycleOptions;
|
||||||
|
|
||||||
private status: TurnDaemonStatus;
|
private status: TurnDaemonStatus;
|
||||||
private pendingRun: PendingRun | null = null;
|
private pendingRun: PendingRun | null = null;
|
||||||
private stopping = false;
|
private stopping = false;
|
||||||
private loopPromise: Promise<void> | null = null;
|
private loopPromise: Promise<void> | null = null;
|
||||||
|
private manualPaused = false;
|
||||||
|
private errorPaused = false;
|
||||||
|
|
||||||
constructor(deps: TurnDaemonLifecycleDeps, options: TurnDaemonLifecycleOptions) {
|
constructor(deps: TurnDaemonLifecycleDeps, options: TurnDaemonLifecycleOptions) {
|
||||||
this.clock = deps.clock;
|
this.clock = deps.clock;
|
||||||
@@ -55,6 +59,7 @@ export class TurnDaemonLifecycle {
|
|||||||
this.stateStore = deps.stateStore;
|
this.stateStore = deps.stateStore;
|
||||||
this.processor = deps.processor;
|
this.processor = deps.processor;
|
||||||
this.hooks = deps.hooks;
|
this.hooks = deps.hooks;
|
||||||
|
this.pauseGate = deps.pauseGate;
|
||||||
this.options = options;
|
this.options = options;
|
||||||
this.status = {
|
this.status = {
|
||||||
state: 'idle',
|
state: 'idle',
|
||||||
@@ -109,10 +114,24 @@ export class TurnDaemonLifecycle {
|
|||||||
if (this.stopping) {
|
if (this.stopping) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
const gatePaused = (await this.pauseGate?.()) ?? false;
|
||||||
|
if (this.errorPaused && !gatePaused) {
|
||||||
|
this.errorPaused = false;
|
||||||
|
this.status.lastError = undefined;
|
||||||
|
}
|
||||||
|
this.status.paused = this.manualPaused || gatePaused || this.errorPaused;
|
||||||
if (this.status.paused) {
|
if (this.status.paused) {
|
||||||
await this.waitForResume();
|
this.status.state = 'paused';
|
||||||
|
if (this.manualPaused) {
|
||||||
|
await this.waitForResume();
|
||||||
|
} else {
|
||||||
|
await this.clock.sleepMs(500);
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (this.status.state === 'paused') {
|
||||||
|
this.status.state = 'idle';
|
||||||
|
}
|
||||||
|
|
||||||
if (this.pendingRun) {
|
if (this.pendingRun) {
|
||||||
await this.runOnce(this.pendingRun);
|
await this.runOnce(this.pendingRun);
|
||||||
@@ -186,11 +205,13 @@ export class TurnDaemonLifecycle {
|
|||||||
private async handleCommand(command: TurnDaemonCommand): Promise<void> {
|
private async handleCommand(command: TurnDaemonCommand): Promise<void> {
|
||||||
switch (command.type) {
|
switch (command.type) {
|
||||||
case 'pause':
|
case 'pause':
|
||||||
|
this.manualPaused = true;
|
||||||
this.status.paused = true;
|
this.status.paused = true;
|
||||||
this.status.state = 'paused';
|
this.status.state = 'paused';
|
||||||
return;
|
return;
|
||||||
case 'resume':
|
case 'resume':
|
||||||
this.status.paused = false;
|
this.manualPaused = false;
|
||||||
|
this.status.paused = this.errorPaused;
|
||||||
this.status.state = 'idle';
|
this.status.state = 'idle';
|
||||||
return;
|
return;
|
||||||
case 'shutdown':
|
case 'shutdown':
|
||||||
@@ -221,6 +242,15 @@ export class TurnDaemonLifecycle {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
result = await this.processor.run(targetTime, budget, checkpoint);
|
result = await this.processor.run(targetTime, budget, checkpoint);
|
||||||
|
} catch (error) {
|
||||||
|
this.status.running = false;
|
||||||
|
this.status.state = 'paused';
|
||||||
|
this.status.paused = true;
|
||||||
|
this.errorPaused = true;
|
||||||
|
this.status.lastError =
|
||||||
|
error instanceof Error ? error.message : 'Unknown turn daemon error.';
|
||||||
|
await this.hooks?.onRunError?.(error);
|
||||||
|
return;
|
||||||
} finally {
|
} finally {
|
||||||
this.status.running = false;
|
this.status.running = false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ export interface TurnDaemonStatus {
|
|||||||
state: TurnDaemonState;
|
state: TurnDaemonState;
|
||||||
running: boolean;
|
running: boolean;
|
||||||
paused: boolean;
|
paused: boolean;
|
||||||
|
lastError?: string;
|
||||||
lastRunAt?: string;
|
lastRunAt?: string;
|
||||||
lastDurationMs?: number;
|
lastDurationMs?: number;
|
||||||
lastTurnTime?: string;
|
lastTurnTime?: string;
|
||||||
@@ -70,4 +71,5 @@ export interface TurnDaemonControlQueue {
|
|||||||
export interface TurnDaemonHooks {
|
export interface TurnDaemonHooks {
|
||||||
flushChanges?(result: TurnRunResult): Promise<void>;
|
flushChanges?(result: TurnRunResult): Promise<void>;
|
||||||
publishEvents?(result: TurnRunResult): Promise<void>;
|
publishEvents?(result: TurnRunResult): Promise<void>;
|
||||||
|
onRunError?(error: unknown): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import { createTurnDaemonRuntime } from './turnDaemon.js';
|
|||||||
|
|
||||||
export interface TurnDaemonCliOptions {
|
export interface TurnDaemonCliOptions {
|
||||||
profile?: string;
|
profile?: string;
|
||||||
|
profileName?: string;
|
||||||
|
scenario?: string;
|
||||||
databaseUrl?: string;
|
databaseUrl?: string;
|
||||||
tickMinutes?: number;
|
tickMinutes?: number;
|
||||||
schedule?: TurnSchedule;
|
schedule?: TurnSchedule;
|
||||||
@@ -71,6 +73,11 @@ export const runTurnDaemonCli = async (
|
|||||||
const env = options.env ?? process.env;
|
const env = options.env ?? process.env;
|
||||||
const profile =
|
const profile =
|
||||||
options.profile ?? env.TURN_PROFILE ?? env.PROFILE ?? 'che';
|
options.profile ?? env.TURN_PROFILE ?? env.PROFILE ?? 'che';
|
||||||
|
const scenario = options.scenario ?? env.TURN_SCENARIO ?? env.SCENARIO;
|
||||||
|
const profileName =
|
||||||
|
options.profileName ??
|
||||||
|
env.TURN_PROFILE_NAME ??
|
||||||
|
(scenario ? `${profile}:${scenario}` : profile);
|
||||||
const databaseUrl =
|
const databaseUrl =
|
||||||
options.databaseUrl ?? (await resolveDatabaseUrl({ env }));
|
options.databaseUrl ?? (await resolveDatabaseUrl({ env }));
|
||||||
const budget = buildBudgetOverride(env, options.budget);
|
const budget = buildBudgetOverride(env, options.budget);
|
||||||
@@ -80,14 +87,17 @@ export const runTurnDaemonCli = async (
|
|||||||
options.enableDatabaseFlush ??
|
options.enableDatabaseFlush ??
|
||||||
parseBoolean(env.TURN_FLUSH_DB) ??
|
parseBoolean(env.TURN_FLUSH_DB) ??
|
||||||
true;
|
true;
|
||||||
|
const pauseGateIntervalMs = parseNumber(env.TURN_PAUSE_GATE_MS);
|
||||||
|
|
||||||
const runtime = await createTurnDaemonRuntime({
|
const runtime = await createTurnDaemonRuntime({
|
||||||
profile,
|
profile,
|
||||||
|
profileName,
|
||||||
databaseUrl,
|
databaseUrl,
|
||||||
defaultBudget: budget,
|
defaultBudget: budget,
|
||||||
tickMinutes,
|
tickMinutes,
|
||||||
schedule: options.schedule,
|
schedule: options.schedule,
|
||||||
enableDatabaseFlush,
|
enableDatabaseFlush,
|
||||||
|
pauseGateIntervalMs,
|
||||||
});
|
});
|
||||||
|
|
||||||
let closed = false;
|
let closed = false;
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
import { createPostgresConnector } from '@sammo-ts/infra';
|
||||||
|
import type { PrismaClient } from '@prisma/client';
|
||||||
|
|
||||||
|
export interface GatewayProfileGateOptions {
|
||||||
|
databaseUrl: string;
|
||||||
|
profileName: string;
|
||||||
|
cacheMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GatewayProfileGate {
|
||||||
|
shouldPause(): Promise<boolean>;
|
||||||
|
markPaused(error?: unknown): Promise<void>;
|
||||||
|
close(): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_CACHE_MS = 2000;
|
||||||
|
|
||||||
|
const isRunningStatus = (status: string | null | undefined): boolean =>
|
||||||
|
status === 'RUNNING';
|
||||||
|
|
||||||
|
export const createGatewayProfileGate = async (
|
||||||
|
options: GatewayProfileGateOptions
|
||||||
|
): Promise<GatewayProfileGate> => {
|
||||||
|
const connector = createPostgresConnector({ url: options.databaseUrl });
|
||||||
|
await connector.connect();
|
||||||
|
const prisma = connector.prisma as PrismaClient;
|
||||||
|
let lastCheckedAt = 0;
|
||||||
|
let cachedPause = false;
|
||||||
|
|
||||||
|
const loadStatus = async (): Promise<boolean> => {
|
||||||
|
try {
|
||||||
|
const profile = await prisma.gatewayProfile.findUnique({
|
||||||
|
where: { profileName: options.profileName },
|
||||||
|
});
|
||||||
|
if (!profile) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return !isRunningStatus(profile.status);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
// 게이트웨이 프로필 상태를 읽어 턴 실행을 멈춰야 하는지 판단한다.
|
||||||
|
async shouldPause(): Promise<boolean> {
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - lastCheckedAt < (options.cacheMs ?? DEFAULT_CACHE_MS)) {
|
||||||
|
return cachedPause;
|
||||||
|
}
|
||||||
|
cachedPause = await loadStatus();
|
||||||
|
lastCheckedAt = now;
|
||||||
|
return cachedPause;
|
||||||
|
},
|
||||||
|
async markPaused(error?: unknown): Promise<void> {
|
||||||
|
const message =
|
||||||
|
error instanceof Error
|
||||||
|
? error.message
|
||||||
|
: error
|
||||||
|
? String(error)
|
||||||
|
: null;
|
||||||
|
try {
|
||||||
|
await prisma.gatewayProfile.update({
|
||||||
|
where: { profileName: options.profileName },
|
||||||
|
data: {
|
||||||
|
status: 'PAUSED',
|
||||||
|
lastError: message,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
async close(): Promise<void> {
|
||||||
|
await connector.disconnect();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -20,12 +20,14 @@ import type {
|
|||||||
import { InMemoryTurnWorld } from './inMemoryWorld.js';
|
import { InMemoryTurnWorld } from './inMemoryWorld.js';
|
||||||
import { InMemoryTurnProcessor } from './inMemoryTurnProcessor.js';
|
import { InMemoryTurnProcessor } from './inMemoryTurnProcessor.js';
|
||||||
import { InMemoryTurnStateStore } from './inMemoryStateStore.js';
|
import { InMemoryTurnStateStore } from './inMemoryStateStore.js';
|
||||||
|
import { createGatewayProfileGate } from './gatewayProfileGate.js';
|
||||||
import { createReservedTurnHandler } from './reservedTurnHandler.js';
|
import { createReservedTurnHandler } from './reservedTurnHandler.js';
|
||||||
import { createReservedTurnStore } from './reservedTurnStore.js';
|
import { createReservedTurnStore } from './reservedTurnStore.js';
|
||||||
import { loadTurnWorldFromDatabase } from './worldLoader.js';
|
import { loadTurnWorldFromDatabase } from './worldLoader.js';
|
||||||
|
|
||||||
export interface TurnDaemonRuntimeOptions {
|
export interface TurnDaemonRuntimeOptions {
|
||||||
profile: string;
|
profile: string;
|
||||||
|
profileName?: string;
|
||||||
databaseUrl: string;
|
databaseUrl: string;
|
||||||
defaultBudget?: TurnRunBudget;
|
defaultBudget?: TurnRunBudget;
|
||||||
clock?: Clock;
|
clock?: Clock;
|
||||||
@@ -36,6 +38,7 @@ export interface TurnDaemonRuntimeOptions {
|
|||||||
generalTurnHandler?: GeneralTurnHandler;
|
generalTurnHandler?: GeneralTurnHandler;
|
||||||
calendarHandler?: TurnCalendarHandler;
|
calendarHandler?: TurnCalendarHandler;
|
||||||
enableDatabaseFlush?: boolean;
|
enableDatabaseFlush?: boolean;
|
||||||
|
pauseGateIntervalMs?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TurnDaemonRuntime {
|
export interface TurnDaemonRuntime {
|
||||||
@@ -119,19 +122,55 @@ export const createTurnDaemonRuntime = async (
|
|||||||
|
|
||||||
let hooks: TurnDaemonHooks | undefined;
|
let hooks: TurnDaemonHooks | undefined;
|
||||||
let close = async () => {};
|
let close = async () => {};
|
||||||
|
let pauseGate: (() => Promise<boolean>) | undefined;
|
||||||
|
const gatewayGate =
|
||||||
|
options.profileName
|
||||||
|
? await createGatewayProfileGate({
|
||||||
|
databaseUrl: options.databaseUrl,
|
||||||
|
profileName: options.profileName,
|
||||||
|
cacheMs: options.pauseGateIntervalMs,
|
||||||
|
})
|
||||||
|
: null;
|
||||||
|
if (gatewayGate) {
|
||||||
|
pauseGate = gatewayGate.shouldPause;
|
||||||
|
}
|
||||||
if (options.enableDatabaseFlush ?? true) {
|
if (options.enableDatabaseFlush ?? true) {
|
||||||
const dbHooks = await createDatabaseTurnHooks(options.databaseUrl, world, {
|
const dbHooks = await createDatabaseTurnHooks(options.databaseUrl, world, {
|
||||||
reservedTurns: reservedTurnStoreHandle?.store,
|
reservedTurns: reservedTurnStoreHandle?.store,
|
||||||
});
|
});
|
||||||
hooks = dbHooks.hooks;
|
hooks = {
|
||||||
|
...dbHooks.hooks,
|
||||||
|
onRunError: async (error) => {
|
||||||
|
await dbHooks.hooks.onRunError?.(error);
|
||||||
|
await gatewayGate?.markPaused(error);
|
||||||
|
},
|
||||||
|
};
|
||||||
close = async () => {
|
close = async () => {
|
||||||
await dbHooks.close();
|
await dbHooks.close();
|
||||||
if (reservedTurnStoreHandle) {
|
if (reservedTurnStoreHandle) {
|
||||||
await reservedTurnStoreHandle.close();
|
await reservedTurnStoreHandle.close();
|
||||||
}
|
}
|
||||||
|
await gatewayGate?.close();
|
||||||
};
|
};
|
||||||
} else if (reservedTurnStoreHandle) {
|
} else if (reservedTurnStoreHandle) {
|
||||||
close = async () => reservedTurnStoreHandle.close();
|
hooks = {
|
||||||
|
onRunError: async (error) => {
|
||||||
|
await gatewayGate?.markPaused(error);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
close = async () => {
|
||||||
|
await reservedTurnStoreHandle.close();
|
||||||
|
await gatewayGate?.close();
|
||||||
|
};
|
||||||
|
} else if (gatewayGate) {
|
||||||
|
hooks = {
|
||||||
|
onRunError: async (error) => {
|
||||||
|
await gatewayGate?.markPaused(error);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
close = async () => {
|
||||||
|
await gatewayGate.close();
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const defaultBudget: TurnRunBudget = options.defaultBudget ?? {
|
const defaultBudget: TurnRunBudget = options.defaultBudget ?? {
|
||||||
@@ -149,6 +188,7 @@ export const createTurnDaemonRuntime = async (
|
|||||||
stateStore,
|
stateStore,
|
||||||
processor,
|
processor,
|
||||||
hooks,
|
hooks,
|
||||||
|
pauseGate,
|
||||||
},
|
},
|
||||||
{ profile: options.profile, defaultBudget }
|
{ profile: options.profile, defaultBudget }
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api",
|
"build": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api",
|
||||||
"dev": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api --watch",
|
"dev": "tsdown -c ../../tsdown.config.ts -F @sammo-ts/gateway-api --watch",
|
||||||
|
"orchestrator": "GATEWAY_ROLE=orchestrator node dist/index.js",
|
||||||
"lint": "node -e \"console.log('lint not configured')\"",
|
"lint": "node -e \"console.log('lint not configured')\"",
|
||||||
"test": "vitest run --config vitest.config.ts",
|
"test": "vitest run --config vitest.config.ts",
|
||||||
"typecheck": "tsc --noEmit"
|
"typecheck": "tsc --noEmit"
|
||||||
|
|||||||
@@ -58,7 +58,10 @@ export const adminRouter = router({
|
|||||||
scenario: z.string().min(1).max(64),
|
scenario: z.string().min(1).max(64),
|
||||||
apiPort: z.number().int().min(1).max(65535),
|
apiPort: z.number().int().min(1).max(65535),
|
||||||
status: zProfileStatus.optional(),
|
status: zProfileStatus.optional(),
|
||||||
|
preopenAt: z.string().datetime().optional(),
|
||||||
|
openAt: z.string().datetime().optional(),
|
||||||
scheduledStartAt: z.string().datetime().optional(),
|
scheduledStartAt: z.string().datetime().optional(),
|
||||||
|
buildCommitSha: z.string().min(7).max(64).optional(),
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
.mutation(async ({ ctx, input }) => {
|
.mutation(async ({ ctx, input }) => {
|
||||||
@@ -68,7 +71,10 @@ export const adminRouter = router({
|
|||||||
scenario: input.scenario,
|
scenario: input.scenario,
|
||||||
apiPort: input.apiPort,
|
apiPort: input.apiPort,
|
||||||
status,
|
status,
|
||||||
|
preopenAt: input.preopenAt,
|
||||||
|
openAt: input.openAt,
|
||||||
scheduledStartAt: input.scheduledStartAt,
|
scheduledStartAt: input.scheduledStartAt,
|
||||||
|
buildCommitSha: input.buildCommitSha,
|
||||||
});
|
});
|
||||||
}),
|
}),
|
||||||
setStatus: adminProcedure
|
setStatus: adminProcedure
|
||||||
@@ -76,21 +82,34 @@ export const adminRouter = router({
|
|||||||
z.object({
|
z.object({
|
||||||
profileName: z.string().min(1),
|
profileName: z.string().min(1),
|
||||||
status: zProfileStatus,
|
status: zProfileStatus,
|
||||||
|
preopenAt: z.string().datetime().optional(),
|
||||||
|
openAt: z.string().datetime().optional(),
|
||||||
scheduledStartAt: z.string().datetime().optional(),
|
scheduledStartAt: z.string().datetime().optional(),
|
||||||
|
buildCommitSha: z.string().min(7).max(64).optional(),
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
.mutation(async ({ ctx, input }) => {
|
.mutation(async ({ ctx, input }) => {
|
||||||
if (input.status === 'RESERVED' && !input.scheduledStartAt) {
|
if (input.status === 'RESERVED' && (!input.preopenAt || !input.openAt)) {
|
||||||
throw new TRPCError({
|
throw new TRPCError({
|
||||||
code: 'BAD_REQUEST',
|
code: 'BAD_REQUEST',
|
||||||
message: 'scheduledStartAt is required for RESERVED status.',
|
message: 'preopenAt and openAt are required for RESERVED status.',
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const result = await ctx.profiles.updateStatus(
|
const result = await ctx.profiles.updateStatus(
|
||||||
input.profileName,
|
input.profileName,
|
||||||
input.status,
|
input.status,
|
||||||
input.status === 'RESERVED' ? input.scheduledStartAt : null
|
{
|
||||||
|
preopenAt: input.preopenAt,
|
||||||
|
openAt: input.openAt,
|
||||||
|
scheduledStartAt:
|
||||||
|
input.status === 'RESERVED' ? input.scheduledStartAt : null,
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
if (input.buildCommitSha) {
|
||||||
|
await ctx.profiles.updateBuildStatus(input.profileName, 'IDLE', {
|
||||||
|
commitSha: input.buildCommitSha,
|
||||||
|
});
|
||||||
|
}
|
||||||
await ctx.orchestrator.reconcileNow();
|
await ctx.orchestrator.reconcileNow();
|
||||||
return result;
|
return result;
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -23,6 +23,16 @@ export interface GatewayApiConfig {
|
|||||||
worktreeRoot: string;
|
worktreeRoot: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface GatewayOrchestratorConfig {
|
||||||
|
redisKeyPrefix: string;
|
||||||
|
gameTokenSecret: string;
|
||||||
|
orchestratorReconcileIntervalMs: number;
|
||||||
|
orchestratorScheduleIntervalMs: number;
|
||||||
|
orchestratorBuildIntervalMs: number;
|
||||||
|
workspaceRootHint: string;
|
||||||
|
worktreeRoot: string;
|
||||||
|
}
|
||||||
|
|
||||||
const parseNumber = (value: string | undefined, fallback: number, label: string): number => {
|
const parseNumber = (value: string | undefined, fallback: number, label: string): number => {
|
||||||
if (!value) {
|
if (!value) {
|
||||||
return fallback;
|
return fallback;
|
||||||
@@ -85,7 +95,40 @@ export const resolveGatewayApiConfigFromEnv = (
|
|||||||
kakaoRedirectUri,
|
kakaoRedirectUri,
|
||||||
publicBaseUrl,
|
publicBaseUrl,
|
||||||
adminToken: env.GATEWAY_ADMIN_TOKEN,
|
adminToken: env.GATEWAY_ADMIN_TOKEN,
|
||||||
orchestratorEnabled: parseBoolean(env.GATEWAY_ORCHESTRATOR_ENABLED, true),
|
orchestratorEnabled: parseBoolean(env.GATEWAY_ORCHESTRATOR_ENABLED, false),
|
||||||
|
orchestratorReconcileIntervalMs: parseNumber(
|
||||||
|
env.GATEWAY_ORCHESTRATOR_RECONCILE_MS,
|
||||||
|
15000,
|
||||||
|
'GATEWAY_ORCHESTRATOR_RECONCILE_MS'
|
||||||
|
),
|
||||||
|
orchestratorScheduleIntervalMs: parseNumber(
|
||||||
|
env.GATEWAY_ORCHESTRATOR_SCHEDULE_MS,
|
||||||
|
5000,
|
||||||
|
'GATEWAY_ORCHESTRATOR_SCHEDULE_MS'
|
||||||
|
),
|
||||||
|
orchestratorBuildIntervalMs: parseNumber(
|
||||||
|
env.GATEWAY_ORCHESTRATOR_BUILD_MS,
|
||||||
|
10000,
|
||||||
|
'GATEWAY_ORCHESTRATOR_BUILD_MS'
|
||||||
|
),
|
||||||
|
workspaceRootHint: env.GATEWAY_WORKSPACE_ROOT ?? process.cwd(),
|
||||||
|
worktreeRoot:
|
||||||
|
env.GATEWAY_WORKTREE_ROOT ??
|
||||||
|
path.resolve(env.GATEWAY_WORKSPACE_ROOT ?? process.cwd(), '.worktrees'),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const resolveGatewayOrchestratorConfigFromEnv = (
|
||||||
|
env: NodeJS.ProcessEnv = process.env
|
||||||
|
): GatewayOrchestratorConfig => {
|
||||||
|
const secret = env.GAME_TOKEN_SECRET ?? env.GATEWAY_TOKEN_SECRET ?? '';
|
||||||
|
if (!secret) {
|
||||||
|
throw new Error('GAME_TOKEN_SECRET is required for game server processes.');
|
||||||
|
}
|
||||||
|
const redisKeyPrefix = env.GATEWAY_REDIS_PREFIX ?? 'sammo:gateway';
|
||||||
|
return {
|
||||||
|
redisKeyPrefix,
|
||||||
|
gameTokenSecret: secret,
|
||||||
orchestratorReconcileIntervalMs: parseNumber(
|
orchestratorReconcileIntervalMs: parseNumber(
|
||||||
env.GATEWAY_ORCHESTRATOR_RECONCILE_MS,
|
env.GATEWAY_ORCHESTRATOR_RECONCILE_MS,
|
||||||
15000,
|
15000,
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import path from 'node:path';
|
|||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
import { runGatewayApiServer } from './server.js';
|
import { runGatewayApiServer } from './server.js';
|
||||||
|
import { runGatewayOrchestrator } from './orchestrator/orchestratorServer.js';
|
||||||
|
|
||||||
export * from './config.js';
|
export * from './config.js';
|
||||||
export * from './context.js';
|
export * from './context.js';
|
||||||
@@ -27,8 +28,11 @@ const isMain = (): boolean => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (isMain()) {
|
if (isMain()) {
|
||||||
runGatewayApiServer().catch((error) => {
|
const role = process.env.GATEWAY_ROLE ?? 'api';
|
||||||
console.error('[gateway-api] failed to start', error);
|
const run = role === 'orchestrator' ? runGatewayOrchestrator : runGatewayApiServer;
|
||||||
|
run().catch((error) => {
|
||||||
|
const prefix = role === 'orchestrator' ? 'gateway-orchestrator' : 'gateway-api';
|
||||||
|
console.error(`[${prefix}] failed to start`, error);
|
||||||
process.exitCode = 1;
|
process.exitCode = 1;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,7 +54,12 @@ export const planProfileReconcile = (
|
|||||||
status: GatewayProfileStatus,
|
status: GatewayProfileStatus,
|
||||||
runtime: ProfileRuntimeState
|
runtime: ProfileRuntimeState
|
||||||
): { shouldStart: boolean; shouldStop: boolean } => {
|
): { shouldStart: boolean; shouldStop: boolean } => {
|
||||||
if (status === 'RUNNING') {
|
if (
|
||||||
|
status === 'RUNNING' ||
|
||||||
|
status === 'PREOPEN' ||
|
||||||
|
status === 'PAUSED' ||
|
||||||
|
status === 'COMPLETED'
|
||||||
|
) {
|
||||||
return {
|
return {
|
||||||
shouldStart: !(runtime.apiRunning && runtime.daemonRunning),
|
shouldStart: !(runtime.apiRunning && runtime.daemonRunning),
|
||||||
shouldStop: false,
|
shouldStop: false,
|
||||||
@@ -94,6 +99,7 @@ const buildProcessDefinitions = (
|
|||||||
TURN_PROFILE: profile.profile,
|
TURN_PROFILE: profile.profile,
|
||||||
PROFILE: profile.profile,
|
PROFILE: profile.profile,
|
||||||
SCENARIO: profile.scenario,
|
SCENARIO: profile.scenario,
|
||||||
|
TURN_PROFILE_NAME: profile.profileName,
|
||||||
};
|
};
|
||||||
return {
|
return {
|
||||||
api: {
|
api: {
|
||||||
@@ -221,18 +227,41 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
const now = this.now();
|
const now = this.now();
|
||||||
const due = await this.repository.listReservedToStart(now);
|
const due = await this.repository.listReservedToStart(now);
|
||||||
for (const profile of due) {
|
for (const profile of due) {
|
||||||
try {
|
if (!profile.preopenAt || !profile.openAt) {
|
||||||
await this.repository.updateStatus(
|
|
||||||
profile.profileName,
|
|
||||||
'RUNNING',
|
|
||||||
null
|
|
||||||
);
|
|
||||||
await this.startProfile(profile);
|
|
||||||
} catch (error) {
|
|
||||||
await this.repository.updateLastError(
|
await this.repository.updateLastError(
|
||||||
profile.profileName,
|
profile.profileName,
|
||||||
error instanceof Error ? error.message : 'Failed to start scheduled profile.'
|
'Reserved profile is missing preopen/open schedule.'
|
||||||
);
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!profile.buildCommitSha) {
|
||||||
|
await this.repository.updateLastError(
|
||||||
|
profile.profileName,
|
||||||
|
'Reserved profile is missing build commit SHA.'
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const queued =
|
||||||
|
profile.buildStatus === 'QUEUED' || profile.buildStatus === 'RUNNING';
|
||||||
|
if (!queued) {
|
||||||
|
await this.repository.updateBuildStatus(profile.profileName, 'QUEUED', {
|
||||||
|
requestedAt: now.toISOString(),
|
||||||
|
error: null,
|
||||||
|
commitSha: profile.buildCommitSha,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const profiles = await this.repository.listProfiles();
|
||||||
|
for (const profile of profiles) {
|
||||||
|
if (
|
||||||
|
profile.status === 'PREOPEN' &&
|
||||||
|
profile.openAt &&
|
||||||
|
new Date(profile.openAt) <= now
|
||||||
|
) {
|
||||||
|
await this.repository.updateStatus(profile.profileName, 'RUNNING', {
|
||||||
|
preopenAt: profile.preopenAt ?? null,
|
||||||
|
openAt: profile.openAt ?? null,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
@@ -304,12 +333,24 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
completedAt,
|
completedAt,
|
||||||
error: null,
|
error: null,
|
||||||
});
|
});
|
||||||
if (queued.status !== 'RUNNING' && queued.status !== 'DISABLED') {
|
if (queued.status === 'RESERVED') {
|
||||||
await this.repository.updateStatus(
|
await this.repository.updateStatus(
|
||||||
queued.profileName,
|
queued.profileName,
|
||||||
'COMPLETED',
|
queued.openAt && new Date(queued.openAt) <= this.now()
|
||||||
queued.scheduledStartAt ? queued.scheduledStartAt : null
|
? 'RUNNING'
|
||||||
|
: 'PREOPEN',
|
||||||
|
{
|
||||||
|
preopenAt: queued.preopenAt ?? null,
|
||||||
|
openAt: queued.openAt ?? null,
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
} else if (queued.status === 'PREOPEN' && queued.openAt) {
|
||||||
|
if (new Date(queued.openAt) <= this.now()) {
|
||||||
|
await this.repository.updateStatus(queued.profileName, 'RUNNING', {
|
||||||
|
preopenAt: queued.preopenAt ?? null,
|
||||||
|
openAt: queued.openAt ?? null,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
await this.repository.updateBuildStatus(queued.profileName, 'FAILED', {
|
await this.repository.updateBuildStatus(queued.profileName, 'FAILED', {
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
import type { PrismaClient } from '@prisma/client';
|
||||||
|
|
||||||
|
import type { GatewayOrchestratorConfig } from '../config.js';
|
||||||
|
import { createGatewayProfileRepository } from './profileRepository.js';
|
||||||
|
import { GatewayOrchestrator } from './gatewayOrchestrator.js';
|
||||||
|
import { Pm2ProcessManager } from './pm2ProcessManager.js';
|
||||||
|
import { PnpmBuildRunner } from './buildRunner.js';
|
||||||
|
import { resolveWorkspaceRoot } from './workspaceRoot.js';
|
||||||
|
import { GitWorkspaceManager } from './workspaceManager.js';
|
||||||
|
|
||||||
|
export const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
|
||||||
|
const entries = Object.entries(env).filter(
|
||||||
|
(entry): entry is [string, string] => typeof entry[1] === 'string'
|
||||||
|
);
|
||||||
|
return Object.fromEntries(entries);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createGatewayOrchestrator = (
|
||||||
|
prisma: PrismaClient,
|
||||||
|
config: GatewayOrchestratorConfig,
|
||||||
|
env: NodeJS.ProcessEnv = process.env
|
||||||
|
): {
|
||||||
|
orchestrator: GatewayOrchestrator;
|
||||||
|
profiles: ReturnType<typeof createGatewayProfileRepository>;
|
||||||
|
} => {
|
||||||
|
const profiles = createGatewayProfileRepository(prisma);
|
||||||
|
const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint);
|
||||||
|
const processManager = new Pm2ProcessManager();
|
||||||
|
const buildRunner = new PnpmBuildRunner();
|
||||||
|
const baseEnv = buildEnvMap(env);
|
||||||
|
const workspaceManager = new GitWorkspaceManager({
|
||||||
|
repoRoot: workspaceRoot,
|
||||||
|
worktreeRoot: config.worktreeRoot,
|
||||||
|
baseEnv,
|
||||||
|
});
|
||||||
|
const orchestrator = new GatewayOrchestrator({
|
||||||
|
repository: profiles,
|
||||||
|
processManager,
|
||||||
|
buildRunner,
|
||||||
|
workspaceManager,
|
||||||
|
processConfig: {
|
||||||
|
workspaceRoot,
|
||||||
|
redisKeyPrefix: config.redisKeyPrefix,
|
||||||
|
gameTokenSecret: config.gameTokenSecret,
|
||||||
|
baseEnv,
|
||||||
|
},
|
||||||
|
reconcileIntervalMs: config.orchestratorReconcileIntervalMs,
|
||||||
|
scheduleIntervalMs: config.orchestratorScheduleIntervalMs,
|
||||||
|
buildIntervalMs: config.orchestratorBuildIntervalMs,
|
||||||
|
});
|
||||||
|
return { orchestrator, profiles };
|
||||||
|
};
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import type { PrismaClient } from '@prisma/client';
|
||||||
|
import {
|
||||||
|
createPostgresConnector,
|
||||||
|
resolvePostgresConfigFromEnv,
|
||||||
|
} from '@sammo-ts/infra';
|
||||||
|
|
||||||
|
import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js';
|
||||||
|
import { createGatewayOrchestrator } from './orchestratorFactory.js';
|
||||||
|
|
||||||
|
export const runGatewayOrchestrator = async (): Promise<void> => {
|
||||||
|
const config = resolveGatewayOrchestratorConfigFromEnv();
|
||||||
|
const postgres = createPostgresConnector(resolvePostgresConfigFromEnv());
|
||||||
|
await postgres.connect();
|
||||||
|
|
||||||
|
const { orchestrator } = createGatewayOrchestrator(
|
||||||
|
postgres.prisma as PrismaClient,
|
||||||
|
config,
|
||||||
|
process.env
|
||||||
|
);
|
||||||
|
|
||||||
|
const stop = async (reason: string): Promise<void> => {
|
||||||
|
console.info(`[gateway-orchestrator] stopping: ${reason}`);
|
||||||
|
await orchestrator.stop();
|
||||||
|
await postgres.disconnect();
|
||||||
|
};
|
||||||
|
|
||||||
|
process.on('SIGINT', () => void stop('SIGINT'));
|
||||||
|
process.on('SIGTERM', () => void stop('SIGTERM'));
|
||||||
|
|
||||||
|
orchestrator.start();
|
||||||
|
console.info('[gateway-orchestrator] started');
|
||||||
|
};
|
||||||
@@ -1,9 +1,11 @@
|
|||||||
import { Prisma, type PrismaClient } from '@prisma/client';
|
import { Prisma, type PrismaClient } from '@prisma/client';
|
||||||
|
|
||||||
export const GATEWAY_PROFILE_STATUSES = [
|
export const GATEWAY_PROFILE_STATUSES = [
|
||||||
'COMPLETED',
|
|
||||||
'RESERVED',
|
'RESERVED',
|
||||||
|
'PREOPEN',
|
||||||
'RUNNING',
|
'RUNNING',
|
||||||
|
'PAUSED',
|
||||||
|
'COMPLETED',
|
||||||
'STOPPED',
|
'STOPPED',
|
||||||
'DISABLED',
|
'DISABLED',
|
||||||
] as const;
|
] as const;
|
||||||
@@ -28,6 +30,8 @@ export interface GatewayProfileRecord {
|
|||||||
buildCommitSha?: string;
|
buildCommitSha?: string;
|
||||||
buildWorkspace?: string;
|
buildWorkspace?: string;
|
||||||
buildLastUsedAt?: string;
|
buildLastUsedAt?: string;
|
||||||
|
preopenAt?: string;
|
||||||
|
openAt?: string;
|
||||||
scheduledStartAt?: string;
|
scheduledStartAt?: string;
|
||||||
buildRequestedAt?: string;
|
buildRequestedAt?: string;
|
||||||
buildStartedAt?: string;
|
buildStartedAt?: string;
|
||||||
@@ -44,7 +48,10 @@ export interface GatewayProfileUpsertInput {
|
|||||||
scenario: string;
|
scenario: string;
|
||||||
apiPort: number;
|
apiPort: number;
|
||||||
status?: GatewayProfileStatus;
|
status?: GatewayProfileStatus;
|
||||||
|
preopenAt?: string;
|
||||||
|
openAt?: string;
|
||||||
scheduledStartAt?: string;
|
scheduledStartAt?: string;
|
||||||
|
buildCommitSha?: string;
|
||||||
meta?: Prisma.JsonObject;
|
meta?: Prisma.JsonObject;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,7 +62,11 @@ export interface GatewayProfileRepository {
|
|||||||
updateStatus(
|
updateStatus(
|
||||||
profileName: string,
|
profileName: string,
|
||||||
status: GatewayProfileStatus,
|
status: GatewayProfileStatus,
|
||||||
scheduledStartAt?: string | null
|
schedule?: {
|
||||||
|
preopenAt?: string | null;
|
||||||
|
openAt?: string | null;
|
||||||
|
scheduledStartAt?: string | null;
|
||||||
|
}
|
||||||
): Promise<GatewayProfileRecord | null>;
|
): Promise<GatewayProfileRecord | null>;
|
||||||
updateBuildStatus(
|
updateBuildStatus(
|
||||||
profileName: string,
|
profileName: string,
|
||||||
@@ -94,6 +105,8 @@ const mapProfile = (row: {
|
|||||||
buildCommitSha: string | null;
|
buildCommitSha: string | null;
|
||||||
buildWorkspace: string | null;
|
buildWorkspace: string | null;
|
||||||
buildLastUsedAt: Date | null;
|
buildLastUsedAt: Date | null;
|
||||||
|
preopenAt: Date | null;
|
||||||
|
openAt: Date | null;
|
||||||
scheduledStartAt: Date | null;
|
scheduledStartAt: Date | null;
|
||||||
buildRequestedAt: Date | null;
|
buildRequestedAt: Date | null;
|
||||||
buildStartedAt: Date | null;
|
buildStartedAt: Date | null;
|
||||||
@@ -113,6 +126,8 @@ const mapProfile = (row: {
|
|||||||
buildCommitSha: row.buildCommitSha ?? undefined,
|
buildCommitSha: row.buildCommitSha ?? undefined,
|
||||||
buildWorkspace: row.buildWorkspace ?? undefined,
|
buildWorkspace: row.buildWorkspace ?? undefined,
|
||||||
buildLastUsedAt: toIso(row.buildLastUsedAt),
|
buildLastUsedAt: toIso(row.buildLastUsedAt),
|
||||||
|
preopenAt: toIso(row.preopenAt),
|
||||||
|
openAt: toIso(row.openAt),
|
||||||
scheduledStartAt: toIso(row.scheduledStartAt),
|
scheduledStartAt: toIso(row.scheduledStartAt),
|
||||||
buildRequestedAt: toIso(row.buildRequestedAt),
|
buildRequestedAt: toIso(row.buildRequestedAt),
|
||||||
buildStartedAt: toIso(row.buildStartedAt),
|
buildStartedAt: toIso(row.buildStartedAt),
|
||||||
@@ -152,19 +167,36 @@ export const createGatewayProfileRepository = (
|
|||||||
scenario: input.scenario,
|
scenario: input.scenario,
|
||||||
apiPort: input.apiPort,
|
apiPort: input.apiPort,
|
||||||
status: input.status ?? 'STOPPED',
|
status: input.status ?? 'STOPPED',
|
||||||
|
preopenAt: input.preopenAt ? new Date(input.preopenAt) : null,
|
||||||
|
openAt: input.openAt ? new Date(input.openAt) : null,
|
||||||
scheduledStartAt: input.scheduledStartAt
|
scheduledStartAt: input.scheduledStartAt
|
||||||
? new Date(input.scheduledStartAt)
|
? new Date(input.scheduledStartAt)
|
||||||
: null,
|
: null,
|
||||||
|
buildCommitSha: input.buildCommitSha ?? null,
|
||||||
meta: (input.meta ?? {}) as Prisma.JsonObject,
|
meta: (input.meta ?? {}) as Prisma.JsonObject,
|
||||||
},
|
},
|
||||||
update: {
|
update: {
|
||||||
apiPort: input.apiPort,
|
apiPort: input.apiPort,
|
||||||
status: input.status,
|
status: input.status,
|
||||||
|
preopenAt: input.preopenAt
|
||||||
|
? new Date(input.preopenAt)
|
||||||
|
: input.preopenAt === null
|
||||||
|
? null
|
||||||
|
: undefined,
|
||||||
|
openAt: input.openAt
|
||||||
|
? new Date(input.openAt)
|
||||||
|
: input.openAt === null
|
||||||
|
? null
|
||||||
|
: undefined,
|
||||||
scheduledStartAt: input.scheduledStartAt
|
scheduledStartAt: input.scheduledStartAt
|
||||||
? new Date(input.scheduledStartAt)
|
? new Date(input.scheduledStartAt)
|
||||||
: input.scheduledStartAt === null
|
: input.scheduledStartAt === null
|
||||||
? null
|
? null
|
||||||
: undefined,
|
: undefined,
|
||||||
|
buildCommitSha:
|
||||||
|
input.buildCommitSha === undefined
|
||||||
|
? undefined
|
||||||
|
: input.buildCommitSha,
|
||||||
meta: input.meta ? (input.meta as Prisma.JsonObject) : undefined,
|
meta: input.meta ? (input.meta as Prisma.JsonObject) : undefined,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -173,17 +205,33 @@ export const createGatewayProfileRepository = (
|
|||||||
async updateStatus(
|
async updateStatus(
|
||||||
profileName: string,
|
profileName: string,
|
||||||
status: GatewayProfileStatus,
|
status: GatewayProfileStatus,
|
||||||
scheduledStartAt?: string | null
|
schedule?: {
|
||||||
|
preopenAt?: string | null;
|
||||||
|
openAt?: string | null;
|
||||||
|
scheduledStartAt?: string | null;
|
||||||
|
}
|
||||||
): Promise<GatewayProfileRecord | null> {
|
): Promise<GatewayProfileRecord | null> {
|
||||||
const row = await prisma.gatewayProfile.update({
|
const row = await prisma.gatewayProfile.update({
|
||||||
where: { profileName },
|
where: { profileName },
|
||||||
data: {
|
data: {
|
||||||
status,
|
status,
|
||||||
scheduledStartAt:
|
preopenAt:
|
||||||
scheduledStartAt === undefined
|
schedule?.preopenAt === undefined
|
||||||
? undefined
|
? undefined
|
||||||
: scheduledStartAt
|
: schedule?.preopenAt
|
||||||
? new Date(scheduledStartAt)
|
? new Date(schedule.preopenAt)
|
||||||
|
: null,
|
||||||
|
openAt:
|
||||||
|
schedule?.openAt === undefined
|
||||||
|
? undefined
|
||||||
|
: schedule?.openAt
|
||||||
|
? new Date(schedule.openAt)
|
||||||
|
: null,
|
||||||
|
scheduledStartAt:
|
||||||
|
schedule?.scheduledStartAt === undefined
|
||||||
|
? undefined
|
||||||
|
: schedule?.scheduledStartAt
|
||||||
|
? new Date(schedule.scheduledStartAt)
|
||||||
: null,
|
: null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -240,7 +288,7 @@ export const createGatewayProfileRepository = (
|
|||||||
const rows = await prisma.gatewayProfile.findMany({
|
const rows = await prisma.gatewayProfile.findMany({
|
||||||
where: {
|
where: {
|
||||||
status: 'RESERVED',
|
status: 'RESERVED',
|
||||||
scheduledStartAt: {
|
preopenAt: {
|
||||||
lte: now,
|
lte: now,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -16,21 +16,9 @@ import { KakaoOAuthClient } from './auth/kakaoClient.js';
|
|||||||
import { RedisOAuthSessionStore } from './auth/oauthSessionStore.js';
|
import { RedisOAuthSessionStore } from './auth/oauthSessionStore.js';
|
||||||
import { createPostgresUserRepository } from './auth/postgresUserRepository.js';
|
import { createPostgresUserRepository } from './auth/postgresUserRepository.js';
|
||||||
import { RedisGatewaySessionService } from './auth/redisSessionService.js';
|
import { RedisGatewaySessionService } from './auth/redisSessionService.js';
|
||||||
import { createGatewayProfileRepository } from './orchestrator/profileRepository.js';
|
import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js';
|
||||||
import { GatewayOrchestrator } from './orchestrator/gatewayOrchestrator.js';
|
|
||||||
import { Pm2ProcessManager } from './orchestrator/pm2ProcessManager.js';
|
|
||||||
import { PnpmBuildRunner } from './orchestrator/buildRunner.js';
|
|
||||||
import { resolveWorkspaceRoot } from './orchestrator/workspaceRoot.js';
|
|
||||||
import { GitWorkspaceManager } from './orchestrator/workspaceManager.js';
|
|
||||||
import { appRouter } from './router.js';
|
import { appRouter } from './router.js';
|
||||||
|
|
||||||
const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
|
|
||||||
const entries = Object.entries(env).filter(
|
|
||||||
(entry): entry is [string, string] => typeof entry[1] === 'string'
|
|
||||||
);
|
|
||||||
return Object.fromEntries(entries);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const createGatewayApiServer = async () => {
|
export const createGatewayApiServer = async () => {
|
||||||
const config = resolveGatewayApiConfigFromEnv();
|
const config = resolveGatewayApiConfigFromEnv();
|
||||||
const postgres = createPostgresConnector(resolvePostgresConfigFromEnv());
|
const postgres = createPostgresConnector(resolvePostgresConfigFromEnv());
|
||||||
@@ -58,33 +46,11 @@ export const createGatewayApiServer = async () => {
|
|||||||
config.oauthSessionTtlSeconds
|
config.oauthSessionTtlSeconds
|
||||||
);
|
);
|
||||||
|
|
||||||
const profiles = createGatewayProfileRepository(
|
const { orchestrator, profiles } = createGatewayOrchestrator(
|
||||||
postgres.prisma as PrismaClient
|
postgres.prisma as PrismaClient,
|
||||||
|
config,
|
||||||
|
process.env
|
||||||
);
|
);
|
||||||
const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint);
|
|
||||||
const processManager = new Pm2ProcessManager();
|
|
||||||
const buildRunner = new PnpmBuildRunner();
|
|
||||||
const baseEnv = buildEnvMap(process.env);
|
|
||||||
const workspaceManager = new GitWorkspaceManager({
|
|
||||||
repoRoot: workspaceRoot,
|
|
||||||
worktreeRoot: config.worktreeRoot,
|
|
||||||
baseEnv,
|
|
||||||
});
|
|
||||||
const orchestrator = new GatewayOrchestrator({
|
|
||||||
repository: profiles,
|
|
||||||
processManager,
|
|
||||||
buildRunner,
|
|
||||||
workspaceManager,
|
|
||||||
processConfig: {
|
|
||||||
workspaceRoot,
|
|
||||||
redisKeyPrefix: config.redisKeyPrefix,
|
|
||||||
gameTokenSecret: config.gameTokenSecret,
|
|
||||||
baseEnv,
|
|
||||||
},
|
|
||||||
reconcileIntervalMs: config.orchestratorReconcileIntervalMs,
|
|
||||||
scheduleIntervalMs: config.orchestratorScheduleIntervalMs,
|
|
||||||
buildIntervalMs: config.orchestratorBuildIntervalMs,
|
|
||||||
});
|
|
||||||
|
|
||||||
const app = fastify({
|
const app = fastify({
|
||||||
logger: true,
|
logger: true,
|
||||||
@@ -121,10 +87,6 @@ export const createGatewayApiServer = async () => {
|
|||||||
ok: true,
|
ok: true,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
if (config.orchestratorEnabled) {
|
|
||||||
orchestrator.start();
|
|
||||||
}
|
|
||||||
|
|
||||||
app.addHook('onClose', async () => {
|
app.addHook('onClose', async () => {
|
||||||
await orchestrator.stop();
|
await orchestrator.stop();
|
||||||
await redis.disconnect();
|
await redis.disconnect();
|
||||||
|
|||||||
@@ -12,6 +12,15 @@ describe('planProfileReconcile', () => {
|
|||||||
).toEqual({ shouldStart: true, shouldStop: false });
|
).toEqual({ shouldStart: true, shouldStop: false });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('starts processes for preopen profiles', () => {
|
||||||
|
expect(
|
||||||
|
planProfileReconcile('PREOPEN', {
|
||||||
|
apiRunning: false,
|
||||||
|
daemonRunning: false,
|
||||||
|
})
|
||||||
|
).toEqual({ shouldStart: true, shouldStop: false });
|
||||||
|
});
|
||||||
|
|
||||||
it('does nothing when running profile is healthy', () => {
|
it('does nothing when running profile is healthy', () => {
|
||||||
expect(
|
expect(
|
||||||
planProfileReconcile('RUNNING', {
|
planProfileReconcile('RUNNING', {
|
||||||
|
|||||||
@@ -20,7 +20,8 @@ selection is required because it drives unit sets and DB settings.
|
|||||||
## Gateway Orchestration (Single Host Draft)
|
## Gateway Orchestration (Single Host Draft)
|
||||||
|
|
||||||
Gateway API is the single source of truth for profile state and reconciles
|
Gateway API is the single source of truth for profile state and reconciles
|
||||||
PM2-managed processes on boot and on a short interval. The DB owns the desired
|
PM2-managed processes on boot and on a short interval. The orchestrator runs
|
||||||
|
as a separate process (`GATEWAY_ROLE=orchestrator`). The DB owns the desired
|
||||||
state; PM2 is treated as the actuator. The runtime state is grouped so that
|
state; PM2 is treated as the actuator. The runtime state is grouped so that
|
||||||
`game-api` + `turn-daemon` are either on together or off together.
|
`game-api` + `turn-daemon` are either on together or off together.
|
||||||
|
|
||||||
@@ -29,10 +30,12 @@ state; PM2 is treated as the actuator. The runtime state is grouped so that
|
|||||||
Profiles are tracked by `profileName` (= `${profile}:${scenario}`).
|
Profiles are tracked by `profileName` (= `${profile}:${scenario}`).
|
||||||
Gateway loads the profile table on boot, then reconciles PM2 to match.
|
Gateway loads the profile table on boot, then reconciles PM2 to match.
|
||||||
|
|
||||||
- `완료됨` (COMPLETED): build output exists; processes should be off.
|
- `예약됨` (RESERVED): preopen/open timestamps are set; processes off.
|
||||||
- `예약됨` (RESERVED): start is scheduled for a timestamp; processes off.
|
- `가오픈` (PREOPEN): build done; API+daemon on, daemon paused.
|
||||||
- `가동중` (RUNNING): both `game-api` and `turn-daemon` should be on.
|
- `가동중` (RUNNING): both `game-api` and `turn-daemon` should be on.
|
||||||
- `정지됨` (STOPPED): processes off; may be resumed later.
|
- `정지됨` (STOPPED): processes off; may be resumed later.
|
||||||
|
- `정지(오류)` (PAUSED): fatal error; daemon paused but process remains on.
|
||||||
|
- `천하통일` (COMPLETED): game finished; API on, daemon paused.
|
||||||
- `비활성화` (DISABLED): excluded from orchestration; start forbidden.
|
- `비활성화` (DISABLED): excluded from orchestration; start forbidden.
|
||||||
|
|
||||||
### Boot Reconciliation
|
### Boot Reconciliation
|
||||||
@@ -40,29 +43,30 @@ Gateway loads the profile table on boot, then reconciles PM2 to match.
|
|||||||
1) Load profile rows from DB.
|
1) Load profile rows from DB.
|
||||||
2) List PM2 processes and map `profileName -> running state`.
|
2) List PM2 processes and map `profileName -> running state`.
|
||||||
3) For each profile:
|
3) For each profile:
|
||||||
- If desired `RUNNING` and any process is missing, start missing processes.
|
- If desired `RUNNING/PREOPEN/PAUSED/COMPLETED` and any process is missing, start.
|
||||||
- If desired not `RUNNING` and any process is running, stop both processes.
|
- If desired `RESERVED/STOPPED/DISABLED` and any process is running, stop both.
|
||||||
4) Persist errors to DB for audit.
|
4) Persist errors to DB for audit.
|
||||||
|
|
||||||
### Internal Scheduler (Gateway Cron)
|
### Internal Scheduler (Gateway Cron)
|
||||||
|
|
||||||
Gateway runs a lightweight cron loop (setInterval) that:
|
Gateway runs a lightweight cron loop (setInterval) that:
|
||||||
- Promotes `RESERVED` profiles whose `scheduledStartAt <= now` to `RUNNING`.
|
- When `RESERVED` and `preopenAt <= now`, queue a build for the reserved commit.
|
||||||
- Triggers a reconcile immediately after promotion.
|
- When build succeeds, status becomes `PREOPEN` (daemon paused).
|
||||||
|
- When `openAt <= now`, status becomes `RUNNING` and daemon resumes.
|
||||||
- Optionally drains a build queue (see build workflow).
|
- Optionally drains a build queue (see build workflow).
|
||||||
|
|
||||||
### Build Workflow (Admin)
|
### Build Workflow (Admin)
|
||||||
|
|
||||||
- Admin triggers a build request for a profile.
|
- Admin triggers a build request for a profile.
|
||||||
- Gateway queues a build job with `(profileName, commitSha)` and prepares a
|
- Gateway queues a build job with `(profileName, commitSha)` and prepares a
|
||||||
per-commit workspace (`/var/sammo/workspaces/{commitSha}` recommended).
|
per-commit workspace (`/.worktrees/{commitSha}` by default).
|
||||||
- Workspace is backed by `git worktree` and is reused across builds for the same commit.
|
- Workspace is backed by `git worktree` and is reused across builds for the same commit.
|
||||||
- Each workspace stores `lastUsedAt` in DB so cleanup can remove stale worktrees.
|
- Each workspace stores `lastUsedAt` in DB so cleanup can remove stale worktrees.
|
||||||
- Cleanup is invoked manually by admin API and removes worktrees unused for 6+ months.
|
- Cleanup is invoked manually by admin API and removes worktrees unused for 6+ months.
|
||||||
- Build runs `pnpm install` when workspace is created, then executes
|
- Build runs `pnpm install` when workspace is created, then executes
|
||||||
`pnpm --filter @sammo-ts/game-api build` and
|
`pnpm --filter @sammo-ts/game-api build` and
|
||||||
`pnpm --filter @sammo-ts/game-engine build`, then marks build success/failure.
|
`pnpm --filter @sammo-ts/game-engine build`, then marks build success/failure.
|
||||||
- On success, profile status remains `COMPLETED` (or stays `RUNNING` if already on).
|
- On success, status moves to `PREOPEN` for reserved builds or stays unchanged for manual builds.
|
||||||
|
|
||||||
## Current Implementation Status
|
## Current Implementation Status
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
-- Create enums for gateway profile tracking
|
-- Create enums for gateway profile tracking
|
||||||
CREATE TYPE "GatewayProfileStatus" AS ENUM (
|
CREATE TYPE "GatewayProfileStatus" AS ENUM (
|
||||||
'COMPLETED',
|
|
||||||
'RESERVED',
|
'RESERVED',
|
||||||
|
'PREOPEN',
|
||||||
'RUNNING',
|
'RUNNING',
|
||||||
|
'PAUSED',
|
||||||
|
'COMPLETED',
|
||||||
'STOPPED',
|
'STOPPED',
|
||||||
'DISABLED'
|
'DISABLED'
|
||||||
);
|
);
|
||||||
@@ -26,6 +28,8 @@ CREATE TABLE "gateway_profile" (
|
|||||||
"build_commit_sha" TEXT,
|
"build_commit_sha" TEXT,
|
||||||
"build_workspace" TEXT,
|
"build_workspace" TEXT,
|
||||||
"build_last_used_at" TIMESTAMP(3),
|
"build_last_used_at" TIMESTAMP(3),
|
||||||
|
"preopen_at" TIMESTAMP(3),
|
||||||
|
"open_at" TIMESTAMP(3),
|
||||||
"scheduled_start_at" TIMESTAMP(3),
|
"scheduled_start_at" TIMESTAMP(3),
|
||||||
"build_requested_at" TIMESTAMP(3),
|
"build_requested_at" TIMESTAMP(3),
|
||||||
"build_started_at" TIMESTAMP(3),
|
"build_started_at" TIMESTAMP(3),
|
||||||
@@ -39,4 +43,3 @@ CREATE TABLE "gateway_profile" (
|
|||||||
CONSTRAINT "gateway_profile_pkey" PRIMARY KEY ("profile_name"),
|
CONSTRAINT "gateway_profile_pkey" PRIMARY KEY ("profile_name"),
|
||||||
CONSTRAINT "gateway_profile_profile_scenario_key" UNIQUE ("profile", "scenario")
|
CONSTRAINT "gateway_profile_profile_scenario_key" UNIQUE ("profile", "scenario")
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -28,9 +28,11 @@ enum OAuthType {
|
|||||||
}
|
}
|
||||||
|
|
||||||
enum GatewayProfileStatus {
|
enum GatewayProfileStatus {
|
||||||
COMPLETED
|
|
||||||
RESERVED
|
RESERVED
|
||||||
|
PREOPEN
|
||||||
RUNNING
|
RUNNING
|
||||||
|
PAUSED
|
||||||
|
COMPLETED
|
||||||
STOPPED
|
STOPPED
|
||||||
DISABLED
|
DISABLED
|
||||||
}
|
}
|
||||||
@@ -72,6 +74,8 @@ model GatewayProfile {
|
|||||||
buildCommitSha String? @map("build_commit_sha")
|
buildCommitSha String? @map("build_commit_sha")
|
||||||
buildWorkspace String? @map("build_workspace")
|
buildWorkspace String? @map("build_workspace")
|
||||||
buildLastUsedAt DateTime? @map("build_last_used_at")
|
buildLastUsedAt DateTime? @map("build_last_used_at")
|
||||||
|
preopenAt DateTime? @map("preopen_at")
|
||||||
|
openAt DateTime? @map("open_at")
|
||||||
scheduledStartAt DateTime? @map("scheduled_start_at")
|
scheduledStartAt DateTime? @map("scheduled_start_at")
|
||||||
buildRequestedAt DateTime? @map("build_requested_at")
|
buildRequestedAt DateTime? @map("build_requested_at")
|
||||||
buildStartedAt DateTime? @map("build_started_at")
|
buildStartedAt DateTime? @map("build_started_at")
|
||||||
|
|||||||
Reference in New Issue
Block a user