feat: Enhance Gateway Orchestrator with preopen and open scheduling, add error handling for paused states, and implement new profile management features

This commit is contained in:
2026-01-01 11:27:10 +00:00
parent e64962c3fd
commit 8c9f61700e
20 changed files with 470 additions and 86 deletions
+1
View File
@@ -15,6 +15,7 @@
"@sammo-ts/common": "workspace:*",
"@sammo-ts/infra": "workspace:*",
"@sammo-ts/logic": "workspace:*",
"@prisma/client": "^7.2.0",
"zod": "^4.2.1"
},
"devDependencies": {
@@ -31,6 +31,7 @@ export interface TurnDaemonLifecycleDeps {
stateStore: TurnStateStore;
processor: TurnProcessor;
hooks?: TurnDaemonHooks;
pauseGate?: () => Promise<boolean>;
}
export class TurnDaemonLifecycle {
@@ -41,12 +42,15 @@ export class TurnDaemonLifecycle {
private readonly stateStore: TurnStateStore;
private readonly processor: TurnProcessor;
private readonly hooks?: TurnDaemonHooks;
private readonly pauseGate?: () => Promise<boolean>;
private readonly options: TurnDaemonLifecycleOptions;
private status: TurnDaemonStatus;
private pendingRun: PendingRun | null = null;
private stopping = false;
private loopPromise: Promise<void> | null = null;
private manualPaused = false;
private errorPaused = false;
constructor(deps: TurnDaemonLifecycleDeps, options: TurnDaemonLifecycleOptions) {
this.clock = deps.clock;
@@ -55,6 +59,7 @@ export class TurnDaemonLifecycle {
this.stateStore = deps.stateStore;
this.processor = deps.processor;
this.hooks = deps.hooks;
this.pauseGate = deps.pauseGate;
this.options = options;
this.status = {
state: 'idle',
@@ -109,10 +114,24 @@ export class TurnDaemonLifecycle {
if (this.stopping) {
break;
}
const gatePaused = (await this.pauseGate?.()) ?? false;
if (this.errorPaused && !gatePaused) {
this.errorPaused = false;
this.status.lastError = undefined;
}
this.status.paused = this.manualPaused || gatePaused || this.errorPaused;
if (this.status.paused) {
await this.waitForResume();
this.status.state = 'paused';
if (this.manualPaused) {
await this.waitForResume();
} else {
await this.clock.sleepMs(500);
}
continue;
}
if (this.status.state === 'paused') {
this.status.state = 'idle';
}
if (this.pendingRun) {
await this.runOnce(this.pendingRun);
@@ -186,11 +205,13 @@ export class TurnDaemonLifecycle {
private async handleCommand(command: TurnDaemonCommand): Promise<void> {
switch (command.type) {
case 'pause':
this.manualPaused = true;
this.status.paused = true;
this.status.state = 'paused';
return;
case 'resume':
this.status.paused = false;
this.manualPaused = false;
this.status.paused = this.errorPaused;
this.status.state = 'idle';
return;
case 'shutdown':
@@ -221,6 +242,15 @@ export class TurnDaemonLifecycle {
try {
result = await this.processor.run(targetTime, budget, checkpoint);
} catch (error) {
this.status.running = false;
this.status.state = 'paused';
this.status.paused = true;
this.errorPaused = true;
this.status.lastError =
error instanceof Error ? error.message : 'Unknown turn daemon error.';
await this.hooks?.onRunError?.(error);
return;
} finally {
this.status.running = false;
}
+2
View File
@@ -28,6 +28,7 @@ export interface TurnDaemonStatus {
state: TurnDaemonState;
running: boolean;
paused: boolean;
lastError?: string;
lastRunAt?: string;
lastDurationMs?: number;
lastTurnTime?: string;
@@ -70,4 +71,5 @@ export interface TurnDaemonControlQueue {
export interface TurnDaemonHooks {
flushChanges?(result: TurnRunResult): Promise<void>;
publishEvents?(result: TurnRunResult): Promise<void>;
onRunError?(error: unknown): Promise<void>;
}
+10
View File
@@ -6,6 +6,8 @@ import { createTurnDaemonRuntime } from './turnDaemon.js';
export interface TurnDaemonCliOptions {
profile?: string;
profileName?: string;
scenario?: string;
databaseUrl?: string;
tickMinutes?: number;
schedule?: TurnSchedule;
@@ -71,6 +73,11 @@ export const runTurnDaemonCli = async (
const env = options.env ?? process.env;
const profile =
options.profile ?? env.TURN_PROFILE ?? env.PROFILE ?? 'che';
const scenario = options.scenario ?? env.TURN_SCENARIO ?? env.SCENARIO;
const profileName =
options.profileName ??
env.TURN_PROFILE_NAME ??
(scenario ? `${profile}:${scenario}` : profile);
const databaseUrl =
options.databaseUrl ?? (await resolveDatabaseUrl({ env }));
const budget = buildBudgetOverride(env, options.budget);
@@ -80,14 +87,17 @@ export const runTurnDaemonCli = async (
options.enableDatabaseFlush ??
parseBoolean(env.TURN_FLUSH_DB) ??
true;
const pauseGateIntervalMs = parseNumber(env.TURN_PAUSE_GATE_MS);
const runtime = await createTurnDaemonRuntime({
profile,
profileName,
databaseUrl,
defaultBudget: budget,
tickMinutes,
schedule: options.schedule,
enableDatabaseFlush,
pauseGateIntervalMs,
});
let closed = false;
@@ -0,0 +1,78 @@
import { createPostgresConnector } from '@sammo-ts/infra';
import type { PrismaClient } from '@prisma/client';
export interface GatewayProfileGateOptions {
databaseUrl: string;
profileName: string;
cacheMs?: number;
}
export interface GatewayProfileGate {
shouldPause(): Promise<boolean>;
markPaused(error?: unknown): Promise<void>;
close(): Promise<void>;
}
const DEFAULT_CACHE_MS = 2000;
const isRunningStatus = (status: string | null | undefined): boolean =>
status === 'RUNNING';
export const createGatewayProfileGate = async (
options: GatewayProfileGateOptions
): Promise<GatewayProfileGate> => {
const connector = createPostgresConnector({ url: options.databaseUrl });
await connector.connect();
const prisma = connector.prisma as PrismaClient;
let lastCheckedAt = 0;
let cachedPause = false;
const loadStatus = async (): Promise<boolean> => {
try {
const profile = await prisma.gatewayProfile.findUnique({
where: { profileName: options.profileName },
});
if (!profile) {
return false;
}
return !isRunningStatus(profile.status);
} catch {
return false;
}
};
return {
// 게이트웨이 프로필 상태를 읽어 턴 실행을 멈춰야 하는지 판단한다.
async shouldPause(): Promise<boolean> {
const now = Date.now();
if (now - lastCheckedAt < (options.cacheMs ?? DEFAULT_CACHE_MS)) {
return cachedPause;
}
cachedPause = await loadStatus();
lastCheckedAt = now;
return cachedPause;
},
async markPaused(error?: unknown): Promise<void> {
const message =
error instanceof Error
? error.message
: error
? String(error)
: null;
try {
await prisma.gatewayProfile.update({
where: { profileName: options.profileName },
data: {
status: 'PAUSED',
lastError: message,
},
});
} catch {
return;
}
},
async close(): Promise<void> {
await connector.disconnect();
},
};
};
+42 -2
View File
@@ -20,12 +20,14 @@ import type {
import { InMemoryTurnWorld } from './inMemoryWorld.js';
import { InMemoryTurnProcessor } from './inMemoryTurnProcessor.js';
import { InMemoryTurnStateStore } from './inMemoryStateStore.js';
import { createGatewayProfileGate } from './gatewayProfileGate.js';
import { createReservedTurnHandler } from './reservedTurnHandler.js';
import { createReservedTurnStore } from './reservedTurnStore.js';
import { loadTurnWorldFromDatabase } from './worldLoader.js';
export interface TurnDaemonRuntimeOptions {
profile: string;
profileName?: string;
databaseUrl: string;
defaultBudget?: TurnRunBudget;
clock?: Clock;
@@ -36,6 +38,7 @@ export interface TurnDaemonRuntimeOptions {
generalTurnHandler?: GeneralTurnHandler;
calendarHandler?: TurnCalendarHandler;
enableDatabaseFlush?: boolean;
pauseGateIntervalMs?: number;
}
export interface TurnDaemonRuntime {
@@ -119,19 +122,55 @@ export const createTurnDaemonRuntime = async (
let hooks: TurnDaemonHooks | undefined;
let close = async () => {};
let pauseGate: (() => Promise<boolean>) | undefined;
const gatewayGate =
options.profileName
? await createGatewayProfileGate({
databaseUrl: options.databaseUrl,
profileName: options.profileName,
cacheMs: options.pauseGateIntervalMs,
})
: null;
if (gatewayGate) {
pauseGate = gatewayGate.shouldPause;
}
if (options.enableDatabaseFlush ?? true) {
const dbHooks = await createDatabaseTurnHooks(options.databaseUrl, world, {
reservedTurns: reservedTurnStoreHandle?.store,
});
hooks = dbHooks.hooks;
hooks = {
...dbHooks.hooks,
onRunError: async (error) => {
await dbHooks.hooks.onRunError?.(error);
await gatewayGate?.markPaused(error);
},
};
close = async () => {
await dbHooks.close();
if (reservedTurnStoreHandle) {
await reservedTurnStoreHandle.close();
}
await gatewayGate?.close();
};
} else if (reservedTurnStoreHandle) {
close = async () => reservedTurnStoreHandle.close();
hooks = {
onRunError: async (error) => {
await gatewayGate?.markPaused(error);
},
};
close = async () => {
await reservedTurnStoreHandle.close();
await gatewayGate?.close();
};
} else if (gatewayGate) {
hooks = {
onRunError: async (error) => {
await gatewayGate?.markPaused(error);
},
};
close = async () => {
await gatewayGate.close();
};
}
const defaultBudget: TurnRunBudget = options.defaultBudget ?? {
@@ -149,6 +188,7 @@ export const createTurnDaemonRuntime = async (
stateStore,
processor,
hooks,
pauseGate,
},
{ profile: options.profile, defaultBudget }
);