feat: Enhance Gateway Orchestrator with preopen and open scheduling, add error handling for paused states, and implement new profile management features
This commit is contained in:
@@ -15,6 +15,7 @@
|
||||
"@sammo-ts/common": "workspace:*",
|
||||
"@sammo-ts/infra": "workspace:*",
|
||||
"@sammo-ts/logic": "workspace:*",
|
||||
"@prisma/client": "^7.2.0",
|
||||
"zod": "^4.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -31,6 +31,7 @@ export interface TurnDaemonLifecycleDeps {
|
||||
stateStore: TurnStateStore;
|
||||
processor: TurnProcessor;
|
||||
hooks?: TurnDaemonHooks;
|
||||
pauseGate?: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export class TurnDaemonLifecycle {
|
||||
@@ -41,12 +42,15 @@ export class TurnDaemonLifecycle {
|
||||
private readonly stateStore: TurnStateStore;
|
||||
private readonly processor: TurnProcessor;
|
||||
private readonly hooks?: TurnDaemonHooks;
|
||||
private readonly pauseGate?: () => Promise<boolean>;
|
||||
private readonly options: TurnDaemonLifecycleOptions;
|
||||
|
||||
private status: TurnDaemonStatus;
|
||||
private pendingRun: PendingRun | null = null;
|
||||
private stopping = false;
|
||||
private loopPromise: Promise<void> | null = null;
|
||||
private manualPaused = false;
|
||||
private errorPaused = false;
|
||||
|
||||
constructor(deps: TurnDaemonLifecycleDeps, options: TurnDaemonLifecycleOptions) {
|
||||
this.clock = deps.clock;
|
||||
@@ -55,6 +59,7 @@ export class TurnDaemonLifecycle {
|
||||
this.stateStore = deps.stateStore;
|
||||
this.processor = deps.processor;
|
||||
this.hooks = deps.hooks;
|
||||
this.pauseGate = deps.pauseGate;
|
||||
this.options = options;
|
||||
this.status = {
|
||||
state: 'idle',
|
||||
@@ -109,10 +114,24 @@ export class TurnDaemonLifecycle {
|
||||
if (this.stopping) {
|
||||
break;
|
||||
}
|
||||
const gatePaused = (await this.pauseGate?.()) ?? false;
|
||||
if (this.errorPaused && !gatePaused) {
|
||||
this.errorPaused = false;
|
||||
this.status.lastError = undefined;
|
||||
}
|
||||
this.status.paused = this.manualPaused || gatePaused || this.errorPaused;
|
||||
if (this.status.paused) {
|
||||
await this.waitForResume();
|
||||
this.status.state = 'paused';
|
||||
if (this.manualPaused) {
|
||||
await this.waitForResume();
|
||||
} else {
|
||||
await this.clock.sleepMs(500);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (this.status.state === 'paused') {
|
||||
this.status.state = 'idle';
|
||||
}
|
||||
|
||||
if (this.pendingRun) {
|
||||
await this.runOnce(this.pendingRun);
|
||||
@@ -186,11 +205,13 @@ export class TurnDaemonLifecycle {
|
||||
private async handleCommand(command: TurnDaemonCommand): Promise<void> {
|
||||
switch (command.type) {
|
||||
case 'pause':
|
||||
this.manualPaused = true;
|
||||
this.status.paused = true;
|
||||
this.status.state = 'paused';
|
||||
return;
|
||||
case 'resume':
|
||||
this.status.paused = false;
|
||||
this.manualPaused = false;
|
||||
this.status.paused = this.errorPaused;
|
||||
this.status.state = 'idle';
|
||||
return;
|
||||
case 'shutdown':
|
||||
@@ -221,6 +242,15 @@ export class TurnDaemonLifecycle {
|
||||
|
||||
try {
|
||||
result = await this.processor.run(targetTime, budget, checkpoint);
|
||||
} catch (error) {
|
||||
this.status.running = false;
|
||||
this.status.state = 'paused';
|
||||
this.status.paused = true;
|
||||
this.errorPaused = true;
|
||||
this.status.lastError =
|
||||
error instanceof Error ? error.message : 'Unknown turn daemon error.';
|
||||
await this.hooks?.onRunError?.(error);
|
||||
return;
|
||||
} finally {
|
||||
this.status.running = false;
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ export interface TurnDaemonStatus {
|
||||
state: TurnDaemonState;
|
||||
running: boolean;
|
||||
paused: boolean;
|
||||
lastError?: string;
|
||||
lastRunAt?: string;
|
||||
lastDurationMs?: number;
|
||||
lastTurnTime?: string;
|
||||
@@ -70,4 +71,5 @@ export interface TurnDaemonControlQueue {
|
||||
export interface TurnDaemonHooks {
|
||||
flushChanges?(result: TurnRunResult): Promise<void>;
|
||||
publishEvents?(result: TurnRunResult): Promise<void>;
|
||||
onRunError?(error: unknown): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import { createTurnDaemonRuntime } from './turnDaemon.js';
|
||||
|
||||
export interface TurnDaemonCliOptions {
|
||||
profile?: string;
|
||||
profileName?: string;
|
||||
scenario?: string;
|
||||
databaseUrl?: string;
|
||||
tickMinutes?: number;
|
||||
schedule?: TurnSchedule;
|
||||
@@ -71,6 +73,11 @@ export const runTurnDaemonCli = async (
|
||||
const env = options.env ?? process.env;
|
||||
const profile =
|
||||
options.profile ?? env.TURN_PROFILE ?? env.PROFILE ?? 'che';
|
||||
const scenario = options.scenario ?? env.TURN_SCENARIO ?? env.SCENARIO;
|
||||
const profileName =
|
||||
options.profileName ??
|
||||
env.TURN_PROFILE_NAME ??
|
||||
(scenario ? `${profile}:${scenario}` : profile);
|
||||
const databaseUrl =
|
||||
options.databaseUrl ?? (await resolveDatabaseUrl({ env }));
|
||||
const budget = buildBudgetOverride(env, options.budget);
|
||||
@@ -80,14 +87,17 @@ export const runTurnDaemonCli = async (
|
||||
options.enableDatabaseFlush ??
|
||||
parseBoolean(env.TURN_FLUSH_DB) ??
|
||||
true;
|
||||
const pauseGateIntervalMs = parseNumber(env.TURN_PAUSE_GATE_MS);
|
||||
|
||||
const runtime = await createTurnDaemonRuntime({
|
||||
profile,
|
||||
profileName,
|
||||
databaseUrl,
|
||||
defaultBudget: budget,
|
||||
tickMinutes,
|
||||
schedule: options.schedule,
|
||||
enableDatabaseFlush,
|
||||
pauseGateIntervalMs,
|
||||
});
|
||||
|
||||
let closed = false;
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { createPostgresConnector } from '@sammo-ts/infra';
|
||||
import type { PrismaClient } from '@prisma/client';
|
||||
|
||||
export interface GatewayProfileGateOptions {
|
||||
databaseUrl: string;
|
||||
profileName: string;
|
||||
cacheMs?: number;
|
||||
}
|
||||
|
||||
export interface GatewayProfileGate {
|
||||
shouldPause(): Promise<boolean>;
|
||||
markPaused(error?: unknown): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
const DEFAULT_CACHE_MS = 2000;
|
||||
|
||||
const isRunningStatus = (status: string | null | undefined): boolean =>
|
||||
status === 'RUNNING';
|
||||
|
||||
export const createGatewayProfileGate = async (
|
||||
options: GatewayProfileGateOptions
|
||||
): Promise<GatewayProfileGate> => {
|
||||
const connector = createPostgresConnector({ url: options.databaseUrl });
|
||||
await connector.connect();
|
||||
const prisma = connector.prisma as PrismaClient;
|
||||
let lastCheckedAt = 0;
|
||||
let cachedPause = false;
|
||||
|
||||
const loadStatus = async (): Promise<boolean> => {
|
||||
try {
|
||||
const profile = await prisma.gatewayProfile.findUnique({
|
||||
where: { profileName: options.profileName },
|
||||
});
|
||||
if (!profile) {
|
||||
return false;
|
||||
}
|
||||
return !isRunningStatus(profile.status);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
// 게이트웨이 프로필 상태를 읽어 턴 실행을 멈춰야 하는지 판단한다.
|
||||
async shouldPause(): Promise<boolean> {
|
||||
const now = Date.now();
|
||||
if (now - lastCheckedAt < (options.cacheMs ?? DEFAULT_CACHE_MS)) {
|
||||
return cachedPause;
|
||||
}
|
||||
cachedPause = await loadStatus();
|
||||
lastCheckedAt = now;
|
||||
return cachedPause;
|
||||
},
|
||||
async markPaused(error?: unknown): Promise<void> {
|
||||
const message =
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: error
|
||||
? String(error)
|
||||
: null;
|
||||
try {
|
||||
await prisma.gatewayProfile.update({
|
||||
where: { profileName: options.profileName },
|
||||
data: {
|
||||
status: 'PAUSED',
|
||||
lastError: message,
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
},
|
||||
async close(): Promise<void> {
|
||||
await connector.disconnect();
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -20,12 +20,14 @@ import type {
|
||||
import { InMemoryTurnWorld } from './inMemoryWorld.js';
|
||||
import { InMemoryTurnProcessor } from './inMemoryTurnProcessor.js';
|
||||
import { InMemoryTurnStateStore } from './inMemoryStateStore.js';
|
||||
import { createGatewayProfileGate } from './gatewayProfileGate.js';
|
||||
import { createReservedTurnHandler } from './reservedTurnHandler.js';
|
||||
import { createReservedTurnStore } from './reservedTurnStore.js';
|
||||
import { loadTurnWorldFromDatabase } from './worldLoader.js';
|
||||
|
||||
export interface TurnDaemonRuntimeOptions {
|
||||
profile: string;
|
||||
profileName?: string;
|
||||
databaseUrl: string;
|
||||
defaultBudget?: TurnRunBudget;
|
||||
clock?: Clock;
|
||||
@@ -36,6 +38,7 @@ export interface TurnDaemonRuntimeOptions {
|
||||
generalTurnHandler?: GeneralTurnHandler;
|
||||
calendarHandler?: TurnCalendarHandler;
|
||||
enableDatabaseFlush?: boolean;
|
||||
pauseGateIntervalMs?: number;
|
||||
}
|
||||
|
||||
export interface TurnDaemonRuntime {
|
||||
@@ -119,19 +122,55 @@ export const createTurnDaemonRuntime = async (
|
||||
|
||||
let hooks: TurnDaemonHooks | undefined;
|
||||
let close = async () => {};
|
||||
let pauseGate: (() => Promise<boolean>) | undefined;
|
||||
const gatewayGate =
|
||||
options.profileName
|
||||
? await createGatewayProfileGate({
|
||||
databaseUrl: options.databaseUrl,
|
||||
profileName: options.profileName,
|
||||
cacheMs: options.pauseGateIntervalMs,
|
||||
})
|
||||
: null;
|
||||
if (gatewayGate) {
|
||||
pauseGate = gatewayGate.shouldPause;
|
||||
}
|
||||
if (options.enableDatabaseFlush ?? true) {
|
||||
const dbHooks = await createDatabaseTurnHooks(options.databaseUrl, world, {
|
||||
reservedTurns: reservedTurnStoreHandle?.store,
|
||||
});
|
||||
hooks = dbHooks.hooks;
|
||||
hooks = {
|
||||
...dbHooks.hooks,
|
||||
onRunError: async (error) => {
|
||||
await dbHooks.hooks.onRunError?.(error);
|
||||
await gatewayGate?.markPaused(error);
|
||||
},
|
||||
};
|
||||
close = async () => {
|
||||
await dbHooks.close();
|
||||
if (reservedTurnStoreHandle) {
|
||||
await reservedTurnStoreHandle.close();
|
||||
}
|
||||
await gatewayGate?.close();
|
||||
};
|
||||
} else if (reservedTurnStoreHandle) {
|
||||
close = async () => reservedTurnStoreHandle.close();
|
||||
hooks = {
|
||||
onRunError: async (error) => {
|
||||
await gatewayGate?.markPaused(error);
|
||||
},
|
||||
};
|
||||
close = async () => {
|
||||
await reservedTurnStoreHandle.close();
|
||||
await gatewayGate?.close();
|
||||
};
|
||||
} else if (gatewayGate) {
|
||||
hooks = {
|
||||
onRunError: async (error) => {
|
||||
await gatewayGate?.markPaused(error);
|
||||
},
|
||||
};
|
||||
close = async () => {
|
||||
await gatewayGate.close();
|
||||
};
|
||||
}
|
||||
|
||||
const defaultBudget: TurnRunBudget = options.defaultBudget ?? {
|
||||
@@ -149,6 +188,7 @@ export const createTurnDaemonRuntime = async (
|
||||
stateStore,
|
||||
processor,
|
||||
hooks,
|
||||
pauseGate,
|
||||
},
|
||||
{ profile: options.profile, defaultBudget }
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user