devsam/core2026
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: VM 시간 변경 후 턴 lease를 안전하게 자동 복구

Browse Source
This commit is contained in:
Hide_D
2026-09-15 23:05:33 +00:00
parent 7dc1670935
commit 4fb1343943
11 changed files with 224 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts
+2 -1
View File
@@ -184,6 +184,7 @@ export class DatabaseTurnDaemonLease {
if (!token || this.lost) {
throw this.getLossError();
}
// CURRENT_TIMESTAMP는 transaction 시작에 고정되어 VM 정지 중 만료를 놓친다.
const db = transaction ?? this.db;
const rows = await db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
SELECT "profile", "owner_id", "fencing_epoch"
@@ -192,7 +193,7 @@ export class DatabaseTurnDaemonLease {
"profile" = ${token.profile}
AND "owner_id" = ${token.ownerId}
AND "fencing_epoch" = ${token.fencingEpoch}
AND "lease_until" > CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
AND "lease_until" > clock_timestamp() AT TIME ZONE 'UTC'
FOR UPDATE
`);
if (rows.length === 0) {
app/game-engine/src/lifecycle/turnDaemonLifecycle.ts
+2 -1
View File
@@ -240,7 +240,8 @@ export class TurnDaemonLifecycle {
const wallDeadline = await this.stateStore.projectGameDeadline?.(nextRunTime);
const command = await this.controlQueue.waitFor(
Math.max(0, wallDeadline ? wallDeadline.getTime() - nowMs : nextTurnMs - gameNowMs)
// 벽시계가 뒤로 이동해도 다음 턴까지 장시간 잠들지 않고 lease/gate를 다시 확인한다.
Math.min(1000, Math.max(0, wallDeadline ? wallDeadline.getTime() - nowMs : nextTurnMs - gameNowMs))
);
if (command) {
await this.handleCommand(command);
app/game-engine/src/turn/cli.ts
+17 -14
View File
@@ -6,6 +6,7 @@ import { resolveDatabaseUrl } from '../scenario/databaseUrl.js';
import { createTurnDaemonRuntime } from './turnDaemon.js';
import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
import { createGatewayProfileGate } from './gatewayProfileGate.js';
import { retryTurnDaemonLeaseStartup } from './leaseStartupRetry.js';
import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
export interface TurnDaemonCliOptions {
@@ -79,19 +80,21 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
}
const gameClockMode = rawGameClockMode as GameClockMode | undefined;
const runtime = await createTurnDaemonRuntime({
profile,
profileName,
databaseUrl,
gatewayDatabaseUrl,
defaultBudget: budget,
tickMinutes,
schedule: options.schedule,
enableDatabaseFlush,
pauseGateIntervalMs,
adminActionIntervalMs,
gameClockMode,
}).catch(async (error: unknown) => {
const runtime = await retryTurnDaemonLeaseStartup(() =>
createTurnDaemonRuntime({
profile,
profileName,
databaseUrl,
gatewayDatabaseUrl,
defaultBudget: budget,
tickMinutes,
schedule: options.schedule,
enableDatabaseFlush,
pauseGateIntervalMs,
adminActionIntervalMs,
gameClockMode,
})
).catch(async (error: unknown) => {
// 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는
// lifecycle hook이 아직 없으므로 여기서 별도로 관리자에게 기록한다.
if (!(error instanceof TurnDaemonLeaseUnavailableError)) {
@@ -103,7 +106,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
incidentContext: () => ({ stage: 'startup' }),
});
try {
await gate.markPaused(error);
await gate.reportFailure(error);
} finally {
await gate.close();
}
app/game-engine/src/turn/gatewayProfileGate.ts
+38 -18
View File
@@ -4,6 +4,8 @@ import { randomUUID } from 'node:crypto';
import { describeRuntimeError, gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common';
import { createGatewayPostgresConnector } from '@sammo-ts/infra';
import { TurnDaemonLeaseLostError } from '../lifecycle/databaseTurnDaemonLease.js';
export interface GatewayProfileGateOptions {
databaseUrl: string;
gatewayDatabaseUrl?: string;
@@ -15,7 +17,7 @@ export interface GatewayProfileGateOptions {
export interface GatewayProfileGate {
shouldPause(): Promise<boolean>;
isExplicitlyPaused(): boolean;
markPaused(error?: unknown): Promise<void>;
reportFailure(error?: unknown): Promise<void>;
close(): Promise<void>;
}
@@ -29,6 +31,7 @@ export const createGatewayProfileGate = async (options: GatewayProfileGateOption
});
await connector.connect();
const prisma = connector.prisma;
const reportedLeaseErrors = new WeakSet<TurnDaemonLeaseLostError>();
let lastCheckedAt = 0;
let cachedPause = false;
let cachedStatus: GatewayProfileStatus | null = null;
@@ -60,26 +63,38 @@ export const createGatewayProfileGate = async (options: GatewayProfileGateOption
lastCheckedAt = now;
return cachedPause;
},
async markPaused(error?: unknown): Promise<void> {
cachedPause = true;
cachedStatus = 'PAUSED';
lastCheckedAt = performance.now();
async reportFailure(error?: unknown): Promise<void> {
// VM 정지/시계 보정으로 lease를 잃은 실행자는 종료하고 새 owner가
// DB를 다시 읽는다. 운영자의 RUNNING/PAUSED/STOPPED 의도는 덮어쓰지 않는다.
const recoverable = error instanceof TurnDaemonLeaseLostError;
if (recoverable && reportedLeaseErrors.has(error)) return;
if (!recoverable) {
cachedPause = true;
cachedStatus = 'PAUSED';
lastCheckedAt = performance.now();
}
const failure = error ? describeRuntimeError(error) : null;
const message = failure?.message ?? null;
try {
await prisma.$transaction(async (tx) => {
const updated = await tx.gatewayProfile.updateMany({
where: {
profileName: options.profileName,
status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
OR: [{ status: { not: 'PAUSED' } }, { lastError: { not: message } }, { lastError: null }],
},
data: {
status: 'PAUSED',
lastError: message,
},
});
if (updated.count && failure) {
const updated = recoverable
? null
: await tx.gatewayProfile.updateMany({
where: {
profileName: options.profileName,
status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
OR: [
{ status: { not: 'PAUSED' } },
{ lastError: { not: message } },
{ lastError: null },
],
},
data: {
status: 'PAUSED',
lastError: message,
},
});
if ((recoverable || updated?.count) && failure) {
// 상태와 이력을 함께 commit한다. 재개가 lastError를 지워도
// 당시 원인과 실행 좌표는 관리자 감사 저장소에 남는다.
await tx.adminAuditEvent.create({
@@ -95,11 +110,16 @@ export const createGatewayProfileGate = async (options: GatewayProfileGateOption
outcome: 'FAILED',
errorCode: failure.code,
errorMessage: failure.message,
summary: { frames: failure.frames, ...options.incidentContext?.() },
summary: {
frames: failure.frames,
...options.incidentContext?.(),
recovery: recoverable ? 'RESTART' : 'OPERATOR',
},
},
});
}
});
if (recoverable) reportedLeaseErrors.add(error);
} catch {
if (failure) console.error('[turn-daemon] failed to persist runtime incident', failure);
return;
app/game-engine/src/turn/leaseStartupRetry.ts
+23
View File
@@ -0,0 +1,23 @@
import { setTimeout } from 'node:timers/promises';
import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
// 역방향 시계 보정 또는 기존 owner의 정상 종료를 기다리는 동안 PM2의
// 짧은 시작 실패 횟수를 소진하지 않는다. 매번 새 runtime/DB snapshot을 만든다.
export const retryTurnDaemonLeaseStartup = async <T>(
create: () => Promise<T>,
wait: () => Promise<void> = () => setTimeout(2000)
): Promise<T> => {
let attempts = 0;
for (;;) {
try {
return await create();
} catch (error) {
if (!(error instanceof TurnDaemonLeaseUnavailableError)) throw error;
if (attempts++ % 15 === 0) {
console.info('[turn-daemon] waiting for the active lease owner; startup will retry.');
}
await wait();
}
}
};
app/game-engine/src/turn/turnDaemon.ts
+3 -3
View File
@@ -942,7 +942,7 @@ const createTurnDaemonRuntimeWithLease = async (
...dbHooks.hooks,
onRunError: async (error) => {
await dbHooks.hooks.onRunError?.(error);
await gatewayGate?.markPaused(error);
await gatewayGate?.reportFailure(error);
if (!turnDaemonLease?.isLost() && world.getGameClockState().phase === 'RUNNING') {
// 같은 command batch의 다음 가입도 정지된 시각을 보게 한다.
await dbHooks.prepareRealtimeRecovery({ paused: true });
@@ -974,7 +974,7 @@ const createTurnDaemonRuntimeWithLease = async (
} else if (reservedTurnStoreHandle) {
hooks = {
onRunError: async (error) => {
await gatewayGate?.markPaused(error);
await gatewayGate?.reportFailure(error);
},
};
close = async () => {
@@ -985,7 +985,7 @@ const createTurnDaemonRuntimeWithLease = async (
} else if (gatewayGate) {
hooks = {
onRunError: async (error) => {
await gatewayGate?.markPaused(error);
await gatewayGate?.reportFailure(error);
},
};
close = async () => {