VM 복귀 후 예약턴 잠금 복구와 Windows 부팅 식별자 확인을 추가한다

This commit is contained in:
Hide_D committed 2026-11-12 09:16:04 +09:00
1 parent fd8145d6eb
commit f36bc2c8d7
24 files changed
+1048 -28

No files matched your search

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
import { performance } from 'node:perf_hooks';
import { createGamePostgresConnector, GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import type { HostBootMonitor } from '../turn/hostBootMonitor.js';
export interface TurnDaemonLeaseToken {
profile: string;
@@ -14,6 +15,7 @@ export interface DatabaseTurnDaemonLeaseOptions {
ownerId?: string;
leaseDurationMs?: number;
heartbeat?: boolean;
hostBootMonitor?: HostBootMonitor;
}
export class TurnDaemonLeaseUnavailableError extends Error {
@@ -45,6 +47,7 @@ export class DatabaseTurnDaemonLease {
private readonly ownerId: string;
private readonly leaseDurationMs: number;
private readonly heartbeatEnabled: boolean;
private readonly hostBootMonitor?: HostBootMonitor;
private token: TurnDaemonLeaseToken | null = null;
private heartbeatTimer: NodeJS.Timeout | null = null;
private expiryTimer: NodeJS.Timeout | null = null;
@@ -63,6 +66,7 @@ export class DatabaseTurnDaemonLease {
this.ownerId = options.ownerId ?? randomUUID();
this.leaseDurationMs = normalizeLeaseDuration(options.leaseDurationMs);
this.heartbeatEnabled = options.heartbeat ?? true;
this.hostBootMonitor = options.hostBootMonitor;
}
static async connect(
@@ -75,6 +79,8 @@ export class DatabaseTurnDaemonLease {
}
async acquire(): Promise<TurnDaemonLeaseToken | null> {
await this.checkHostBoot();
if (this.lost) throw this.getLossError();
const requestStartedAt = performance.now();
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
INSERT INTO "turn_daemon_lease" (
@@ -145,6 +151,12 @@ export class DatabaseTurnDaemonLease {
return new TurnDaemonLeaseLostError(this.profile, this.lossReason);
}
async checkHostBoot(): Promise<void> {
if (!this.lost && (await this.hostBootMonitor?.hasChanged())) {
this.markLost('Host boot identity changed; restart with a fresh database snapshot.');
}
}
async renew(): Promise<boolean> {
const token = this.token;
if (!token || this.lost || this.renewalInFlight) {
@@ -153,6 +165,8 @@ export class DatabaseTurnDaemonLease {
const requestStartedAt = performance.now();
this.renewalInFlight = true;
try {
await this.checkHostBoot();
if (this.lost) return false;
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
UPDATE "turn_daemon_lease"
SET
@@ -180,6 +194,7 @@ export class DatabaseTurnDaemonLease {
}
async assertActive(transaction?: GamePrisma.TransactionClient): Promise<void> {
await this.checkHostBoot();
const token = this.token;
if (!token || this.lost) {
throw this.getLossError();
@@ -207,12 +222,13 @@ export class DatabaseTurnDaemonLease {
this.stopExpiryWatchdog();
const token = this.token;
this.token = null;
if (!token || this.lost) {
if (!token) {
return;
}
await this.db.$executeRaw(GamePrisma.sql`
UPDATE "turn_daemon_lease"
SET "lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
SET "clock_ready" = FALSE,
"lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
"heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
WHERE
"profile" = ${token.profile}
@@ -45,6 +45,7 @@ export interface TurnDaemonLifecycleDeps {
commandResponder?: TurnDaemonCommandResponder;
pauseGate?: () => Promise<boolean>;
stateManager?: Pick<EngineStateManager, 'transaction'>;
runErrorRetry?: { wait(error: unknown): Promise<boolean>; reset(): void };
}
export class TurnDaemonLifecycle {
@@ -59,6 +60,7 @@ export class TurnDaemonLifecycle {
private readonly commandResponder?: TurnDaemonCommandResponder;
private readonly pauseGate?: () => Promise<boolean>;
private readonly stateManager?: Pick<EngineStateManager, 'transaction'>;
private readonly runErrorRetry?: TurnDaemonLifecycleDeps['runErrorRetry'];
private readonly options: TurnDaemonLifecycleOptions;
private status: TurnDaemonStatus;
@@ -79,6 +81,7 @@ export class TurnDaemonLifecycle {
this.commandResponder = deps.commandResponder;
this.pauseGate = deps.pauseGate;
this.stateManager = deps.stateManager;
this.runErrorRetry = deps.runErrorRetry;
this.options = options;
this.status = {
state: 'idle',
@@ -192,8 +195,9 @@ export class TurnDaemonLifecycle {
// 수동 실행 요청도 가오픈·정지·재조정의 턴 실행 gate를 통과해야 한다.
// 사용자 명령 처리는 루프 시작에서 계속하되 시간 진행은 여기서 분리한다.
if (this.pendingRun) {
await this.runOnce(this.pendingRun);
const pending = this.pendingRun;
this.pendingRun = null;
await this.runOnce(pending);
continue;
}
@@ -447,6 +451,15 @@ export class TurnDaemonLifecycle {
};
result = this.stateManager ? await this.stateManager.transaction(runAndFlush) : await runAndFlush();
} catch (error) {
// savepoint 복원이 성공한 획득 충돌만 대기한다. 복원 실패/AggregateError나
// 저장 중 소유권 상실을 자동 재실행하면 중복 효과를 숨길 수 있다.
if (this.stateManager && (await this.runErrorRetry?.wait(error))) {
this.status.running = false;
this.status.state = 'idle';
this.pendingRun = pending;
return;
}
this.runErrorRetry?.reset();
this.status.running = false;
this.status.state = 'paused';
this.status.paused = true;
@@ -459,6 +472,7 @@ export class TurnDaemonLifecycle {
}
await this.applyRunResult(result, startMs);
this.runErrorRetry?.reset();
this.status.state = 'idle';
try {
await this.hooks?.publishEvents?.(result);
+3
View File
@@ -8,6 +8,7 @@ import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
import { createGatewayProfileGate } from './gatewayProfileGate.js';
import { retryTurnDaemonLeaseStartup } from './leaseStartupRetry.js';
import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
import { loadHostBootMonitor } from './hostBootMonitor.js';
export interface TurnDaemonCliOptions {
profile?: string;
@@ -80,6 +81,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`);
}
const gameClockMode = rawGameClockMode as GameClockMode | undefined;
const hostBootMonitor = await loadHostBootMonitor(env);
const runtime = await retryTurnDaemonLeaseStartup(() =>
createTurnDaemonRuntime({
@@ -95,6 +97,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
pauseGateIntervalMs,
adminActionIntervalMs,
gameClockMode,
hostBootMonitor,
})
).catch(async (error: unknown) => {
// 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는
+142
View File
@@ -0,0 +1,142 @@
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
import { readFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks';
import { isRecord } from '@sammo-ts/common';
export interface HostBootMonitor {
hasChanged(): Promise<boolean>;
}
const mac = (key: Buffer, value: string): string => createHmac('sha256', key).update(value).digest('hex');
/** 부팅 세대는 첫 인증 응답에 고정한다. 통신 실패는 기존 lease 복구를 막지 않는다. */
export const createHostBootMonitor = (options: {
url: string;
key: Buffer;
pollMs?: number;
timeoutMs?: number;
now?: () => number;
wallNow?: () => number;
onUnavailable?: () => void;
onVerified?: () => void;
}): HostBootMonitor => {
const url = new URL(options.url);
if (
!['http:', 'https:'].includes(url.protocol) ||
url.username ||
url.password ||
url.search ||
url.hash ||
url.pathname !== '/v1/boot-id'
) {
throw new Error('Host boot URL must use HTTP(S) and /v1/boot-id without credentials or query.');
}
if (options.key.length !== 32) throw new Error('Host boot key must contain exactly 32 bytes.');
const now = options.now ?? (() => performance.now());
const wallNow = options.wallNow ?? (() => Date.now());
let baseline: string | null = null;
let changed = false;
let lastAttempt = -Infinity;
let lastWall = 0;
let lastWarning = -Infinity;
let inFlight: Promise<boolean> | null = null;
const sample = async (): Promise<boolean> => {
const nonce = randomBytes(32).toString('hex');
const response = await fetch(url, {
method: 'GET',
redirect: 'error',
signal: AbortSignal.timeout(options.timeoutMs ?? 1_000),
headers: {
'x-sammo-challenge': nonce,
'x-sammo-signature': mac(options.key, `sammo-host-boot-request-v1\n${nonce}`),
},
});
if (!response.ok || !response.body) {
await response.body?.cancel();
throw new Error('Host boot response unavailable.');
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
size += value.length;
if (size > 2_048) throw new Error('Host boot response exceeds its limit.');
chunks.push(value);
}
} finally {
await reader.cancel();
}
const body: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'));
if (
!isRecord(body) ||
body.version !== 1 ||
body.challenge !== nonce ||
typeof body.bootId !== 'string' ||
!/^[a-f0-9]{64}$/.test(body.bootId) ||
typeof body.signature !== 'string' ||
!/^[a-f0-9]{64}$/.test(body.signature)
) {
throw new Error('Host boot response is invalid.');
}
const expected = mac(options.key, `sammo-host-boot-response-v1\n${nonce}\n${body.bootId}`);
if (!timingSafeEqual(Buffer.from(expected, 'hex'), Buffer.from(body.signature, 'hex'))) {
throw new Error('Host boot response signature is invalid.');
}
if (baseline === null) {
baseline = body.bootId;
options.onVerified?.();
}
changed ||= body.bootId !== baseline;
return changed;
};
return {
hasChanged(): Promise<boolean> {
if (changed) return Promise.resolve(true);
if (inFlight) return inFlight;
const current = now();
const wall = wallNow();
const discontinuity = Math.abs(wall - lastWall - (current - lastAttempt)) > 1_000;
if (current - lastAttempt < (options.pollMs ?? 2_000) && !discontinuity) return Promise.resolve(false);
lastAttempt = current;
lastWall = wall;
inFlight = sample()
.catch(() => {
if (current - lastWarning >= 60_000) {
options.onUnavailable?.();
lastWarning = current;
}
return false;
})
.finally(() => {
inFlight = null;
});
return inFlight;
},
};
};
export const loadHostBootMonitor = async (env: NodeJS.ProcessEnv): Promise<HostBootMonitor | undefined> => {
const url = env.TURN_HOST_BOOT_URL?.trim();
const keyFile = env.TURN_HOST_BOOT_KEY_FILE?.trim();
if (!url && !keyFile) return undefined;
try {
if (!url || !keyFile) throw new Error('Incomplete optional configuration.');
const encoded = (await readFile(keyFile, 'utf8')).trim();
if (!/^[a-f0-9]{64}$/.test(encoded)) throw new Error('Invalid optional key file.');
return createHostBootMonitor({
url,
key: Buffer.from(encoded, 'hex'),
onVerified: () => console.info('[turn-daemon] host boot identity verified.'),
onUnavailable: () => {
console.warn('[turn-daemon] host boot identity unavailable; ordinary lease recovery remains active.');
},
});
} catch {
// 보조 관측기의 설정/파일 부재도 게임 기동의 필수 의존성이 되지 않는다.
console.warn('[turn-daemon] host boot observation disabled; check optional configuration and key file.');
return undefined;
}
};
@@ -0,0 +1,38 @@
import { performance } from 'node:perf_hooks';
import { setTimeout } from 'node:timers/promises';
import { ReservedTurnLeaseBusyError } from './reservedTurnStore.js';
/** rollback 완료 뒤에만 대기한다. 다음 loop에서 pause/lease gate와 최신 큐를 다시 읽는다. */
export const createReservedTurnLeaseRetry = (
options: {
now?: () => number;
sleep?: (ms: number) => Promise<void>;
maxWaitMs?: number;
onWait?: (queueKey: string) => void;
} = {}
) => {
const now = options.now ?? (() => performance.now());
const sleep = options.sleep ?? ((ms: number) => setTimeout(ms));
const maxWaitMs = options.maxWaitMs ?? 5 * 60_000 + 1_000;
let startedAt: number | null = null;
let lastReportedAt = -Infinity;
return {
reset(): void {
startedAt = null;
lastReportedAt = -Infinity;
},
async wait(error: unknown): Promise<boolean> {
if (!(error instanceof ReservedTurnLeaseBusyError)) return false;
const current = now();
startedAt ??= current;
const remaining = maxWaitMs - (current - startedAt);
if (remaining <= 0) return false;
if (current - lastReportedAt >= 30_000) {
options.onWait?.(error.queueKey);
lastReportedAt = current;
}
await sleep(Math.min(1_000, remaining));
return true;
},
};
};
+36 -8
View File
@@ -129,6 +129,14 @@ export class ReservedTurnLeaseConflictError extends Error {
}
}
/** 실행 전 획득 충돌만 재시도한다. flush/finalize의 소유권 상실과 구분한다. */
export class ReservedTurnLeaseBusyError extends ReservedTurnLeaseConflictError {
constructor(queueKey: string) {
super(queueKey);
this.name = 'ReservedTurnLeaseBusyError';
}
}
export class InMemoryReservedTurnStore {
private readonly generalTurns = new Map<number, ReservedTurnEntry[]>();
private readonly nationTurns = new Map<string, ReservedTurnEntry[]>();
@@ -249,11 +257,7 @@ export class InMemoryReservedTurnStore {
}
};
pruneNationKeys(Array.from(this.nationTurns.keys()), (key) => this.nationTurns.delete(key));
for (const keys of [
this.dirtyNationKeys,
this.pendingNationInitializationKeys,
this.leasedNationKeys,
]) {
for (const keys of [this.dirtyNationKeys, this.pendingNationInitializationKeys, this.leasedNationKeys]) {
for (const key of Array.from(keys)) {
const nationId = Number(key.split(':', 1)[0]);
if (deletedNations.has(nationId)) {
@@ -376,7 +380,7 @@ export class InMemoryReservedTurnStore {
}
}
if (claimed.count !== 1) {
throw new ReservedTurnLeaseConflictError(`general:${generalId}`);
throw new ReservedTurnLeaseBusyError(`general:${generalId}`);
}
this.leasedGeneralIds.add(generalId);
return !retainedExistingLease;
@@ -427,7 +431,7 @@ export class InMemoryReservedTurnStore {
}
}
if (claimed.count !== 1) {
throw new ReservedTurnLeaseConflictError(`nation:${nationId}:${officerLevel}`);
throw new ReservedTurnLeaseBusyError(`nation:${nationId}:${officerLevel}`);
}
this.leasedNationKeys.add(buildNationKey(nationId, officerLevel));
return !retainedExistingLease;
@@ -904,6 +908,24 @@ export class InMemoryReservedTurnStore {
await this.persistChanges(this.prisma, changes);
this.acknowledgeDirtyState(changes);
}
/** 실행 루프 종료 뒤 호출한다. rollback으로 journal에서 빠진 DB 임대도 해제한다. */
async releaseOwnedLeases(): Promise<void> {
const results = await Promise.allSettled([
this.prisma.generalTurnRevision?.updateMany({
where: { leaseOwner: this.leaseOwner },
data: { leaseOwner: null, leaseExpiresAt: null },
}),
this.prisma.nationTurnRevision?.updateMany({
where: { leaseOwner: this.leaseOwner },
data: { leaseOwner: null, leaseExpiresAt: null },
}),
]);
const errors = results.flatMap((result) => (result.status === 'rejected' ? [result.reason] : []));
if (errors.length) throw new AggregateError(errors, 'Reserved turn lease cleanup failed.');
this.leasedGeneralIds.clear();
this.leasedNationKeys.clear();
}
}
export const createReservedTurnStore = async (options: ReservedTurnStoreOptions): Promise<ReservedTurnStoreHandle> => {
@@ -918,6 +940,12 @@ export const createReservedTurnStore = async (options: ReservedTurnStoreOptions)
await store.loadAll();
return {
store,
close: () => connector.disconnect(),
close: async () => {
try {
await store.releaseOwnedLeases();
} finally {
await connector.disconnect();
}
},
};
};
+2 -2
View File
@@ -2,7 +2,7 @@ import type { GameClockPhase } from '@sammo-ts/common';
/** Gateway의 실행 gate와 durable 시계를 명령 claim 전에 맞춘다. */
export const createRuntimePauseGate = (options: {
assertLease(): void;
assertLease(): void | Promise<void>;
shouldPause(): Promise<boolean>;
getPhase(): GameClockPhase;
isExplicitlyPaused(): boolean;
@@ -11,7 +11,7 @@ export const createRuntimePauseGate = (options: {
}): (() => Promise<boolean>) => {
let lastPaused: boolean | null = null;
return async () => {
options.assertLease();
await options.assertLease();
const paused = await options.shouldPause();
const phase = options.getPhase();
// 오류 정지는 Gateway 상태만 PAUSED로 바꿀 수 있다. 그대로 두면 가입은
+34 -14
View File
@@ -108,6 +108,8 @@ import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lif
import { EngineStateManager } from './engineStateManager.js';
import { applyRuntimeClockShift } from './runtimeClockShift.js';
import { applyRuntimeGameSettings } from './runtimeGameSettings.js';
import { createReservedTurnLeaseRetry } from './reservedTurnLeaseRetry.js';
import type { HostBootMonitor } from './hostBootMonitor.js';
export interface TurnDaemonRuntimeOptions {
profile: string;
@@ -134,6 +136,7 @@ export interface TurnDaemonRuntimeOptions {
leaseDurationMs?: number;
leaseOwnerId?: string;
enableLeaseHeartbeat?: boolean;
hostBootMonitor?: HostBootMonitor;
/**
* Isolated, single-process fixture acceleration only. Reserved turns are
* loaded once and no concurrent API writer may touch this database.
@@ -985,6 +988,7 @@ const createTurnDaemonRuntimeWithLease = async (
prepareClockRecovery = dbHooks.prepareRealtimeRecovery;
prunePreviousAudit = dbHooks.prunePreviousAudit;
close = async () => {
await adminActionConsumer?.stop();
if (auctionBidder) {
await auctionBidder.close();
}
@@ -995,12 +999,15 @@ const createTurnDaemonRuntimeWithLease = async (
await tournamentRewardFinalizer.close();
}
await dbHooks.close();
if (reservedTurnStoreHandle) {
await reservedTurnStoreHandle.close();
try {
await reservedTurnStoreHandle?.close();
} finally {
try {
await gatewayGate?.close();
} finally {
await turnDaemonLease?.close();
}
}
await gatewayGate?.close();
await adminActionConsumer?.stop();
await turnDaemonLease?.close();
};
} else if (reservedTurnStoreHandle) {
hooks = {
@@ -1009,9 +1016,12 @@ const createTurnDaemonRuntimeWithLease = async (
},
};
close = async () => {
await reservedTurnStoreHandle.close();
await gatewayGate?.close();
await adminActionConsumer?.stop();
try {
await reservedTurnStoreHandle.close();
} finally {
await gatewayGate?.close();
}
};
} else if (gatewayGate) {
hooks = {
@@ -1070,12 +1080,17 @@ const createTurnDaemonRuntimeWithLease = async (
const baseClose = close;
close = async () => {
stopClockProjectionWorker();
await baseClose();
await neutralAuctionRegistrar.close();
if (redisConnector) {
await redisConnector.disconnect();
}
await commandConnector?.disconnect();
const baseResults = await Promise.allSettled([baseClose()]);
// queue 임대 정리 실패도 다른 연결/worker의 종료를 막지 않게 한다.
const results = await Promise.allSettled([
neutralAuctionRegistrar.close(),
redisConnector?.disconnect(),
commandConnector?.disconnect(),
]);
const failures = [...baseResults, ...results].flatMap((result) =>
result.status === 'rejected' ? [result.reason] : []
);
if (failures.length) throw new AggregateError(failures, 'Turn daemon connection cleanup failed.');
};
const resolvedControlQueue = options.controlQueue ?? databaseCommandQueue ?? controlQueue;
@@ -1109,7 +1124,8 @@ const createTurnDaemonRuntimeWithLease = async (
processor,
hooks,
pauseGate: createRuntimePauseGate({
assertLease: () => {
assertLease: async () => {
await turnDaemonLease?.checkHostBoot();
if (turnDaemonLease?.isLost()) throw turnDaemonLease.getLossError();
},
shouldPause: async () => (await pauseGate?.()) ?? false,
@@ -1127,6 +1143,9 @@ const createTurnDaemonRuntimeWithLease = async (
// accumulated world before every due tick in that isolated mode;
// production and gateway-managed runtimes keep rollback savepoints.
stateManager: options.exclusiveFastForward ? undefined : stateManager,
runErrorRetry: createReservedTurnLeaseRetry({
onWait: (queueKey) => console.info(`[turn-daemon] waiting for reserved turn lease: ${queueKey}`),
}),
},
{ profile: options.profile, defaultBudget }
);
@@ -1179,6 +1198,7 @@ export const createTurnDaemonRuntime = async (options: TurnDaemonRuntimeOptions)
ownerId: options.leaseOwnerId,
leaseDurationMs: options.leaseDurationMs,
heartbeat: options.enableLeaseHeartbeat,
hostBootMonitor: options.hostBootMonitor,
})
: null;
if (turnDaemonLease && !(await turnDaemonLease.acquire())) {