From 1345aa25a1c4331dd8dc7379095408114dd87c01 Mon Sep 17 00:00:00 2001 From: hided62 Date: Fri, 31 Jul 2026 11:59:02 +0000 Subject: [PATCH] fix: recover auction finalization across worker restarts --- app/game-api/src/auction/scheduler.ts | 6 +- app/game-api/src/auction/worker.ts | 234 ++++-- .../src/services/pollingWorkerLifecycle.ts | 46 ++ app/game-api/src/tournament/worker.ts | 114 +-- .../test/auctionWorker.integration.test.ts | 668 ++++++++++++++++++ app/game-api/test/auctionWorker.test.ts | 207 +++++- ...pollingWorkerLifecycle.integration.test.ts | 38 + .../test/pollingWorkerLifecycle.test.ts | 35 + 8 files changed, 1202 insertions(+), 146 deletions(-) create mode 100644 app/game-api/src/services/pollingWorkerLifecycle.ts create mode 100644 app/game-api/test/auctionWorker.integration.test.ts create mode 100644 app/game-api/test/pollingWorkerLifecycle.integration.test.ts create mode 100644 app/game-api/test/pollingWorkerLifecycle.test.ts diff --git a/app/game-api/src/auction/scheduler.ts b/app/game-api/src/auction/scheduler.ts index 958dadc..65d8fbd 100644 --- a/app/game-api/src/auction/scheduler.ts +++ b/app/game-api/src/auction/scheduler.ts @@ -22,7 +22,11 @@ export const seedAuctionTimers = async ( keys: AuctionTimerKeys ): Promise => { const rows = await db.$queryRaw( - GamePrisma.sql`SELECT id, close_at as "closeAt", status FROM auction WHERE status = 'OPEN'` + GamePrisma.sql` + SELECT id, close_at as "closeAt", status + FROM auction + WHERE status IN ('OPEN', 'FINALIZING') + ` ); if (!rows.length) { return 0; diff --git a/app/game-api/src/auction/worker.ts b/app/game-api/src/auction/worker.ts index 4eaa8d7..2b44763 100644 --- a/app/game-api/src/auction/worker.ts +++ b/app/game-api/src/auction/worker.ts @@ -8,8 +8,8 @@ import { } from '@sammo-ts/infra'; import { resolveGameApiConfigFromEnv } from '../config.js'; -import { RedisTurnDaemonTransport } from '../daemon/redisTransport.js'; -import { buildTurnDaemonStreamKeys } from '../daemon/streamKeys.js'; +import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js'; +import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js'; import { buildAuctionTimerKeys } from './keys.js'; import { seedAuctionTimers } from './scheduler.js'; @@ -26,7 +26,37 @@ interface RedisTimerClient { zRemRangeByScore(key: string, min: number, max: number): Promise; } -const sleepMs = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +const AUCTION_FINALIZE_RECOVERY_LIMIT = 1; + +const buildAuctionFinalizeRequestId = (auctionId: number, closeAt: Date, retry = 0): string => { + const generation = closeAt.getTime(); + const base = `auction:finalize:${auctionId}:${generation}`; + return retry > 0 ? `${base}:retry:${retry}` : base; +}; + +const isMatchingAuctionFinalizeEvent = ( + event: { target: string; eventType: string; payload: unknown }, + command: { type: 'auctionFinalize'; requestId: string; auctionId: number } +): boolean => { + const payload = event.payload; + const payloadRecord = + payload !== null && typeof payload === 'object' && !Array.isArray(payload) + ? (payload as Record) + : null; + return ( + event.target === 'ENGINE' && + event.eventType === command.type && + payloadRecord?.type === command.type && + payloadRecord.requestId === command.requestId && + payloadRecord.auctionId === command.auctionId + ); +}; + +const isSuccessfulAuctionFinalizeResult = (result: unknown, auctionId: number): boolean => { + if (result === null || typeof result !== 'object' || Array.isArray(result)) return false; + const resultRecord = result as Record; + return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId; +}; const popDueAuctionIds = async ( redis: RedisTimerClient, @@ -56,44 +86,90 @@ export const processDueAuctionId = async (options: { historyKey: string; id: string; nowMs: number; - sendCommand: (command: { type: 'auctionFinalize'; auctionId: number }) => Promise; }): Promise<'FINALIZING' | 'RESCHEDULED' | 'IGNORED'> => { - const { db, redis, timerKey, historyKey, id, nowMs, sendCommand } = options; + const { db, redis, timerKey, historyKey, id, nowMs } = options; const auctionId = Number(id); - if (!Number.isFinite(auctionId)) { + if (!Number.isSafeInteger(auctionId) || auctionId < 1) { return 'IGNORED'; } const now = new Date(nowMs); - const updated = await db.$executeRaw( - GamePrisma.sql` - UPDATE auction - SET status = 'FINALIZING', - finalizing_at = ${now}, - updated_at = ${now} - WHERE id = ${auctionId} - AND status = 'OPEN' - AND close_at <= ${now} - ` - ); + const outcome = await db.$transaction(async (transaction) => { + const updated = await transaction.$executeRaw( + GamePrisma.sql` + UPDATE auction + SET status = 'FINALIZING', + finalizing_at = ${now}, + updated_at = ${now} + WHERE id = ${auctionId} + AND status = 'OPEN' + AND close_at <= ${now} + ` + ); - if (updated > 0) { + const current = await transaction.auction.findUnique({ + where: { id: auctionId }, + select: { status: true, closeAt: true }, + }); + if (!current) { + if (updated > 0) { + throw new Error(`Auction disappeared after FINALIZING transition: ${auctionId}`); + } + return { status: 'IGNORED' as const }; + } + if (current.status === 'OPEN') { + return { status: 'RESCHEDULED' as const, closeAt: current.closeAt }; + } + if (current.status !== 'FINALIZING') { + return { status: 'IGNORED' as const }; + } + + for (let retry = 0; retry <= AUCTION_FINALIZE_RECOVERY_LIMIT; retry += 1) { + const requestId = buildAuctionFinalizeRequestId(auctionId, current.closeAt, retry); + const command = { type: 'auctionFinalize' as const, requestId, auctionId }; + const existing = await transaction.inputEvent.findUnique({ + where: { requestId }, + select: { target: true, eventType: true, payload: true, status: true, result: true }, + }); + if (!existing) { + await transaction.inputEvent.create({ + data: { + requestId, + target: 'ENGINE', + eventType: command.type, + payload: command, + }, + }); + return { status: 'FINALIZING' as const }; + } + if (!isMatchingAuctionFinalizeEvent(existing, command)) { + throw new Error(`Conflicting durable auction finalization event: ${requestId}`); + } + if (existing.status === 'PENDING' || existing.status === 'PROCESSING') { + return { status: 'FINALIZING' as const }; + } + if (existing.status === 'SUCCEEDED' && isSuccessfulAuctionFinalizeResult(existing.result, auctionId)) { + throw new Error(`Auction remained FINALIZING after successful durable event: ${requestId}`); + } + } + throw new Error(`Auction finalization recovery exhausted: ${auctionId}`); + }); + + if (outcome.status === 'FINALIZING') { await redis.zAdd(historyKey, [{ score: nowMs, value: id }]); - await sendCommand({ type: 'auctionFinalize', auctionId }); return 'FINALIZING'; } - - const current = await db.auction.findFirst({ - where: { id: auctionId, status: 'OPEN' }, - select: { closeAt: true }, - }); - if (!current) { - return 'IGNORED'; + if (outcome.status === 'RESCHEDULED') { + await redis.zAdd(timerKey, [{ score: outcome.closeAt.getTime(), value: String(auctionId) }]); + return 'RESCHEDULED'; } - await redis.zAdd(timerKey, [{ score: current.closeAt.getTime(), value: String(auctionId) }]); - return 'RESCHEDULED'; + return 'IGNORED'; }; -export const runAuctionWorker = async (): Promise => { +export interface AuctionWorkerOptions { + signal?: AbortSignal; +} + +export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Promise => { const config = resolveGameApiConfigFromEnv(); const postgres = createGamePostgresConnector(resolvePostgresConfigFromEnv({ schema: config.profile })); const redis = createRedisConnector(resolveRedisConfigFromEnv()); @@ -102,54 +178,68 @@ export const runAuctionWorker = async (): Promise => { await redis.connect(); const keys = buildAuctionTimerKeys(config.profileName); - const daemonTransport = new RedisTurnDaemonTransport(redis.client, { - keys: buildTurnDaemonStreamKeys(config.profileName), - requestTimeoutMs: config.daemonRequestTimeoutMs, - }); - - const handleExit = async () => { - await redis.disconnect(); - await postgres.disconnect(); - }; - process.on('SIGINT', handleExit); - process.on('SIGTERM', handleExit); + const control = createPollingWorkerControl(options.signal); + const closeResources = createBestEffortResourceCloser([ + { name: 'auction-worker-redis', run: () => redis.disconnect() }, + { name: 'auction-worker-postgres', run: () => postgres.disconnect() }, + ]); let nextResyncAt = Date.now(); - while (true) { - const nowMs = Date.now(); - const historyTrimBefore = nowMs - config.auctionTimerRetentionSeconds * 1000; - if (historyTrimBefore > 0) { - await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore); - } - if (nowMs >= nextResyncAt) { - await seedAuctionTimers(postgres.prisma, redis.client, keys); - nextResyncAt = nowMs + config.auctionTimerResyncMs; - } - - const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, nowMs, 100); - if (dueIds.length > 0) { - for (const id of dueIds) { - await processDueAuctionId({ - db: postgres.prisma, - redis: redis.client, - timerKey: keys.timerKey, - historyKey: keys.historyKey, - id, - nowMs, - sendCommand: (command) => daemonTransport.sendCommand(command), - }); + try { + while (!control.signal.aborted) { + const nowMs = Date.now(); + const historyTrimBefore = nowMs - config.auctionTimerRetentionSeconds * 1000; + if (historyTrimBefore > 0) { + await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore); + } + if (nowMs >= nextResyncAt) { + await seedAuctionTimers(postgres.prisma, redis.client, keys); + nextResyncAt = nowMs + config.auctionTimerResyncMs; } - continue; - } - const nextDueMs = await getNextDueMs(redis.client, keys.timerKey); - if (nextDueMs === null) { - await sleepMs(config.auctionTimerPollMs); - continue; - } + const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, nowMs, 100); + if (dueIds.length > 0) { + for (const id of dueIds) { + try { + await processDueAuctionId({ + db: postgres.prisma, + redis: redis.client, + timerKey: keys.timerKey, + historyKey: keys.historyKey, + id, + nowMs, + }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown auction worker error'; + const trace = error instanceof Error ? error.stack : undefined; + try { + await postgres.prisma.errorLog.create({ + data: { + category: 'AUCTION', + source: 'auction-worker', + message, + trace, + context: { auctionId: id }, + }, + }); + } catch (logError) { + console.error('[auction-worker] failed to record auction error', logError); + } + } + } + continue; + } - const waitMs = Math.max(0, Math.min(config.auctionTimerPollMs, nextDueMs - Date.now())); - await sleepMs(waitMs); + const nextDueMs = await getNextDueMs(redis.client, keys.timerKey); + const waitMs = + nextDueMs === null + ? config.auctionTimerPollMs + : Math.max(0, Math.min(config.auctionTimerPollMs, nextDueMs - Date.now())); + await waitForWorkerPoll(control.signal, waitMs); + } + } finally { + control.dispose(); + await closeResources(); } }; diff --git a/app/game-api/src/services/pollingWorkerLifecycle.ts b/app/game-api/src/services/pollingWorkerLifecycle.ts new file mode 100644 index 0000000..675d86f --- /dev/null +++ b/app/game-api/src/services/pollingWorkerLifecycle.ts @@ -0,0 +1,46 @@ +export interface PollingWorkerControl { + readonly signal: AbortSignal; + dispose(): void; +} + +/** + * SIGINT/SIGTERM 또는 test AbortSignal을 하나의 idempotent stop signal로 + * 합칩니다. Signal은 새 poll만 막고 현재 처리 중인 작업은 caller가 끝낸 뒤 + * finally에서 resource를 닫게 합니다. + */ +export const createPollingWorkerControl = (externalSignal?: AbortSignal): PollingWorkerControl => { + const controller = new AbortController(); + const stop = () => controller.abort(); + + process.on('SIGINT', stop); + process.on('SIGTERM', stop); + externalSignal?.addEventListener('abort', stop, { once: true }); + if (externalSignal?.aborted) { + stop(); + } + + let disposed = false; + return { + signal: controller.signal, + dispose: () => { + if (disposed) return; + disposed = true; + process.off('SIGINT', stop); + process.off('SIGTERM', stop); + externalSignal?.removeEventListener('abort', stop); + }, + }; +}; + +export const waitForWorkerPoll = async (signal: AbortSignal, delayMs: number): Promise => { + if (signal.aborted || delayMs <= 0) return; + await new Promise((resolve) => { + const timeout = setTimeout(finish, delayMs); + function finish() { + clearTimeout(timeout); + signal.removeEventListener('abort', finish); + resolve(); + } + signal.addEventListener('abort', finish, { once: true }); + }); +}; diff --git a/app/game-api/src/tournament/worker.ts b/app/game-api/src/tournament/worker.ts index 5b95e5f..4530c6b 100644 --- a/app/game-api/src/tournament/worker.ts +++ b/app/game-api/src/tournament/worker.ts @@ -10,6 +10,8 @@ import { import { resolveGameApiConfigFromEnv } from '../config.js'; import { DatabaseTurnDaemonTransport } from '../daemon/databaseTransport.js'; +import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js'; +import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js'; import type { TurnDaemonTransport } from '../daemon/transport.js'; import { buildTournamentKeys } from './keys.js'; import { TournamentStore } from './store.js'; @@ -31,7 +33,6 @@ import { resolveGroupPair, resolveNextAt, seedNpcBets, - sleepMs, sortByRanking, type TournamentMatchOutcome, type TournamentPrismaClient, @@ -590,7 +591,11 @@ export const processTournamentTick = async (options: { return processedState; }; -export const runTournamentWorker = async (): Promise => { +export interface TournamentWorkerOptions { + signal?: AbortSignal; +} + +export const runTournamentWorker = async (options: TournamentWorkerOptions = {}): Promise => { const config = resolveGameApiConfigFromEnv(); const postgres = createGamePostgresConnector(resolvePostgresConfigFromEnv({ schema: config.profile })); const redis = createRedisConnector(resolveRedisConfigFromEnv()); @@ -600,61 +605,64 @@ export const runTournamentWorker = async (): Promise => { const store = new TournamentStore(redis.client, buildTournamentKeys(config.profileName)); const daemonTransport = new DatabaseTurnDaemonTransport(postgres.prisma, config.daemonRequestTimeoutMs); + const control = createPollingWorkerControl(options.signal); + const closeResources = createBestEffortResourceCloser([ + { name: 'tournament-worker-redis', run: () => redis.disconnect() }, + { name: 'tournament-worker-postgres', run: () => postgres.disconnect() }, + ]); - const handleExit = async () => { - await redis.disconnect(); - await postgres.disconnect(); - }; - process.on('SIGINT', handleExit); - process.on('SIGTERM', handleExit); + try { + while (!control.signal.aborted) { + const state = await store.getState(); + if (!state || (!state.auto && !needsSettlement(state))) { + await waitForWorkerPoll(control.signal, config.tournamentPollMs); + continue; + } - while (true) { - const state = await store.getState(); - if (!state || (!state.auto && !needsSettlement(state))) { - await sleepMs(config.tournamentPollMs); - continue; - } + const nextAt = new Date(state.nextAt).getTime(); + const now = Date.now(); + if (state.auto && Number.isFinite(nextAt) && nextAt > now) { + await waitForWorkerPoll(control.signal, Math.min(config.tournamentPollMs, nextAt - now)); + continue; + } - const nextAt = new Date(state.nextAt).getTime(); - const now = Date.now(); - if (state.auto && Number.isFinite(nextAt) && nextAt > now) { - await sleepMs(Math.min(config.tournamentPollMs, nextAt - now)); - continue; - } - - try { - await processTournamentTick({ - store, - prisma: postgres.prisma, - daemonTransport, - }); - } catch (error) { - const message = error instanceof Error ? error.message : 'Unknown error'; - const trace = error instanceof Error ? error.stack : undefined; - const now = new Date().toISOString(); - await postgres.prisma.errorLog.create({ - data: { - category: 'TOURNAMENT', - source: 'tournament-worker', - message, - trace, - context: { - stage: state.stage, - phase: state.phase, - bettingId: state.bettingId ?? null, - nextAt: state.nextAt, + try { + await processTournamentTick({ + store, + prisma: postgres.prisma, + daemonTransport, + }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + const trace = error instanceof Error ? error.stack : undefined; + const failedAt = new Date().toISOString(); + await postgres.prisma.errorLog.create({ + data: { + category: 'TOURNAMENT', + source: 'tournament-worker', + message, + trace, + context: { + stage: state.stage, + phase: state.phase, + bettingId: state.bettingId ?? null, + nextAt: state.nextAt, + }, }, - }, - }); - const currentState = (await store.getState()) ?? state; - const nextState: TournamentState = { - ...currentState, - lastError: message, - lastErrorAt: now, - }; - await store.setState(nextState); - } + }); + const currentState = (await store.getState()) ?? state; + const nextState: TournamentState = { + ...currentState, + lastError: message, + lastErrorAt: failedAt, + }; + await store.setState(nextState); + } - await sleepMs(config.tournamentPollMs); + await waitForWorkerPoll(control.signal, config.tournamentPollMs); + } + } finally { + control.dispose(); + await closeResources(); } }; diff --git a/app/game-api/test/auctionWorker.integration.test.ts b/app/game-api/test/auctionWorker.integration.test.ts new file mode 100644 index 0000000..7e15e98 --- /dev/null +++ b/app/game-api/test/auctionWorker.integration.test.ts @@ -0,0 +1,668 @@ +import { randomUUID } from 'node:crypto'; + +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { SystemClock } from '@sammo-ts/common'; +import { + createDatabaseTurnHooks, + DatabaseTurnDaemonCommandQueue, + EngineStateManager, + InMemoryTurnStateStore, + InMemoryTurnWorld, + loadTurnWorldFromDatabase, + TurnDaemonLifecycle, + type TurnGeneral, + type TurnWorldSnapshot, + type TurnWorldState, +} from '@sammo-ts/game-engine'; +import { createAuctionFinalizer } from '@sammo-ts/game-engine/auction/finalizer.js'; +import { createTurnDaemonCommandHandler } from '@sammo-ts/game-engine/turn/worldCommandHandler.js'; +import { createGamePostgresConnector, createRedisConnector, resolveRedisConfigFromEnv } from '@sammo-ts/infra'; +import type { GamePrisma } from '@sammo-ts/infra'; +import type { MapDefinition, ScenarioConfig, ScenarioMeta, TurnSchedule } from '@sammo-ts/logic'; + +import { buildAuctionTimerKeys } from '../src/auction/keys.js'; +import { processDueAuctionId, runAuctionWorker } from '../src/auction/worker.js'; + +const databaseUrl = process.env.INPUT_EVENT_DATABASE_URL; +const liveDescribe = databaseUrl && process.env.REDIS_URL ? describe : describe.skip; + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +liveDescribe('auction worker durable recovery', () => { + const connector = createGamePostgresConnector({ url: databaseUrl! }); + const createdAuctionIds: number[] = []; + const createdRequestPrefixes: string[] = []; + const createdWorldIds: number[] = []; + const createdGeneralIds: number[] = []; + + beforeAll(async () => { + await connector.connect(); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + afterAll(async () => { + if (createdRequestPrefixes.length > 0) { + await connector.prisma.inputEvent.deleteMany({ + where: { OR: createdRequestPrefixes.map((prefix) => ({ requestId: { startsWith: prefix } })) }, + }); + } + if (createdAuctionIds.length > 0) { + await connector.prisma.auction.deleteMany({ where: { id: { in: createdAuctionIds } } }); + } + if (createdWorldIds.length > 0) { + await connector.prisma.worldState.deleteMany({ where: { id: { in: createdWorldIds } } }); + } + if (createdGeneralIds.length > 0) { + await connector.prisma.logEntry.deleteMany({ where: { generalId: { in: createdGeneralIds } } }); + await connector.prisma.general.deleteMany({ where: { id: { in: createdGeneralIds } } }); + } + await connector.disconnect(); + }); + + const createAuction = async (status: 'OPEN' | 'FINALIZING') => { + const closeAt = new Date(Date.now() - 60_000); + const auction = await connector.prisma.auction.create({ + data: { + type: 'BUY_RICE', + hostGeneralId: 0, + hostName: 'worker-test', + detail: { amount: 100 }, + status, + closeAt, + ...(status === 'FINALIZING' ? { finalizingAt: new Date(Date.now() - 30_000) } : {}), + }, + }); + createdAuctionIds.push(auction.id); + createdRequestPrefixes.push(`auction:finalize:${auction.id}:`); + return auction; + }; + + const requestIdFor = (auction: { id: number; closeAt: Date }): string => + `auction:finalize:${auction.id}:${auction.closeAt.getTime()}`; + + const memoryRedis = () => ({ + zRangeByScore: vi.fn(async () => []), + zRangeWithScores: vi.fn(async () => []), + zAdd: vi.fn(async () => 1), + zRem: vi.fn(async () => 0), + zRemRangeByScore: vi.fn(async () => 0), + }); + + it('atomically moves OPEN to FINALIZING and creates one deterministic input event', async () => { + const auction = await createAuction('OPEN'); + const redis = memoryRedis(); + + await expect( + processDueAuctionId({ + db: connector.prisma, + redis, + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }) + ).resolves.toBe('FINALIZING'); + await expect( + processDueAuctionId({ + db: connector.prisma, + redis, + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }) + ).resolves.toBe('FINALIZING'); + + const [storedAuction, events] = await Promise.all([ + connector.prisma.auction.findUniqueOrThrow({ where: { id: auction.id } }), + connector.prisma.inputEvent.findMany({ where: { requestId: requestIdFor(auction) } }), + ]); + expect(storedAuction.status).toBe('FINALIZING'); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + target: 'ENGINE', + eventType: 'auctionFinalize', + status: 'PENDING', + payload: { + type: 'auctionFinalize', + requestId: requestIdFor(auction), + auctionId: auction.id, + }, + }); + }); + + it('rolls the OPEN transition back when the deterministic request ID conflicts', async () => { + const auction = await createAuction('OPEN'); + const requestId = requestIdFor(auction); + await connector.prisma.inputEvent.create({ + data: { + requestId, + target: 'ENGINE', + eventType: 'getStatus', + payload: { type: 'getStatus', requestId }, + }, + }); + + await expect( + processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }) + ).rejects.toThrow(`Conflicting durable auction finalization event: ${requestId}`); + + await expect(connector.prisma.auction.findUniqueOrThrow({ where: { id: auction.id } })).resolves.toMatchObject({ + status: 'OPEN', + finalizingAt: null, + }); + await connector.prisma.inputEvent.delete({ where: { requestId } }); + await connector.prisma.auction.delete({ where: { id: auction.id } }); + }); + + it('creates one bounded recovery event after terminal failure and then stops retrying', async () => { + const auction = await createAuction('FINALIZING'); + const requestId = requestIdFor(auction); + const retryRequestId = `${requestId}:retry:1`; + await connector.prisma.inputEvent.create({ + data: { + requestId, + target: 'ENGINE', + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId, auctionId: auction.id }, + status: 'FAILED', + attempts: 3, + error: 'simulated terminal failure', + completedAt: new Date(), + }, + }); + + await expect( + processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }) + ).resolves.toBe('FINALIZING'); + await expect( + processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }) + ).resolves.toBe('FINALIZING'); + await expect( + connector.prisma.inputEvent.findMany({ + where: { requestId: { startsWith: requestId } }, + orderBy: { sequence: 'asc' }, + }) + ).resolves.toEqual([ + expect.objectContaining({ requestId, status: 'FAILED' }), + expect.objectContaining({ requestId: retryRequestId, status: 'PENDING', attempts: 0 }), + ]); + + await connector.prisma.inputEvent.update({ + where: { requestId: retryRequestId }, + data: { status: 'FAILED', attempts: 3, error: 'simulated recovery failure', completedAt: new Date() }, + }); + await expect( + processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }) + ).rejects.toThrow(`Auction finalization recovery exhausted: ${auction.id}`); + }); + + it('creates a new generation after an earlier close was extended', async () => { + const auction = await createAuction('OPEN'); + const priorRequestId = `auction:finalize:${auction.id}:${auction.closeAt.getTime() - 300_000}`; + await connector.prisma.inputEvent.create({ + data: { + requestId: priorRequestId, + target: 'ENGINE', + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId: priorRequestId, auctionId: auction.id }, + status: 'SUCCEEDED', + attempts: 1, + result: { + type: 'auctionFinalize', + ok: false, + auctionId: auction.id, + reason: 'extended', + }, + completedAt: new Date(), + }, + }); + + await expect( + processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }) + ).resolves.toBe('FINALIZING'); + + await expect( + connector.prisma.inputEvent.findUnique({ where: { requestId: requestIdFor(auction) } }) + ).resolves.toMatchObject({ status: 'PENDING', eventType: 'auctionFinalize' }); + }); + + it( + 'flows the durable event through the actual daemon finalizer and reloads one FINISHED settlement', + { timeout: 15_000 }, + async () => { + const worldId = 992_031; + const schedule: TurnSchedule = { entries: [{ startMinute: 0, tickMinutes: 10 }] }; + const scenarioMeta: ScenarioMeta = { + title: '경매 durable lifecycle 통합', + startYear: 190, + life: null, + fiction: null, + history: [], + ignoreDefaultEvents: false, + }; + const scenarioConfig: ScenarioConfig = { + stat: { total: 300, min: 10, max: 100, npcTotal: 150, npcMax: 50, npcMin: 10, chiefMin: 65 }, + iconPath: '', + map: {}, + const: {}, + environment: { mapName: 'che', unitSet: 'che' }, + }; + const map: MapDefinition = { id: 'auction-durable-lifecycle', name: scenarioMeta.title, cities: [] }; + const state: TurnWorldState = { + id: worldId, + currentYear: 190, + currentMonth: 1, + tickSeconds: 600, + lastTurnTime: new Date('2026-07-31T11:00:00.000Z'), + meta: { killturn: 24, scenarioMeta }, + }; + const buildGeneral = (options: { + id: number; + userId: string; + name: string; + gold: number; + rice: number; + }): TurnGeneral => ({ + id: options.id, + userId: options.userId, + name: options.name, + nationId: 0, + cityId: 0, + troopId: 0, + stats: { leadership: 50, strength: 50, intelligence: 50 }, + turnTime: new Date('2026-07-31T12:00:00.000Z'), + recentWarTime: null, + role: { + items: { horse: null, weapon: null, book: null, item: null }, + personality: null, + specialDomestic: null, + specialWar: null, + }, + triggerState: { flags: {}, counters: {}, modifiers: {}, meta: {} }, + meta: { killturn: 24 }, + penalty: {}, + officerLevel: 0, + experience: 0, + dedication: 0, + injury: 0, + gold: options.gold, + rice: options.rice, + crew: 0, + crewTypeId: 0, + train: 0, + atmos: 0, + age: 20, + npcState: 0, + picture: 'default.jpg', + imageServer: 0, + }); + const host = buildGeneral({ + id: 992_032, + userId: 'auction-durable-host', + name: '경매주최자', + gold: 1_000, + rice: 900, + }); + const bidder = buildGeneral({ + id: 992_033, + userId: 'auction-durable-bidder', + name: '경매입찰자', + gold: 800, + rice: 1_000, + }); + const snapshot: TurnWorldSnapshot = { + generals: [host, bidder], + cities: [], + nations: [], + troops: [], + diplomacy: [], + events: [], + initialEvents: [], + scenarioConfig, + scenarioMeta, + map, + }; + + const lifecycleGeneralIds = [992_032, 992_033]; + await connector.prisma.logEntry.deleteMany({ where: { generalId: { in: lifecycleGeneralIds } } }); + await connector.prisma.general.deleteMany({ where: { id: { in: lifecycleGeneralIds } } }); + await connector.prisma.worldState.deleteMany({ where: { id: worldId } }); + await connector.prisma.worldState.create({ + data: { + id: worldId, + scenarioCode: map.id, + currentYear: state.currentYear, + currentMonth: state.currentMonth, + tickSeconds: state.tickSeconds, + config: JSON.parse(JSON.stringify(scenarioConfig)) as GamePrisma.InputJsonValue, + meta: state.meta as GamePrisma.InputJsonValue, + }, + }); + createdWorldIds.push(worldId); + for (const general of [host, bidder]) { + await connector.prisma.general.create({ + data: { + id: general.id, + userId: general.userId, + name: general.name, + turnTime: general.turnTime, + gold: general.gold, + rice: general.rice, + picture: general.picture, + imageServer: general.imageServer, + meta: general.meta, + }, + }); + createdGeneralIds.push(general.id); + } + const auction = await createAuction('OPEN'); + await connector.prisma.auction.update({ + where: { id: auction.id }, + data: { hostGeneralId: host.id, hostName: host.name }, + }); + await connector.prisma.auctionBid.create({ + data: { + auctionId: auction.id, + generalId: bidder.id, + amount: 200, + eventId: `auction-durable-bid:${auction.id}`, + eventAt: new Date(), + }, + }); + const requestId = requestIdFor(auction); + await processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(auction.id), + nowMs: Date.now(), + }); + + const world = new InMemoryTurnWorld(state, snapshot, { schedule }); + const queue = new DatabaseTurnDaemonCommandQueue(connector.prisma); + await queue.initialize(); + const hooks = await createDatabaseTurnHooks(databaseUrl!, world); + const auctionFinalizer = await createAuctionFinalizer({ databaseUrl: databaseUrl!, world }); + const stateManager = new EngineStateManager(); + stateManager.register('world', { + capture: () => world.captureState(), + restore: (captured) => world.restoreState(captured), + }); + const lifecycle = new TurnDaemonLifecycle( + { + clock: new SystemClock(), + controlQueue: queue, + commandResponder: queue, + getNextTickTime: () => new Date(Date.now() + 3_600_000), + stateStore: new InMemoryTurnStateStore(world), + processor: { + run: async () => { + throw new Error('scheduled turn must not run in auction lifecycle integration'); + }, + }, + commandHandler: createTurnDaemonCommandHandler({ world, auctionFinalizer }), + hooks: hooks.hooks, + stateManager, + }, + { + profile: 'auction-durable-lifecycle', + defaultBudget: { budgetMs: 100, maxGenerals: 1, catchUpCap: 1 }, + } + ); + const waitForSettlement = async () => { + for (let attempt = 0; attempt < 200; attempt += 1) { + const [event, storedAuction] = await Promise.all([ + connector.prisma.inputEvent.findUnique({ where: { requestId } }), + connector.prisma.auction.findUnique({ where: { id: auction.id } }), + ]); + if (event?.status === 'SUCCEEDED' && storedAuction?.status === 'FINISHED') { + return { event, storedAuction }; + } + await delay(25); + } + throw new Error(`Timed out waiting for auction ${auction.id} settlement`); + }; + let extensionAuctionId: number; + + let loop: Promise | undefined; + try { + loop = lifecycle.start(); + await expect(waitForSettlement()).resolves.toMatchObject({ + event: { + attempts: 1, + result: { type: 'auctionFinalize', ok: true, auctionId: auction.id }, + }, + storedAuction: { status: 'FINISHED' }, + }); + + const extensionAuction = await connector.prisma.auction.create({ + data: { + type: 'UNIQUE_ITEM', + targetCode: 'integration-invalid-unique-item', + hostGeneralId: 0, + hostName: '(상인)', + detail: { remainCloseDateExtensionCnt: 1 }, + status: 'OPEN', + closeAt: new Date(Date.now() - 60_000), + }, + }); + extensionAuctionId = extensionAuction.id; + createdAuctionIds.push(extensionAuction.id); + createdRequestPrefixes.push(`auction:finalize:${extensionAuction.id}:`); + await connector.prisma.auctionBid.create({ + data: { + auctionId: extensionAuction.id, + generalId: bidder.id, + amount: 50, + eventId: `auction-extension-bid:${extensionAuction.id}`, + eventAt: new Date(), + meta: { tryExtendCloseDate: true }, + }, + }); + const firstExtensionRequestId = requestIdFor(extensionAuction); + await processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(extensionAuction.id), + nowMs: Date.now(), + }); + + let reopened: { status: string; closeAt: Date } | null = null; + for (let attempt = 0; attempt < 200; attempt += 1) { + const [event, storedAuction] = await Promise.all([ + connector.prisma.inputEvent.findUnique({ where: { requestId: firstExtensionRequestId } }), + connector.prisma.auction.findUnique({ where: { id: extensionAuction.id } }), + ]); + if (event?.status === 'SUCCEEDED' && storedAuction?.status === 'OPEN') { + reopened = storedAuction; + break; + } + await delay(25); + } + expect(reopened).toMatchObject({ status: 'OPEN' }); + expect(reopened!.closeAt.getTime()).toBeGreaterThan(extensionAuction.closeAt.getTime()); + + const secondCloseAt = new Date(Date.now() - 1_000); + await connector.prisma.auction.update({ + where: { id: extensionAuction.id }, + data: { closeAt: secondCloseAt }, + }); + const secondExtensionRequestId = requestIdFor({ id: extensionAuction.id, closeAt: secondCloseAt }); + await processDueAuctionId({ + db: connector.prisma, + redis: memoryRedis(), + timerKey: 'timer', + historyKey: 'history', + id: String(extensionAuction.id), + nowMs: Date.now(), + }); + + for (let attempt = 0; attempt < 200; attempt += 1) { + const [event, storedAuction] = await Promise.all([ + connector.prisma.inputEvent.findUnique({ where: { requestId: secondExtensionRequestId } }), + connector.prisma.auction.findUnique({ where: { id: extensionAuction.id } }), + ]); + if (event?.status === 'SUCCEEDED' && storedAuction?.status === 'CANCELED') break; + await delay(25); + } + await expect( + connector.prisma.auction.findUniqueOrThrow({ where: { id: extensionAuction.id } }) + ).resolves.toMatchObject({ status: 'CANCELED' }); + await expect( + connector.prisma.inputEvent.count({ + where: { requestId: { startsWith: `auction:finalize:${extensionAuction.id}:` } }, + }) + ).resolves.toBe(2); + } finally { + await lifecycle.stop('auction durable lifecycle integration finished'); + await loop; + await auctionFinalizer.close(); + await hooks.close(); + } + + const freshConnector = createGamePostgresConnector({ url: databaseUrl! }); + await freshConnector.connect(); + try { + await expect( + freshConnector.prisma.auction.findUniqueOrThrow({ where: { id: auction.id } }) + ).resolves.toMatchObject({ status: 'FINISHED' }); + await expect( + freshConnector.prisma.inputEvent.count({ + where: { requestId: { startsWith: `auction:finalize:${auction.id}:` } }, + }) + ).resolves.toBe(1); + await expect( + freshConnector.prisma.logEntry.count({ + where: { generalId: { in: [host.id, bidder.id] } }, + }) + ).resolves.toBe(2); + await expect( + freshConnector.prisma.auction.findUniqueOrThrow({ where: { id: extensionAuctionId } }) + ).resolves.toMatchObject({ status: 'CANCELED' }); + } finally { + await freshConnector.disconnect(); + } + const reloaded = await loadTurnWorldFromDatabase({ databaseUrl: databaseUrl! }); + expect(reloaded.snapshot.generals.find((general) => general.id === host.id)).toMatchObject({ + gold: 1_200, + rice: 900, + }); + expect(reloaded.snapshot.generals.find((general) => general.id === bidder.id)).toMatchObject({ + gold: 800, + rice: 1_100, + }); + } + ); + + it( + 'restarts from a stranded FINALIZING row, creates its event, and stops cleanly', + { timeout: 15_000 }, + async () => { + const poisonedAuction = await createAuction('OPEN'); + const poisonedRequestId = requestIdFor(poisonedAuction); + await connector.prisma.inputEvent.create({ + data: { + requestId: poisonedRequestId, + target: 'ENGINE', + eventType: 'getStatus', + payload: { type: 'getStatus', requestId: poisonedRequestId }, + }, + }); + const auction = await createAuction('FINALIZING'); + const schema = new URL(databaseUrl!).searchParams.get('schema'); + if (!schema) throw new Error('integration database URL must include a schema'); + const profileName = `auction-recovery:${randomUUID()}`; + vi.stubEnv('DATABASE_URL', databaseUrl!); + vi.stubEnv('PROFILE', schema); + vi.stubEnv('SCENARIO', 'worker-recovery'); + vi.stubEnv('GAME_PROFILE_NAME', profileName); + vi.stubEnv('GAME_TOKEN_SECRET', 'auction-worker-integration-only'); + vi.stubEnv('AUCTION_TIMER_POLL_MS', '25'); + vi.stubEnv('AUCTION_TIMER_RESYNC_MS', '50'); + + const redisConnector = createRedisConnector(resolveRedisConfigFromEnv()); + await redisConnector.connect(); + const keys = buildAuctionTimerKeys(profileName); + const abortController = new AbortController(); + const worker = runAuctionWorker({ signal: abortController.signal }); + + try { + const deadline = Date.now() + 5_000; + let event = null; + while (Date.now() < deadline) { + event = await connector.prisma.inputEvent.findUnique({ + where: { requestId: requestIdFor(auction) }, + }); + if (event) break; + await delay(25); + } + expect(event).toMatchObject({ + status: 'PENDING', + eventType: 'auctionFinalize', + }); + expect(await redisConnector.client.zScore(keys.historyKey, String(auction.id))).not.toBeNull(); + expect( + await connector.prisma.errorLog.count({ + where: { + source: 'auction-worker', + message: { contains: poisonedRequestId }, + }, + }) + ).toBeGreaterThan(0); + } finally { + abortController.abort(); + await worker; + await redisConnector.client.del([keys.timerKey, keys.historyKey]); + await redisConnector.disconnect(); + await connector.prisma.errorLog.deleteMany({ + where: { + source: 'auction-worker', + message: { contains: poisonedRequestId }, + }, + }); + } + } + ); +}); diff --git a/app/game-api/test/auctionWorker.test.ts b/app/game-api/test/auctionWorker.test.ts index 713676d..3282374 100644 --- a/app/game-api/test/auctionWorker.test.ts +++ b/app/game-api/test/auctionWorker.test.ts @@ -12,17 +12,46 @@ const buildRedis = () => ({ zRemRangeByScore: vi.fn(async () => 0), }); +const buildDb = (options: { + updated: number; + auction?: { status: 'OPEN' | 'FINALIZING' | 'FINISHED' | 'CANCELED'; closeAt: Date } | null; + existingEvents?: Array<{ + requestId: string; + target: 'ENGINE'; + eventType: string; + payload: Record; + status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED'; + result?: Record | null; + }>; +}) => { + const transaction = { + $executeRaw: vi.fn(async () => options.updated), + auction: { + findUnique: vi.fn(async () => options.auction ?? null), + }, + inputEvent: { + findUnique: vi.fn( + async ({ where }: { where: { requestId: string } }) => + options.existingEvents?.find((event) => event.requestId === where.requestId) ?? null + ), + create: vi.fn(async () => ({ sequence: 1n })), + }, + }; + return { + db: { + $transaction: vi.fn(async (callback: (tx: typeof transaction) => Promise) => + callback(transaction) + ), + } as unknown as GamePrismaClient, + transaction, + }; +}; + describe('auction worker clock-shift race', () => { it('requeues an OPEN auction at its current DB deadline when an old due score loses the race', async () => { const redis = buildRedis(); const closeAt = new Date('2026-07-30T12:15:00.000Z'); - const db = { - $executeRaw: vi.fn(async () => 0), - auction: { - findFirst: vi.fn(async () => ({ closeAt })), - }, - } as unknown as GamePrismaClient; - const sendCommand = vi.fn(async () => {}); + const { db, transaction } = buildDb({ updated: 0, auction: { status: 'OPEN', closeAt } }); await expect( processDueAuctionId({ @@ -32,24 +61,19 @@ describe('auction worker clock-shift race', () => { historyKey: 'history', id: '7', nowMs: new Date('2026-07-30T12:00:00.000Z').getTime(), - sendCommand, }) ).resolves.toBe('RESCHEDULED'); expect(redis.zAdd).toHaveBeenCalledTimes(1); expect(redis.zAdd).toHaveBeenCalledWith('timer', [{ score: closeAt.getTime(), value: '7' }]); - expect(sendCommand).not.toHaveBeenCalled(); + expect(transaction.inputEvent.create).not.toHaveBeenCalled(); }); - it('records history and finalizes only after the guarded DB transition succeeds', async () => { + it('commits the FINALIZING transition and durable command in one transaction before recording history', async () => { const redis = buildRedis(); - const db = { - $executeRaw: vi.fn(async () => 1), - auction: { - findFirst: vi.fn(), - }, - } as unknown as GamePrismaClient; - const sendCommand = vi.fn(async () => {}); + const closeAt = new Date('2026-07-30T11:00:00.000Z'); + const requestId = `auction:finalize:7:${closeAt.getTime()}`; + const { db, transaction } = buildDb({ updated: 1, auction: { status: 'FINALIZING', closeAt } }); const nowMs = new Date('2026-07-30T12:00:00.000Z').getTime(); await expect( @@ -60,12 +84,155 @@ describe('auction worker clock-shift race', () => { historyKey: 'history', id: '7', nowMs, - sendCommand, }) ).resolves.toBe('FINALIZING'); expect(redis.zAdd).toHaveBeenCalledWith('history', [{ score: nowMs, value: '7' }]); - expect(db.auction.findFirst).not.toHaveBeenCalled(); - expect(sendCommand).toHaveBeenCalledWith({ type: 'auctionFinalize', auctionId: 7 }); + expect(transaction.inputEvent.create).toHaveBeenCalledWith({ + data: { + requestId, + target: 'ENGINE', + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId, auctionId: 7 }, + }, + }); + }); + + it('repairs a pre-existing FINALIZING auction without creating a duplicate command', async () => { + const redis = buildRedis(); + const closeAt = new Date('2026-07-30T11:00:00.000Z'); + const requestId = `auction:finalize:7:${closeAt.getTime()}`; + const existingEvent = { + requestId, + target: 'ENGINE' as const, + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId, auctionId: 7 }, + status: 'PENDING' as const, + result: null, + }; + const { db, transaction } = buildDb({ + updated: 0, + auction: { status: 'FINALIZING', closeAt }, + existingEvents: [existingEvent], + }); + + await expect( + processDueAuctionId({ + db, + redis, + timerKey: 'timer', + historyKey: 'history', + id: '7', + nowMs: new Date('2026-07-30T12:00:00.000Z').getTime(), + }) + ).resolves.toBe('FINALIZING'); + + expect(transaction.inputEvent.create).not.toHaveBeenCalled(); + expect(redis.zAdd).toHaveBeenCalledWith('history', [ + { score: new Date('2026-07-30T12:00:00.000Z').getTime(), value: '7' }, + ]); + }); + + it('creates one bounded successor after a terminal event failure', async () => { + const redis = buildRedis(); + const closeAt = new Date('2026-07-30T11:00:00.000Z'); + const requestId = `auction:finalize:7:${closeAt.getTime()}`; + const retryRequestId = `${requestId}:retry:1`; + const { db, transaction } = buildDb({ + updated: 0, + auction: { status: 'FINALIZING', closeAt }, + existingEvents: [ + { + requestId, + target: 'ENGINE', + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId, auctionId: 7 }, + status: 'FAILED', + result: null, + }, + ], + }); + + await expect( + processDueAuctionId({ + db, + redis, + timerKey: 'timer', + historyKey: 'history', + id: '7', + nowMs: new Date('2026-07-30T12:00:00.000Z').getTime(), + }) + ).resolves.toBe('FINALIZING'); + + expect(transaction.inputEvent.create).toHaveBeenCalledWith({ + data: { + requestId: retryRequestId, + target: 'ENGINE', + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId: retryRequestId, auctionId: 7 }, + }, + }); + }); + + it('uses the close deadline as the generation so a reopened auction gets a new command', async () => { + const redis = buildRedis(); + const previousCloseAt = new Date('2026-07-30T11:00:00.000Z'); + const closeAt = new Date('2026-07-30T11:30:00.000Z'); + const previousRequestId = `auction:finalize:7:${previousCloseAt.getTime()}`; + const requestId = `auction:finalize:7:${closeAt.getTime()}`; + const { db, transaction } = buildDb({ + updated: 1, + auction: { status: 'FINALIZING', closeAt }, + existingEvents: [ + { + requestId: previousRequestId, + target: 'ENGINE', + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId: previousRequestId, auctionId: 7 }, + status: 'SUCCEEDED', + result: { type: 'auctionFinalize', ok: false, auctionId: 7 }, + }, + ], + }); + + await expect( + processDueAuctionId({ + db, + redis, + timerKey: 'timer', + historyKey: 'history', + id: '7', + nowMs: new Date('2026-07-30T12:00:00.000Z').getTime(), + }) + ).resolves.toBe('FINALIZING'); + + expect(transaction.inputEvent.create).toHaveBeenCalledWith({ + data: { + requestId, + target: 'ENGINE', + eventType: 'auctionFinalize', + payload: { type: 'auctionFinalize', requestId, auctionId: 7 }, + }, + }); + }); + + it('rolls the auction transition back when durable event creation fails', async () => { + const redis = buildRedis(); + const closeAt = new Date('2026-07-30T11:00:00.000Z'); + const { db, transaction } = buildDb({ updated: 1, auction: { status: 'FINALIZING', closeAt } }); + transaction.inputEvent.create.mockRejectedValueOnce(new Error('event insert failed')); + + await expect( + processDueAuctionId({ + db, + redis, + timerKey: 'timer', + historyKey: 'history', + id: '7', + nowMs: new Date('2026-07-30T12:00:00.000Z').getTime(), + }) + ).rejects.toThrow('event insert failed'); + + expect(redis.zAdd).not.toHaveBeenCalled(); }); }); diff --git a/app/game-api/test/pollingWorkerLifecycle.integration.test.ts b/app/game-api/test/pollingWorkerLifecycle.integration.test.ts new file mode 100644 index 0000000..2beb171 --- /dev/null +++ b/app/game-api/test/pollingWorkerLifecycle.integration.test.ts @@ -0,0 +1,38 @@ +import { randomUUID } from 'node:crypto'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { runTournamentWorker } from '../src/tournament/worker.js'; + +const databaseUrl = process.env.INPUT_EVENT_DATABASE_URL; +const liveDescribe = databaseUrl && process.env.REDIS_URL ? describe : describe.skip; +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +liveDescribe('polling worker graceful shutdown', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('interrupts an idle tournament poll, tolerates repeated abort, and disposes signal listeners', async () => { + const schema = new URL(databaseUrl!).searchParams.get('schema'); + if (!schema) throw new Error('integration database URL must include a schema'); + vi.stubEnv('DATABASE_URL', databaseUrl!); + vi.stubEnv('PROFILE', schema); + vi.stubEnv('SCENARIO', 'worker-shutdown'); + vi.stubEnv('GAME_PROFILE_NAME', `worker-shutdown:${randomUUID()}`); + vi.stubEnv('GAME_TOKEN_SECRET', 'worker-shutdown-integration-only'); + vi.stubEnv('TOURNAMENT_POLL_MS', '60000'); + + const sigintListeners = process.listenerCount('SIGINT'); + const sigtermListeners = process.listenerCount('SIGTERM'); + const abortController = new AbortController(); + const worker = runTournamentWorker({ signal: abortController.signal }); + await delay(100); + + abortController.abort(); + abortController.abort(); + await expect(worker).resolves.toBeUndefined(); + expect(process.listenerCount('SIGINT')).toBe(sigintListeners); + expect(process.listenerCount('SIGTERM')).toBe(sigtermListeners); + }); +}); diff --git a/app/game-api/test/pollingWorkerLifecycle.test.ts b/app/game-api/test/pollingWorkerLifecycle.test.ts new file mode 100644 index 0000000..be0a092 --- /dev/null +++ b/app/game-api/test/pollingWorkerLifecycle.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createPollingWorkerControl, waitForWorkerPoll } from '../src/services/pollingWorkerLifecycle.js'; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('polling worker lifecycle', () => { + it('merges repeated process signals and external abort into one idempotent stop signal', () => { + const external = new AbortController(); + const control = createPollingWorkerControl(external.signal); + const aborts = vi.fn(); + control.signal.addEventListener('abort', aborts); + + process.emit('SIGTERM'); + process.emit('SIGINT'); + external.abort(); + + expect(control.signal.aborted).toBe(true); + expect(aborts).toHaveBeenCalledTimes(1); + control.dispose(); + }); + + it('interrupts an idle poll without canceling caller-owned in-flight work', async () => { + vi.useFakeTimers(); + const external = new AbortController(); + const control = createPollingWorkerControl(external.signal); + const wait = waitForWorkerPoll(control.signal, 60_000); + + external.abort(); + await expect(wait).resolves.toBeUndefined(); + control.dispose(); + }); +});