From ab5a1d240c69d5650ace887654802b66b82fe945 Mon Sep 17 00:00:00 2001 From: hided62 Date: Tue, 15 Sep 2026 23:36:50 +0000 Subject: [PATCH] =?UTF-8?q?fix:=2010=EB=B6=84=20=EC=A4=91=EB=8B=A8=20?= =?UTF-8?q?=EC=A6=89=EC=8B=9C=20=EB=B3=B5=EA=B5=AC=EC=99=80=20=EB=8C=80?= =?UTF-8?q?=EA=B8=B0=20=EC=A4=91=20=ED=86=A0=EB=84=88=EB=A8=BC=ED=8A=B8=20?= =?UTF-8?q?=EC=B0=B8=EA=B0=80=20=ED=97=88=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/game-api/src/router/tournament/index.ts | 136 +++++------ app/game-api/src/services/redisClockFence.ts | 7 + ...tournamentRecoveryWait.integration.test.ts | 214 ++++++++++++++++++ app/game-api/test/tournamentRouter.test.ts | 20 +- .../src/turn/prepareRealtimeRecovery.ts | 7 +- .../clockReconciliation.integration.test.ts | 67 +++++- .../test/runtimeClockShift.test.ts | 46 ++++ .../architecture/game-clock-reconciliation.md | 15 +- docs/release-operations.md | 5 + packages/common/src/time/TurnRecovery.ts | 8 +- packages/common/test/turnRecovery.test.ts | 8 +- tools/conditional-integration-registry.tsv | 1 + 12 files changed, 447 insertions(+), 87 deletions(-) create mode 100644 app/game-api/test/tournamentRecoveryWait.integration.test.ts diff --git a/app/game-api/src/router/tournament/index.ts b/app/game-api/src/router/tournament/index.ts index b8cc4e23..8f575e13 100644 --- a/app/game-api/src/router/tournament/index.ts +++ b/app/game-api/src/router/tournament/index.ts @@ -11,7 +11,11 @@ import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js'; import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js'; import { getMyGeneral } from '../shared/general.js'; import { loadCurrentGameTime } from '../../services/gameClock.js'; -import { ensureActiveRedisClockFence, ensureBettingRedisClockFence } from '../../services/redisClockFence.js'; +import { + ensureActiveRedisClockFence, + ensureBettingRedisClockFence, + ensureTournamentParticipationRedisClockFence, +} from '../../services/redisClockFence.js'; import { loadClockAdminStatus } from '../../services/clockReadiness.js'; const hasAdminRole = (roles: string[], profileName: string): boolean => { @@ -63,10 +67,11 @@ const withTournamentClockMutation = async ( tournamentMutationLockHeld?: boolean; }, store: TournamentStore, - operation: () => Promise + operation: () => Promise, + ensureFence = ensureActiveRedisClockFence ): Promise => { const gameTime = await loadCurrentGameTime(ctx.db); - const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime); + const fence = await ensureFence(ctx.redis, ctx.profile.name, gameTime); if (!fence) { throw new TRPCError({ code: 'PRECONDITION_FAILED', @@ -473,70 +478,75 @@ export const tournamentRouter = router({ join: engineAuthedProcedure.mutation(async ({ ctx }) => { const general = await getMyGeneral(ctx); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return withTournamentClockMutation(ctx, store, async () => { - const state = await store.getState(); - if (!state || state.stage !== 1 || state.participantsLockedAt) { - throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' }); - } + return withTournamentClockMutation( + ctx, + store, + async () => { + const state = await store.getState(); + if (!state || state.stage !== 1 || state.participantsLockedAt) { + throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' }); + } - const [participants, worldState] = await Promise.all([ - store.getParticipants(), - ctx.db.worldState.findFirst(), - ]); - if (participants.some((entry) => entry.id === general.id)) { - return { ok: true, count: participants.length }; - } - if (participants.length >= 64) { - throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' }); - } + const [participants, worldState] = await Promise.all([ + store.getParticipants(), + ctx.db.worldState.findFirst(), + ]); + if (participants.some((entry) => entry.id === general.id)) { + return { ok: true, count: participants.length }; + } + if (participants.length >= 64) { + throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' }); + } - const develCost = resolveCurrentDevelCost(worldState); - const feeResult = await ctx.turnDaemon.requestCommand({ - type: 'adjustGeneralResources', - requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'), - reason: 'tournamentJoin', - adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }], - }); - if (!feeResult || feeResult.type !== 'adjustGeneralResources') { - throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' }); - } - if (!feeResult.ok || feeResult.processed !== 1) { - throw new TRPCError({ - code: 'BAD_REQUEST', - message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason, - }); - } - - const meta = asRecord(general.meta); - const level = typeof meta.explevel === 'number' ? meta.explevel : 0; - const applicant = assignManualApplicantGroup({ - state, - baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'), - current: participants, - applicant: { - id: general.id, - name: general.name, - leadership: general.leadership, - strength: general.strength, - intel: general.intel, - level, - }, - }); - const next = participants.concat(applicant); - - try { - await store.setParticipants(next); - } catch (error) { - await ctx.turnDaemon.requestCommand({ + const develCost = resolveCurrentDevelCost(worldState); + const feeResult = await ctx.turnDaemon.requestCommand({ type: 'adjustGeneralResources', - requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'), - reason: 'tournamentJoinRollback', - adjustments: [{ generalId: general.id, goldDelta: develCost }], + requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'), + reason: 'tournamentJoin', + adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }], }); - throw error; - } - return { ok: true, count: next.length }; - }); + if (!feeResult || feeResult.type !== 'adjustGeneralResources') { + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' }); + } + if (!feeResult.ok || feeResult.processed !== 1) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason, + }); + } + + const meta = asRecord(general.meta); + const level = typeof meta.explevel === 'number' ? meta.explevel : 0; + const applicant = assignManualApplicantGroup({ + state, + baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'), + current: participants, + applicant: { + id: general.id, + name: general.name, + leadership: general.leadership, + strength: general.strength, + intel: general.intel, + level, + }, + }); + const next = participants.concat(applicant); + + try { + await store.setParticipants(next); + } catch (error) { + await ctx.turnDaemon.requestCommand({ + type: 'adjustGeneralResources', + requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'), + reason: 'tournamentJoinRollback', + adjustments: [{ generalId: general.id, goldDelta: develCost }], + }); + throw error; + } + return { ok: true, count: next.length }; + }, + ensureTournamentParticipationRedisClockFence + ); }), cancel: adminProcedure.mutation(async ({ ctx }) => { const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); diff --git a/app/game-api/src/services/redisClockFence.ts b/app/game-api/src/services/redisClockFence.ts index e0067975..4bfa7967 100644 --- a/app/game-api/src/services/redisClockFence.ts +++ b/app/game-api/src/services/redisClockFence.ts @@ -73,6 +73,13 @@ export const ensureActiveRedisClockFence = async ( return ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']); }; +/** 복구 대기 중에도 참가 신청은 가능하다. 자동 진행/정산에는 사용하지 않는다. */ +export const ensureTournamentParticipationRedisClockFence = async ( + redis: ClockFenceRedis, + profileName: string, + gameTime: CurrentGameTime +): Promise => ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']); + /** * User betting is allowed against a frozen tournament deadline while the game * clock is suspended. Stage progression and settlement continue to use the diff --git a/app/game-api/test/tournamentRecoveryWait.integration.test.ts b/app/game-api/test/tournamentRecoveryWait.integration.test.ts new file mode 100644 index 00000000..42e66528 --- /dev/null +++ b/app/game-api/test/tournamentRecoveryWait.integration.test.ts @@ -0,0 +1,214 @@ +import { expect, it } from 'vitest'; +import fastify from 'fastify'; +import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify'; +import { createGamePostgresConnector, createRedisConnector } from '@sammo-ts/infra'; +import { DatabaseTurnDaemonCommandQueue, DatabaseTurnDaemonLease } from '@sammo-ts/game-engine'; +import { appRouter } from '../src/router.js'; +import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js'; +import { RedisAccessTokenStore } from '../src/auth/accessTokenStore.js'; +import { InMemoryFlushStore } from '../src/auth/flushStore.js'; +import type { GameApiContext } from '../src/context.js'; +import { loadCurrentGameTime } from '../src/services/gameClock.js'; + +const databaseUrl = process.env.TOURNAMENT_RECOVERY_DATABASE_URL; +const integration = it.skipIf(!databaseUrl || !process.env.REDIS_URL); + +integration( + 'accepts an authenticated HTTP tournament join during recovery wait with durable fee and Redis fences', + async () => { + const schema = new URL(databaseUrl!).searchParams.get('schema'); + if (!schema?.endsWith('_tournament_recovery_integration')) + throw new Error('Dedicated tournament fixture schema required'); + const connector = createGamePostgresConnector({ url: databaseUrl! }); + const redisConnector = createRedisConnector({ url: process.env.REDIS_URL! }); + await connector.connect(); + await redisConnector.connect(); + const db = connector.prisma; + const redis = redisConnector.client; + const profile = 'test:recovery-wait'; + const prefix = `sammo:${profile}`; + const tokenStore = new RedisAccessTokenStore(redis, profile); + const transport = new DatabaseTurnDaemonTransport(db, 4000); + const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false }); + const queue = new DatabaseTurnDaemonCommandQueue(db); + const app = fastify(); + let workerError: unknown; + let workerBusy = false; + let timer: ReturnType | undefined; + try { + await db.inputEvent.deleteMany(); + await db.general.deleteMany(); + await db.turnDaemonLease.deleteMany(); + await db.worldState.deleteMany(); + const baseTime = new Date('2026-01-01T00:00:00Z'); + const waitAt = new Date(Date.now() + 1_800_000); + const world = await db.worldState.create({ + data: { + scenarioCode: 'recovery-wait', + currentYear: 200, + currentMonth: 1, + tickSeconds: 3600, + clockBaseTime: baseTime, + clockTick: 0n, + lastTurnTick: 0n, + clockWallAnchor: waitAt, + clockRecoveryStartTick: 0n, + clockRecoveryEndTick: 36_000_000n, + clockRecoveryStartWallAt: waitAt, + clockMode: 'realtime', + clockPhase: 'RUNNING', + clockRevision: 1n, + deadlineGeneration: 1n, + config: { const: { develCost: 100 } }, + }, + }); + await db.general.create({ + data: { id: 901, userId: 'fixture-user', name: 'fixture', gold: 1000, turnTime: baseTime }, + }); + await lease.acquire(); + await lease.markClockReady(); + await queue.initialize(); + await redis.set( + `${prefix}:tournament:state`, + JSON.stringify({ + stage: 1, + phase: 0, + type: 0, + auto: true, + openYear: 200, + openMonth: 1, + termSeconds: 60, + nextAt: baseTime.toISOString(), + }) + ); + await redis.set(`${prefix}:tournament:participants`, '[]'); + await redis.del([ + `${prefix}:clock:active-revision`, + `${prefix}:clock:deadline-generation`, + `${prefix}:clock:phase`, + ]); + const auth: NonNullable = { + version: 1, + profile, + sessionId: 'fixture-session', + issuedAt: new Date().toISOString(), + expiresAt: '2999-01-01T00:00:00Z', + user: { id: 'fixture-user', username: 'fixture', displayName: 'fixture', roles: [] }, + sanctions: {}, + }; + await app.register(fastifyTRPCPlugin, { + prefix: '/trpc', + trpcOptions: { + router: appRouter, + createContext: ({ + req, + }: { + req: { headers: Record }; + }): GameApiContext => ({ + db, + redis, + profile: { id: 'test', scenario: 'recovery-wait', name: profile }, + auth: req.headers.authorization === 'Bearer fixture-auth' ? auth : null, + requestId: + typeof req.headers['x-test-request'] === 'string' + ? req.headers['x-test-request'] + : undefined, + turnDaemon: transport, + accessTokenStore: tokenStore, + flushStore: new InMemoryFlushStore(), + battleSim: {} as GameApiContext['battleSim'], + uploadDir: '/tmp', + uploadPath: '/uploads', + uploadPublicUrl: null, + gameTokenSecret: 'fixture-only', + }), + }, + }); + await app.listen({ host: '127.0.0.1', port: 0 }); + const address = app.server.address(); + if (!address || typeof address === 'string') throw new Error('Missing fixture listener'); + const url = `http://127.0.0.1:${address.port}/trpc/tournament.join`; + const post = (authenticated = true) => + fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(authenticated ? { authorization: 'Bearer fixture-auth' } : {}), + 'x-test-request': 'recovery-join', + }, + body: '{}', + }); + expect((await post(false)).status).toBe(401); + expect(await loadCurrentGameTime(db)).toMatchObject({ + phase: 'RUNNING', + running: false, + runtimeReady: true, + }); + // Fixture worker: production queue/lease and real PostgreSQL transaction; no auto turns. + timer = setInterval(() => { + if (workerBusy) return; + workerBusy = true; + void (async () => { + for (const command of await queue.drain()) { + if (command.type !== 'adjustGeneralResources' || !command.requestId) + throw new Error('Unexpected fixture command'); + await db.$transaction(async (tx) => { + await lease.assertActive(tx); + const adjustment = command.adjustments[0]!; + await tx.general.update({ + where: { id: adjustment.generalId }, + data: { gold: { increment: adjustment.goldDelta ?? 0 } }, + }); + await tx.inputEvent.update({ + where: { requestId: command.requestId }, + data: { + status: 'SUCCEEDED', + result: { + type: 'adjustGeneralResources', + ok: true, + processed: 1, + missing: 0, + totalGoldDelta: adjustment.goldDelta ?? 0, + totalRiceDelta: 0, + }, + }, + }); + }); + } + })() + .catch((error: unknown) => { + workerError = error; + }) + .finally(() => { + workerBusy = false; + }); + }, 10); + expect((await post()).status).toBe(200); + expect((await post()).status).toBe(200); + expect(workerError).toBeUndefined(); + expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900); + expect(JSON.parse((await redis.get(`${prefix}:tournament:participants`))!)).toHaveLength(1); + const events = await db.inputEvent.findMany({ where: { eventType: 'adjustGeneralResources' } }); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ status: 'SUCCEEDED', acceptedGameTick: 0n, acceptedClockRevision: 1n }); + await redis.set(`${prefix}:clock:active-revision`, '2'); + expect((await post()).status).toBe(412); + await redis.set(`${prefix}:clock:active-revision`, '1'); + await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RECONCILING' } }); + expect((await post()).status).toBe(412); + await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } }); + await lease.release(); + expect((await post()).status).toBe(412); + expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900); + expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockTick).toBe(0n); + } finally { + if (timer) clearInterval(timer); + while (workerBusy) await new Promise((resolve) => setTimeout(resolve, 10)); + await app.close(); + await lease.close(); + await connector.disconnect(); + await redisConnector.disconnect(); + } + }, + 15000 +); diff --git a/app/game-api/test/tournamentRouter.test.ts b/app/game-api/test/tournamentRouter.test.ts index c9639d1e..3a5b4de3 100644 --- a/app/game-api/test/tournamentRouter.test.ts +++ b/app/game-api/test/tournamentRouter.test.ts @@ -153,6 +153,7 @@ const buildContext = (options: { rankRows?: Array<{ generalId: number; type: string; value: number }>; clockPhase?: 'PREOPEN' | 'RUNNING' | 'MANUAL' | 'SUSPENDED' | 'RECONCILING'; requestId?: string; + clockWallAnchor?: Date; }): GameApiContext => { const db = { general: { @@ -169,7 +170,7 @@ const buildContext = (options: { clockBaseTime: new Date('2026-01-01T00:00:00.000Z'), clockTick: 0n, clockMode: 'realtime', - clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'), + clockWallAnchor: options.clockWallAnchor ?? new Date('2026-01-01T00:00:00.000Z'), clockPhase: options.clockPhase ?? 'RUNNING', clockRevision: 1n, deadlineGeneration: 1n, @@ -261,7 +262,21 @@ describe('tournament router permissions and mutations', () => { ]); }); - it('charges the authenticated general once when joining', async () => { + it.each(['PREOPEN', 'SUSPENDED', 'RECONCILING'] as const)( + 'rejects joining in %s without charging', + async (clockPhase) => { + const redis = new MemoryRedis(); + const transport = new TournamentTransport(); + const general = buildGeneral(1, 'user-1'); + const caller = appRouter.createCaller( + buildContext({ redis, transport, generals: [general], userId: 'user-1', clockPhase }) + ); + await expect(caller.tournament.join()).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' }); + expect(transport.commands).toHaveLength(0); + } + ); + + it.each([false, true])('charges the authenticated general once when joining; recovery wait=%s', async (waiting) => { const redis = new MemoryRedis(); const transport = new TournamentTransport(); const general = buildGeneral(1, 'user-1'); @@ -284,6 +299,7 @@ describe('tournament router permissions and mutations', () => { userId: 'user-1', develCost: 200, requestId: 'http:tournament-join', + ...(waiting ? { clockWallAnchor: new Date(Date.now() + 1_800_000) } : {}), }); const outerApiTransaction = vi.fn(async () => { throw new Error('tournament join must not hold an API transaction while waiting for the daemon'); diff --git a/app/game-engine/src/turn/prepareRealtimeRecovery.ts b/app/game-engine/src/turn/prepareRealtimeRecovery.ts index ae768624..7893cb2e 100644 --- a/app/game-engine/src/turn/prepareRealtimeRecovery.ts +++ b/app/game-engine/src/turn/prepareRealtimeRecovery.ts @@ -28,11 +28,8 @@ export const prepareRealtimeRecovery = async ( if (world.clockPhase !== 'RUNNING' || !world.clockWallAnchor || world.clockTick === null) return; const now = await readClockDatabaseWall(db); // 가속 중 정상적인 프로세스 교체는 기존 창을 그대로 재사용한다. - // 짧은 중단만 즉시 처리한다. 기준값과 같으면 대기 후 복구한다. - if ( - !options.paused && - now.getTime() - world.clockWallAnchor.getTime() < immediateRecoveryLimitSeconds(world.tickSeconds) * 1_000 - ) + // 전체 중단 10분까지는 기존 엔진이 밀린 턴을 순서대로 처리한다. + if (!options.paused && now.getTime() - world.clockWallAnchor.getTime() <= immediateRecoveryLimitSeconds() * 1_000) return; const suspensionId = `recovery-${randomUUID()}`; await startClockSuspension({ diff --git a/app/game-engine/test/clockReconciliation.integration.test.ts b/app/game-engine/test/clockReconciliation.integration.test.ts index aa524e54..e041a176 100644 --- a/app/game-engine/test/clockReconciliation.integration.test.ts +++ b/app/game-engine/test/clockReconciliation.integration.test.ts @@ -231,7 +231,7 @@ describeIntegration('durable clock reconciliation', () => { expect((await db.worldState.findUniqueOrThrow({ where: { id: row.id } })).clockRecoveryStartTick).toBe(1n); }); - it.each([300, 420])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => { + it.each([180, 420, 590, 610])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => { const profile = 'short-startup'; await db.worldState.create({ data: { @@ -259,8 +259,8 @@ describeIntegration('durable clock reconciliation', () => { fencingEpoch: token.fencingEpoch, }); const world = await db.worldState.findFirstOrThrow(); - expect(world.clockPhase).toBe(delay < 360 ? 'RUNNING' : 'RECONCILING'); - expect(readTurnRecovery(world) === null).toBe(delay < 360); + expect(world.clockPhase).toBe(delay <= 600 ? 'RUNNING' : 'RECONCILING'); + expect(readTurnRecovery(world) === null).toBe(delay <= 600); } finally { await lease.close(); } @@ -345,6 +345,61 @@ describeIntegration('durable clock reconciliation', () => { } }); + it.each([300, 1200, 3600])( + 'keeps pending turns and operator pause across a short VM outage (%i second turn)', + async (turnSeconds) => { + const profile = 'short-vm-outage'; + const anchor = new Date(Date.now() - 590_000); + await db.worldState.create({ + data: { + scenarioCode: profile, + currentYear: 199, + currentMonth: 1, + tickSeconds: turnSeconds, + clockBaseTime: anchor, + clockWallAnchor: anchor, + clockTick: 0n, + lastTurnTick: 0n, + clockMode: 'realtime', + clockPhase: 'RUNNING', + clockRevision: 1n, + deadlineGeneration: 1n, + }, + }); + const general = await db.general.create({ + data: { id: 901, name: 'pending-turn', turnTick: 12345n, turnTime: anchor }, + }); + const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false }); + try { + const token = await lease.acquire(); + const authority = { + kind: 'DAEMON' as const, + profileName: profile, + ownerId: token!.ownerId, + fencingEpoch: token!.fencingEpoch, + }; + await prepareRealtimeRecovery(db, authority); + expect(await db.worldState.findFirstOrThrow()).toMatchObject({ + clockPhase: 'RUNNING', + clockRevision: 1n, + clockTick: 0n, + lastTurnTick: 0n, + clockRecoveryStartTick: null, + }); + expect(await db.general.findUniqueOrThrow({ where: { id: general.id } })).toMatchObject({ + turnTick: general.turnTick, + turnTime: general.turnTime, + }); + expect(await db.clockSuspension.count()).toBe(0); + // 짧은 host 중단도 운영자가 정지한 서버를 자동 재개하는 근거가 되지 않는다. + await prepareRealtimeRecovery(db, authority, { paused: true }); + expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'SUSPENDED' }); + } finally { + await lease.close(); + } + } + ); + it.each([false, true])('fences outage recovery and reuses its window; repeated outage=%s', async (repeated) => { const profile = 'recovery-startup'; await db.worldState.create({ @@ -513,8 +568,8 @@ describeIntegration('durable clock reconciliation', () => { } ); - it.each([359999, 360000, 360001, 840000, 12 * 3600000 + 1000])( - 'persists strict recovery boundaries for %i ms', + it.each([180000, 599999, 600000, 600001, 840000, 12 * 3600000 + 1000])( + 'persists the inclusive ten-minute recovery boundary for %i ms', async (gap) => { const observed = T / 6; const future = new Date(Date.now() + 3600000); @@ -549,7 +604,7 @@ describeIntegration('durable clock reconciliation', () => { authority, testResumeWallAt: now, }); - expect(plan.recovery === null).toBe(gap < 360000); + expect(plan.recovery === null).toBe(gap <= 600000); expect(await db.message.count()).toBe(0); // Redis 장애 후에도 알림은 DB의 RUNNING 전이와 함께 한 번만 저장한다. await expect( diff --git a/app/game-engine/test/runtimeClockShift.test.ts b/app/game-engine/test/runtimeClockShift.test.ts index 998c2002..56578d31 100644 --- a/app/game-engine/test/runtimeClockShift.test.ts +++ b/app/game-engine/test/runtimeClockShift.test.ts @@ -84,6 +84,52 @@ const buildWorld = (stateOverride: Partial = {}): InMemoryTurnWo }; describe('runtime clock shift', () => { + it('executes ten minutes of pending work in bounded batches without skipping or repeating generals', async () => { + const base = new Date('2026-07-30T10:00:00Z'); + const resumed = new Date(base.getTime() + 600_000); + const plan = planTurnRecovery({ + observedTick: 0, + normalTick: GAME_TICKS_PER_TURN, + wallNow: resumed, + turnSeconds: 600, + }); + expect(plan).toMatchObject({ skippedTurns: 0, recovery: null }); + const world = buildWorld({ + clockBaseTime: base, + clockTick: 0, + clockWallAnchor: base, + clockMode: 'realtime', + clockPhase: 'RUNNING', + lastTurnTick: 0, + }); + for (const [id, offset] of [ + [1, 19_902], + [2, 42_001], + ] as const) { + world.updateGeneral(id, { turnTime: new Date(base.getTime() + offset), turnTick: offset * 60 }); + } + const executed: number[] = []; + const processor = new InMemoryTurnProcessor(world, { + afterExecuteGeneral: async (general) => { + executed.push(general.id); + }, + }); + const target = world.getGameNow(resumed); + world.advanceGameClockTo(target, resumed); + const budget = { budgetMs: 10_000, maxGenerals: 1, catchUpCap: 1 }; + let result = await processor.run(target, budget); + expect(result.processedGenerals).toBe(1); + for (let batch = 0; result.partial && batch < 4; batch++) { + result = await processor.run(target, budget, result.checkpoint); + expect(result.processedGenerals).toBeLessThanOrEqual(1); + } + expect(result.partial).toBe(false); + expect(executed).toEqual([1, 2]); + expect(world.getState().currentMonth).toBe(2); + expect(world.getGeneralById(1)!.turnTick).toBe(GAME_TICKS_PER_TURN + 19_902 * 60); + expect(world.getGeneralById(2)!.turnTick).toBe(GAME_TICKS_PER_TURN + 42_001 * 60); + }); + it('runs two real monthly cycles per normal interval, survives reload, and returns to one cycle', async () => { const base = new Date('2026-07-30T10:00:00Z'); const wallAt = (minutes: number) => new Date(base.getTime() + minutes * 60_000); diff --git a/docs/architecture/game-clock-reconciliation.md b/docs/architecture/game-clock-reconciliation.md index d0d54e5c..8492fe3d 100644 --- a/docs/architecture/game-clock-reconciliation.md +++ b/docs/architecture/game-clock-reconciliation.md @@ -44,9 +44,18 @@ From 2026-09-07, maintenance and crash recovery use the following `RECOVER_TURNS` policy. The base turn length does not change. One turn remains 36,000,000 ticks; a persisted `TurnRecoveryWindow` changes only wall execution. -- An entire delay strictly below `min(600 seconds, turnSeconds / 10)` catches - up immediately through the ordinary engine. Equality uses recovery. The - limit is 30 seconds on a 5-minute server and 6 minutes on a 60-minute server. +- From 2026-09-15, an entire delay up to and including 600 seconds catches + up immediately through the ordinary engine, independently of the server's + turn length. This accommodates host updates that pause and resume a VM. + Pending turns are executed in order with the ordinary per-run budget and + month boundary cap; they are not marked complete or skipped. The lease still + expires after 30 seconds and a fresh owner must fence the old process and + reload durable state before catching up. Operator pauses remain pauses. +- During the persisted wait before 2x recovery, tournament participation is + allowed when the phase is `RUNNING`, the daemon lease is ready, and DB/Redis + revision and deadline generation agree. Registration stage, capacity, + duplicate entry, fee and ENGINE transaction checks still apply. Tournament + stage advancement and settlement continue to require a ticking clock. - For longer delays, skip only complete 12-turn blocks, moving future schedules and the execution cursor by the same integer delta. Never apply the short-delay exception again to the remainder. An exact multiple of diff --git a/docs/release-operations.md b/docs/release-operations.md index 7253f71e..fdc19a86 100644 --- a/docs/release-operations.md +++ b/docs/release-operations.md @@ -336,6 +336,11 @@ PM2가 새 owner/epoch와 DB snapshot으로 재시작합니다. lease 오류만 않습니다. 이전 버전에서 이미 lease 오류로 PAUSED가 된 서버는 원인을 확인한 뒤 한 번 `START`/재개해야 합니다. +Host 업데이트 등으로 VM이 최대 10분 중단되면 새 runtime은 밀린 턴을 기존 엔진의 +순서·batch budget으로 즉시 따라잡습니다(정확히 10분 포함). 10분 초과는 기존의 +12턴 묶음 이동 및 대기 후 2배속 복구 정책을 사용합니다. 이미 저장한 복구 창과 +운영자의 PAUSED/STOPPED는 변경하지 않습니다. + 30초 lease와 이전 owner의 fencing은 유지합니다. transaction의 만료 검사는 시작에 고정된 `CURRENT_TIMESTAMP`가 아닌 검사 순간의 `clock_timestamp()`를 사용합니다. 역방향 시계 보정으로 기존 lease가 아직 유효하면 startup은 2초 간격으로 기다리며 diff --git a/packages/common/src/time/TurnRecovery.ts b/packages/common/src/time/TurnRecovery.ts index b2ef5388..217bf488 100644 --- a/packages/common/src/time/TurnRecovery.ts +++ b/packages/common/src/time/TurnRecovery.ts @@ -68,8 +68,8 @@ export const turnShiftTicks = (turns: number): GameTick => { return asGameTick(turns * GAME_TICKS_PER_TURN); }; -/** 즉시 처리 여부는 12턴 묶음 생략 전의 전체 지연으로 판정한다. */ -export const immediateRecoveryLimitSeconds = (turnSeconds: number): number => Math.min(600, turnSeconds / 10); +/** VM/host 업데이트의 전체 중단 10분까지는 기존 엔진 순서와 budget으로 따라잡는다. */ +export const immediateRecoveryLimitSeconds = (): number => 600; /** * observedTick은 중단 전에 저장한 관측 지점, normalTick은 기존 시간표의 현재 지점이다. @@ -91,8 +91,8 @@ export const planTurnRecovery = (input: { if (!Number.isFinite(wallNow.getTime())) throw new Error('Recovery wall instant is invalid.'); const gap = Math.max(0, normalTick - observedTick); const ticksPerSecond = GAME_TICKS_PER_TURN / turnSeconds; - const immediateLimit = immediateRecoveryLimitSeconds(turnSeconds) * ticksPerSecond; - if (gap < immediateLimit) { + const immediateLimit = immediateRecoveryLimitSeconds() * ticksPerSecond; + if (gap <= immediateLimit) { return { skippedTurns: 0, recoveryTurns: 0, diff --git a/packages/common/test/turnRecovery.test.ts b/packages/common/test/turnRecovery.test.ts index 11aee019..a2112680 100644 --- a/packages/common/test/turnRecovery.test.ts +++ b/packages/common/test/turnRecovery.test.ts @@ -160,9 +160,9 @@ describe('turn-aligned double-speed recovery', () => { expect(clock.executionRate(end)).toBe(1); }); - it.each([300, 3600, 6000, 7200])('uses the strict whole-delay threshold for a %i second turn', (turnSeconds) => { + it.each([60, 300, 1200, 3600, 6000, 7200])('catches up through ten minutes for a %i second turn', (turnSeconds) => { const rate = T / turnSeconds; - const limitMs = Math.min(600, turnSeconds / 10) * 1000; + const limitMs = 600_000; for (const delta of [-1, 0, 1]) { const delayMs = limitMs + delta; const normal = Math.trunc((delayMs * rate) / 1000); @@ -172,8 +172,8 @@ describe('turn-aligned double-speed recovery', () => { wallNow: new Date(base + delayMs), turnSeconds, }); - expect(plan.recovery === null).toBe(delta < 0); - expect(plan.initialTick).toBe(delta < 0 ? normal : 0); + expect(plan.recovery === null).toBe(delta <= 0); + expect(plan.initialTick).toBe(delta <= 0 ? normal : 0); } // 장시간 중단의 작은 나머지에는 즉시 처리 예외를 다시 적용하지 않는다. const long = planTurnRecovery({ observedTick: 0, normalTick: 12 * T + rate, wallNow: wall(20), turnSeconds }); diff --git a/tools/conditional-integration-registry.tsv b/tools/conditional-integration-registry.tsv index 74fe9f77..d69227f6 100644 --- a/tools/conditional-integration-registry.tsv +++ b/tools/conditional-integration-registry.tsv @@ -25,3 +25,4 @@ TURN_COMMAND_DURABLE_MATRIX_DATABASE_URL reference_command_durable_matrix TURN_DIFFERENTIAL_DATABASE_URL core TURN_FULL_LIFECYCLE_PERSISTENCE_DATABASE_URL reference_full_lifecycle WEB_PUSH_GATEWAY_DATABASE_URL web_push_gateway +TOURNAMENT_RECOVERY_DATABASE_URL external_fixture