fix: 10분 중단 즉시 복구와 대기 중 토너먼트 참가 허용

2026-09-15 23:36:50 +00:00
parent 4fb1343943
commit ab5a1d240c
12 changed files with 447 additions and 87 deletions
@@ -11,7 +11,11 @@ import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
 import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js';
 import { getMyGeneral } from '../shared/general.js';
 import { loadCurrentGameTime } from '../../services/gameClock.js';
-import { ensureActiveRedisClockFence, ensureBettingRedisClockFence } from '../../services/redisClockFence.js';
+import {
+    ensureActiveRedisClockFence,
+    ensureBettingRedisClockFence,
+    ensureTournamentParticipationRedisClockFence,
+} from '../../services/redisClockFence.js';
 import { loadClockAdminStatus } from '../../services/clockReadiness.js';

 const hasAdminRole = (roles: string[], profileName: string): boolean => {
@@ -63,10 +67,11 @@ const withTournamentClockMutation = async <T>(
        tournamentMutationLockHeld?: boolean;
    },
    store: TournamentStore,
-    operation: () => Promise<T>
+    operation: () => Promise<T>,
+    ensureFence = ensureActiveRedisClockFence
 ): Promise<T> => {
    const gameTime = await loadCurrentGameTime(ctx.db);
-    const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime);
+    const fence = await ensureFence(ctx.redis, ctx.profile.name, gameTime);
    if (!fence) {
        throw new TRPCError({
            code: 'PRECONDITION_FAILED',
@@ -473,70 +478,75 @@ export const tournamentRouter = router({
    join: engineAuthedProcedure.mutation(async ({ ctx }) => {
        const general = await getMyGeneral(ctx);
        const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
-        return withTournamentClockMutation(ctx, store, async () => {
-            const state = await store.getState();
-            if (!state || state.stage !== 1 || state.participantsLockedAt) {
-                throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
-            }
+        return withTournamentClockMutation(
+            ctx,
+            store,
+            async () => {
+                const state = await store.getState();
+                if (!state || state.stage !== 1 || state.participantsLockedAt) {
+                    throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
+                }

-            const [participants, worldState] = await Promise.all([
-                store.getParticipants(),
-                ctx.db.worldState.findFirst(),
-            ]);
-            if (participants.some((entry) => entry.id === general.id)) {
-                return { ok: true, count: participants.length };
-            }
-            if (participants.length >= 64) {
-                throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' });
-            }
+                const [participants, worldState] = await Promise.all([
+                    store.getParticipants(),
+                    ctx.db.worldState.findFirst(),
+                ]);
+                if (participants.some((entry) => entry.id === general.id)) {
+                    return { ok: true, count: participants.length };
+                }
+                if (participants.length >= 64) {
+                    throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' });
+                }

-            const develCost = resolveCurrentDevelCost(worldState);
-            const feeResult = await ctx.turnDaemon.requestCommand({
-                type: 'adjustGeneralResources',
-                requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'),
-                reason: 'tournamentJoin',
-                adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }],
-            });
-            if (!feeResult || feeResult.type !== 'adjustGeneralResources') {
-                throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' });
-            }
-            if (!feeResult.ok || feeResult.processed !== 1) {
-                throw new TRPCError({
-                    code: 'BAD_REQUEST',
-                    message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason,
-                });
-            }
-
-            const meta = asRecord(general.meta);
-            const level = typeof meta.explevel === 'number' ? meta.explevel : 0;
-            const applicant = assignManualApplicantGroup({
-                state,
-                baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'),
-                current: participants,
-                applicant: {
-                    id: general.id,
-                    name: general.name,
-                    leadership: general.leadership,
-                    strength: general.strength,
-                    intel: general.intel,
-                    level,
-                },
-            });
-            const next = participants.concat(applicant);
-
-            try {
-                await store.setParticipants(next);
-            } catch (error) {
-                await ctx.turnDaemon.requestCommand({
+                const develCost = resolveCurrentDevelCost(worldState);
+                const feeResult = await ctx.turnDaemon.requestCommand({
                    type: 'adjustGeneralResources',
-                    requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'),
-                    reason: 'tournamentJoinRollback',
-                    adjustments: [{ generalId: general.id, goldDelta: develCost }],
+                    requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'),
+                    reason: 'tournamentJoin',
+                    adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }],
                });
-                throw error;
-            }
-            return { ok: true, count: next.length };
-        });
+                if (!feeResult || feeResult.type !== 'adjustGeneralResources') {
+                    throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' });
+                }
+                if (!feeResult.ok || feeResult.processed !== 1) {
+                    throw new TRPCError({
+                        code: 'BAD_REQUEST',
+                        message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason,
+                    });
+                }
+
+                const meta = asRecord(general.meta);
+                const level = typeof meta.explevel === 'number' ? meta.explevel : 0;
+                const applicant = assignManualApplicantGroup({
+                    state,
+                    baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'),
+                    current: participants,
+                    applicant: {
+                        id: general.id,
+                        name: general.name,
+                        leadership: general.leadership,
+                        strength: general.strength,
+                        intel: general.intel,
+                        level,
+                    },
+                });
+                const next = participants.concat(applicant);
+
+                try {
+                    await store.setParticipants(next);
+                } catch (error) {
+                    await ctx.turnDaemon.requestCommand({
+                        type: 'adjustGeneralResources',
+                        requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'),
+                        reason: 'tournamentJoinRollback',
+                        adjustments: [{ generalId: general.id, goldDelta: develCost }],
+                    });
+                    throw error;
+                }
+                return { ok: true, count: next.length };
+            },
+            ensureTournamentParticipationRedisClockFence
+        );
    }),
    cancel: adminProcedure.mutation(async ({ ctx }) => {
        const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
@@ -73,6 +73,13 @@ export const ensureActiveRedisClockFence = async (
    return ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']);
 };

+/** 복구 대기 중에도 참가 신청은 가능하다. 자동 진행/정산에는 사용하지 않는다. */
+export const ensureTournamentParticipationRedisClockFence = async (
+    redis: ClockFenceRedis,
+    profileName: string,
+    gameTime: CurrentGameTime
+): Promise<ActiveRedisClockFence | null> => ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']);
+
 /**
 * User betting is allowed against a frozen tournament deadline while the game
 * clock is suspended. Stage progression and settlement continue to use the
@@ -0,0 +1,214 @@
+import { expect, it } from 'vitest';
+import fastify from 'fastify';
+import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
+import { createGamePostgresConnector, createRedisConnector } from '@sammo-ts/infra';
+import { DatabaseTurnDaemonCommandQueue, DatabaseTurnDaemonLease } from '@sammo-ts/game-engine';
+import { appRouter } from '../src/router.js';
+import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js';
+import { RedisAccessTokenStore } from '../src/auth/accessTokenStore.js';
+import { InMemoryFlushStore } from '../src/auth/flushStore.js';
+import type { GameApiContext } from '../src/context.js';
+import { loadCurrentGameTime } from '../src/services/gameClock.js';
+
+const databaseUrl = process.env.TOURNAMENT_RECOVERY_DATABASE_URL;
+const integration = it.skipIf(!databaseUrl || !process.env.REDIS_URL);
+
+integration(
+    'accepts an authenticated HTTP tournament join during recovery wait with durable fee and Redis fences',
+    async () => {
+        const schema = new URL(databaseUrl!).searchParams.get('schema');
+        if (!schema?.endsWith('_tournament_recovery_integration'))
+            throw new Error('Dedicated tournament fixture schema required');
+        const connector = createGamePostgresConnector({ url: databaseUrl! });
+        const redisConnector = createRedisConnector({ url: process.env.REDIS_URL! });
+        await connector.connect();
+        await redisConnector.connect();
+        const db = connector.prisma;
+        const redis = redisConnector.client;
+        const profile = 'test:recovery-wait';
+        const prefix = `sammo:${profile}`;
+        const tokenStore = new RedisAccessTokenStore(redis, profile);
+        const transport = new DatabaseTurnDaemonTransport(db, 4000);
+        const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false });
+        const queue = new DatabaseTurnDaemonCommandQueue(db);
+        const app = fastify();
+        let workerError: unknown;
+        let workerBusy = false;
+        let timer: ReturnType<typeof setInterval> | undefined;
+        try {
+            await db.inputEvent.deleteMany();
+            await db.general.deleteMany();
+            await db.turnDaemonLease.deleteMany();
+            await db.worldState.deleteMany();
+            const baseTime = new Date('2026-01-01T00:00:00Z');
+            const waitAt = new Date(Date.now() + 1_800_000);
+            const world = await db.worldState.create({
+                data: {
+                    scenarioCode: 'recovery-wait',
+                    currentYear: 200,
+                    currentMonth: 1,
+                    tickSeconds: 3600,
+                    clockBaseTime: baseTime,
+                    clockTick: 0n,
+                    lastTurnTick: 0n,
+                    clockWallAnchor: waitAt,
+                    clockRecoveryStartTick: 0n,
+                    clockRecoveryEndTick: 36_000_000n,
+                    clockRecoveryStartWallAt: waitAt,
+                    clockMode: 'realtime',
+                    clockPhase: 'RUNNING',
+                    clockRevision: 1n,
+                    deadlineGeneration: 1n,
+                    config: { const: { develCost: 100 } },
+                },
+            });
+            await db.general.create({
+                data: { id: 901, userId: 'fixture-user', name: 'fixture', gold: 1000, turnTime: baseTime },
+            });
+            await lease.acquire();
+            await lease.markClockReady();
+            await queue.initialize();
+            await redis.set(
+                `${prefix}:tournament:state`,
+                JSON.stringify({
+                    stage: 1,
+                    phase: 0,
+                    type: 0,
+                    auto: true,
+                    openYear: 200,
+                    openMonth: 1,
+                    termSeconds: 60,
+                    nextAt: baseTime.toISOString(),
+                })
+            );
+            await redis.set(`${prefix}:tournament:participants`, '[]');
+            await redis.del([
+                `${prefix}:clock:active-revision`,
+                `${prefix}:clock:deadline-generation`,
+                `${prefix}:clock:phase`,
+            ]);
+            const auth: NonNullable<GameApiContext['auth']> = {
+                version: 1,
+                profile,
+                sessionId: 'fixture-session',
+                issuedAt: new Date().toISOString(),
+                expiresAt: '2999-01-01T00:00:00Z',
+                user: { id: 'fixture-user', username: 'fixture', displayName: 'fixture', roles: [] },
+                sanctions: {},
+            };
+            await app.register(fastifyTRPCPlugin, {
+                prefix: '/trpc',
+                trpcOptions: {
+                    router: appRouter,
+                    createContext: ({
+                        req,
+                    }: {
+                        req: { headers: Record<string, string | string[] | undefined> };
+                    }): GameApiContext => ({
+                        db,
+                        redis,
+                        profile: { id: 'test', scenario: 'recovery-wait', name: profile },
+                        auth: req.headers.authorization === 'Bearer fixture-auth' ? auth : null,
+                        requestId:
+                            typeof req.headers['x-test-request'] === 'string'
+                                ? req.headers['x-test-request']
+                                : undefined,
+                        turnDaemon: transport,
+                        accessTokenStore: tokenStore,
+                        flushStore: new InMemoryFlushStore(),
+                        battleSim: {} as GameApiContext['battleSim'],
+                        uploadDir: '/tmp',
+                        uploadPath: '/uploads',
+                        uploadPublicUrl: null,
+                        gameTokenSecret: 'fixture-only',
+                    }),
+                },
+            });
+            await app.listen({ host: '127.0.0.1', port: 0 });
+            const address = app.server.address();
+            if (!address || typeof address === 'string') throw new Error('Missing fixture listener');
+            const url = `http://127.0.0.1:${address.port}/trpc/tournament.join`;
+            const post = (authenticated = true) =>
+                fetch(url, {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        ...(authenticated ? { authorization: 'Bearer fixture-auth' } : {}),
+                        'x-test-request': 'recovery-join',
+                    },
+                    body: '{}',
+                });
+            expect((await post(false)).status).toBe(401);
+            expect(await loadCurrentGameTime(db)).toMatchObject({
+                phase: 'RUNNING',
+                running: false,
+                runtimeReady: true,
+            });
+            // Fixture worker: production queue/lease and real PostgreSQL transaction; no auto turns.
+            timer = setInterval(() => {
+                if (workerBusy) return;
+                workerBusy = true;
+                void (async () => {
+                    for (const command of await queue.drain()) {
+                        if (command.type !== 'adjustGeneralResources' || !command.requestId)
+                            throw new Error('Unexpected fixture command');
+                        await db.$transaction(async (tx) => {
+                            await lease.assertActive(tx);
+                            const adjustment = command.adjustments[0]!;
+                            await tx.general.update({
+                                where: { id: adjustment.generalId },
+                                data: { gold: { increment: adjustment.goldDelta ?? 0 } },
+                            });
+                            await tx.inputEvent.update({
+                                where: { requestId: command.requestId },
+                                data: {
+                                    status: 'SUCCEEDED',
+                                    result: {
+                                        type: 'adjustGeneralResources',
+                                        ok: true,
+                                        processed: 1,
+                                        missing: 0,
+                                        totalGoldDelta: adjustment.goldDelta ?? 0,
+                                        totalRiceDelta: 0,
+                                    },
+                                },
+                            });
+                        });
+                    }
+                })()
+                    .catch((error: unknown) => {
+                        workerError = error;
+                    })
+                    .finally(() => {
+                        workerBusy = false;
+                    });
+            }, 10);
+            expect((await post()).status).toBe(200);
+            expect((await post()).status).toBe(200);
+            expect(workerError).toBeUndefined();
+            expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900);
+            expect(JSON.parse((await redis.get(`${prefix}:tournament:participants`))!)).toHaveLength(1);
+            const events = await db.inputEvent.findMany({ where: { eventType: 'adjustGeneralResources' } });
+            expect(events).toHaveLength(1);
+            expect(events[0]).toMatchObject({ status: 'SUCCEEDED', acceptedGameTick: 0n, acceptedClockRevision: 1n });
+            await redis.set(`${prefix}:clock:active-revision`, '2');
+            expect((await post()).status).toBe(412);
+            await redis.set(`${prefix}:clock:active-revision`, '1');
+            await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RECONCILING' } });
+            expect((await post()).status).toBe(412);
+            await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } });
+            await lease.release();
+            expect((await post()).status).toBe(412);
+            expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900);
+            expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockTick).toBe(0n);
+        } finally {
+            if (timer) clearInterval(timer);
+            while (workerBusy) await new Promise((resolve) => setTimeout(resolve, 10));
+            await app.close();
+            await lease.close();
+            await connector.disconnect();
+            await redisConnector.disconnect();
+        }
+    },
+    15000
+);
@@ -153,6 +153,7 @@ const buildContext = (options: {
    rankRows?: Array<{ generalId: number; type: string; value: number }>;
    clockPhase?: 'PREOPEN' | 'RUNNING' | 'MANUAL' | 'SUSPENDED' | 'RECONCILING';
    requestId?: string;
+    clockWallAnchor?: Date;
 }): GameApiContext => {
    const db = {
        general: {
@@ -169,7 +170,7 @@ const buildContext = (options: {
                clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
                clockTick: 0n,
                clockMode: 'realtime',
-                clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'),
+                clockWallAnchor: options.clockWallAnchor ?? new Date('2026-01-01T00:00:00.000Z'),
                clockPhase: options.clockPhase ?? 'RUNNING',
                clockRevision: 1n,
                deadlineGeneration: 1n,
@@ -261,7 +262,21 @@ describe('tournament router permissions and mutations', () => {
        ]);
    });

-    it('charges the authenticated general once when joining', async () => {
+    it.each(['PREOPEN', 'SUSPENDED', 'RECONCILING'] as const)(
+        'rejects joining in %s without charging',
+        async (clockPhase) => {
+            const redis = new MemoryRedis();
+            const transport = new TournamentTransport();
+            const general = buildGeneral(1, 'user-1');
+            const caller = appRouter.createCaller(
+                buildContext({ redis, transport, generals: [general], userId: 'user-1', clockPhase })
+            );
+            await expect(caller.tournament.join()).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' });
+            expect(transport.commands).toHaveLength(0);
+        }
+    );
+
+    it.each([false, true])('charges the authenticated general once when joining; recovery wait=%s', async (waiting) => {
        const redis = new MemoryRedis();
        const transport = new TournamentTransport();
        const general = buildGeneral(1, 'user-1');
@@ -284,6 +299,7 @@ describe('tournament router permissions and mutations', () => {
            userId: 'user-1',
            develCost: 200,
            requestId: 'http:tournament-join',
+            ...(waiting ? { clockWallAnchor: new Date(Date.now() + 1_800_000) } : {}),
        });
        const outerApiTransaction = vi.fn(async () => {
            throw new Error('tournament join must not hold an API transaction while waiting for the daemon');
@@ -28,11 +28,8 @@ export const prepareRealtimeRecovery = async (
    if (world.clockPhase !== 'RUNNING' || !world.clockWallAnchor || world.clockTick === null) return;
    const now = await readClockDatabaseWall(db);
    // 가속 중 정상적인 프로세스 교체는 기존 창을 그대로 재사용한다.
-    // 짧은 중단만 즉시 처리한다. 기준값과 같으면 대기 후 복구한다.
-    if (
-        !options.paused &&
-        now.getTime() - world.clockWallAnchor.getTime() < immediateRecoveryLimitSeconds(world.tickSeconds) * 1_000
-    )
+    // 전체 중단 10분까지는 기존 엔진이 밀린 턴을 순서대로 처리한다.
+    if (!options.paused && now.getTime() - world.clockWallAnchor.getTime() <= immediateRecoveryLimitSeconds() * 1_000)
        return;
    const suspensionId = `recovery-${randomUUID()}`;
    await startClockSuspension({
@@ -231,7 +231,7 @@ describeIntegration('durable clock reconciliation', () => {
        expect((await db.worldState.findUniqueOrThrow({ where: { id: row.id } })).clockRecoveryStartTick).toBe(1n);
    });

-    it.each([300, 420])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => {
+    it.each([180, 420, 590, 610])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => {
        const profile = 'short-startup';
        await db.worldState.create({
            data: {
@@ -259,8 +259,8 @@ describeIntegration('durable clock reconciliation', () => {
                fencingEpoch: token.fencingEpoch,
            });
            const world = await db.worldState.findFirstOrThrow();
-            expect(world.clockPhase).toBe(delay < 360 ? 'RUNNING' : 'RECONCILING');
-            expect(readTurnRecovery(world) === null).toBe(delay < 360);
+            expect(world.clockPhase).toBe(delay <= 600 ? 'RUNNING' : 'RECONCILING');
+            expect(readTurnRecovery(world) === null).toBe(delay <= 600);
        } finally {
            await lease.close();
        }
@@ -345,6 +345,61 @@ describeIntegration('durable clock reconciliation', () => {
        }
    });

+    it.each([300, 1200, 3600])(
+        'keeps pending turns and operator pause across a short VM outage (%i second turn)',
+        async (turnSeconds) => {
+            const profile = 'short-vm-outage';
+            const anchor = new Date(Date.now() - 590_000);
+            await db.worldState.create({
+                data: {
+                    scenarioCode: profile,
+                    currentYear: 199,
+                    currentMonth: 1,
+                    tickSeconds: turnSeconds,
+                    clockBaseTime: anchor,
+                    clockWallAnchor: anchor,
+                    clockTick: 0n,
+                    lastTurnTick: 0n,
+                    clockMode: 'realtime',
+                    clockPhase: 'RUNNING',
+                    clockRevision: 1n,
+                    deadlineGeneration: 1n,
+                },
+            });
+            const general = await db.general.create({
+                data: { id: 901, name: 'pending-turn', turnTick: 12345n, turnTime: anchor },
+            });
+            const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false });
+            try {
+                const token = await lease.acquire();
+                const authority = {
+                    kind: 'DAEMON' as const,
+                    profileName: profile,
+                    ownerId: token!.ownerId,
+                    fencingEpoch: token!.fencingEpoch,
+                };
+                await prepareRealtimeRecovery(db, authority);
+                expect(await db.worldState.findFirstOrThrow()).toMatchObject({
+                    clockPhase: 'RUNNING',
+                    clockRevision: 1n,
+                    clockTick: 0n,
+                    lastTurnTick: 0n,
+                    clockRecoveryStartTick: null,
+                });
+                expect(await db.general.findUniqueOrThrow({ where: { id: general.id } })).toMatchObject({
+                    turnTick: general.turnTick,
+                    turnTime: general.turnTime,
+                });
+                expect(await db.clockSuspension.count()).toBe(0);
+                // 짧은 host 중단도 운영자가 정지한 서버를 자동 재개하는 근거가 되지 않는다.
+                await prepareRealtimeRecovery(db, authority, { paused: true });
+                expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'SUSPENDED' });
+            } finally {
+                await lease.close();
+            }
+        }
+    );
+
    it.each([false, true])('fences outage recovery and reuses its window; repeated outage=%s', async (repeated) => {
        const profile = 'recovery-startup';
        await db.worldState.create({
@@ -513,8 +568,8 @@ describeIntegration('durable clock reconciliation', () => {
        }
    );

-    it.each([359999, 360000, 360001, 840000, 12 * 3600000 + 1000])(
-        'persists strict recovery boundaries for %i ms',
+    it.each([180000, 599999, 600000, 600001, 840000, 12 * 3600000 + 1000])(
+        'persists the inclusive ten-minute recovery boundary for %i ms',
        async (gap) => {
            const observed = T / 6;
            const future = new Date(Date.now() + 3600000);
@@ -549,7 +604,7 @@ describeIntegration('durable clock reconciliation', () => {
                authority,
                testResumeWallAt: now,
            });
-            expect(plan.recovery === null).toBe(gap < 360000);
+            expect(plan.recovery === null).toBe(gap <= 600000);
            expect(await db.message.count()).toBe(0);
            // Redis 장애 후에도 알림은 DB의 RUNNING 전이와 함께 한 번만 저장한다.
            await expect(
@@ -84,6 +84,52 @@ const buildWorld = (stateOverride: Partial<TurnWorldState> = {}): InMemoryTurnWo
 };

 describe('runtime clock shift', () => {
+    it('executes ten minutes of pending work in bounded batches without skipping or repeating generals', async () => {
+        const base = new Date('2026-07-30T10:00:00Z');
+        const resumed = new Date(base.getTime() + 600_000);
+        const plan = planTurnRecovery({
+            observedTick: 0,
+            normalTick: GAME_TICKS_PER_TURN,
+            wallNow: resumed,
+            turnSeconds: 600,
+        });
+        expect(plan).toMatchObject({ skippedTurns: 0, recovery: null });
+        const world = buildWorld({
+            clockBaseTime: base,
+            clockTick: 0,
+            clockWallAnchor: base,
+            clockMode: 'realtime',
+            clockPhase: 'RUNNING',
+            lastTurnTick: 0,
+        });
+        for (const [id, offset] of [
+            [1, 19_902],
+            [2, 42_001],
+        ] as const) {
+            world.updateGeneral(id, { turnTime: new Date(base.getTime() + offset), turnTick: offset * 60 });
+        }
+        const executed: number[] = [];
+        const processor = new InMemoryTurnProcessor(world, {
+            afterExecuteGeneral: async (general) => {
+                executed.push(general.id);
+            },
+        });
+        const target = world.getGameNow(resumed);
+        world.advanceGameClockTo(target, resumed);
+        const budget = { budgetMs: 10_000, maxGenerals: 1, catchUpCap: 1 };
+        let result = await processor.run(target, budget);
+        expect(result.processedGenerals).toBe(1);
+        for (let batch = 0; result.partial && batch < 4; batch++) {
+            result = await processor.run(target, budget, result.checkpoint);
+            expect(result.processedGenerals).toBeLessThanOrEqual(1);
+        }
+        expect(result.partial).toBe(false);
+        expect(executed).toEqual([1, 2]);
+        expect(world.getState().currentMonth).toBe(2);
+        expect(world.getGeneralById(1)!.turnTick).toBe(GAME_TICKS_PER_TURN + 19_902 * 60);
+        expect(world.getGeneralById(2)!.turnTick).toBe(GAME_TICKS_PER_TURN + 42_001 * 60);
+    });
+
    it('runs two real monthly cycles per normal interval, survives reload, and returns to one cycle', async () => {
        const base = new Date('2026-07-30T10:00:00Z');
        const wallAt = (minutes: number) => new Date(base.getTime() + minutes * 60_000);
@@ -44,9 +44,18 @@ From 2026-09-07, maintenance and crash recovery use the following
 `RECOVER_TURNS` policy. The base turn length does not change. One turn remains
 36,000,000 ticks; a persisted `TurnRecoveryWindow` changes only wall execution.

- An entire delay strictly below `min(600 seconds, turnSeconds / 10)` catches
-  up immediately through the ordinary engine. Equality uses recovery. The
-  limit is 30 seconds on a 5-minute server and 6 minutes on a 60-minute server.
+- From 2026-09-15, an entire delay up to and including 600 seconds catches
+  up immediately through the ordinary engine, independently of the server's
+  turn length. This accommodates host updates that pause and resume a VM.
+  Pending turns are executed in order with the ordinary per-run budget and
+  month boundary cap; they are not marked complete or skipped. The lease still
+  expires after 30 seconds and a fresh owner must fence the old process and
+  reload durable state before catching up. Operator pauses remain pauses.
+- During the persisted wait before 2x recovery, tournament participation is
+  allowed when the phase is `RUNNING`, the daemon lease is ready, and DB/Redis
+  revision and deadline generation agree. Registration stage, capacity,
+  duplicate entry, fee and ENGINE transaction checks still apply. Tournament
+  stage advancement and settlement continue to require a ticking clock.
 - For longer delays, skip only complete 12-turn blocks, moving future
  schedules and the execution cursor by the same integer delta. Never apply
  the short-delay exception again to the remainder. An exact multiple of
@@ -336,6 +336,11 @@ PM2가 새 owner/epoch와 DB snapshot으로 재시작합니다. lease 오류만
 않습니다. 이전 버전에서 이미 lease 오류로 PAUSED가 된 서버는 원인을 확인한 뒤
 한 번 `START`/재개해야 합니다.

+Host 업데이트 등으로 VM이 최대 10분 중단되면 새 runtime은 밀린 턴을 기존 엔진의
+순서·batch budget으로 즉시 따라잡습니다(정확히 10분 포함). 10분 초과는 기존의
+12턴 묶음 이동 및 대기 후 2배속 복구 정책을 사용합니다. 이미 저장한 복구 창과
+운영자의 PAUSED/STOPPED는 변경하지 않습니다.
+
 30초 lease와 이전 owner의 fencing은 유지합니다. transaction의 만료 검사는 시작에
 고정된 `CURRENT_TIMESTAMP`가 아닌 검사 순간의 `clock_timestamp()`를 사용합니다.
 역방향 시계 보정으로 기존 lease가 아직 유효하면 startup은 2초 간격으로 기다리며
@@ -68,8 +68,8 @@ export const turnShiftTicks = (turns: number): GameTick => {
    return asGameTick(turns * GAME_TICKS_PER_TURN);
 };

-/** 즉시 처리 여부는 12턴 묶음 생략 전의 전체 지연으로 판정한다. */
-export const immediateRecoveryLimitSeconds = (turnSeconds: number): number => Math.min(600, turnSeconds / 10);
+/** VM/host 업데이트의 전체 중단 10분까지는 기존 엔진 순서와 budget으로 따라잡는다. */
+export const immediateRecoveryLimitSeconds = (): number => 600;

 /**
 * observedTick은 중단 전에 저장한 관측 지점, normalTick은 기존 시간표의 현재 지점이다.
@@ -91,8 +91,8 @@ export const planTurnRecovery = (input: {
    if (!Number.isFinite(wallNow.getTime())) throw new Error('Recovery wall instant is invalid.');
    const gap = Math.max(0, normalTick - observedTick);
    const ticksPerSecond = GAME_TICKS_PER_TURN / turnSeconds;
-    const immediateLimit = immediateRecoveryLimitSeconds(turnSeconds) * ticksPerSecond;
-    if (gap < immediateLimit) {
+    const immediateLimit = immediateRecoveryLimitSeconds() * ticksPerSecond;
+    if (gap <= immediateLimit) {
        return {
            skippedTurns: 0,
            recoveryTurns: 0,
@@ -160,9 +160,9 @@ describe('turn-aligned double-speed recovery', () => {
        expect(clock.executionRate(end)).toBe(1);
    });

-    it.each([300, 3600, 6000, 7200])('uses the strict whole-delay threshold for a %i second turn', (turnSeconds) => {
+    it.each([60, 300, 1200, 3600, 6000, 7200])('catches up through ten minutes for a %i second turn', (turnSeconds) => {
        const rate = T / turnSeconds;
-        const limitMs = Math.min(600, turnSeconds / 10) * 1000;
+        const limitMs = 600_000;
        for (const delta of [-1, 0, 1]) {
            const delayMs = limitMs + delta;
            const normal = Math.trunc((delayMs * rate) / 1000);
@@ -172,8 +172,8 @@ describe('turn-aligned double-speed recovery', () => {
                wallNow: new Date(base + delayMs),
                turnSeconds,
            });
-            expect(plan.recovery === null).toBe(delta < 0);
-            expect(plan.initialTick).toBe(delta < 0 ? normal : 0);
+            expect(plan.recovery === null).toBe(delta <= 0);
+            expect(plan.initialTick).toBe(delta <= 0 ? normal : 0);
        }
        // 장시간 중단의 작은 나머지에는 즉시 처리 예외를 다시 적용하지 않는다.
        const long = planTurnRecovery({ observedTick: 0, normalTick: 12 * T + rate, wallNow: wall(20), turnSeconds });
@@ -25,3 +25,4 @@ TURN_COMMAND_DURABLE_MATRIX_DATABASE_URL	reference_command_durable_matrix
 TURN_DIFFERENTIAL_DATABASE_URL	core
 TURN_FULL_LIFECYCLE_PERSISTENCE_DATABASE_URL	reference_full_lifecycle
 WEB_PUSH_GATEWAY_DATABASE_URL	web_push_gateway
+TOURNAMENT_RECOVERY_DATABASE_URL	external_fixture