feat(admin): apply durable runtime clock shifts

2026-07-30 17:21:01 +00:00
parent d6904f2c9d
commit 7f31459385
23 changed files with 1889 additions and 93 deletions
@@ -276,30 +276,36 @@ export class TurnDaemonLifecycle {
        const executeHandler = async (
            context?: TurnDaemonCommandExecutionContext
        ): Promise<TurnDaemonCommandResult> => {
-            const execute = async (): Promise<TurnDaemonCommandResult> => {
-                const handled = this.commandHandler ? await this.commandHandler.handle(command, context) : null;
-                return (
-                    handled ?? {
-                        type: 'commandRejected',
-                        ok: false,
-                        commandType: command.type,
-                        reason: '턴 데몬이 명령을 처리할 수 없습니다.',
-                    }
-                );
-            };
-            return this.stateManager ? this.stateManager.transaction(execute) : execute();
+            const handled = this.commandHandler ? await this.commandHandler.handle(command, context) : null;
+            return (
+                handled ?? {
+                    type: 'commandRejected',
+                    ok: false,
+                    commandType: command.type,
+                    reason: '턴 데몬이 명령을 처리할 수 없습니다.',
+                }
+            );
        };
        try {
-            if (command.requestId && this.hooks?.executeCommand) {
-                result = await this.hooks.executeCommand(command.requestId, executeHandler);
-                committedByExecutionBoundary = true;
-            } else {
-                result = await executeHandler();
-            }
+            const executeAndCommit = async (): Promise<TurnDaemonCommandResult> => {
+                let nextResult: TurnDaemonCommandResult;
+                if (command.requestId && this.hooks?.executeCommand) {
+                    nextResult = await this.hooks.executeCommand(command.requestId, executeHandler);
+                    committedByExecutionBoundary = true;
+                } else {
+                    nextResult = await executeHandler();
+                }
+                if (!committedByExecutionBoundary && command.requestId && this.hooks?.commitCommand) {
+                    await this.hooks.commitCommand(command.requestId, nextResult);
+                }
+                return nextResult;
+            };
+            result = this.stateManager
+                ? await this.stateManager.transaction(executeAndCommit)
+                : await executeAndCommit();
        } catch (error) {
-            // A handler may already have changed the in-memory world. Do not commit
-            // either those changes or the inbox completion marker after an exception.
-            // Pausing forces a reload/retry instead of acknowledging a partial event.
+            // The state-manager boundary includes the durable command commit, so a
+            // database/fencing failure restores every in-memory mutation as well.
            this.status.state = 'paused';
            this.status.paused = true;
            this.errorPaused = true;
@@ -308,17 +314,11 @@ export class TurnDaemonLifecycle {
            return;
        }

-        if (!committedByExecutionBoundary && command.requestId && this.hooks?.commitCommand) {
-            try {
-                await this.hooks.commitCommand(command.requestId, result);
-            } catch (error) {
-                this.status.state = 'paused';
-                this.status.paused = true;
-                this.errorPaused = true;
-                this.status.lastError = error instanceof Error ? error.message : 'Unknown input event commit error.';
-                await this.hooks.onRunError?.(error);
-                return;
-            }
+        if (result.type === 'shiftSchedule' && result.ok) {
+            this.status.lastTurnTime = result.lastTurnTime;
+            this.status.checkpoint = result.checkpoint;
+            await this.stateStore.saveCheckpoint(result.checkpoint);
+            await this.resolveNextRunTime();
        }

        if (this.commandResponder && command.requestId) {
@@ -270,6 +270,12 @@ const zShutdown = z.object({
    reason: z.string().optional(),
 });

+const zShiftSchedule = z.object({
+    type: z.literal('shiftSchedule'),
+    actionId: z.string().uuid(),
+    deltaMinutes: z.number().int().min(-1440).max(1440).refine((value) => value !== 0),
+});
+
 const normalizeAuctionFinalize: CommandNormalizer<'auctionFinalize'> = (envelope) => {
    const command = parseWith(zAuctionFinalize, envelope.command);
    if (!command) {
@@ -506,6 +512,14 @@ const normalizeShutdown: CommandNormalizer<'shutdown'> = (envelope) => {
    return command ? { ...command, requestId: envelope.requestId } : null;
 };

+const normalizeShiftSchedule: CommandNormalizer<'shiftSchedule'> = (envelope) => {
+    const command = parseWith(zShiftSchedule, envelope.command);
+    if (!command) {
+        return null;
+    }
+    return { ...command, requestId: envelope.requestId };
+};
+
 const normalizers: CommandNormalizerMap = {
    auctionFinalize: normalizeAuctionFinalize,
    auctionOpen: normalizeAuctionOpen,
@@ -538,6 +552,7 @@ const normalizers: CommandNormalizerMap = {
    pause: normalizePause,
    resume: normalizeResume,
    shutdown: normalizeShutdown,
+    shiftSchedule: normalizeShiftSchedule,
 };

 export const normalizeTurnDaemonCommand = (envelope: TurnDaemonCommandEnvelope): TurnDaemonCommand | null => {
@@ -1,9 +1,11 @@
 import { createGatewayPostgresConnector } from '@sammo-ts/infra';
 import { isRecord } from '@sammo-ts/common';

-export type GatewayAdminActionStatus = 'REQUESTED' | 'APPLIED' | 'FAILED' | 'IGNORED';
+export type GatewayAdminActionStatus = 'REQUESTED' | 'PARTIAL' | 'APPLIED' | 'FAILED' | 'IGNORED';

 export interface GatewayAdminActionRecord {
+    id?: string;
+    profileName?: string;
    action?: string;
    requestedAt?: string;
    durationMinutes?: number | null;
@@ -79,12 +81,71 @@ export const createGatewayAdminActionConsumer = async (
    let timer: NodeJS.Timeout | null = null;
    let inFlight = false;

+    const pollRuntimeActions = async (): Promise<void> => {
+        const pending = await prisma.gatewayRuntimeAction.findMany({
+            where: {
+                profileName: options.profileName,
+                status: { in: ['REQUESTED', 'PARTIAL'] },
+                OR: [{ nextAttemptAt: null }, { nextAttemptAt: { lte: new Date() } }],
+            },
+            orderBy: { createdAt: 'asc' },
+        });
+        for (const action of pending) {
+            const actionRecord: GatewayAdminActionRecord = {
+                id: action.id,
+                profileName: action.profileName,
+                action: action.action,
+                requestedAt: action.createdAt.toISOString(),
+                durationMinutes: action.durationMinutes,
+                scheduledAt: action.scheduledAt?.toISOString() ?? null,
+                reason: action.reason,
+                status: action.status,
+                handledAt: action.handledAt?.toISOString() ?? null,
+                handler: action.handler,
+                detail: action.detail,
+            };
+            let result: GatewayAdminActionResult;
+            try {
+                result = await options.handler(actionRecord);
+            } catch (error) {
+                result = {
+                    status: 'PARTIAL',
+                    detail: error instanceof Error ? error.message : String(error),
+                };
+            }
+            if (result.status === 'REQUESTED') {
+                continue;
+            }
+            const terminal = result.status !== 'PARTIAL';
+            const updated = await prisma.gatewayRuntimeAction.updateMany({
+                where: {
+                    id: action.id,
+                    status: { in: ['REQUESTED', 'PARTIAL'] },
+                },
+                data: {
+                    status: result.status,
+                    detail: result.detail ?? null,
+                    handler: 'turn-daemon',
+                    handledAt: terminal ? new Date() : null,
+                    attempts: { increment: 1 },
+                    nextAttemptAt: terminal
+                        ? null
+                        : new Date(Date.now() + Math.min(60_000, 1_000 * 2 ** Math.min(action.attempts, 6))),
+                },
+            });
+            if (terminal && updated.count > 0) {
+                await options.onActionApplied?.(actionRecord, result);
+            }
+        }
+    };
+
    const pollOnce = async (): Promise<void> => {
        if (inFlight) {
            return;
        }
        inFlight = true;
        try {
+            await pollRuntimeActions();
            const profile = await prisma.gatewayProfile.findUnique({
                where: { profileName: options.profileName },
            });
@@ -782,6 +782,78 @@ export class InMemoryTurnWorld {
        };
    }

+    shiftSchedule(deltaMinutes: number): { shiftedGenerals: number; lastTurnTime: string } {
+        if (!Number.isInteger(deltaMinutes) || deltaMinutes === 0) {
+            throw new Error('Schedule shift must be a non-zero integer number of minutes.');
+        }
+        const deltaMs = deltaMinutes * 60_000;
+        const shiftDate = (date: Date): Date => new Date(date.getTime() + deltaMs);
+        const shiftMetaDate = (value: unknown): unknown => {
+            if (typeof value !== 'string' || !value.trim()) {
+                return value;
+            }
+            if (value.includes('T')) {
+                const shifted = shiftDate(new Date(value));
+                return Number.isNaN(shifted.getTime()) ? value : shifted.toISOString();
+            }
+            const match = /^(\d{4})-(\d{2})-(\d{2})[ ](\d{2}):(\d{2}):(\d{2})(\.\d{1,6})?$/.exec(value);
+            if (!match) {
+                return value;
+            }
+            const parts = match.slice(1).map(Number);
+            const shifted = new Date(
+                Date.UTC(parts[0]!, parts[1]! - 1, parts[2]!, parts[3]!, parts[4]!, parts[5]!) + deltaMs
+            );
+            return (
+                [
+                    shifted.getUTCFullYear().toString().padStart(4, '0'),
+                    (shifted.getUTCMonth() + 1).toString().padStart(2, '0'),
+                    shifted.getUTCDate().toString().padStart(2, '0'),
+                ].join('-') +
+                ' ' +
+                [
+                    shifted.getUTCHours().toString().padStart(2, '0'),
+                    shifted.getUTCMinutes().toString().padStart(2, '0'),
+                    shifted.getUTCSeconds().toString().padStart(2, '0'),
+                ].join(':') +
+                (match[7] ?? '')
+            );
+        };
+
+        const nextLastTurnTime = shiftDate(this.state.lastTurnTime);
+        const nextMeta = {
+            ...this.state.meta,
+            lastTurnTime: nextLastTurnTime.toISOString(),
+            turntime: shiftMetaDate(this.state.meta.turntime),
+            starttime: shiftMetaDate(this.state.meta.starttime),
+            tnmt_time: shiftMetaDate(this.state.meta.tnmt_time),
+        };
+        this.state = {
+            ...this.state,
+            lastTurnTime: nextLastTurnTime,
+            meta: nextMeta,
+        };
+
+        for (const general of this.generals.values()) {
+            this.updateGeneral(general.id, { turnTime: shiftDate(general.turnTime) });
+        }
+        for (const auction of this.pendingNeutralAuctions) {
+            auction.closeAt = shiftDate(auction.closeAt);
+        }
+        if (this.checkpoint) {
+            const checkpointTime = shiftDate(new Date(this.checkpoint.turnTime));
+            this.checkpoint = {
+                ...this.checkpoint,
+                turnTime: checkpointTime.toISOString(),
+            };
+        }
+
+        return {
+            shiftedGenerals: this.generals.size,
+            lastTurnTime: nextLastTurnTime.toISOString(),
+        };
+    }
+
    getNextNationId(): number {
        const meta = this.state.meta as Record<string, unknown>;
        let lastId = (meta.lastNationId as number | undefined) ?? 0;
@@ -0,0 +1,211 @@
+import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra';
+import { randomUUID } from 'node:crypto';
+
+import type { TurnDaemonCommand, TurnDaemonCommandResult } from '@sammo-ts/common';
+
+import type { GatewayAdminActionRecord, GatewayAdminActionResult } from './gatewayAdminActions.js';
+
+interface RuntimeRedisClient {
+    get(key: string): Promise<string | null>;
+    set(
+        key: string,
+        value: string,
+        options?: {
+            NX?: boolean;
+            PX?: number;
+        }
+    ): Promise<unknown>;
+    del(key: string): Promise<unknown>;
+    zAdd(key: string, values: Array<{ score: number; value: string }>): Promise<number>;
+}
+
+type TournamentClockState = {
+    nextAt?: string;
+    bettingCloseAt?: string;
+    runtimeClockShiftActionIds?: string[];
+    [key: string]: unknown;
+};
+
+const asJson = (value: unknown): GamePrisma.InputJsonValue => value as GamePrisma.InputJsonValue;
+const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+
+const isUniqueConflict = (error: unknown): boolean =>
+    typeof error === 'object' && error !== null && 'code' in error && error.code === 'P2002';
+
+const shiftDateText = (value: unknown, deltaMinutes: number): unknown => {
+    if (typeof value !== 'string' || !value.trim()) {
+        return value;
+    }
+    const shifted = new Date(new Date(value).getTime() + deltaMinutes * 60_000);
+    return Number.isNaN(shifted.getTime()) ? value : shifted.toISOString();
+};
+
+const syncAuctionTimers = async (
+    db: GamePrismaClient,
+    redis: RuntimeRedisClient,
+    profileName: string
+): Promise<number> => {
+    const auctions = await db.auction.findMany({
+        where: { status: 'OPEN' },
+        select: { id: true, closeAt: true },
+    });
+    if (auctions.length > 0) {
+        await redis.zAdd(
+            `sammo:${profileName}:auction:timer`,
+            auctions.map((auction) => ({
+                score: auction.closeAt.getTime(),
+                value: String(auction.id),
+            }))
+        );
+    }
+    return auctions.length;
+};
+
+const shiftTournamentClock = async (
+    redis: RuntimeRedisClient,
+    profileName: string,
+    actionId: string,
+    deltaMinutes: number
+): Promise<boolean> => {
+    const stateKey = `sammo:${profileName}:tournament:state`;
+    const lockKey = `${stateKey}:mutation-lock`;
+    const token = randomUUID();
+    const deadline = Date.now() + 2_000;
+    while (Date.now() < deadline) {
+        const acquired = await redis.set(lockKey, token, { NX: true, PX: 30_000 });
+        if (acquired) {
+            try {
+                const rawState = await redis.get(stateKey);
+                if (!rawState) {
+                    return false;
+                }
+                const state = JSON.parse(rawState) as TournamentClockState;
+                const applied = Array.isArray(state.runtimeClockShiftActionIds)
+                    ? state.runtimeClockShiftActionIds.filter((entry): entry is string => typeof entry === 'string')
+                    : [];
+                if (applied.includes(actionId)) {
+                    return true;
+                }
+                const nextState: TournamentClockState = {
+                    ...state,
+                    nextAt: shiftDateText(state.nextAt, deltaMinutes) as string | undefined,
+                    bettingCloseAt: shiftDateText(state.bettingCloseAt, deltaMinutes) as string | undefined,
+                    runtimeClockShiftActionIds: [...applied, actionId],
+                };
+                await redis.set(stateKey, JSON.stringify(nextState));
+                return true;
+            } finally {
+                if ((await redis.get(lockKey)) === token) {
+                    await redis.del(lockKey);
+                }
+            }
+        }
+        await sleep(10);
+    }
+    throw new Error('토너먼트 시간 조정 lock을 획득하지 못했습니다.');
+};
+
+const ensureEngineCommand = async (
+    db: GamePrismaClient,
+    actionId: string,
+    deltaMinutes: number
+): Promise<{ requestId: string; result?: TurnDaemonCommandResult; failed?: string }> => {
+    const requestId = `gateway-runtime:${actionId}`;
+    const command: TurnDaemonCommand = {
+        type: 'shiftSchedule',
+        requestId,
+        actionId,
+        deltaMinutes,
+    };
+    try {
+        await db.inputEvent.create({
+            data: {
+                requestId,
+                target: 'ENGINE',
+                eventType: command.type,
+                payload: asJson(command),
+            },
+        });
+    } catch (error) {
+        if (!isUniqueConflict(error)) {
+            throw error;
+        }
+        const existing = await db.inputEvent.findUniqueOrThrow({
+            where: { requestId },
+            select: { eventType: true, payload: true },
+        });
+        const payload = existing.payload as Partial<TurnDaemonCommand>;
+        if (
+            existing.eventType !== command.type ||
+            payload.type !== command.type ||
+            payload.actionId !== actionId ||
+            payload.deltaMinutes !== deltaMinutes
+        ) {
+            return { requestId, failed: '같은 action ID에 다른 시간 조정 payload가 이미 존재합니다.' };
+        }
+    }
+
+    const event = await db.inputEvent.findUniqueOrThrow({
+        where: { requestId },
+        select: { status: true, result: true, error: true },
+    });
+    if (event.status === 'FAILED') {
+        return { requestId, failed: event.error ?? '게임 엔진 시간 조정이 실패했습니다.' };
+    }
+    if (event.status !== 'SUCCEEDED') {
+        return { requestId };
+    }
+    return { requestId, result: event.result as TurnDaemonCommandResult };
+};
+
+export const applyRuntimeClockShift = async (options: {
+    action: GatewayAdminActionRecord;
+    profileName: string;
+    db: GamePrismaClient;
+    redis?: RuntimeRedisClient;
+}): Promise<GatewayAdminActionResult> => {
+    const { action, profileName, db, redis } = options;
+    if (!action.id) {
+        return { status: 'FAILED', detail: '시간 조정 action ID가 없습니다.' };
+    }
+    if (!Number.isInteger(action.durationMinutes) || (action.durationMinutes ?? 0) < 1) {
+        return { status: 'FAILED', detail: '시간 조정 분은 1 이상의 정수여야 합니다.' };
+    }
+    const direction = action.action === 'ACCELERATE' ? -1 : action.action === 'DELAY' ? 1 : 0;
+    if (direction === 0) {
+        return { status: 'IGNORED', detail: `지원하지 않는 시간 조정 action입니다: ${action.action ?? ''}` };
+    }
+    const deltaMinutes = direction * action.durationMinutes!;
+    const engine = await ensureEngineCommand(db, action.id, deltaMinutes);
+    if (engine.failed) {
+        return { status: 'FAILED', detail: engine.failed };
+    }
+    if (!engine.result) {
+        return { status: 'REQUESTED', detail: `게임 엔진 처리 대기 중: ${engine.requestId}` };
+    }
+    if (engine.result.type !== 'shiftSchedule' || !engine.result.ok) {
+        return {
+            status: 'FAILED',
+            detail:
+                engine.result.type === 'shiftSchedule' ? engine.result.reason : '게임 엔진이 다른 결과를 반환했습니다.',
+        };
+    }
+    if (!redis) {
+        return {
+            status: 'PARTIAL',
+            detail: `DB 시간 조정은 적용됐지만 Redis timer 동기화를 기다리는 중입니다: ${engine.requestId}`,
+        };
+    }
+
+    const syncedAuctions = await syncAuctionTimers(db, redis, profileName);
+    const shiftedTournament = await shiftTournamentClock(redis, profileName, action.id, deltaMinutes);
+    return {
+        status: 'APPLIED',
+        detail: [
+            `${Math.abs(deltaMinutes)}분 ${deltaMinutes < 0 ? '가속' : '연기'}`,
+            `장수 ${engine.result.shiftedGenerals}명`,
+            `경매 ${engine.result.shiftedAuctions}건(DB)/${syncedAuctions}건(timer)`,
+            shiftedTournament ? '토너먼트 적용' : '활성 토너먼트 없음',
+        ].join(' · '),
+    };
+};
@@ -76,6 +76,7 @@ import {
 import { buildCommandEnv } from './reservedTurnCommands.js';
 import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
 import { EngineStateManager } from './engineStateManager.js';
+import { applyRuntimeClockShift } from './runtimeClockShift.js';

 export interface TurnDaemonRuntimeOptions {
    profile: string;
@@ -755,10 +756,24 @@ const createTurnDaemonRuntimeWithLease = async (
            pollIntervalMs: options.adminActionIntervalMs,
            handler: async (action) => {
                const reason = action.reason ?? `admin:${action.action ?? 'action'}`;
+                if (turnDaemonLease?.isLost()) {
+                    return { status: 'REQUESTED', detail: 'turn-daemon lease 재획득을 기다리는 중입니다.' };
+                }
                if (action.action === 'RESET_NOW' || action.action === 'RESET_SCHEDULED') {
                    // 리셋은 오케스트레이터에서 빌드+재기동으로 처리한다.
                    return { status: 'REQUESTED', detail: 'waiting for orchestrator reset' };
                }
+                if (action.action === 'ACCELERATE' || action.action === 'DELAY') {
+                    if (!commandConnector) {
+                        return { status: 'FAILED', detail: '게임 command database 연결이 없습니다.' };
+                    }
+                    return applyRuntimeClockShift({
+                        action,
+                        profileName: options.profileName!,
+                        db: commandConnector.prisma,
+                        redis: redisConnector?.client,
+                    });
+                }
                switch (action.action) {
                    case 'RESUME':
                        resolvedControlQueue.enqueue({ type: 'resume', reason });
@@ -4,7 +4,7 @@ import type {
    TurnDaemonCommandExecutionContext,
    TurnDaemonCommandResult,
 } from '../lifecycle/types.js';
-import type { GamePrisma } from '@sammo-ts/infra';
+import { GamePrisma } from '@sammo-ts/infra';
 import { asRecord, JosaUtil, LiteHashDRBG, RandUtil } from '@sammo-ts/common';
 import {
    LogCategory,
@@ -442,6 +442,41 @@ async function handlePatchGeneral(
    return { type: 'patchGeneral', ok: true, generalId: command.generalId };
 }

+async function handleShiftSchedule(
+    ctx: CommandHandlerContext,
+    command: Extract<TurnDaemonCommand, { type: 'shiftSchedule' }>
+): Promise<TurnDaemonCommandResult> {
+    if (!ctx.commandDb) {
+        return {
+            type: 'shiftSchedule',
+            ok: false,
+            actionId: command.actionId,
+            reason: '시간 조정은 데이터베이스 transaction 경계에서만 실행할 수 있습니다.',
+        };
+    }
+
+    const shifted = ctx.world.shiftSchedule(command.deltaMinutes);
+    const shiftedAuctions = await ctx.commandDb.$executeRaw(
+        GamePrisma.sql`
+            UPDATE auction
+            SET close_at = close_at + (${command.deltaMinutes} * INTERVAL '1 minute'),
+                updated_at = NOW()
+            WHERE status = 'OPEN'
+        `
+    );
+
+    return {
+        type: 'shiftSchedule',
+        ok: true,
+        actionId: command.actionId,
+        deltaMinutes: command.deltaMinutes,
+        lastTurnTime: shifted.lastTurnTime,
+        shiftedGenerals: shifted.shiftedGenerals,
+        shiftedAuctions,
+        checkpoint: ctx.world.getCheckpoint(),
+    };
+}
+
 async function handleTroopJoin(
    ctx: CommandHandlerContext,
    command: Extract<TurnDaemonCommand, { type: 'troopJoin' }>
@@ -1812,6 +1847,8 @@ export const createTurnDaemonCommandHandler = (options: {
            handleTournamentMatchResult(ctx, command as Extract<TurnDaemonCommand, { type: 'tournamentMatchResult' }>),
        patchGeneral: (command) =>
            handlePatchGeneral(ctx, command as Extract<TurnDaemonCommand, { type: 'patchGeneral' }>),
+        shiftSchedule: (command) =>
+            handleShiftSchedule(ctx, command as Extract<TurnDaemonCommand, { type: 'shiftSchedule' }>),
    };

    return {