관리자 서버 진단에 lease 상태와 영속 장애 이력 추가

app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts

@@ -24,8 +24,8 @@ export class TurnDaemonLeaseUnavailableError extends Error {
 }
 
 export class TurnDaemonLeaseLostError extends Error {
-    constructor(profile: string) {
+    constructor(profile: string, reason?: string) {
-        super(`Turn daemon lease was lost for profile "${profile}".`);
+        super(`Turn daemon lease was lost for profile "${profile}".${reason ? ` ${reason}` : ''}`);
         this.name = 'TurnDaemonLeaseLostError';
     }
 }
@@ -50,6 +50,7 @@ export class DatabaseTurnDaemonLease {
     private expiryTimer: NodeJS.Timeout | null = null;
     private renewalInFlight = false;
     private lost = false;
+    private lossReason: string | undefined;
 
     private constructor(
         db: GamePrismaClient,
@@ -116,6 +117,7 @@ export class DatabaseTurnDaemonLease {
             fencingEpoch: BigInt(row.fencing_epoch),
         };
         this.lost = false;
+        this.lossReason = undefined;
         this.scheduleExpiryWatchdog(requestStartedAt);
         if (this.heartbeatEnabled) {
             this.startHeartbeat();
@@ -139,6 +141,10 @@ export class DatabaseTurnDaemonLease {
         return this.lost;
     }
 
+    getLossError(): TurnDaemonLeaseLostError {
+        return new TurnDaemonLeaseLostError(this.profile, this.lossReason);
+    }
+
     async renew(): Promise<boolean> {
         const token = this.token;
         if (!token || this.lost || this.renewalInFlight) {
@@ -160,7 +166,7 @@ export class DatabaseTurnDaemonLease {
                 RETURNING "profile", "owner_id", "fencing_epoch"
             `);
             if (rows.length === 0) {
-                this.markLost();
+                this.markLost('Heartbeat renewal rejected: lease expired or owner/epoch changed.');
                 return false;
             }
             if (this.lost) {
@@ -176,7 +182,7 @@ export class DatabaseTurnDaemonLease {
     async assertActive(transaction?: GamePrisma.TransactionClient): Promise<void> {
         const token = this.token;
         if (!token || this.lost) {
-            throw new TurnDaemonLeaseLostError(this.profile);
+            throw this.getLossError();
         }
         const db = transaction ?? this.db;
         const rows = await db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
@@ -190,8 +196,8 @@ export class DatabaseTurnDaemonLease {
             FOR UPDATE
         `);
         if (rows.length === 0) {
-            this.markLost();
+            this.markLost('Transaction fencing rejected: lease expired or owner/epoch changed.');
-            throw new TurnDaemonLeaseLostError(this.profile);
+            throw this.getLossError();
         }
     }
 
@@ -228,8 +234,10 @@ export class DatabaseTurnDaemonLease {
         }
         const intervalMs = Math.max(250, Math.floor(this.leaseDurationMs / 3));
         this.heartbeatTimer = setInterval(() => {
-            void this.renew().catch(() => {
+            void this.renew().catch((error: unknown) => {
-                this.markLost();
+                this.markLost(
+                    `Heartbeat database request failed (${error instanceof Error ? error.name : 'unknown error'}).`
+                );
             });
         }, intervalMs);
         this.heartbeatTimer.unref();
@@ -248,7 +256,9 @@ export class DatabaseTurnDaemonLease {
         this.stopExpiryWatchdog();
         const remainingMs = Math.max(0, this.leaseDurationMs - (performance.now() - requestStartedAt));
         this.expiryTimer = setTimeout(() => {
-            this.markLost();
+            this.markLost(
+                `Heartbeat deadline exceeded (${this.leaseDurationMs}ms; renewal in flight: ${this.renewalInFlight}).`
+            );
         }, remainingMs);
         this.expiryTimer.unref();
     }
@@ -260,7 +270,9 @@ export class DatabaseTurnDaemonLease {
         }
     }
 
-    private markLost(): void {
+    private markLost(reason: string): void {
+        if (this.lost) return;
+        this.lossReason = reason;
         this.lost = true;
         this.stopHeartbeat();
         this.stopExpiryWatchdog();

app/game-engine/src/turn/cli.ts

@@ -5,6 +5,8 @@ import type { TurnRunBudget } from '../lifecycle/types.js';
 import { resolveDatabaseUrl } from '../scenario/databaseUrl.js';
 import { createTurnDaemonRuntime } from './turnDaemon.js';
 import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
+import { createGatewayProfileGate } from './gatewayProfileGate.js';
+import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
 
 export interface TurnDaemonCliOptions {
     profile?: string;
@@ -89,6 +91,27 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
         pauseGateIntervalMs,
         adminActionIntervalMs,
         gameClockMode,
+    }).catch(async (error: unknown) => {
+        // 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는
+        // lifecycle hook이 아직 없으므로 여기서 별도로 관리자에게 기록한다.
+        if (!(error instanceof TurnDaemonLeaseUnavailableError)) {
+            try {
+                const gate = await createGatewayProfileGate({
+                    databaseUrl,
+                    gatewayDatabaseUrl,
+                    profileName,
+                    incidentContext: () => ({ stage: 'startup' }),
+                });
+                try {
+                    await gate.markPaused(error);
+                } finally {
+                    await gate.close();
+                }
+            } catch {
+                /* 원래 시작 실패를 보존한다. */
+            }
+        }
+        throw error;
     });
 
     const memoryReporter = createTurnDaemonMemoryReporter({

app/game-engine/src/turn/gatewayProfileGate.ts

@@ -1,6 +1,7 @@
 import { performance } from 'node:perf_hooks';
+import { randomUUID } from 'node:crypto';
 
-import { gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common';
+import { describeRuntimeError, gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common';
 import { createGatewayPostgresConnector } from '@sammo-ts/infra';
 
 export interface GatewayProfileGateOptions {
@@ -8,6 +9,7 @@ export interface GatewayProfileGateOptions {
     gatewayDatabaseUrl?: string;
     profileName: string;
     cacheMs?: number;
+    incidentContext?: () => Record<string, string | number | boolean | null>;
 }
 
 export interface GatewayProfileGate {
@@ -22,6 +24,7 @@ const PROFILE_STATUSES_MARKABLE_AS_PAUSED = ['PREOPEN', 'RUNNING', 'PAUSED'] as
 export const createGatewayProfileGate = async (options: GatewayProfileGateOptions): Promise<GatewayProfileGate> => {
     const connector = createGatewayPostgresConnector({
         url: options.gatewayDatabaseUrl ?? options.databaseUrl,
+        connectionTimeoutMillis: 3000,
     });
     await connector.connect();
     const prisma = connector.prisma;
@@ -54,19 +57,44 @@ export const createGatewayProfileGate = async (options: GatewayProfileGateOption
             return cachedPause;
         },
         async markPaused(error?: unknown): Promise<void> {
-            const message = error instanceof Error ? error.message : error ? String(error) : null;
+            const failure = error ? describeRuntimeError(error) : null;
+            const message = failure?.message ?? null;
             try {
-                await prisma.gatewayProfile.updateMany({
+                await prisma.$transaction(async (tx) => {
-                    where: {
+                    const updated = await tx.gatewayProfile.updateMany({
-                        profileName: options.profileName,
+                        where: {
-                        status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
+                            profileName: options.profileName,
-                    },
+                            status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
-                    data: {
+                            OR: [{ status: { not: 'PAUSED' } }, { lastError: { not: message } }, { lastError: null }],
-                        status: 'PAUSED',
+                        },
-                        lastError: message,
+                        data: {
-                    },
+                            status: 'PAUSED',
+                            lastError: message,
+                        },
+                    });
+                    if (updated.count && failure) {
+                        // 상태와 이력을 함께 commit한다. 재개가 lastError를 지워도
+                        // 당시 원인과 실행 좌표는 관리자 감사 저장소에 남는다.
+                        await tx.adminAuditEvent.create({
+                            data: {
+                                correlationId: randomUUID(),
+                                actorUserId: 'system:turn-daemon',
+                                actorUsername: 'turn-daemon',
+                                credentialKind: 'DAEMON',
+                                action: 'runtime.failure',
+                                targetType: 'profile-runtime',
+                                targetId: options.profileName,
+                                profileName: options.profileName,
+                                outcome: 'FAILED',
+                                errorCode: failure.code,
+                                errorMessage: failure.message,
+                                summary: { frames: failure.frames, ...options.incidentContext?.() },
+                            },
+                        });
+                    }
                 });
             } catch {
+                if (failure) console.error('[turn-daemon] failed to persist runtime incident', failure);
                 return;
             }
         },

app/game-engine/src/turn/turnDaemon.ts

@@ -93,11 +93,7 @@ import {
     createResetOfficerLockHandler,
 } from './monthlyCoreEventAction.js';
 import { buildCommandEnv } from './reservedTurnCommands.js';
-import {
+import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
-    DatabaseTurnDaemonLease,
-    TurnDaemonLeaseLostError,
-    TurnDaemonLeaseUnavailableError,
-} from '../lifecycle/databaseTurnDaemonLease.js';
 import { EngineStateManager } from './engineStateManager.js';
 import { applyRuntimeClockShift } from './runtimeClockShift.js';
 import { applyRuntimeGameSettings } from './runtimeGameSettings.js';
@@ -892,6 +888,19 @@ const createTurnDaemonRuntimeWithLease = async (
               gatewayDatabaseUrl: options.gatewayDatabaseUrl,
               profileName: options.profileName,
               cacheMs: options.pauseGateIntervalMs,
+              incidentContext: () => {
+                  const state = world.getState();
+                  const clock = world.getGameClockState();
+                  const token = turnDaemonLease?.getToken();
+                  return {
+                      year: state.currentYear,
+                      month: state.currentMonth,
+                      clockPhase: clock.phase,
+                      clockTick: clock.tick,
+                      ownerId: token?.ownerId ?? null,
+                      fencingEpoch: token?.fencingEpoch.toString() ?? null,
+                  };
+              },
           })
         : null;
     if (gatewayGate) {
@@ -1066,7 +1075,7 @@ const createTurnDaemonRuntimeWithLease = async (
                 if (turnDaemonLease?.isLost()) {
                     // 만료된 owner는 재개 명령도 처리할 수 없다. 현재 runtime을
                     // 끝내 PM2가 새 owner와 DB snapshot으로 시작하도록 한다.
-                    throw new TurnDaemonLeaseLostError(options.profileName ?? options.profile);
+                    throw turnDaemonLease.getLossError();
                 }
                 const gatewayPaused = (await pauseGate?.()) ?? false;
                 const phase = world.getGameClockState().phase;

app/game-engine/test/gatewayRuntimeAction.integration.test.ts

@@ -112,6 +112,7 @@ integration('gateway runtime action consumer', () => {
     });
 
     it('does not overwrite a terminal operator status while reporting a daemon error', async () => {
+        const existingIncidents = await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } });
         const gate = await createGatewayProfileGate({
             databaseUrl: databaseUrl!,
             gatewayDatabaseUrl: databaseUrl!,
@@ -127,6 +128,19 @@ integration('gateway runtime action consumer', () => {
                 status: 'PAUSED',
                 lastError: 'running failure',
             });
+            await gate.markPaused(new Error('running failure'));
+            const incidents = await db.adminAuditEvent.findMany({ where: { profileName, action: 'runtime.failure' } });
+            expect(incidents).toHaveLength(existingIncidents + 1);
+            expect(incidents[0]).toMatchObject({
+                credentialKind: 'DAEMON',
+                errorCode: 'Error',
+                errorMessage: 'running failure',
+            });
+
+            await db.gatewayProfile.update({ where: { profileName }, data: { status: 'RUNNING', lastError: null } });
+            expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe(
+                existingIncidents + 1
+            );
 
             await db.gatewayProfile.update({
                 where: { profileName },
@@ -137,6 +151,9 @@ integration('gateway runtime action consumer', () => {
                 status: 'STOPPED',
                 lastError: null,
             });
+            expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe(
+                existingIncidents + 1
+            );
         } finally {
             await gate.close();
         }

app/gateway-api/src/adminRouter.ts

@@ -2078,6 +2078,26 @@ export const adminRouter = router({
         }),
     }),
     profiles: router({
+        diagnostics: adminProcedure
+            .input(z.object({ profileName: z.string().min(1).max(100) }))
+            .query(async ({ ctx, input }) => {
+                const auth = requireAdminAuth(ctx);
+                if (!canReadProfile(auth, input.profileName)) throw new TRPCError({ code: 'FORBIDDEN' });
+                const profile = await ctx.profiles.getProfile(input.profileName);
+                if (!profile) throw new TRPCError({ code: 'NOT_FOUND' });
+                const [observation, incidents, processes] = await Promise.all([
+                    ctx.orchestrator.inspectRuntime?.(input.profileName) ?? Promise.resolve(null),
+                    ctx.adminAudit.list({ profileName: input.profileName, targetType: 'profile-runtime', limit: 20 }),
+                    ctx.orchestrator.listRuntimeStates([input.profileName]).catch(() => []),
+                ]);
+                return {
+                    profileName: input.profileName,
+                    status: profile.status,
+                    observation,
+                    runtime: processes[0] ?? null,
+                    incidents,
+                };
+            }),
         getResetDefaults: adminProcedure
             .input(z.object({ profileName: z.string().min(1) }))
             .query(async ({ ctx, input }) => {

app/gateway-api/src/orchestrator/gatewayOrchestrator.ts

@@ -20,7 +20,7 @@ import {
     type GameCancellationHistoryMode,
     type GameCancellationResult,
 } from '@sammo-ts/game-engine/scenario/gameCancellation.js';
-import { gatewayProfileCapabilities } from '@sammo-ts/common';
+import { gatewayProfileCapabilities, type ProfileRuntimeDiagnostics } from '@sammo-ts/common';
 import {
     createGamePostgresConnector,
     createRedisConnector,
@@ -177,6 +177,7 @@ export interface GatewayOrchestratorHandle {
     }>;
     listRuntimeStates(profileNames: string[]): Promise<ProfileRuntimeSnapshot[]>;
     listRuntimeSettings?(profileNames: string[]): Promise<ProfileRuntimeSettingsSnapshot[]>;
+    inspectRuntime?(profileName: string): Promise<ProfileRuntimeDiagnostics>;
     transitionProfileClock(
         profileName: string,
         action: 'SUSPEND' | 'RESUME',
@@ -1164,6 +1165,94 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
         return snapshots;
     }
 
+    async inspectRuntime(profileName: string): Promise<ProfileRuntimeDiagnostics> {
+        const processes = await this.processManager.list().catch(() => null);
+        const empty: ProfileRuntimeDiagnostics = {
+            profileName,
+            checkedAt: new Date().toISOString(),
+            database: 'UNINITIALIZED',
+            processObservation: processes ? 'AVAILABLE' : 'UNAVAILABLE',
+            processes: (processes ?? [])
+                .filter((process) => process.name.startsWith(`sammo:${profileName}:`))
+                .map((process) => ({
+                    name: process.name,
+                    status: process.status,
+                    restartCount: process.restartCount ?? 0,
+                    exitCode: process.exitCode ?? null,
+                })),
+            lease: null,
+            clock: null,
+        };
+        const profile = await this.repository.getProfile(profileName);
+        if (!profile || profile.currentScenario === null) return empty;
+        const connector = createGamePostgresConnector({
+            url: this.resolveProfileDatabaseUrl(profile),
+            maxConnections: 1,
+            connectionTimeoutMillis: 3000,
+        });
+        try {
+            await connector.connect();
+            return await connector.prisma.$transaction(
+                async (db) => {
+                    await db.$executeRaw`SET LOCAL statement_timeout = '3000ms'`;
+                    const [time] = await db.$queryRaw<
+                        Array<{ now: Date }>
+                    >`SELECT clock_timestamp() AT TIME ZONE 'UTC' AS now`;
+                    const lease = await db.turnDaemonLease.findUnique({ where: { profile: profileName } });
+                    const clock = await db.worldState.findFirst({
+                        orderBy: { id: 'asc' },
+                        select: {
+                            clockPhase: true,
+                            clockRevision: true,
+                            clockTick: true,
+                            lastTurnTick: true,
+                            currentYear: true,
+                            currentMonth: true,
+                            clockWallAnchor: true,
+                            clockRecoveryStartWallAt: true,
+                            clockRecoveryEndTick: true,
+                        },
+                    });
+                    const now = time!.now;
+                    return {
+                        ...empty,
+                        checkedAt: now.toISOString(),
+                        database: 'AVAILABLE' as const,
+                        lease: lease
+                            ? {
+                                  ownerId: lease.ownerId,
+                                  fencingEpoch: lease.fencingEpoch.toString(),
+                                  heartbeatAt: lease.heartbeatAt.toISOString(),
+                                  leaseUntil: lease.leaseUntil.toISOString(),
+                                  heartbeatAgeMs: Math.max(0, now.getTime() - lease.heartbeatAt.getTime()),
+                                  valid: lease.leaseUntil > now,
+                                  clockReady: lease.clockReady,
+                              }
+                            : null,
+                        clock: clock
+                            ? {
+                                  phase: clock.clockPhase,
+                                  revision: clock.clockRevision.toString(),
+                                  tick: clock.clockTick?.toString() ?? null,
+                                  lastTurnTick: clock.lastTurnTick?.toString() ?? null,
+                                  year: clock.currentYear,
+                                  month: clock.currentMonth,
+                                  wallAnchor: clock.clockWallAnchor?.toISOString() ?? null,
+                                  recoveryStartWallAt: clock.clockRecoveryStartWallAt?.toISOString() ?? null,
+                                  recoveryEndTick: clock.clockRecoveryEndTick?.toString() ?? null,
+                              }
+                            : null,
+                    };
+                },
+                { timeout: 5000, maxWait: 3000 }
+            );
+        } catch {
+            return { ...empty, checkedAt: new Date().toISOString(), database: 'UNAVAILABLE' };
+        } finally {
+            await connector.disconnect().catch(() => undefined);
+        }
+    }
+
     async listRuntimeSettings(profileNames: string[]): Promise<ProfileRuntimeSettingsSnapshot[]> {
         const allowedAutorunOptions = new Set<RuntimeAutorunOption>([
             'develop',

app/gateway-api/src/orchestrator/pm2ProcessManager.ts

@@ -123,6 +123,12 @@ export class Pm2ProcessManager implements ProcessManager {
                                 cwd: item.pm2_env?.pm_cwd ?? undefined,
                                 script: item.pm2_env?.pm_exec_path ?? undefined,
                                 restartCount: item.pm2_env?.restart_time ?? 0,
+                                exitCode:
+                                    item.pm2_env &&
+                                    'exit_code' in item.pm2_env &&
+                                    typeof item.pm2_env.exit_code === 'number'
+                                        ? item.pm2_env.exit_code
+                                        : undefined,
                             })) ?? [];
                         resolve(normalized);
                     });
@@ -145,16 +151,13 @@ export class Pm2ProcessManager implements ProcessManager {
                             reject(new Error(`PM2 process name already exists: ${definition.name}`));
                             return;
                         }
-                        pm2.start(
+                        pm2.start(buildPm2StartOptions(definition), (error) => {
-                            buildPm2StartOptions(definition),
+                            if (error) {
-                            (error) => {
+                                reject(error);
-                                if (error) {
+                                return;
-                                    reject(error);
-                                    return;
-                                }
-                                resolve();
                             }
-                        );
+                            resolve();
+                        });
                     });
                 })
         );

app/gateway-api/src/orchestrator/processManager.ts

@@ -5,6 +5,7 @@ export interface ManagedProcessInfo {
     cwd?: string;
     script?: string;
     restartCount?: number;
+    exitCode?: number;
 }
 
 export interface ProcessDefinition {

app/gateway-api/test/adminOperations.test.ts

@@ -435,6 +435,35 @@ describe('admin profile navigation API', () => {
     });
 });
 
+describe('runtime diagnostics authorization', () => {
+    it('allows the scoped administrator and rejects another profile scope', async () => {
+        const harness = await buildCaller(
+            async () => {
+                throw new Error('not used');
+            },
+            { adminRoles: ['admin.profiles.runtime:che:2'], firstUserIsAdmin: false }
+        );
+        await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'che:2' })).resolves.toMatchObject({
+            profileName: 'che:2',
+            incidents: [],
+        });
+        await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'kwe:2' })).rejects.toMatchObject({
+            code: 'FORBIDDEN',
+        });
+    });
+    it('rejects users without profile administration permission', async () => {
+        const harness = await buildCaller(
+            async () => {
+                throw new Error('not used');
+            },
+            { adminRoles: [], firstUserIsAdmin: false }
+        );
+        await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'che:2' })).rejects.toMatchObject({
+            code: 'FORBIDDEN',
+        });
+    });
+});
+
 describe('admin scenario catalog API', () => {
     it('marks scenario zero as the current selectable scenario', async () => {
         const harness = await buildCaller(

app/gateway-api/test/adminSecurityTransport.e2e.test.ts

@@ -174,6 +174,36 @@ const postTrpc = async (
 };
 
 describe('admin security over HTTP transport', () => {
+    it('protects runtime diagnostics at the HTTP authentication and profile scope boundaries', async () => {
+        const harness = await createHarness(['admin.profiles.runtime:che:default']);
+        const input = { profileName: 'che:default' };
+        expect((await postTrpc(harness.baseUrl, 'admin.profiles.diagnostics', input)).response.status).toBe(401);
+        expect(
+            (await postTrpc(harness.baseUrl, 'admin.profiles.diagnostics', input, harness.adminSessionToken)).response
+                .status
+        ).toBe(200);
+        expect(
+            (
+                await postTrpc(
+                    harness.baseUrl,
+                    'admin.profiles.diagnostics',
+                    { profileName: 'kwe:default' },
+                    harness.adminSessionToken
+                )
+            ).response.status
+        ).toBe(403);
+        expect(
+            (
+                await postTrpc(
+                    harness.baseUrl,
+                    'admin.profiles.diagnostics',
+                    { profileName: '' },
+                    harness.adminSessionToken
+                )
+            ).response.status
+        ).toBe(400);
+    });
+
     it('accepts query input from a POST JSON body but still rejects a mutation sent as GET', async () => {
         const harness = await createHarness();
 

app/gateway-frontend/e2e/admin-runtime-actions.spec.ts

@@ -42,9 +42,11 @@ const installFixture = async (
         currentScenario?: string | null;
         gameIsUnited?: number;
         openerOnly?: boolean;
+        diagnosticsFixture?: boolean;
     } = {}
 ) => {
     let requested = false;
+    let diagnosticReads = 0;
     let installRequested = false;
     let installActive = false;
     let postRequestProfileReads = 0;
@@ -84,6 +86,57 @@ const installFixture = async (
             installActive = true;
         }
         const results = operations.map((operation) => {
+            if (operation === 'admin.profiles.diagnostics' && options.diagnosticsFixture) {
+                diagnosticReads += 1;
+                return response({
+                    profileName: 'che:default',
+                    status: diagnosticReads > 1 ? 'RUNNING' : 'PAUSED',
+                    runtime: { daemonRunning: true },
+                    observation: {
+                        profileName: 'che:default',
+                        checkedAt: '2026-09-09T18:00:00.000Z',
+                        database: 'AVAILABLE',
+                        processObservation: 'AVAILABLE',
+                        processes: [
+                            { name: 'sammo:hwe:default:turn-daemon', status: 'online', restartCount: 1, exitCode: 1 },
+                        ],
+                        lease: {
+                            ownerId: 'owner-0123456789-0123456789-0123456789',
+                            fencingEpoch: '72',
+                            heartbeatAt: '2026-09-09T17:30:00.000Z',
+                            leaseUntil: '2026-09-09T17:30:30.000Z',
+                            heartbeatAgeMs: 1800000,
+                            valid: diagnosticReads > 1,
+                            clockReady: true,
+                        },
+                        clock: {
+                            phase: 'RUNNING',
+                            revision: '6',
+                            tick: '11286094560',
+                            lastTurnTick: '11268000000',
+                            year: 206,
+                            month: 2,
+                            wallAnchor: '2026-09-09T17:00:00.000Z',
+                            recoveryStartWallAt: null,
+                            recoveryEndTick: null,
+                        },
+                    },
+                    incidents: [
+                        {
+                            id: 'incident-1',
+                            createdAt: '2026-09-09T17:30:31.000Z',
+                            errorCode: 'TurnDaemonLeaseLostError',
+                            errorMessage: 'Heartbeat deadline exceeded (30000ms; renewal in flight: true).',
+                            summary: {
+                                year: 206,
+                                month: 2,
+                                clockPhase: 'RUNNING',
+                                frames: ['at flush (/srv/app/flush.ts:42:7)'],
+                            },
+                        },
+                    ],
+                });
+            }
             if (operation === 'me') {
                 return response({
                     id: 'admin-user',
@@ -390,7 +443,9 @@ test('updates live game options from the authoritative database snapshot', async
     await page.screenshot({ path: testInfo.outputPath('runtime-settings-mobile.png'), fullPage: true });
 });
 
-test('distinguishes a turn pause from an inaccessible stopped server in operator controls', async ({ page }, testInfo) => {
+test('distinguishes a turn pause from an inaccessible stopped server in operator controls', async ({
+    page,
+}, testInfo) => {
     await installFixture(page, {
         profileStatus: 'PAUSED',
         pauseReason: "Cannot assign to read only property 'charges' of object '#<Object>'",
@@ -583,3 +638,48 @@ test('directs profile deployment to the selected server version tab', async ({ p
     expect(linkGeometry.right).toBeLessThanOrEqual(linkGeometry.viewportWidth);
     await page.screenshot({ path: testInfo.outputPath('status-tabs-mobile.png'), fullPage: true });
 });
+
+test('runtime diagnostics shows expired lease and retains history after recovery on desktop and mobile', async ({
+    page,
+}, testInfo) => {
+    await installFixture(page, { profileStatus: 'PAUSED', diagnosticsFixture: true });
+    await page.goto('/gateway/admin/servers/hwe%3Adefault');
+    await page.getByRole('button', { name: '장애 진단과 이력' }).click();
+    const panel = page.getByTestId('runtime-diagnostics');
+    await expect(panel).toContainText('턴 실행 권한 만료');
+    await panel.locator('summary').filter({ hasText: 'TurnDaemonLeaseLostError' }).click();
+    await expect(panel).toContainText('Heartbeat deadline exceeded');
+    await expect(panel).toContainText('flush.ts:42:7');
+    await panel.getByText('프로세스 상태와 종료 코드', { exact: true }).click();
+    await expect(panel).toContainText('마지막 종료 코드 1');
+    for (const width of [1200, 390]) {
+        await page.setViewportSize({ width, height: 900 });
+        await page.evaluate(() => document.fonts.ready);
+        const measured = await panel.evaluate((element) => {
+            const rect = element.getBoundingClientRect();
+            const style = getComputedStyle(element);
+            return {
+                left: rect.left,
+                right: rect.right,
+                width: rect.width,
+                viewport: innerWidth,
+                scrollWidth: element.scrollWidth,
+                clientWidth: element.clientWidth,
+                font: style.font,
+                border: style.border,
+                html: element.outerHTML,
+            };
+        });
+        expect(measured.left).toBeGreaterThanOrEqual(0);
+        expect(measured.right).toBeLessThanOrEqual(width);
+        expect(measured.scrollWidth).toBeLessThanOrEqual(measured.clientWidth);
+        await testInfo.attach(`diagnostics-${width}.json`, {
+            body: JSON.stringify(measured),
+            contentType: 'application/json',
+        });
+        await page.screenshot({ path: testInfo.outputPath(`diagnostics-${width}.png`), fullPage: true });
+    }
+    await page.getByRole('button', { name: '장애 진단 새로고침' }).click();
+    await expect(panel).toContainText('턴 프로세스와 실행 권한 정상');
+    await expect(panel).toContainText('Heartbeat deadline exceeded');
+});

app/gateway-frontend/src/components/ProfileRuntimeDiagnostics.vue

@@ -0,0 +1,115 @@
+<script setup lang="ts">
+import { computed, ref } from 'vue';
+import { formatServerDateTime } from '@sammo-ts/common/time/ServerDateTime';
+import { trpc } from '../utils/trpc';
+
+const props = defineProps<{ profileName: string }>();
+type Diagnostics = Awaited<ReturnType<typeof trpc.admin.profiles.diagnostics.query>>;
+const result = ref<Diagnostics | null>(null);
+const opened = ref(false);
+const loading = ref(false);
+const error = ref('');
+const inspect = async (): Promise<void> => {
+    opened.value = true;
+    loading.value = true;
+    error.value = '';
+    try {
+        result.value = await trpc.admin.profiles.diagnostics.query({ profileName: props.profileName });
+    } catch {
+        error.value = '진단 정보를 가져오지 못했습니다. 연결과 조회 권한을 확인한 뒤 다시 시도하세요.';
+    } finally {
+        loading.value = false;
+    }
+};
+const assessment = computed(() => {
+    const data = result.value;
+    if (!data) return '';
+    if (['STOPPED', 'CANCELLED'].includes(data.status)) return '서버 중지 상태';
+    if (!data.observation || data.observation.database === 'UNAVAILABLE') return 'DB 상태 확인 실패';
+    if (data.observation.database === 'UNINITIALIZED') return '게임 DB 초기화 대기';
+    if (data.observation.processObservation === 'UNAVAILABLE') return '프로세스 상태 확인 실패';
+    if (!data.runtime?.daemonRunning) return '턴 데몬 프로세스 중지';
+    if (!data.observation.lease) return '턴 실행 권한 없음';
+    if (!data.observation.lease.valid) return '턴 실행 권한 만료';
+    if (!data.observation.lease.clockReady) return '게임 시계 준비 중';
+    if (data.status === 'PAUSED') return '턴 일시정지 상태';
+    if (data.observation.clock?.phase === 'COMPLETED') return '시즌 종료';
+    if (data.observation.clock?.phase === 'PREOPEN') return '가오픈 대기';
+    const anchor = data.observation.clock?.wallAnchor;
+    if (anchor && anchor > data.observation.checkedAt) return '예정된 시각까지 복구 대기';
+    return '턴 프로세스와 실행 권한 정상';
+});
+const displayTime = (value: string | null | undefined): string => (value ? formatServerDateTime(value) : '없음');
+</script>
+
+<template>
+    <section class="min-w-0 rounded border border-zinc-700 p-3 text-sm" data-testid="runtime-diagnostics">
+        <button
+            type="button"
+            class="rounded border border-zinc-600 px-3 py-1 disabled:opacity-50"
+            :disabled="loading"
+            @click="inspect"
+        >
+            {{ loading ? '조회 중…' : opened ? '장애 진단 새로고침' : '장애 진단과 이력' }}
+        </button>
+        <p v-if="error" role="alert" class="mt-2 text-red-200">{{ error }}</p>
+        <div v-if="opened && result" class="mt-3 min-w-0 space-y-2" data-testid="runtime-diagnostics-result">
+            <p class="font-semibold">{{ assessment }}</p>
+            <p class="text-xs text-zinc-400">
+                조회 시각: {{ displayTime(result.observation?.checkedAt) }} · 상태는 조회 시점 기준입니다.
+            </p>
+            <dl v-if="result.observation" class="grid min-w-0 grid-cols-[auto_minmax(0,1fr)] gap-x-3 gap-y-1 text-xs">
+                <dt>DB 연결</dt>
+                <dd>{{ result.observation.database }}</dd>
+                <dt>마지막 heartbeat</dt>
+                <dd>{{ displayTime(result.observation.lease?.heartbeatAt) }}</dd>
+                <dt>실행 권한 만료</dt>
+                <dd>{{ displayTime(result.observation.lease?.leaseUntil) }}</dd>
+                <dt>시계 상태</dt>
+                <dd>
+                    {{ result.observation.clock?.phase ?? '없음' }} / revision
+                    {{ result.observation.clock?.revision ?? '없음' }}
+                </dd>
+                <dt>게임 연월</dt>
+                <dd>{{ result.observation.clock?.year ?? '-' }}년 {{ result.observation.clock?.month ?? '-' }}월</dd>
+                <dt>마지막 처리 tick</dt>
+                <dd class="break-all">{{ result.observation.clock?.lastTurnTick ?? '없음' }}</dd>
+                <dt>복구 시작</dt>
+                <dd>{{ displayTime(result.observation.clock?.recoveryStartWallAt) }}</dd>
+                <dt>owner / epoch</dt>
+                <dd class="break-all">
+                    {{ result.observation.lease?.ownerId ?? '없음' }} /
+                    {{ result.observation.lease?.fencingEpoch ?? '-' }}
+                </dd>
+            </dl>
+            <p class="text-xs text-zinc-300">
+                정지 원인을 해결한 뒤 턴을 재개하세요. 배포 작업 중이거나 복구 시작 시각을 기다리는 경우에는 해당 작업과
+                일정을 먼저 확인하세요.
+            </p>
+            <details v-if="result.observation?.processes?.length" class="min-w-0 text-xs">
+                <summary class="cursor-pointer">프로세스 상태와 종료 코드</summary>
+                <p v-for="process in result.observation.processes" :key="process.name" class="mt-1 break-all">
+                    {{ process.name }}: {{ process.status }} · 재시작 {{ process.restartCount }}회 · 마지막 종료 코드
+                    {{ process.exitCode ?? '없음' }}
+                </p>
+            </details>
+            <h4 class="pt-2 font-semibold">최근 장애 이력</h4>
+            <p v-if="!result.incidents.length" class="text-xs text-zinc-400">
+                저장된 장애 이력이 없습니다. 이력 수집 이전의 오류는 현재 정지 사유와 운영 로그에서 확인하세요.
+            </p>
+            <details
+                v-for="incident in result.incidents"
+                :key="incident.id"
+                class="min-w-0 rounded border border-red-900/60 p-2"
+            >
+                <summary class="cursor-pointer break-words">
+                    {{ displayTime(incident.createdAt) }} · {{ incident.errorCode }}
+                </summary>
+                <p class="mt-2 whitespace-pre-wrap break-words text-xs text-red-200">{{ incident.errorMessage }}</p>
+                <pre class="mt-2 whitespace-pre-wrap break-all text-xs text-zinc-400">{{
+                    JSON.stringify(incident.summary, null, 2)
+                }}</pre>
+            </details>
+        </div>
+    </section>
+</template>

app/gateway-frontend/src/views/AdminView.vue

@@ -7,6 +7,7 @@ import {
 import { gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common/gateway/profileStatus';
 import { computed, onMounted, ref, watch } from 'vue';
 import ServerProfileTabs from '../components/ServerProfileTabs.vue';
+import ProfileRuntimeDiagnostics from '../components/ProfileRuntimeDiagnostics.vue';
 import AdminConsoleLayout from '../layouts/AdminConsoleLayout.vue';
 import { useToast } from '../composables/useToast';
 import {
@@ -2443,6 +2444,7 @@ onMounted(() => {
                             </div>
 
                             <div class="text-xs text-zinc-400">빌드 커밋: {{ profile.buildCommitSha ?? '미지정' }}</div>
+                            <ProfileRuntimeDiagnostics :profile-name="profile.profileName" />
 
                             <div
                                 v-if="profile.status === 'PAUSED' && profile.lastError"

docs/release-operations.md

@@ -306,6 +306,27 @@ process 복구를 시도합니다. 관리자 화면의 오류와 PM2 process 상
 뒤 원인을 해결하고 실패한 작업을 재시도해 주세요. 재시도는 처음 고정된 commit을
 사용합니다.
 
+### 서버 장애 진단
+
+`/gateway/admin/servers/:profileName`의 **장애 진단과 이력**에서 직접 조회합니다.
+프로필 읽기 권한을 서버에서 검사하며 다른 profile의 이력은 반환하지 않습니다.
+
+- DB 시각으로 계산한 lease 유효성·마지막 heartbeat·만료 시각과 owner/epoch,
+  시계 phase/revision·마지막 처리 tick·복구 시작 시각을 보여줍니다.
+- PM2 상태·재시작 횟수·마지막 종료 코드를 별도로 표시합니다. 프로세스가 online이어도
+  lease가 만료되면 실행 권한 만료로 분류합니다. DB 또는 PM2 조회 실패는 확인 실패로
+  표시하며 정상이나 중지로 단정하지 않습니다.
+- 실행·명령·lifecycle 및 초기화 실패는 `admin_audit_event`의
+  `action=runtime.failure`, `credentialKind=DAEMON`으로 저장합니다. profile PAUSED
+  전환과 같은 transaction이며 현재 오류와 같은 반복 보고는 중복 저장하지 않습니다.
+  재개·배포로 `lastError`가 사라져도 append-only 이력은 남습니다.
+- 오류 종류·정화한 메시지·최대 8개 stack frame과 게임 연월/시계/lease 좌표를
+  저장합니다. 연결 URL과 인증값은 제거합니다. Gateway DB 자체에 기록할 수 없는
+  장애는 정화한 오류를 프로세스 로그에 남깁니다. OOM·강제 종료처럼 hook을 실행하지
+  못한 경우에는 PM2 종료 정보와 호스트 로그를 함께 확인합니다.
+- 화면은 조회 시점의 snapshot입니다. 원인을 해결한 뒤 재개하고 새로고침하여
+  lease와 시계를 다시 확인합니다. 복구 시작 시각 전의 대기를 장애로 오인하지 않습니다.
+
 VM 중단이나 DB 연결 장애로 turn-daemon lease가 만료되면 기존 owner는 턴과
 관리자 mutation을 처리할 수 없습니다. Lifecycle은 이를 즉시 `lastError`와
 `PAUSED`로 기록하고 종료합니다. PM2가 새 runtime과 DB snapshot으로 시작한 뒤

packages/common/src/gateway/runtimeDiagnostics.ts

@@ -0,0 +1,51 @@
+/** 관리자 장애 기록에서도 연결 URL과 인증값을 보존하지 않는다. */
+export const sanitizeRuntimeErrorText = (text: string): string =>
+    text
+        .replace(/\b(?:https?|postgres(?:ql)?|rediss?):\/\/[^\s"'<>]+/gi, '[REDACTED_URL]')
+        .replace(/\bBearer\s+[^\s"',;]+/gi, 'Bearer [REDACTED]')
+        .replace(
+            /((?:password|passwd|token|secret|authorization|cookie|api[_-]?key)["']?\s*[:=]\s*)(?:"[^"\n]*"|'[^'\n]*'|[^\s,;]+)/gi,
+            '$1[REDACTED]'
+        )
+        .slice(0, 2000);
+
+export const describeRuntimeError = (error: unknown): { code: string; message: string; frames: string[] } => ({
+    code: error instanceof Error ? error.name.slice(0, 100) : 'RuntimeError',
+    message: sanitizeRuntimeErrorText(error instanceof Error ? error.message : String(error)),
+    frames:
+        error instanceof Error
+            ? (error.stack ?? '')
+                  .split('\n')
+                  .filter((line) => /^\s*at\s/.test(line))
+                  .slice(0, 8)
+                  .map(sanitizeRuntimeErrorText)
+            : [],
+});
+
+export interface ProfileRuntimeDiagnostics {
+    profileName: string;
+    checkedAt: string;
+    database: 'AVAILABLE' | 'UNAVAILABLE' | 'UNINITIALIZED';
+    processObservation: 'AVAILABLE' | 'UNAVAILABLE';
+    processes: Array<{ name: string; status: string; restartCount: number; exitCode: number | null }>;
+    lease: {
+        ownerId: string;
+        fencingEpoch: string;
+        heartbeatAt: string;
+        leaseUntil: string;
+        heartbeatAgeMs: number;
+        valid: boolean;
+        clockReady: boolean;
+    } | null;
+    clock: {
+        phase: string;
+        revision: string;
+        tick: string | null;
+        lastTurnTick: string | null;
+        year: number;
+        month: number;
+        wallAnchor: string | null;
+        recoveryStartWallAt: string | null;
+        recoveryEndTick: string | null;
+    } | null;
+}

packages/common/src/index.ts

@@ -28,6 +28,7 @@ export * from './auth/accountIconProjection.js';
 export * from './logging/formatLegacyLogHtml.js';
 export * from './legacyArchive/ArchivedGeneralSnapshot.js';
 export * from './gateway/profileStatus.js';
+export * from './gateway/runtimeDiagnostics.js';
 export * from './game/accessPenalty.js';
 export * from './http/trpcTransport.js';
 export * from './webPush/types.js';

packages/common/test/runtimeDiagnostics.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { describeRuntimeError } from '../src/gateway/runtimeDiagnostics.js';
+
+describe('runtime failure records', () => {
+    it('keeps the cause and frames while removing connection and authentication values', () => {
+        const error = new Error(
+            'database failed postgresql://admin:private@host/db password="hidden value" token=abc Bearer xyz'
+        );
+        error.stack = `${error.message}\n    at flush (/srv/app/flush.ts:42:7)`;
+        const record = describeRuntimeError(error);
+        expect(record.code).toBe('Error');
+        expect(record.message).toContain('database failed');
+        for (const secret of ['private', 'hidden value', 'abc', 'xyz'])
+            expect(JSON.stringify(record)).not.toContain(secret);
+        expect(record.frames).toEqual(['    at flush (/srv/app/flush.ts:42:7)']);
+    });
+    it('bounds untrusted messages and stack depth', () => {
+        const error = new Error('x'.repeat(4000));
+        error.stack = Array.from({ length: 30 }, () => ' at run (/app/run.ts:1:1)').join('\n');
+        expect(describeRuntimeError(error).message).toHaveLength(2000);
+        expect(describeRuntimeError(error).frames).toHaveLength(8);
+    });
+});

packages/infra/src/postgres.ts

@@ -15,6 +15,7 @@ export interface PostgresConfig {
     log?: PostgresLogOption[];
     maxConnections?: number;
     sessionTimezone?: 'UTC';
+    connectionTimeoutMillis?: number;
 }
 
 export interface PostgresPoolStats {
@@ -65,16 +66,18 @@ const buildSharedPoolKey = (
     url: string,
     schema: string | undefined,
     maxConnections: number,
-    sessionTimezone: 'UTC' | undefined
+    sessionTimezone: 'UTC' | undefined,
-): string => JSON.stringify([url, schema ?? '', maxConnections, sessionTimezone ?? '']);
+    connectionTimeoutMillis: number | undefined
+): string => JSON.stringify([url, schema ?? '', maxConnections, sessionTimezone ?? '', connectionTimeoutMillis ?? 0]);
 
 const acquireSharedPool = (
     url: string,
     schema: string | undefined,
     maxConnections: number,
-    sessionTimezone: 'UTC' | undefined
+    sessionTimezone: 'UTC' | undefined,
+    connectionTimeoutMillis: number | undefined
 ): { entry: SharedPoolEntry; release: () => Promise<void> } => {
-    const key = buildSharedPoolKey(url, schema, maxConnections, sessionTimezone);
+    const key = buildSharedPoolKey(url, schema, maxConnections, sessionTimezone, connectionTimeoutMillis);
     let entry = sharedPools.get(key);
     if (!entry) {
         const connectionOptions = [
@@ -86,6 +89,7 @@ const acquireSharedPool = (
         const pool = new pg.Pool({
             connectionString: url,
             max: maxConnections,
+            ...(connectionTimeoutMillis !== undefined ? { connectionTimeoutMillis } : {}),
             ...(connectionOptions ? { options: connectionOptions } : {}),
         });
         entry = { pool, references: 0, maxConnections };
@@ -173,7 +177,13 @@ export const createPostgresConnector = <TClient>(
     const schema =
         extractSchemaFromDatabaseUrl(config.url) ?? process.env.POSTGRES_SCHEMA ?? process.env.DATABASE_SCHEMA;
     const maxConnections = resolvePostgresPoolMax(config.maxConnections ?? process.env.POSTGRES_POOL_MAX);
-    const sharedPool = acquireSharedPool(config.url, schema, maxConnections, config.sessionTimezone);
+    const sharedPool = acquireSharedPool(
+        config.url,
+        schema,
+        maxConnections,
+        config.sessionTimezone,
+        config.connectionTimeoutMillis
+    );
     const adapter = new PrismaPg(sharedPool.entry.pool, schema ? { schema } : undefined);
     const prisma = createClient({
         adapter,