devsam/core2026
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

관리자 서버 진단에 lease 상태와 영속 장애 이력 추가

Browse Source
This commit is contained in:
Hide_D
2026-09-09 23:36:16 +00:00
parent 9c73085353
commit a00f46dfc8
19 changed files with 627 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts
+22 -10
View File
@@ -24,8 +24,8 @@ export class TurnDaemonLeaseUnavailableError extends Error {
}
export class TurnDaemonLeaseLostError extends Error {
constructor(profile: string) {
super(`Turn daemon lease was lost for profile "${profile}".`);
constructor(profile: string, reason?: string) {
super(`Turn daemon lease was lost for profile "${profile}".${reason ? ` ${reason}` : ''}`);
this.name = 'TurnDaemonLeaseLostError';
}
}
@@ -50,6 +50,7 @@ export class DatabaseTurnDaemonLease {
private expiryTimer: NodeJS.Timeout | null = null;
private renewalInFlight = false;
private lost = false;
private lossReason: string | undefined;
private constructor(
db: GamePrismaClient,
@@ -116,6 +117,7 @@ export class DatabaseTurnDaemonLease {
fencingEpoch: BigInt(row.fencing_epoch),
};
this.lost = false;
this.lossReason = undefined;
this.scheduleExpiryWatchdog(requestStartedAt);
if (this.heartbeatEnabled) {
this.startHeartbeat();
@@ -139,6 +141,10 @@ export class DatabaseTurnDaemonLease {
return this.lost;
}
getLossError(): TurnDaemonLeaseLostError {
return new TurnDaemonLeaseLostError(this.profile, this.lossReason);
}
async renew(): Promise<boolean> {
const token = this.token;
if (!token || this.lost || this.renewalInFlight) {
@@ -160,7 +166,7 @@ export class DatabaseTurnDaemonLease {
RETURNING "profile", "owner_id", "fencing_epoch"
`);
if (rows.length === 0) {
this.markLost();
this.markLost('Heartbeat renewal rejected: lease expired or owner/epoch changed.');
return false;
}
if (this.lost) {
@@ -176,7 +182,7 @@ export class DatabaseTurnDaemonLease {
async assertActive(transaction?: GamePrisma.TransactionClient): Promise<void> {
const token = this.token;
if (!token || this.lost) {
throw new TurnDaemonLeaseLostError(this.profile);
throw this.getLossError();
}
const db = transaction ?? this.db;
const rows = await db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
@@ -190,8 +196,8 @@ export class DatabaseTurnDaemonLease {
FOR UPDATE
`);
if (rows.length === 0) {
this.markLost();
throw new TurnDaemonLeaseLostError(this.profile);
this.markLost('Transaction fencing rejected: lease expired or owner/epoch changed.');
throw this.getLossError();
}
}
@@ -228,8 +234,10 @@ export class DatabaseTurnDaemonLease {
}
const intervalMs = Math.max(250, Math.floor(this.leaseDurationMs / 3));
this.heartbeatTimer = setInterval(() => {
void this.renew().catch(() => {
this.markLost();
void this.renew().catch((error: unknown) => {
this.markLost(
`Heartbeat database request failed (${error instanceof Error ? error.name : 'unknown error'}).`
);
});
}, intervalMs);
this.heartbeatTimer.unref();
@@ -248,7 +256,9 @@ export class DatabaseTurnDaemonLease {
this.stopExpiryWatchdog();
const remainingMs = Math.max(0, this.leaseDurationMs - (performance.now() - requestStartedAt));
this.expiryTimer = setTimeout(() => {
this.markLost();
this.markLost(
`Heartbeat deadline exceeded (${this.leaseDurationMs}ms; renewal in flight: ${this.renewalInFlight}).`
);
}, remainingMs);
this.expiryTimer.unref();
}
@@ -260,7 +270,9 @@ export class DatabaseTurnDaemonLease {
}
}
private markLost(): void {
private markLost(reason: string): void {
if (this.lost) return;
this.lossReason = reason;
this.lost = true;
this.stopHeartbeat();
this.stopExpiryWatchdog();
app/game-engine/src/turn/cli.ts
+23
View File
@@ -5,6 +5,8 @@ import type { TurnRunBudget } from '../lifecycle/types.js';
import { resolveDatabaseUrl } from '../scenario/databaseUrl.js';
import { createTurnDaemonRuntime } from './turnDaemon.js';
import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
import { createGatewayProfileGate } from './gatewayProfileGate.js';
import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
export interface TurnDaemonCliOptions {
profile?: string;
@@ -89,6 +91,27 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
pauseGateIntervalMs,
adminActionIntervalMs,
gameClockMode,
}).catch(async (error: unknown) => {
// 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는
// lifecycle hook이 아직 없으므로 여기서 별도로 관리자에게 기록한다.
if (!(error instanceof TurnDaemonLeaseUnavailableError)) {
try {
const gate = await createGatewayProfileGate({
databaseUrl,
gatewayDatabaseUrl,
profileName,
incidentContext: () => ({ stage: 'startup' }),
});
try {
await gate.markPaused(error);
} finally {
await gate.close();
}
} catch {
/* 원래 시작 실패를 보존한다. */
}
}
throw error;
});
const memoryReporter = createTurnDaemonMemoryReporter({
app/game-engine/src/turn/gatewayProfileGate.ts
+39 -11
View File
@@ -1,6 +1,7 @@
import { performance } from 'node:perf_hooks';
import { randomUUID } from 'node:crypto';
import { gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common';
import { describeRuntimeError, gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common';
import { createGatewayPostgresConnector } from '@sammo-ts/infra';
export interface GatewayProfileGateOptions {
@@ -8,6 +9,7 @@ export interface GatewayProfileGateOptions {
gatewayDatabaseUrl?: string;
profileName: string;
cacheMs?: number;
incidentContext?: () => Record<string, string | number | boolean | null>;
}
export interface GatewayProfileGate {
@@ -22,6 +24,7 @@ const PROFILE_STATUSES_MARKABLE_AS_PAUSED = ['PREOPEN', 'RUNNING', 'PAUSED'] as
export const createGatewayProfileGate = async (options: GatewayProfileGateOptions): Promise<GatewayProfileGate> => {
const connector = createGatewayPostgresConnector({
url: options.gatewayDatabaseUrl ?? options.databaseUrl,
connectionTimeoutMillis: 3000,
});
await connector.connect();
const prisma = connector.prisma;
@@ -54,19 +57,44 @@ export const createGatewayProfileGate = async (options: GatewayProfileGateOption
return cachedPause;
},
async markPaused(error?: unknown): Promise<void> {
const message = error instanceof Error ? error.message : error ? String(error) : null;
const failure = error ? describeRuntimeError(error) : null;
const message = failure?.message ?? null;
try {
await prisma.gatewayProfile.updateMany({
where: {
profileName: options.profileName,
status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
},
data: {
status: 'PAUSED',
lastError: message,
},
await prisma.$transaction(async (tx) => {
const updated = await tx.gatewayProfile.updateMany({
where: {
profileName: options.profileName,
status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
OR: [{ status: { not: 'PAUSED' } }, { lastError: { not: message } }, { lastError: null }],
},
data: {
status: 'PAUSED',
lastError: message,
},
});
if (updated.count && failure) {
// 상태와 이력을 함께 commit한다. 재개가 lastError를 지워도
// 당시 원인과 실행 좌표는 관리자 감사 저장소에 남는다.
await tx.adminAuditEvent.create({
data: {
correlationId: randomUUID(),
actorUserId: 'system:turn-daemon',
actorUsername: 'turn-daemon',
credentialKind: 'DAEMON',
action: 'runtime.failure',
targetType: 'profile-runtime',
targetId: options.profileName,
profileName: options.profileName,
outcome: 'FAILED',
errorCode: failure.code,
errorMessage: failure.message,
summary: { frames: failure.frames, ...options.incidentContext?.() },
},
});
}
});
} catch {
if (failure) console.error('[turn-daemon] failed to persist runtime incident', failure);
return;
}
},
app/game-engine/src/turn/turnDaemon.ts
+15 -6
View File
@@ -93,11 +93,7 @@ import {
createResetOfficerLockHandler,
} from './monthlyCoreEventAction.js';
import { buildCommandEnv } from './reservedTurnCommands.js';
import {
DatabaseTurnDaemonLease,
TurnDaemonLeaseLostError,
TurnDaemonLeaseUnavailableError,
} from '../lifecycle/databaseTurnDaemonLease.js';
import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
import { EngineStateManager } from './engineStateManager.js';
import { applyRuntimeClockShift } from './runtimeClockShift.js';
import { applyRuntimeGameSettings } from './runtimeGameSettings.js';
@@ -892,6 +888,19 @@ const createTurnDaemonRuntimeWithLease = async (
gatewayDatabaseUrl: options.gatewayDatabaseUrl,
profileName: options.profileName,
cacheMs: options.pauseGateIntervalMs,
incidentContext: () => {
const state = world.getState();
const clock = world.getGameClockState();
const token = turnDaemonLease?.getToken();
return {
year: state.currentYear,
month: state.currentMonth,
clockPhase: clock.phase,
clockTick: clock.tick,
ownerId: token?.ownerId ?? null,
fencingEpoch: token?.fencingEpoch.toString() ?? null,
};
},
})
: null;
if (gatewayGate) {
@@ -1066,7 +1075,7 @@ const createTurnDaemonRuntimeWithLease = async (
if (turnDaemonLease?.isLost()) {
// 만료된 owner는 재개 명령도 처리할 수 없다. 현재 runtime을
// 끝내 PM2가 새 owner와 DB snapshot으로 시작하도록 한다.
throw new TurnDaemonLeaseLostError(options.profileName ?? options.profile);
throw turnDaemonLease.getLossError();
}
const gatewayPaused = (await pauseGate?.()) ?? false;
const phase = world.getGameClockState().phase;
app/game-engine/test/gatewayRuntimeAction.integration.test.ts
+17
View File
@@ -112,6 +112,7 @@ integration('gateway runtime action consumer', () => {
});
it('does not overwrite a terminal operator status while reporting a daemon error', async () => {
const existingIncidents = await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } });
const gate = await createGatewayProfileGate({
databaseUrl: databaseUrl!,
gatewayDatabaseUrl: databaseUrl!,
@@ -127,6 +128,19 @@ integration('gateway runtime action consumer', () => {
status: 'PAUSED',
lastError: 'running failure',
});
await gate.markPaused(new Error('running failure'));
const incidents = await db.adminAuditEvent.findMany({ where: { profileName, action: 'runtime.failure' } });
expect(incidents).toHaveLength(existingIncidents + 1);
expect(incidents[0]).toMatchObject({
credentialKind: 'DAEMON',
errorCode: 'Error',
errorMessage: 'running failure',
});
await db.gatewayProfile.update({ where: { profileName }, data: { status: 'RUNNING', lastError: null } });
expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe(
existingIncidents + 1
);
await db.gatewayProfile.update({
where: { profileName },
@@ -137,6 +151,9 @@ integration('gateway runtime action consumer', () => {
status: 'STOPPED',
lastError: null,
});
expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe(
existingIncidents + 1
);
} finally {
await gate.close();
}
app/gateway-api/src/adminRouter.ts
+20
View File
@@ -2078,6 +2078,26 @@ export const adminRouter = router({
}),
}),
profiles: router({
diagnostics: adminProcedure
.input(z.object({ profileName: z.string().min(1).max(100) }))
.query(async ({ ctx, input }) => {
const auth = requireAdminAuth(ctx);
if (!canReadProfile(auth, input.profileName)) throw new TRPCError({ code: 'FORBIDDEN' });
const profile = await ctx.profiles.getProfile(input.profileName);
if (!profile) throw new TRPCError({ code: 'NOT_FOUND' });
const [observation, incidents, processes] = await Promise.all([
ctx.orchestrator.inspectRuntime?.(input.profileName) ?? Promise.resolve(null),
ctx.adminAudit.list({ profileName: input.profileName, targetType: 'profile-runtime', limit: 20 }),
ctx.orchestrator.listRuntimeStates([input.profileName]).catch(() => []),
]);
return {
profileName: input.profileName,
status: profile.status,
observation,
runtime: processes[0] ?? null,
incidents,
};
}),
getResetDefaults: adminProcedure
.input(z.object({ profileName: z.string().min(1) }))
.query(async ({ ctx, input }) => {
app/gateway-api/src/orchestrator/gatewayOrchestrator.ts
+90 -1
View File
@@ -20,7 +20,7 @@ import {
type GameCancellationHistoryMode,
type GameCancellationResult,
} from '@sammo-ts/game-engine/scenario/gameCancellation.js';
import { gatewayProfileCapabilities } from '@sammo-ts/common';
import { gatewayProfileCapabilities, type ProfileRuntimeDiagnostics } from '@sammo-ts/common';
import {
createGamePostgresConnector,
createRedisConnector,
@@ -177,6 +177,7 @@ export interface GatewayOrchestratorHandle {
}>;
listRuntimeStates(profileNames: string[]): Promise<ProfileRuntimeSnapshot[]>;
listRuntimeSettings?(profileNames: string[]): Promise<ProfileRuntimeSettingsSnapshot[]>;
inspectRuntime?(profileName: string): Promise<ProfileRuntimeDiagnostics>;
transitionProfileClock(
profileName: string,
action: 'SUSPEND' | 'RESUME',
@@ -1164,6 +1165,94 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
return snapshots;
}
async inspectRuntime(profileName: string): Promise<ProfileRuntimeDiagnostics> {
const processes = await this.processManager.list().catch(() => null);
const empty: ProfileRuntimeDiagnostics = {
profileName,
checkedAt: new Date().toISOString(),
database: 'UNINITIALIZED',
processObservation: processes ? 'AVAILABLE' : 'UNAVAILABLE',
processes: (processes ?? [])
.filter((process) => process.name.startsWith(`sammo:${profileName}:`))
.map((process) => ({
name: process.name,
status: process.status,
restartCount: process.restartCount ?? 0,
exitCode: process.exitCode ?? null,
})),
lease: null,
clock: null,
};
const profile = await this.repository.getProfile(profileName);
if (!profile || profile.currentScenario === null) return empty;
const connector = createGamePostgresConnector({
url: this.resolveProfileDatabaseUrl(profile),
maxConnections: 1,
connectionTimeoutMillis: 3000,
});
try {
await connector.connect();
return await connector.prisma.$transaction(
async (db) => {
await db.$executeRaw`SET LOCAL statement_timeout = '3000ms'`;
const [time] = await db.$queryRaw<
Array<{ now: Date }>
>`SELECT clock_timestamp() AT TIME ZONE 'UTC' AS now`;
const lease = await db.turnDaemonLease.findUnique({ where: { profile: profileName } });
const clock = await db.worldState.findFirst({
orderBy: { id: 'asc' },
select: {
clockPhase: true,
clockRevision: true,
clockTick: true,
lastTurnTick: true,
currentYear: true,
currentMonth: true,
clockWallAnchor: true,
clockRecoveryStartWallAt: true,
clockRecoveryEndTick: true,
},
});
const now = time!.now;
return {
...empty,
checkedAt: now.toISOString(),
database: 'AVAILABLE' as const,
lease: lease
? {
ownerId: lease.ownerId,
fencingEpoch: lease.fencingEpoch.toString(),
heartbeatAt: lease.heartbeatAt.toISOString(),
leaseUntil: lease.leaseUntil.toISOString(),
heartbeatAgeMs: Math.max(0, now.getTime() - lease.heartbeatAt.getTime()),
valid: lease.leaseUntil > now,
clockReady: lease.clockReady,
}
: null,
clock: clock
? {
phase: clock.clockPhase,
revision: clock.clockRevision.toString(),
tick: clock.clockTick?.toString() ?? null,
lastTurnTick: clock.lastTurnTick?.toString() ?? null,
year: clock.currentYear,
month: clock.currentMonth,
wallAnchor: clock.clockWallAnchor?.toISOString() ?? null,
recoveryStartWallAt: clock.clockRecoveryStartWallAt?.toISOString() ?? null,
recoveryEndTick: clock.clockRecoveryEndTick?.toString() ?? null,
}
: null,
};
},
{ timeout: 5000, maxWait: 3000 }
);
} catch {
return { ...empty, checkedAt: new Date().toISOString(), database: 'UNAVAILABLE' };
} finally {
await connector.disconnect().catch(() => undefined);
}
}
async listRuntimeSettings(profileNames: string[]): Promise<ProfileRuntimeSettingsSnapshot[]> {
const allowedAutorunOptions = new Set<RuntimeAutorunOption>([
'develop',
app/gateway-api/src/orchestrator/pm2ProcessManager.ts
+12 -9
View File
@@ -123,6 +123,12 @@ export class Pm2ProcessManager implements ProcessManager {
cwd: item.pm2_env?.pm_cwd ?? undefined,
script: item.pm2_env?.pm_exec_path ?? undefined,
restartCount: item.pm2_env?.restart_time ?? 0,
exitCode:
item.pm2_env &&
'exit_code' in item.pm2_env &&
typeof item.pm2_env.exit_code === 'number'
? item.pm2_env.exit_code
: undefined,
})) ?? [];
resolve(normalized);
});
@@ -145,16 +151,13 @@ export class Pm2ProcessManager implements ProcessManager {
reject(new Error(`PM2 process name already exists: ${definition.name}`));
return;
}
pm2.start(
buildPm2StartOptions(definition),
(error) => {
if (error) {
reject(error);
return;
}
resolve();
pm2.start(buildPm2StartOptions(definition), (error) => {
if (error) {
reject(error);
return;
}
);
resolve();
});
});
})
);
app/gateway-api/src/orchestrator/processManager.ts
+1
View File
@@ -5,6 +5,7 @@ export interface ManagedProcessInfo {
cwd?: string;
script?: string;
restartCount?: number;
exitCode?: number;
}
export interface ProcessDefinition {
app/gateway-api/test/adminOperations.test.ts
+29
View File
@@ -435,6 +435,35 @@ describe('admin profile navigation API', () => {
});
});
describe('runtime diagnostics authorization', () => {
it('allows the scoped administrator and rejects another profile scope', async () => {
const harness = await buildCaller(
async () => {
throw new Error('not used');
},
{ adminRoles: ['admin.profiles.runtime:che:2'], firstUserIsAdmin: false }
);
await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'che:2' })).resolves.toMatchObject({
profileName: 'che:2',
incidents: [],
});
await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'kwe:2' })).rejects.toMatchObject({
code: 'FORBIDDEN',
});
});
it('rejects users without profile administration permission', async () => {
const harness = await buildCaller(
async () => {
throw new Error('not used');
},
{ adminRoles: [], firstUserIsAdmin: false }
);
await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'che:2' })).rejects.toMatchObject({
code: 'FORBIDDEN',
});
});
});
describe('admin scenario catalog API', () => {
it('marks scenario zero as the current selectable scenario', async () => {
const harness = await buildCaller(
app/gateway-api/test/adminSecurityTransport.e2e.test.ts
+30
View File
@@ -174,6 +174,36 @@ const postTrpc = async (
};
describe('admin security over HTTP transport', () => {
it('protects runtime diagnostics at the HTTP authentication and profile scope boundaries', async () => {
const harness = await createHarness(['admin.profiles.runtime:che:default']);
const input = { profileName: 'che:default' };
expect((await postTrpc(harness.baseUrl, 'admin.profiles.diagnostics', input)).response.status).toBe(401);
expect(
(await postTrpc(harness.baseUrl, 'admin.profiles.diagnostics', input, harness.adminSessionToken)).response
.status
).toBe(200);
expect(
(
await postTrpc(
harness.baseUrl,
'admin.profiles.diagnostics',
{ profileName: 'kwe:default' },
harness.adminSessionToken
)
).response.status
).toBe(403);
expect(
(
await postTrpc(
harness.baseUrl,
'admin.profiles.diagnostics',
{ profileName: '' },
harness.adminSessionToken
)
).response.status
).toBe(400);
});
it('accepts query input from a POST JSON body but still rejects a mutation sent as GET', async () => {
const harness = await createHarness();
app/gateway-frontend/e2e/admin-runtime-actions.spec.ts
+101 -1
View File
@@ -42,9 +42,11 @@ const installFixture = async (
currentScenario?: string | null;
gameIsUnited?: number;
openerOnly?: boolean;
diagnosticsFixture?: boolean;
} = {}
) => {
let requested = false;
let diagnosticReads = 0;
let installRequested = false;
let installActive = false;
let postRequestProfileReads = 0;
@@ -84,6 +86,57 @@ const installFixture = async (
installActive = true;
}
const results = operations.map((operation) => {
if (operation === 'admin.profiles.diagnostics' && options.diagnosticsFixture) {
diagnosticReads += 1;
return response({
profileName: 'che:default',
status: diagnosticReads > 1 ? 'RUNNING' : 'PAUSED',
runtime: { daemonRunning: true },
observation: {
profileName: 'che:default',
checkedAt: '2026-09-09T18:00:00.000Z',
database: 'AVAILABLE',
processObservation: 'AVAILABLE',
processes: [
{ name: 'sammo:hwe:default:turn-daemon', status: 'online', restartCount: 1, exitCode: 1 },
],
lease: {
ownerId: 'owner-0123456789-0123456789-0123456789',
fencingEpoch: '72',
heartbeatAt: '2026-09-09T17:30:00.000Z',
leaseUntil: '2026-09-09T17:30:30.000Z',
heartbeatAgeMs: 1800000,
valid: diagnosticReads > 1,
clockReady: true,
},
clock: {
phase: 'RUNNING',
revision: '6',
tick: '11286094560',
lastTurnTick: '11268000000',
year: 206,
month: 2,
wallAnchor: '2026-09-09T17:00:00.000Z',
recoveryStartWallAt: null,
recoveryEndTick: null,
},
},
incidents: [
{
id: 'incident-1',
createdAt: '2026-09-09T17:30:31.000Z',
errorCode: 'TurnDaemonLeaseLostError',
errorMessage: 'Heartbeat deadline exceeded (30000ms; renewal in flight: true).',
summary: {
year: 206,
month: 2,
clockPhase: 'RUNNING',
frames: ['at flush (/srv/app/flush.ts:42:7)'],
},
},
],
});
}
if (operation === 'me') {
return response({
id: 'admin-user',
@@ -390,7 +443,9 @@ test('updates live game options from the authoritative database snapshot', async
await page.screenshot({ path: testInfo.outputPath('runtime-settings-mobile.png'), fullPage: true });
});
test('distinguishes a turn pause from an inaccessible stopped server in operator controls', async ({ page }, testInfo) => {
test('distinguishes a turn pause from an inaccessible stopped server in operator controls', async ({
page,
}, testInfo) => {
await installFixture(page, {
profileStatus: 'PAUSED',
pauseReason: "Cannot assign to read only property 'charges' of object '#<Object>'",
@@ -583,3 +638,48 @@ test('directs profile deployment to the selected server version tab', async ({ p
expect(linkGeometry.right).toBeLessThanOrEqual(linkGeometry.viewportWidth);
await page.screenshot({ path: testInfo.outputPath('status-tabs-mobile.png'), fullPage: true });
});
test('runtime diagnostics shows expired lease and retains history after recovery on desktop and mobile', async ({
page,
}, testInfo) => {
await installFixture(page, { profileStatus: 'PAUSED', diagnosticsFixture: true });
await page.goto('/gateway/admin/servers/hwe%3Adefault');
await page.getByRole('button', { name: '장애 진단과 이력' }).click();
const panel = page.getByTestId('runtime-diagnostics');
await expect(panel).toContainText('턴 실행 권한 만료');
await panel.locator('summary').filter({ hasText: 'TurnDaemonLeaseLostError' }).click();
await expect(panel).toContainText('Heartbeat deadline exceeded');
await expect(panel).toContainText('flush.ts:42:7');
await panel.getByText('프로세스 상태와 종료 코드', { exact: true }).click();
await expect(panel).toContainText('마지막 종료 코드 1');
for (const width of [1200, 390]) {
await page.setViewportSize({ width, height: 900 });
await page.evaluate(() => document.fonts.ready);
const measured = await panel.evaluate((element) => {
const rect = element.getBoundingClientRect();
const style = getComputedStyle(element);
return {
left: rect.left,
right: rect.right,
width: rect.width,
viewport: innerWidth,
scrollWidth: element.scrollWidth,
clientWidth: element.clientWidth,
font: style.font,
border: style.border,
html: element.outerHTML,
};
});
expect(measured.left).toBeGreaterThanOrEqual(0);
expect(measured.right).toBeLessThanOrEqual(width);
expect(measured.scrollWidth).toBeLessThanOrEqual(measured.clientWidth);
await testInfo.attach(`diagnostics-${width}.json`, {
body: JSON.stringify(measured),
contentType: 'application/json',
});
await page.screenshot({ path: testInfo.outputPath(`diagnostics-${width}.png`), fullPage: true });
}
await page.getByRole('button', { name: '장애 진단 새로고침' }).click();
await expect(panel).toContainText('턴 프로세스와 실행 권한 정상');
await expect(panel).toContainText('Heartbeat deadline exceeded');
});
app/gateway-frontend/src/components/ProfileRuntimeDiagnostics.vue
+115
View File
@@ -0,0 +1,115 @@
<script setup lang="ts">
import { computed, ref } from 'vue';
import { formatServerDateTime } from '@sammo-ts/common/time/ServerDateTime';
import { trpc } from '../utils/trpc';
const props = defineProps<{ profileName: string }>();
type Diagnostics = Awaited<ReturnType<typeof trpc.admin.profiles.diagnostics.query>>;
const result = ref<Diagnostics | null>(null);
const opened = ref(false);
const loading = ref(false);
const error = ref('');
const inspect = async (): Promise<void> => {
opened.value = true;
loading.value = true;
error.value = '';
try {
result.value = await trpc.admin.profiles.diagnostics.query({ profileName: props.profileName });
} catch {
error.value = '진단 정보를 가져오지 못했습니다. 연결과 조회 권한을 확인한 뒤 다시 시도하세요.';
} finally {
loading.value = false;
}
};
const assessment = computed(() => {
const data = result.value;
if (!data) return '';
if (['STOPPED', 'CANCELLED'].includes(data.status)) return '서버 중지 상태';
if (!data.observation || data.observation.database === 'UNAVAILABLE') return 'DB 상태 확인 실패';
if (data.observation.database === 'UNINITIALIZED') return '게임 DB 초기화 대기';
if (data.observation.processObservation === 'UNAVAILABLE') return '프로세스 상태 확인 실패';
if (!data.runtime?.daemonRunning) return '턴 데몬 프로세스 중지';
if (!data.observation.lease) return '턴 실행 권한 없음';
if (!data.observation.lease.valid) return '턴 실행 권한 만료';
if (!data.observation.lease.clockReady) return '게임 시계 준비 중';
if (data.status === 'PAUSED') return '턴 일시정지 상태';
if (data.observation.clock?.phase === 'COMPLETED') return '시즌 종료';
if (data.observation.clock?.phase === 'PREOPEN') return '가오픈 대기';
const anchor = data.observation.clock?.wallAnchor;
if (anchor && anchor > data.observation.checkedAt) return '예정된 시각까지 복구 대기';
return '턴 프로세스와 실행 권한 정상';
});
const displayTime = (value: string | null | undefined): string => (value ? formatServerDateTime(value) : '없음');
</script>
<template>
<section class="min-w-0 rounded border border-zinc-700 p-3 text-sm" data-testid="runtime-diagnostics">
<button
type="button"
class="rounded border border-zinc-600 px-3 py-1 disabled:opacity-50"
:disabled="loading"
@click="inspect"
>
{{ loading ? '조회 중…' : opened ? '장애 진단 새로고침' : '장애 진단과 이력' }}
</button>
<p v-if="error" role="alert" class="mt-2 text-red-200">{{ error }}</p>
<div v-if="opened && result" class="mt-3 min-w-0 space-y-2" data-testid="runtime-diagnostics-result">
<p class="font-semibold">{{ assessment }}</p>
<p class="text-xs text-zinc-400">
조회 시각: {{ displayTime(result.observation?.checkedAt) }} · 상태는 조회 시점 기준입니다.
</p>
<dl v-if="result.observation" class="grid min-w-0 grid-cols-[auto_minmax(0,1fr)] gap-x-3 gap-y-1 text-xs">
<dt>DB 연결</dt>
<dd>{{ result.observation.database }}</dd>
<dt>마지막 heartbeat</dt>
<dd>{{ displayTime(result.observation.lease?.heartbeatAt) }}</dd>
<dt>실행 권한 만료</dt>
<dd>{{ displayTime(result.observation.lease?.leaseUntil) }}</dd>
<dt>시계 상태</dt>
<dd>
{{ result.observation.clock?.phase ?? '없음' }} / revision
{{ result.observation.clock?.revision ?? '없음' }}
</dd>
<dt>게임 연월</dt>
<dd>{{ result.observation.clock?.year ?? '-' }}년 {{ result.observation.clock?.month ?? '-' }}월</dd>
<dt>마지막 처리 tick</dt>
<dd class="break-all">{{ result.observation.clock?.lastTurnTick ?? '없음' }}</dd>
<dt>복구 시작</dt>
<dd>{{ displayTime(result.observation.clock?.recoveryStartWallAt) }}</dd>
<dt>owner / epoch</dt>
<dd class="break-all">
{{ result.observation.lease?.ownerId ?? '없음' }} /
{{ result.observation.lease?.fencingEpoch ?? '-' }}
</dd>
</dl>
<p class="text-xs text-zinc-300">
정지 원인을 해결한 뒤 턴을 재개하세요. 배포 작업 중이거나 복구 시작 시각을 기다리는 경우에는 해당 작업과
일정을 먼저 확인하세요.
</p>
<details v-if="result.observation?.processes?.length" class="min-w-0 text-xs">
<summary class="cursor-pointer">프로세스 상태와 종료 코드</summary>
<p v-for="process in result.observation.processes" :key="process.name" class="mt-1 break-all">
{{ process.name }}: {{ process.status }} · 재시작 {{ process.restartCount }}회 · 마지막 종료 코드
{{ process.exitCode ?? '없음' }}
</p>
</details>
<h4 class="pt-2 font-semibold">최근 장애 이력</h4>
<p v-if="!result.incidents.length" class="text-xs text-zinc-400">
저장된 장애 이력이 없습니다. 이력 수집 이전의 오류는 현재 정지 사유와 운영 로그에서 확인하세요.
</p>
<details
v-for="incident in result.incidents"
:key="incident.id"
class="min-w-0 rounded border border-red-900/60 p-2"
>
<summary class="cursor-pointer break-words">
{{ displayTime(incident.createdAt) }} · {{ incident.errorCode }}
</summary>
<p class="mt-2 whitespace-pre-wrap break-words text-xs text-red-200">{{ incident.errorMessage }}</p>
<pre class="mt-2 whitespace-pre-wrap break-all text-xs text-zinc-400">{{
JSON.stringify(incident.summary, null, 2)
}}</pre>
</details>
</div>
</section>
</template>
app/gateway-frontend/src/views/AdminView.vue
+2
View File
@@ -7,6 +7,7 @@ import {
import { gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common/gateway/profileStatus';
import { computed, onMounted, ref, watch } from 'vue';
import ServerProfileTabs from '../components/ServerProfileTabs.vue';
import ProfileRuntimeDiagnostics from '../components/ProfileRuntimeDiagnostics.vue';
import AdminConsoleLayout from '../layouts/AdminConsoleLayout.vue';
import { useToast } from '../composables/useToast';
import {
@@ -2443,6 +2444,7 @@ onMounted(() => {
</div>
<div class="text-xs text-zinc-400">빌드 커밋: {{ profile.buildCommitSha ?? '미지정' }}</div>
<ProfileRuntimeDiagnostics :profile-name="profile.profileName" />
<div
v-if="profile.status === 'PAUSED' && profile.lastError"