merge: 턴 데몬 메모리 재발 방지 계측을 반영한다
This commit is contained in:
@@ -24,6 +24,7 @@ export * from './turn/joinCreateGeneralService.js';
|
||||
export * from './turn/npcPossessionService.js';
|
||||
export * from './turn/selectPoolService.js';
|
||||
export * from './turn/turnDaemon.js';
|
||||
export * from './turn/turnDaemonMemoryReporter.js';
|
||||
export * from './turn/cli.js';
|
||||
|
||||
export const shouldRunTurnDaemon = (role: string | undefined): boolean => role === 'turn-daemon';
|
||||
|
||||
@@ -4,6 +4,7 @@ import { parseOptionalBoolean, parseOptionalNumber, type GameClockMode } from '@
|
||||
import type { TurnRunBudget } from '../lifecycle/types.js';
|
||||
import { resolveDatabaseUrl } from '../scenario/databaseUrl.js';
|
||||
import { createTurnDaemonRuntime } from './turnDaemon.js';
|
||||
import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
|
||||
|
||||
export interface TurnDaemonCliOptions {
|
||||
profile?: string;
|
||||
@@ -16,6 +17,7 @@ export interface TurnDaemonCliOptions {
|
||||
budget?: Partial<TurnRunBudget>;
|
||||
enableDatabaseFlush?: boolean;
|
||||
adminActionIntervalMs?: number;
|
||||
memoryReportIntervalMs?: number;
|
||||
gameClockMode?: GameClockMode;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}
|
||||
@@ -26,6 +28,9 @@ const DEFAULT_BUDGET: TurnRunBudget = {
|
||||
catchUpCap: 1,
|
||||
};
|
||||
|
||||
const DEFAULT_MEMORY_REPORT_INTERVAL_MS = 5 * 60 * 1000;
|
||||
const MIN_MEMORY_REPORT_INTERVAL_MS = 10 * 1000;
|
||||
|
||||
const buildBudgetOverride = (env: NodeJS.ProcessEnv, override?: Partial<TurnRunBudget>): TurnRunBudget | undefined => {
|
||||
const budgetOverride: Partial<TurnRunBudget> = {
|
||||
budgetMs: parseOptionalNumber(env.TURN_BUDGET_MS),
|
||||
@@ -59,6 +64,13 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
|
||||
const enableDatabaseFlush = options.enableDatabaseFlush ?? parseOptionalBoolean(env.TURN_FLUSH_DB) ?? true;
|
||||
const pauseGateIntervalMs = parseOptionalNumber(env.TURN_PAUSE_GATE_MS);
|
||||
const adminActionIntervalMs = options.adminActionIntervalMs ?? parseOptionalNumber(env.TURN_ADMIN_ACTION_MS);
|
||||
const memoryReportIntervalMs =
|
||||
options.memoryReportIntervalMs ??
|
||||
parseOptionalNumber(env.TURN_MEMORY_REPORT_INTERVAL_MS) ??
|
||||
DEFAULT_MEMORY_REPORT_INTERVAL_MS;
|
||||
if (!Number.isFinite(memoryReportIntervalMs) || memoryReportIntervalMs < MIN_MEMORY_REPORT_INTERVAL_MS) {
|
||||
throw new Error(`TURN_MEMORY_REPORT_INTERVAL_MS must be at least ${MIN_MEMORY_REPORT_INTERVAL_MS}.`);
|
||||
}
|
||||
const rawGameClockMode = options.gameClockMode ?? env.GAME_CLOCK_MODE;
|
||||
if (rawGameClockMode && rawGameClockMode !== 'realtime' && rawGameClockMode !== 'manual') {
|
||||
throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`);
|
||||
@@ -79,12 +91,28 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
|
||||
gameClockMode,
|
||||
});
|
||||
|
||||
const memoryReporter = createTurnDaemonMemoryReporter({
|
||||
profile,
|
||||
intervalMs: memoryReportIntervalMs,
|
||||
getContext: () => {
|
||||
const state = runtime.world.getState();
|
||||
return {
|
||||
year: state.currentYear,
|
||||
month: state.currentMonth,
|
||||
...runtime.world.getEntityCounts(),
|
||||
lifecycleState: runtime.lifecycle.getStatus().state,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
let closed = false;
|
||||
const closeOnce = async (): Promise<void> => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
memoryReporter.report('shutdown');
|
||||
memoryReporter.stop();
|
||||
await runtime.close();
|
||||
};
|
||||
|
||||
@@ -104,6 +132,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
|
||||
|
||||
const activeTickMinutes = tickMinutes ?? Math.max(1, Math.round(runtime.world.getState().tickSeconds / 60));
|
||||
console.info(`[turn-daemon] started profile=${profile} tickMinutes=${activeTickMinutes}`);
|
||||
memoryReporter.report('startup');
|
||||
|
||||
try {
|
||||
await runtime.lifecycle.start();
|
||||
|
||||
@@ -881,6 +881,22 @@ export class InMemoryTurnWorld {
|
||||
return { ...this.state };
|
||||
}
|
||||
|
||||
getEntityCounts(): {
|
||||
generals: number;
|
||||
cities: number;
|
||||
nations: number;
|
||||
troops: number;
|
||||
events: number;
|
||||
} {
|
||||
return {
|
||||
generals: this.generals.size,
|
||||
cities: this.cities.size,
|
||||
nations: this.nations.size,
|
||||
troops: this.troops.size,
|
||||
events: this.events.size,
|
||||
};
|
||||
}
|
||||
|
||||
updateWorldMeta(patch: Record<string, unknown>): void {
|
||||
this.state = {
|
||||
...this.state,
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { getHeapStatistics } from 'node:v8';
|
||||
|
||||
export interface TurnDaemonMemoryContext {
|
||||
year: number;
|
||||
month: number;
|
||||
generals: number;
|
||||
cities: number;
|
||||
nations: number;
|
||||
troops: number;
|
||||
events: number;
|
||||
lifecycleState: string;
|
||||
}
|
||||
|
||||
export interface TurnDaemonMemoryReporterOptions {
|
||||
profile: string;
|
||||
intervalMs: number;
|
||||
getContext(): TurnDaemonMemoryContext;
|
||||
info?: (message: string) => void;
|
||||
warn?: (message: string) => void;
|
||||
}
|
||||
|
||||
const BYTES_PER_MIB = 1024 * 1024;
|
||||
const HEAP_WARNING_RATIO = 0.8;
|
||||
|
||||
const toMiB = (value: number): number => Math.round((value / BYTES_PER_MIB) * 10) / 10;
|
||||
|
||||
export const buildTurnDaemonMemoryReport = (
|
||||
profile: string,
|
||||
reason: string,
|
||||
context: TurnDaemonMemoryContext,
|
||||
memory = process.memoryUsage(),
|
||||
heapLimitBytes = getHeapStatistics().heap_size_limit
|
||||
): { message: string; warning: boolean } => {
|
||||
const heapRatio = heapLimitBytes > 0 ? memory.heapUsed / heapLimitBytes : 0;
|
||||
const message = [
|
||||
`[turn-daemon:memory] profile=${profile}`,
|
||||
`reason=${reason}`,
|
||||
`rssMiB=${toMiB(memory.rss)}`,
|
||||
`heapUsedMiB=${toMiB(memory.heapUsed)}`,
|
||||
`heapTotalMiB=${toMiB(memory.heapTotal)}`,
|
||||
`heapLimitMiB=${toMiB(heapLimitBytes)}`,
|
||||
`externalMiB=${toMiB(memory.external)}`,
|
||||
`arrayBuffersMiB=${toMiB(memory.arrayBuffers)}`,
|
||||
`year=${context.year}`,
|
||||
`month=${context.month}`,
|
||||
`generals=${context.generals}`,
|
||||
`cities=${context.cities}`,
|
||||
`nations=${context.nations}`,
|
||||
`troops=${context.troops}`,
|
||||
`events=${context.events}`,
|
||||
`lifecycle=${context.lifecycleState}`,
|
||||
].join(' ');
|
||||
return { message, warning: heapRatio >= HEAP_WARNING_RATIO };
|
||||
};
|
||||
|
||||
export const createTurnDaemonMemoryReporter = (
|
||||
options: TurnDaemonMemoryReporterOptions
|
||||
): { report(reason: string): void; stop(): void } => {
|
||||
const info = options.info ?? console.info;
|
||||
const warn = options.warn ?? console.warn;
|
||||
const report = (reason: string): void => {
|
||||
const result = buildTurnDaemonMemoryReport(options.profile, reason, options.getContext());
|
||||
(result.warning ? warn : info)(result.message);
|
||||
};
|
||||
const timer = setInterval(() => report('interval'), options.intervalMs);
|
||||
timer.unref();
|
||||
return {
|
||||
report,
|
||||
stop: () => clearInterval(timer),
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { buildTurnDaemonMemoryReport } from '../src/turn/turnDaemonMemoryReporter.js';
|
||||
|
||||
const context = {
|
||||
year: 214,
|
||||
month: 12,
|
||||
generals: 2461,
|
||||
cities: 78,
|
||||
nations: 3,
|
||||
troops: 15,
|
||||
events: 4,
|
||||
lifecycleState: 'paused',
|
||||
};
|
||||
|
||||
describe('turn daemon memory reporting', () => {
|
||||
it('reports bounded process and world-size fields without inspecting or cloning entities', () => {
|
||||
const result = buildTurnDaemonMemoryReport(
|
||||
'hwe',
|
||||
'interval',
|
||||
context,
|
||||
{
|
||||
rss: 1_258_291_200,
|
||||
heapTotal: 1_100_000_000,
|
||||
heapUsed: 900_000_000,
|
||||
external: 20_000_000,
|
||||
arrayBuffers: 10_000_000,
|
||||
},
|
||||
3_221_225_472
|
||||
);
|
||||
|
||||
expect(result.warning).toBe(false);
|
||||
expect(result.message).toContain('profile=hwe reason=interval');
|
||||
expect(result.message).toContain('heapLimitMiB=3072');
|
||||
expect(result.message).toContain('year=214 month=12 generals=2461');
|
||||
expect(result.message).toContain('lifecycle=paused');
|
||||
});
|
||||
|
||||
it('marks samples at or above 80 percent of the V8 heap limit as warnings', () => {
|
||||
const result = buildTurnDaemonMemoryReport(
|
||||
'hwe',
|
||||
'interval',
|
||||
context,
|
||||
{
|
||||
rss: 1_500_000_000,
|
||||
heapTotal: 1_400_000_000,
|
||||
heapUsed: 1_288_490_189,
|
||||
external: 0,
|
||||
arrayBuffers: 0,
|
||||
},
|
||||
1_610_612_736
|
||||
);
|
||||
|
||||
expect(result.warning).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user