feat: 턴 데몬 메모리 계측을 추가한다
RSS와 V8 heap, 월드 entity 수를 bounded 로그로 기록하고 heap 상한 80% 이상을 경고한다.
This commit is contained in:
@@ -24,6 +24,7 @@ export * from './turn/joinCreateGeneralService.js';
|
|||||||
export * from './turn/npcPossessionService.js';
|
export * from './turn/npcPossessionService.js';
|
||||||
export * from './turn/selectPoolService.js';
|
export * from './turn/selectPoolService.js';
|
||||||
export * from './turn/turnDaemon.js';
|
export * from './turn/turnDaemon.js';
|
||||||
|
export * from './turn/turnDaemonMemoryReporter.js';
|
||||||
export * from './turn/cli.js';
|
export * from './turn/cli.js';
|
||||||
|
|
||||||
export const shouldRunTurnDaemon = (role: string | undefined): boolean => role === 'turn-daemon';
|
export const shouldRunTurnDaemon = (role: string | undefined): boolean => role === 'turn-daemon';
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { parseOptionalBoolean, parseOptionalNumber, type GameClockMode } from '@
|
|||||||
import type { TurnRunBudget } from '../lifecycle/types.js';
|
import type { TurnRunBudget } from '../lifecycle/types.js';
|
||||||
import { resolveDatabaseUrl } from '../scenario/databaseUrl.js';
|
import { resolveDatabaseUrl } from '../scenario/databaseUrl.js';
|
||||||
import { createTurnDaemonRuntime } from './turnDaemon.js';
|
import { createTurnDaemonRuntime } from './turnDaemon.js';
|
||||||
|
import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
|
||||||
|
|
||||||
export interface TurnDaemonCliOptions {
|
export interface TurnDaemonCliOptions {
|
||||||
profile?: string;
|
profile?: string;
|
||||||
@@ -16,6 +17,7 @@ export interface TurnDaemonCliOptions {
|
|||||||
budget?: Partial<TurnRunBudget>;
|
budget?: Partial<TurnRunBudget>;
|
||||||
enableDatabaseFlush?: boolean;
|
enableDatabaseFlush?: boolean;
|
||||||
adminActionIntervalMs?: number;
|
adminActionIntervalMs?: number;
|
||||||
|
memoryReportIntervalMs?: number;
|
||||||
gameClockMode?: GameClockMode;
|
gameClockMode?: GameClockMode;
|
||||||
env?: NodeJS.ProcessEnv;
|
env?: NodeJS.ProcessEnv;
|
||||||
}
|
}
|
||||||
@@ -26,6 +28,9 @@ const DEFAULT_BUDGET: TurnRunBudget = {
|
|||||||
catchUpCap: 1,
|
catchUpCap: 1,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const DEFAULT_MEMORY_REPORT_INTERVAL_MS = 5 * 60 * 1000;
|
||||||
|
const MIN_MEMORY_REPORT_INTERVAL_MS = 10 * 1000;
|
||||||
|
|
||||||
const buildBudgetOverride = (env: NodeJS.ProcessEnv, override?: Partial<TurnRunBudget>): TurnRunBudget | undefined => {
|
const buildBudgetOverride = (env: NodeJS.ProcessEnv, override?: Partial<TurnRunBudget>): TurnRunBudget | undefined => {
|
||||||
const budgetOverride: Partial<TurnRunBudget> = {
|
const budgetOverride: Partial<TurnRunBudget> = {
|
||||||
budgetMs: parseOptionalNumber(env.TURN_BUDGET_MS),
|
budgetMs: parseOptionalNumber(env.TURN_BUDGET_MS),
|
||||||
@@ -59,6 +64,13 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
|
|||||||
const enableDatabaseFlush = options.enableDatabaseFlush ?? parseOptionalBoolean(env.TURN_FLUSH_DB) ?? true;
|
const enableDatabaseFlush = options.enableDatabaseFlush ?? parseOptionalBoolean(env.TURN_FLUSH_DB) ?? true;
|
||||||
const pauseGateIntervalMs = parseOptionalNumber(env.TURN_PAUSE_GATE_MS);
|
const pauseGateIntervalMs = parseOptionalNumber(env.TURN_PAUSE_GATE_MS);
|
||||||
const adminActionIntervalMs = options.adminActionIntervalMs ?? parseOptionalNumber(env.TURN_ADMIN_ACTION_MS);
|
const adminActionIntervalMs = options.adminActionIntervalMs ?? parseOptionalNumber(env.TURN_ADMIN_ACTION_MS);
|
||||||
|
const memoryReportIntervalMs =
|
||||||
|
options.memoryReportIntervalMs ??
|
||||||
|
parseOptionalNumber(env.TURN_MEMORY_REPORT_INTERVAL_MS) ??
|
||||||
|
DEFAULT_MEMORY_REPORT_INTERVAL_MS;
|
||||||
|
if (!Number.isFinite(memoryReportIntervalMs) || memoryReportIntervalMs < MIN_MEMORY_REPORT_INTERVAL_MS) {
|
||||||
|
throw new Error(`TURN_MEMORY_REPORT_INTERVAL_MS must be at least ${MIN_MEMORY_REPORT_INTERVAL_MS}.`);
|
||||||
|
}
|
||||||
const rawGameClockMode = options.gameClockMode ?? env.GAME_CLOCK_MODE;
|
const rawGameClockMode = options.gameClockMode ?? env.GAME_CLOCK_MODE;
|
||||||
if (rawGameClockMode && rawGameClockMode !== 'realtime' && rawGameClockMode !== 'manual') {
|
if (rawGameClockMode && rawGameClockMode !== 'realtime' && rawGameClockMode !== 'manual') {
|
||||||
throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`);
|
throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`);
|
||||||
@@ -79,12 +91,28 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
|
|||||||
gameClockMode,
|
gameClockMode,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const memoryReporter = createTurnDaemonMemoryReporter({
|
||||||
|
profile,
|
||||||
|
intervalMs: memoryReportIntervalMs,
|
||||||
|
getContext: () => {
|
||||||
|
const state = runtime.world.getState();
|
||||||
|
return {
|
||||||
|
year: state.currentYear,
|
||||||
|
month: state.currentMonth,
|
||||||
|
...runtime.world.getEntityCounts(),
|
||||||
|
lifecycleState: runtime.lifecycle.getStatus().state,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
let closed = false;
|
let closed = false;
|
||||||
const closeOnce = async (): Promise<void> => {
|
const closeOnce = async (): Promise<void> => {
|
||||||
if (closed) {
|
if (closed) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
closed = true;
|
closed = true;
|
||||||
|
memoryReporter.report('shutdown');
|
||||||
|
memoryReporter.stop();
|
||||||
await runtime.close();
|
await runtime.close();
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -104,6 +132,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
|
|||||||
|
|
||||||
const activeTickMinutes = tickMinutes ?? Math.max(1, Math.round(runtime.world.getState().tickSeconds / 60));
|
const activeTickMinutes = tickMinutes ?? Math.max(1, Math.round(runtime.world.getState().tickSeconds / 60));
|
||||||
console.info(`[turn-daemon] started profile=${profile} tickMinutes=${activeTickMinutes}`);
|
console.info(`[turn-daemon] started profile=${profile} tickMinutes=${activeTickMinutes}`);
|
||||||
|
memoryReporter.report('startup');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await runtime.lifecycle.start();
|
await runtime.lifecycle.start();
|
||||||
|
|||||||
@@ -881,6 +881,22 @@ export class InMemoryTurnWorld {
|
|||||||
return { ...this.state };
|
return { ...this.state };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getEntityCounts(): {
|
||||||
|
generals: number;
|
||||||
|
cities: number;
|
||||||
|
nations: number;
|
||||||
|
troops: number;
|
||||||
|
events: number;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
generals: this.generals.size,
|
||||||
|
cities: this.cities.size,
|
||||||
|
nations: this.nations.size,
|
||||||
|
troops: this.troops.size,
|
||||||
|
events: this.events.size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
updateWorldMeta(patch: Record<string, unknown>): void {
|
updateWorldMeta(patch: Record<string, unknown>): void {
|
||||||
this.state = {
|
this.state = {
|
||||||
...this.state,
|
...this.state,
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
import { getHeapStatistics } from 'node:v8';
|
||||||
|
|
||||||
|
export interface TurnDaemonMemoryContext {
|
||||||
|
year: number;
|
||||||
|
month: number;
|
||||||
|
generals: number;
|
||||||
|
cities: number;
|
||||||
|
nations: number;
|
||||||
|
troops: number;
|
||||||
|
events: number;
|
||||||
|
lifecycleState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TurnDaemonMemoryReporterOptions {
|
||||||
|
profile: string;
|
||||||
|
intervalMs: number;
|
||||||
|
getContext(): TurnDaemonMemoryContext;
|
||||||
|
info?: (message: string) => void;
|
||||||
|
warn?: (message: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const BYTES_PER_MIB = 1024 * 1024;
|
||||||
|
const HEAP_WARNING_RATIO = 0.8;
|
||||||
|
|
||||||
|
const toMiB = (value: number): number => Math.round((value / BYTES_PER_MIB) * 10) / 10;
|
||||||
|
|
||||||
|
export const buildTurnDaemonMemoryReport = (
|
||||||
|
profile: string,
|
||||||
|
reason: string,
|
||||||
|
context: TurnDaemonMemoryContext,
|
||||||
|
memory = process.memoryUsage(),
|
||||||
|
heapLimitBytes = getHeapStatistics().heap_size_limit
|
||||||
|
): { message: string; warning: boolean } => {
|
||||||
|
const heapRatio = heapLimitBytes > 0 ? memory.heapUsed / heapLimitBytes : 0;
|
||||||
|
const message = [
|
||||||
|
`[turn-daemon:memory] profile=${profile}`,
|
||||||
|
`reason=${reason}`,
|
||||||
|
`rssMiB=${toMiB(memory.rss)}`,
|
||||||
|
`heapUsedMiB=${toMiB(memory.heapUsed)}`,
|
||||||
|
`heapTotalMiB=${toMiB(memory.heapTotal)}`,
|
||||||
|
`heapLimitMiB=${toMiB(heapLimitBytes)}`,
|
||||||
|
`externalMiB=${toMiB(memory.external)}`,
|
||||||
|
`arrayBuffersMiB=${toMiB(memory.arrayBuffers)}`,
|
||||||
|
`year=${context.year}`,
|
||||||
|
`month=${context.month}`,
|
||||||
|
`generals=${context.generals}`,
|
||||||
|
`cities=${context.cities}`,
|
||||||
|
`nations=${context.nations}`,
|
||||||
|
`troops=${context.troops}`,
|
||||||
|
`events=${context.events}`,
|
||||||
|
`lifecycle=${context.lifecycleState}`,
|
||||||
|
].join(' ');
|
||||||
|
return { message, warning: heapRatio >= HEAP_WARNING_RATIO };
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createTurnDaemonMemoryReporter = (
|
||||||
|
options: TurnDaemonMemoryReporterOptions
|
||||||
|
): { report(reason: string): void; stop(): void } => {
|
||||||
|
const info = options.info ?? console.info;
|
||||||
|
const warn = options.warn ?? console.warn;
|
||||||
|
const report = (reason: string): void => {
|
||||||
|
const result = buildTurnDaemonMemoryReport(options.profile, reason, options.getContext());
|
||||||
|
(result.warning ? warn : info)(result.message);
|
||||||
|
};
|
||||||
|
const timer = setInterval(() => report('interval'), options.intervalMs);
|
||||||
|
timer.unref();
|
||||||
|
return {
|
||||||
|
report,
|
||||||
|
stop: () => clearInterval(timer),
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { buildTurnDaemonMemoryReport } from '../src/turn/turnDaemonMemoryReporter.js';
|
||||||
|
|
||||||
|
const context = {
|
||||||
|
year: 214,
|
||||||
|
month: 12,
|
||||||
|
generals: 2461,
|
||||||
|
cities: 78,
|
||||||
|
nations: 3,
|
||||||
|
troops: 15,
|
||||||
|
events: 4,
|
||||||
|
lifecycleState: 'paused',
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('turn daemon memory reporting', () => {
|
||||||
|
it('reports bounded process and world-size fields without inspecting or cloning entities', () => {
|
||||||
|
const result = buildTurnDaemonMemoryReport(
|
||||||
|
'hwe',
|
||||||
|
'interval',
|
||||||
|
context,
|
||||||
|
{
|
||||||
|
rss: 1_258_291_200,
|
||||||
|
heapTotal: 1_100_000_000,
|
||||||
|
heapUsed: 900_000_000,
|
||||||
|
external: 20_000_000,
|
||||||
|
arrayBuffers: 10_000_000,
|
||||||
|
},
|
||||||
|
3_221_225_472
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.warning).toBe(false);
|
||||||
|
expect(result.message).toContain('profile=hwe reason=interval');
|
||||||
|
expect(result.message).toContain('heapLimitMiB=3072');
|
||||||
|
expect(result.message).toContain('year=214 month=12 generals=2461');
|
||||||
|
expect(result.message).toContain('lifecycle=paused');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('marks samples at or above 80 percent of the V8 heap limit as warnings', () => {
|
||||||
|
const result = buildTurnDaemonMemoryReport(
|
||||||
|
'hwe',
|
||||||
|
'interval',
|
||||||
|
context,
|
||||||
|
{
|
||||||
|
rss: 1_500_000_000,
|
||||||
|
heapTotal: 1_400_000_000,
|
||||||
|
heapUsed: 1_288_490_189,
|
||||||
|
external: 0,
|
||||||
|
arrayBuffers: 0,
|
||||||
|
},
|
||||||
|
1_610_612_736
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.warning).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user