perf: NPC 생명주기 메모리 프로파일과 큐 정리를 추가한다
This commit is contained in:
@@ -107,6 +107,7 @@
|
|||||||
"lint:fix": "eslint . --fix",
|
"lint:fix": "eslint . --fix",
|
||||||
"profile:npc-unification-memory": "node scripts/profile-npc-unification-memory.mjs",
|
"profile:npc-unification-memory": "node scripts/profile-npc-unification-memory.mjs",
|
||||||
"profile:npc-unification-timing": "node scripts/profile-npc-unification-timing.mjs",
|
"profile:npc-unification-timing": "node scripts/profile-npc-unification-timing.mjs",
|
||||||
|
"profile:npc-lifecycle-memory": "node scripts/profile-npc-lifecycle-memory.mjs",
|
||||||
"profile:npc-capacity-1200": "node scripts/profile-npc-capacity-1200.mjs",
|
"profile:npc-capacity-1200": "node scripts/profile-npc-capacity-1200.mjs",
|
||||||
"test": "vitest run --config vitest.config.ts",
|
"test": "vitest run --config vitest.config.ts",
|
||||||
"typecheck": "pnpm -w tsc7 -b app/game-engine/tsconfig.json"
|
"typecheck": "pnpm -w tsc7 -b app/game-engine/tsconfig.json"
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||||
|
const vitestPath = path.join(packageRoot, 'node_modules', 'vitest', 'vitest.mjs');
|
||||||
|
const defaultRuns = [
|
||||||
|
{ scenario: 'steady-state', pruneDeletedQueues: true },
|
||||||
|
{ scenario: 'growth', pruneDeletedQueues: true },
|
||||||
|
{ scenario: 'death-drain', pruneDeletedQueues: false },
|
||||||
|
{ scenario: 'death-drain', pruneDeletedQueues: true },
|
||||||
|
{ scenario: 'balanced-churn', pruneDeletedQueues: false },
|
||||||
|
{ scenario: 'balanced-churn', pruneDeletedQueues: true },
|
||||||
|
{ scenario: 'rollback-churn', pruneDeletedQueues: true },
|
||||||
|
];
|
||||||
|
|
||||||
|
const parseRuns = () => {
|
||||||
|
const raw = process.env.NPC_LIFECYCLE_MEMORY_SCENARIOS;
|
||||||
|
if (!raw) return defaultRuns;
|
||||||
|
return raw.split(',').map((entry) => {
|
||||||
|
const [scenario, variant = 'prune'] = entry.trim().split('@');
|
||||||
|
if (!scenario) throw new Error(`invalid scenario entry: ${entry}`);
|
||||||
|
if (variant !== 'prune' && variant !== 'retain') {
|
||||||
|
throw new Error(`scenario variant must be prune or retain: ${entry}`);
|
||||||
|
}
|
||||||
|
return { scenario, pruneDeletedQueues: variant === 'prune' };
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const runChild = (run, reportPath, repetition) =>
|
||||||
|
new Promise((resolve, reject) => {
|
||||||
|
const child = spawn(
|
||||||
|
process.execPath,
|
||||||
|
[
|
||||||
|
'--expose-gc',
|
||||||
|
vitestPath,
|
||||||
|
'run',
|
||||||
|
'--config',
|
||||||
|
'vitest.config.ts',
|
||||||
|
'--pool=threads',
|
||||||
|
'--maxWorkers=1',
|
||||||
|
'test/npcLifecycleMemoryProfile.test.ts',
|
||||||
|
],
|
||||||
|
{
|
||||||
|
cwd: packageRoot,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
NPC_LIFECYCLE_MEMORY_PROFILE: '1',
|
||||||
|
NPC_LIFECYCLE_MEMORY_SCENARIO: run.scenario,
|
||||||
|
NPC_LIFECYCLE_MEMORY_PRUNE_DELETED: run.pruneDeletedQueues ? '1' : '0',
|
||||||
|
NPC_LIFECYCLE_MEMORY_CHILD_REPORT_PATH: reportPath,
|
||||||
|
NPC_LIFECYCLE_MEMORY_REPETITION: String(repetition),
|
||||||
|
},
|
||||||
|
stdio: 'inherit',
|
||||||
|
}
|
||||||
|
);
|
||||||
|
child.once('error', reject);
|
||||||
|
child.once('exit', (code, signal) => {
|
||||||
|
if (signal) {
|
||||||
|
reject(new Error(`${run.scenario} terminated by ${signal}`));
|
||||||
|
} else if (code !== 0) {
|
||||||
|
reject(new Error(`${run.scenario} exited with code ${code}`));
|
||||||
|
} else {
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const runs = parseRuns();
|
||||||
|
const repetitions = Number(process.env.NPC_LIFECYCLE_MEMORY_REPETITIONS ?? 1);
|
||||||
|
if (!Number.isSafeInteger(repetitions) || repetitions <= 0) {
|
||||||
|
throw new Error('NPC_LIFECYCLE_MEMORY_REPETITIONS must be a positive integer.');
|
||||||
|
}
|
||||||
|
const reportPath = path.resolve(
|
||||||
|
packageRoot,
|
||||||
|
process.env.NPC_LIFECYCLE_MEMORY_REPORT_PATH ?? 'test-results/npc-lifecycle-memory.json'
|
||||||
|
);
|
||||||
|
const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'sammo-npc-lifecycle-memory-'));
|
||||||
|
const reports = [];
|
||||||
|
try {
|
||||||
|
for (let repetition = 1; repetition <= repetitions; repetition += 1) {
|
||||||
|
for (const [index, run] of runs.entries()) {
|
||||||
|
const childReportPath = path.join(
|
||||||
|
temporaryDirectory,
|
||||||
|
`${String(repetition).padStart(2, '0')}-${String(index).padStart(2, '0')}.json`
|
||||||
|
);
|
||||||
|
await runChild(run, childReportPath, repetition);
|
||||||
|
reports.push({
|
||||||
|
repetition,
|
||||||
|
...JSON.parse(readFileSync(childReportPath, 'utf8')),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mkdirSync(path.dirname(reportPath), { recursive: true });
|
||||||
|
writeFileSync(
|
||||||
|
reportPath,
|
||||||
|
`${JSON.stringify(
|
||||||
|
{
|
||||||
|
schemaVersion: 1,
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
inputs: {
|
||||||
|
cycles: Number(process.env.NPC_LIFECYCLE_MEMORY_CYCLES ?? 80),
|
||||||
|
batchSize: Number(process.env.NPC_LIFECYCLE_MEMORY_BATCH_SIZE ?? 100),
|
||||||
|
sampleEvery: Number(process.env.NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY ?? 5),
|
||||||
|
baseGenerals: Number(process.env.NPC_LIFECYCLE_MEMORY_BASE_GENERALS ?? 1_200),
|
||||||
|
repetitions,
|
||||||
|
},
|
||||||
|
reports,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}\n`,
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
console.log(`[npc-lifecycle-memory] wrote ${reports.length} scenario reports to ${reportPath}`);
|
||||||
|
} finally {
|
||||||
|
rmSync(temporaryDirectory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
@@ -96,10 +96,17 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
|
|||||||
intervalMs: memoryReportIntervalMs,
|
intervalMs: memoryReportIntervalMs,
|
||||||
getContext: () => {
|
getContext: () => {
|
||||||
const state = runtime.world.getState();
|
const state = runtime.world.getState();
|
||||||
|
const queueCounts = runtime.reservedTurns?.getQueueCounts();
|
||||||
return {
|
return {
|
||||||
year: state.currentYear,
|
year: state.currentYear,
|
||||||
month: state.currentMonth,
|
month: state.currentMonth,
|
||||||
...runtime.world.getEntityCounts(),
|
...runtime.world.getEntityCounts(),
|
||||||
|
...(queueCounts
|
||||||
|
? {
|
||||||
|
generalTurnQueues: queueCounts.generalQueues,
|
||||||
|
nationTurnQueues: queueCounts.nationQueues,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
lifecycleState: runtime.lifecycle.getStatus().state,
|
lifecycleState: runtime.lifecycle.getStatus().state,
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1787,6 +1787,7 @@ export const createDatabaseTurnHooks = async (
|
|||||||
world.acknowledgeDirtyState(changes);
|
world.acknowledgeDirtyState(changes);
|
||||||
if (options?.reservedTurns && reservedTurnChanges) {
|
if (options?.reservedTurns && reservedTurnChanges) {
|
||||||
options.reservedTurns.acknowledgeDirtyState(reservedTurnChanges);
|
options.reservedTurns.acknowledgeDirtyState(reservedTurnChanges);
|
||||||
|
options.reservedTurns.pruneDeletedEntityQueues(deletedGenerals, deletedNations);
|
||||||
}
|
}
|
||||||
applyRealtimeReadModelBaseline(readModelBaseline, changes);
|
applyRealtimeReadModelBaseline(readModelBaseline, changes);
|
||||||
worldReadModelBaseline = persisted.worldReadModelSignature;
|
worldReadModelBaseline = persisted.worldReadModelSignature;
|
||||||
|
|||||||
@@ -105,6 +105,17 @@ export interface InMemoryReservedTurnStateSnapshot {
|
|||||||
leasedNationKeys: string[];
|
leasedNationKeys: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ReservedTurnQueueCounts {
|
||||||
|
generalQueues: number;
|
||||||
|
nationQueues: number;
|
||||||
|
dirtyGeneralQueues: number;
|
||||||
|
dirtyNationQueues: number;
|
||||||
|
pendingGeneralInitializations: number;
|
||||||
|
pendingNationInitializations: number;
|
||||||
|
leasedGeneralQueues: number;
|
||||||
|
leasedNationQueues: number;
|
||||||
|
}
|
||||||
|
|
||||||
export class ReservedTurnLeaseConflictError extends Error {
|
export class ReservedTurnLeaseConflictError extends Error {
|
||||||
constructor(readonly queueKey: string) {
|
constructor(readonly queueKey: string) {
|
||||||
super(`Reserved turn queue lease conflict: ${queueKey}.`);
|
super(`Reserved turn queue lease conflict: ${queueKey}.`);
|
||||||
@@ -189,6 +200,64 @@ export class InMemoryReservedTurnStore {
|
|||||||
return this.captureState();
|
return this.captureState();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getQueueCounts(): ReservedTurnQueueCounts {
|
||||||
|
return {
|
||||||
|
generalQueues: this.generalTurns.size,
|
||||||
|
nationQueues: this.nationTurns.size,
|
||||||
|
dirtyGeneralQueues: this.dirtyGeneralIds.size,
|
||||||
|
dirtyNationQueues: this.dirtyNationKeys.size,
|
||||||
|
pendingGeneralInitializations: this.pendingGeneralInitializationIds.size,
|
||||||
|
pendingNationInitializations: this.pendingNationInitializationKeys.size,
|
||||||
|
leasedGeneralQueues: this.leasedGeneralIds.size,
|
||||||
|
leasedNationQueues: this.leasedNationKeys.size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drops queues whose owning rows were deleted by a successful world flush.
|
||||||
|
* The daemon calls this inside EngineStateManager.transaction(), so a later
|
||||||
|
* failure still restores these maps and journals from the transaction savepoint.
|
||||||
|
*/
|
||||||
|
pruneDeletedEntityQueues(
|
||||||
|
generalIds: readonly number[],
|
||||||
|
nationIds: readonly number[]
|
||||||
|
): { generalQueues: number; nationQueues: number } {
|
||||||
|
let generalQueues = 0;
|
||||||
|
for (const generalId of new Set(generalIds)) {
|
||||||
|
if (this.generalTurns.delete(generalId)) {
|
||||||
|
generalQueues += 1;
|
||||||
|
}
|
||||||
|
this.dirtyGeneralIds.delete(generalId);
|
||||||
|
this.pendingGeneralInitializationIds.delete(generalId);
|
||||||
|
this.leasedGeneralIds.delete(generalId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const deletedNations = new Set(nationIds);
|
||||||
|
let nationQueues = 0;
|
||||||
|
const pruneNationKeys = (keys: Iterable<string>, remove: (key: string) => boolean | void): void => {
|
||||||
|
for (const key of keys) {
|
||||||
|
const nationId = Number(key.split(':', 1)[0]);
|
||||||
|
if (deletedNations.has(nationId) && remove(key) !== false) {
|
||||||
|
nationQueues += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
pruneNationKeys(Array.from(this.nationTurns.keys()), (key) => this.nationTurns.delete(key));
|
||||||
|
for (const keys of [
|
||||||
|
this.dirtyNationKeys,
|
||||||
|
this.pendingNationInitializationKeys,
|
||||||
|
this.leasedNationKeys,
|
||||||
|
]) {
|
||||||
|
for (const key of Array.from(keys)) {
|
||||||
|
const nationId = Number(key.split(':', 1)[0]);
|
||||||
|
if (deletedNations.has(nationId)) {
|
||||||
|
keys.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { generalQueues, nationQueues };
|
||||||
|
}
|
||||||
|
|
||||||
inspectGeneralTurnActivity(): Array<[number, boolean]> {
|
inspectGeneralTurnActivity(): Array<[number, boolean]> {
|
||||||
return Array.from(this.generalTurns, ([generalId, turns]) => [
|
return Array.from(this.generalTurns, ([generalId, turns]) => [
|
||||||
generalId,
|
generalId,
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ import {
|
|||||||
} from './monthlyNationStatsHandler.js';
|
} from './monthlyNationStatsHandler.js';
|
||||||
import { createFrontStateHandler } from './frontStateHandler.js';
|
import { createFrontStateHandler } from './frontStateHandler.js';
|
||||||
import { createReservedTurnHandler } from './reservedTurnHandler.js';
|
import { createReservedTurnHandler } from './reservedTurnHandler.js';
|
||||||
import { createReservedTurnStore } from './reservedTurnStore.js';
|
import { createReservedTurnStore, type InMemoryReservedTurnStore } from './reservedTurnStore.js';
|
||||||
import { createTurnDaemonCommandHandler } from './worldCommandHandler.js';
|
import { createTurnDaemonCommandHandler } from './worldCommandHandler.js';
|
||||||
import { loadTurnCommandProfile } from './turnCommandProfile.js';
|
import { loadTurnCommandProfile } from './turnCommandProfile.js';
|
||||||
import { loadTurnWorldFromDatabase } from './worldLoader.js';
|
import { loadTurnWorldFromDatabase } from './worldLoader.js';
|
||||||
@@ -132,6 +132,7 @@ export interface TurnDaemonRuntime {
|
|||||||
stateStore: InMemoryTurnStateStore;
|
stateStore: InMemoryTurnStateStore;
|
||||||
stateManager: EngineStateManager;
|
stateManager: EngineStateManager;
|
||||||
processor: InMemoryTurnProcessor;
|
processor: InMemoryTurnProcessor;
|
||||||
|
reservedTurns: InMemoryReservedTurnStore | null;
|
||||||
hooks?: TurnDaemonHooks;
|
hooks?: TurnDaemonHooks;
|
||||||
close(): Promise<void>;
|
close(): Promise<void>;
|
||||||
}
|
}
|
||||||
@@ -1012,6 +1013,7 @@ const createTurnDaemonRuntimeWithLease = async (
|
|||||||
stateStore,
|
stateStore,
|
||||||
stateManager,
|
stateManager,
|
||||||
processor,
|
processor,
|
||||||
|
reservedTurns: reservedTurnStoreHandle?.store ?? null,
|
||||||
hooks,
|
hooks,
|
||||||
close,
|
close,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ export interface TurnDaemonMemoryContext {
|
|||||||
nations: number;
|
nations: number;
|
||||||
troops: number;
|
troops: number;
|
||||||
events: number;
|
events: number;
|
||||||
|
generalTurnQueues?: number;
|
||||||
|
nationTurnQueues?: number;
|
||||||
lifecycleState: string;
|
lifecycleState: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,6 +50,10 @@ export const buildTurnDaemonMemoryReport = (
|
|||||||
`nations=${context.nations}`,
|
`nations=${context.nations}`,
|
||||||
`troops=${context.troops}`,
|
`troops=${context.troops}`,
|
||||||
`events=${context.events}`,
|
`events=${context.events}`,
|
||||||
|
...(context.generalTurnQueues === undefined
|
||||||
|
? []
|
||||||
|
: [`generalTurnQueues=${context.generalTurnQueues}`]),
|
||||||
|
...(context.nationTurnQueues === undefined ? [] : [`nationTurnQueues=${context.nationTurnQueues}`]),
|
||||||
`lifecycle=${context.lifecycleState}`,
|
`lifecycle=${context.lifecycleState}`,
|
||||||
].join(' ');
|
].join(' ');
|
||||||
return { message, warning: heapRatio >= HEAP_WARNING_RATIO };
|
return { message, warning: heapRatio >= HEAP_WARNING_RATIO };
|
||||||
|
|||||||
@@ -474,6 +474,12 @@ integration('general turn lifecycle persistence', () => {
|
|||||||
events: [],
|
events: [],
|
||||||
initialEvents: [],
|
initialEvents: [],
|
||||||
};
|
};
|
||||||
|
const reservedTurns = new InMemoryReservedTurnStore(db, {
|
||||||
|
maxGeneralTurns: 3,
|
||||||
|
maxNationTurns: 3,
|
||||||
|
});
|
||||||
|
reservedTurns.ensureGeneralTurns(general.id);
|
||||||
|
expect(reservedTurns.getQueueCounts().generalQueues).toBe(1);
|
||||||
const world = new InMemoryTurnWorld(state, snapshot, {
|
const world = new InMemoryTurnWorld(state, snapshot, {
|
||||||
schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] },
|
schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] },
|
||||||
generalTurnHandler: {
|
generalTurnHandler: {
|
||||||
@@ -517,7 +523,7 @@ integration('general turn lifecycle persistence', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
world.executeGeneralTurn(world.getGeneralById(general.id)!);
|
world.executeGeneralTurn(world.getGeneralById(general.id)!);
|
||||||
const hooks = await createDatabaseTurnHooks(databaseUrl!, world);
|
const hooks = await createDatabaseTurnHooks(databaseUrl!, world, { reservedTurns });
|
||||||
try {
|
try {
|
||||||
await hooks.hooks.flushChanges?.({
|
await hooks.hooks.flushChanges?.({
|
||||||
lastTurnTime: state.lastTurnTime.toISOString(),
|
lastTurnTime: state.lastTurnTime.toISOString(),
|
||||||
@@ -529,6 +535,10 @@ integration('general turn lifecycle persistence', () => {
|
|||||||
} finally {
|
} finally {
|
||||||
await hooks.close();
|
await hooks.close();
|
||||||
}
|
}
|
||||||
|
expect(reservedTurns.getQueueCounts()).toMatchObject({
|
||||||
|
generalQueues: 0,
|
||||||
|
pendingGeneralInitializations: 0,
|
||||||
|
});
|
||||||
|
|
||||||
const archived = await db.oldGeneral.findUniqueOrThrow({
|
const archived = await db.oldGeneral.findUniqueOrThrow({
|
||||||
where: { by_no: { serverId, generalNo: general.id } },
|
where: { by_no: { serverId, generalNo: general.id } },
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
import { performance } from 'node:perf_hooks';
|
||||||
|
import { serialize } from 'node:v8';
|
||||||
|
|
||||||
|
import type { InMemoryTurnWorld } from '../../src/turn/inMemoryWorld.js';
|
||||||
|
import type {
|
||||||
|
InMemoryReservedTurnStore,
|
||||||
|
ReservedTurnQueueCounts,
|
||||||
|
} from '../../src/turn/reservedTurnStore.js';
|
||||||
|
|
||||||
|
export type NpcLifecycleMemoryScenario =
|
||||||
|
| 'steady-state'
|
||||||
|
| 'growth'
|
||||||
|
| 'death-drain'
|
||||||
|
| 'balanced-churn'
|
||||||
|
| 'rollback-churn';
|
||||||
|
|
||||||
|
export interface ProcessMemorySnapshot {
|
||||||
|
rssBytes: number;
|
||||||
|
heapTotalBytes: number;
|
||||||
|
heapUsedBytes: number;
|
||||||
|
externalBytes: number;
|
||||||
|
arrayBuffersBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NpcLifecycleMemorySample {
|
||||||
|
cycle: number;
|
||||||
|
phase: 'initial' | 'in-transaction' | 'post-flush';
|
||||||
|
elapsedMs: number;
|
||||||
|
liveGeneralCount: number;
|
||||||
|
queueCounts: ReservedTurnQueueCounts;
|
||||||
|
process: ProcessMemorySnapshot;
|
||||||
|
pending?: {
|
||||||
|
createdGenerals: number;
|
||||||
|
deletedGenerals: number;
|
||||||
|
lifecycleEvents: number;
|
||||||
|
reservedGeneralQueues: number;
|
||||||
|
};
|
||||||
|
snapshot?: {
|
||||||
|
worldBytes: number;
|
||||||
|
reservedTurnBytes: number;
|
||||||
|
totalBytes: number;
|
||||||
|
cloneAndSerializeMs: number;
|
||||||
|
heapUsedAfterReleaseBytes: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const readProcessMemory = (): ProcessMemorySnapshot => {
|
||||||
|
const usage = process.memoryUsage();
|
||||||
|
return {
|
||||||
|
rssBytes: usage.rss,
|
||||||
|
heapTotalBytes: usage.heapTotal,
|
||||||
|
heapUsedBytes: usage.heapUsed,
|
||||||
|
externalBytes: usage.external,
|
||||||
|
arrayBuffersBytes: usage.arrayBuffers,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const linearRegressionSlope = (points: ReadonlyArray<{ x: number; y: number }>): number => {
|
||||||
|
if (points.length < 2) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const meanX = points.reduce((sum, point) => sum + point.x, 0) / points.length;
|
||||||
|
const meanY = points.reduce((sum, point) => sum + point.y, 0) / points.length;
|
||||||
|
let numerator = 0;
|
||||||
|
let denominator = 0;
|
||||||
|
for (const point of points) {
|
||||||
|
const xDelta = point.x - meanX;
|
||||||
|
numerator += xDelta * (point.y - meanY);
|
||||||
|
denominator += xDelta * xDelta;
|
||||||
|
}
|
||||||
|
return denominator === 0 ? 0 : numerator / denominator;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const captureLifecycleMemorySample = (input: {
|
||||||
|
world: InMemoryTurnWorld;
|
||||||
|
reservedTurns: InMemoryReservedTurnStore;
|
||||||
|
startedAtMs: number;
|
||||||
|
cycle: number;
|
||||||
|
phase: NpcLifecycleMemorySample['phase'];
|
||||||
|
includePending: boolean;
|
||||||
|
includeSnapshot: boolean;
|
||||||
|
}): NpcLifecycleMemorySample => {
|
||||||
|
globalThis.gc?.();
|
||||||
|
const processSnapshot = readProcessMemory();
|
||||||
|
const pending = input.includePending
|
||||||
|
? (() => {
|
||||||
|
const worldChanges = input.world.peekDirtyState();
|
||||||
|
const reservedChanges = input.reservedTurns.peekDirtyState();
|
||||||
|
return {
|
||||||
|
createdGenerals: worldChanges.createdGenerals.length,
|
||||||
|
deletedGenerals: worldChanges.deletedGenerals.length,
|
||||||
|
lifecycleEvents: worldChanges.lifecycleEvents.length,
|
||||||
|
reservedGeneralQueues: reservedChanges.generalIds.length,
|
||||||
|
};
|
||||||
|
})()
|
||||||
|
: undefined;
|
||||||
|
const sample: NpcLifecycleMemorySample = {
|
||||||
|
cycle: input.cycle,
|
||||||
|
phase: input.phase,
|
||||||
|
elapsedMs: performance.now() - input.startedAtMs,
|
||||||
|
liveGeneralCount: input.world.getEntityCounts().generals,
|
||||||
|
queueCounts: input.reservedTurns.getQueueCounts(),
|
||||||
|
process: processSnapshot,
|
||||||
|
...(pending ? { pending } : {}),
|
||||||
|
};
|
||||||
|
if (input.includeSnapshot) {
|
||||||
|
const snapshotMetrics = (() => {
|
||||||
|
const snapshotStartedAt = performance.now();
|
||||||
|
const worldSnapshot = input.world.captureState();
|
||||||
|
const reservedSnapshot = input.reservedTurns.captureTransactionState();
|
||||||
|
const worldBytes = serialize(worldSnapshot).byteLength;
|
||||||
|
const reservedTurnBytes = serialize(reservedSnapshot).byteLength;
|
||||||
|
return {
|
||||||
|
worldBytes,
|
||||||
|
reservedTurnBytes,
|
||||||
|
cloneAndSerializeMs: performance.now() - snapshotStartedAt,
|
||||||
|
};
|
||||||
|
})();
|
||||||
|
globalThis.gc?.();
|
||||||
|
sample.snapshot = {
|
||||||
|
...snapshotMetrics,
|
||||||
|
totalBytes: snapshotMetrics.worldBytes + snapshotMetrics.reservedTurnBytes,
|
||||||
|
heapUsedAfterReleaseBytes: readProcessMemory().heapUsedBytes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return sample;
|
||||||
|
};
|
||||||
|
|
||||||
|
const maxValue = (values: readonly number[]): number => Math.max(0, ...values);
|
||||||
|
|
||||||
|
export const buildNpcLifecycleMemoryReport = (input: {
|
||||||
|
scenario: NpcLifecycleMemoryScenario;
|
||||||
|
pruneDeletedQueues: boolean;
|
||||||
|
initialGeneralCount: number;
|
||||||
|
cycles: number;
|
||||||
|
batchSize: number;
|
||||||
|
sampleEvery: number;
|
||||||
|
createdTotal: number;
|
||||||
|
deletedTotal: number;
|
||||||
|
rolledBackCycles: number;
|
||||||
|
startedAtMs: number;
|
||||||
|
samples: NpcLifecycleMemorySample[];
|
||||||
|
}) => {
|
||||||
|
const retained = input.samples.filter(
|
||||||
|
(sample) => sample.phase === 'initial' || sample.phase === 'post-flush'
|
||||||
|
);
|
||||||
|
const warmSampleIndex = Math.floor(retained.length / 3);
|
||||||
|
const trendSamples = retained.slice(warmSampleIndex);
|
||||||
|
const first = retained[0];
|
||||||
|
const final = retained.at(-1);
|
||||||
|
const heapSlope = linearRegressionSlope(
|
||||||
|
trendSamples.map((sample) => ({ x: sample.cycle, y: sample.process.heapUsedBytes }))
|
||||||
|
);
|
||||||
|
const snapshotSlope = linearRegressionSlope(
|
||||||
|
trendSamples.flatMap((sample) =>
|
||||||
|
sample.snapshot ? [{ x: sample.cycle, y: sample.snapshot.totalBytes }] : []
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const queueSlope = linearRegressionSlope(
|
||||||
|
trendSamples.map((sample) => ({ x: sample.cycle, y: sample.queueCounts.generalQueues }))
|
||||||
|
);
|
||||||
|
const lifecycleOperations = input.createdTotal + input.deletedTotal;
|
||||||
|
|
||||||
|
return {
|
||||||
|
schemaVersion: 1,
|
||||||
|
runtime: {
|
||||||
|
node: process.version,
|
||||||
|
platform: process.platform,
|
||||||
|
arch: process.arch,
|
||||||
|
explicitGc: typeof globalThis.gc === 'function',
|
||||||
|
},
|
||||||
|
scenario: {
|
||||||
|
name: input.scenario,
|
||||||
|
pruneDeletedQueues: input.pruneDeletedQueues,
|
||||||
|
initialGeneralCount: input.initialGeneralCount,
|
||||||
|
cycles: input.cycles,
|
||||||
|
batchSize: input.batchSize,
|
||||||
|
sampleEvery: input.sampleEvery,
|
||||||
|
},
|
||||||
|
result: {
|
||||||
|
createdTotal: input.createdTotal,
|
||||||
|
deletedTotal: input.deletedTotal,
|
||||||
|
rolledBackCycles: input.rolledBackCycles,
|
||||||
|
finalGeneralCount: final?.liveGeneralCount ?? 0,
|
||||||
|
finalGeneralQueueCount: final?.queueCounts.generalQueues ?? 0,
|
||||||
|
deadQueueRetentionCount:
|
||||||
|
(final?.queueCounts.generalQueues ?? 0) - (final?.liveGeneralCount ?? 0),
|
||||||
|
wallDurationMs: performance.now() - input.startedAtMs,
|
||||||
|
},
|
||||||
|
memory: {
|
||||||
|
retainedHeapStartBytes: first?.process.heapUsedBytes ?? 0,
|
||||||
|
retainedHeapFinalBytes: final?.process.heapUsedBytes ?? 0,
|
||||||
|
retainedHeapDeltaBytes:
|
||||||
|
(final?.process.heapUsedBytes ?? 0) - (first?.process.heapUsedBytes ?? 0),
|
||||||
|
retainedHeapSlopeBytesPerCycle: heapSlope,
|
||||||
|
retainedHeapSlopeBytesPerLifecycleOperation:
|
||||||
|
lifecycleOperations === 0 ? 0 : (heapSlope * input.cycles) / lifecycleOperations,
|
||||||
|
retainedSnapshotStartBytes: first?.snapshot?.totalBytes ?? 0,
|
||||||
|
retainedSnapshotFinalBytes: final?.snapshot?.totalBytes ?? 0,
|
||||||
|
retainedSnapshotDeltaBytes:
|
||||||
|
(final?.snapshot?.totalBytes ?? 0) - (first?.snapshot?.totalBytes ?? 0),
|
||||||
|
retainedSnapshotSlopeBytesPerCycle: snapshotSlope,
|
||||||
|
generalQueueSlopePerCycle: queueSlope,
|
||||||
|
maxObservedHeapUsedBytes: maxValue(input.samples.map((sample) => sample.process.heapUsedBytes)),
|
||||||
|
maxObservedRssBytes: maxValue(input.samples.map((sample) => sample.process.rssBytes)),
|
||||||
|
processResourceMaxRssBytes: process.resourceUsage().maxRSS * 1024,
|
||||||
|
},
|
||||||
|
samples: input.samples,
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -0,0 +1,388 @@
|
|||||||
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||||
|
import { dirname, resolve } from 'node:path';
|
||||||
|
import { performance } from 'node:perf_hooks';
|
||||||
|
|
||||||
|
import { buildScenarioBootstrap, type TurnSchedule } from '@sammo-ts/logic';
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { loadMapDefinitionByName } from '../src/scenario/mapLoader.js';
|
||||||
|
import { loadScenarioDefinitionById } from '../src/scenario/scenarioLoader.js';
|
||||||
|
import { loadUnitSetDefinitionByName } from '../src/scenario/unitSetLoader.js';
|
||||||
|
import { EngineStateManager } from '../src/turn/engineStateManager.js';
|
||||||
|
import { InMemoryTurnWorld } from '../src/turn/inMemoryWorld.js';
|
||||||
|
import { InMemoryReservedTurnStore } from '../src/turn/reservedTurnStore.js';
|
||||||
|
import type { TurnGeneral, TurnWorldSnapshot, TurnWorldState } from '../src/turn/types.js';
|
||||||
|
import {
|
||||||
|
buildNpcLifecycleMemoryReport,
|
||||||
|
captureLifecycleMemorySample,
|
||||||
|
type NpcLifecycleMemorySample,
|
||||||
|
type NpcLifecycleMemoryScenario,
|
||||||
|
} from './helpers/npcLifecycleMemoryProfiler.js';
|
||||||
|
|
||||||
|
const profileEnabled = process.env.NPC_LIFECYCLE_MEMORY_PROFILE === '1';
|
||||||
|
const profileDescribe = describe.runIf(profileEnabled);
|
||||||
|
const SCENARIO_ID = 2601;
|
||||||
|
const HIDDEN_SEED = 'scenario-2601-npc-lifecycle-memory-v1';
|
||||||
|
const ROLLBACK_SENTINEL = new Error('npc-lifecycle-memory-rollback');
|
||||||
|
const VALID_SCENARIOS = new Set<NpcLifecycleMemoryScenario>([
|
||||||
|
'steady-state',
|
||||||
|
'growth',
|
||||||
|
'death-drain',
|
||||||
|
'balanced-churn',
|
||||||
|
'rollback-churn',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const readPositiveInteger = (name: string, fallback: number): number => {
|
||||||
|
const raw = process.env[name];
|
||||||
|
if (raw === undefined) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
const value = Number(raw);
|
||||||
|
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||||
|
throw new Error(`${name} must be a positive integer: ${raw}`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createGameDate = (year: number, month: number): Date => {
|
||||||
|
const date = new Date(0);
|
||||||
|
date.setUTCFullYear(year, month - 1, 1);
|
||||||
|
date.setUTCHours(0, 0, 0, 0);
|
||||||
|
return date;
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildTurnGeneral = (
|
||||||
|
domainGeneral: ReturnType<typeof buildScenarioBootstrap>['snapshot']['generals'][number],
|
||||||
|
seedGeneral: ReturnType<typeof buildScenarioBootstrap>['seed']['generals'][number],
|
||||||
|
startTime: Date,
|
||||||
|
startYear: number,
|
||||||
|
startMonth: number
|
||||||
|
): TurnGeneral => {
|
||||||
|
const deathMonthRaw = seedGeneral.meta.deathMonth;
|
||||||
|
const deathMonth =
|
||||||
|
typeof deathMonthRaw === 'number' && Number.isInteger(deathMonthRaw) ? deathMonthRaw : startMonth;
|
||||||
|
const killturn = Math.max(0, (seedGeneral.deathYear - startYear) * 12 + deathMonth - startMonth);
|
||||||
|
return {
|
||||||
|
...domainGeneral,
|
||||||
|
userId: null,
|
||||||
|
bornYear: seedGeneral.birthYear,
|
||||||
|
deadYear: seedGeneral.deathYear,
|
||||||
|
affinity: seedGeneral.affinity,
|
||||||
|
picture: seedGeneral.picture === null ? null : String(seedGeneral.picture),
|
||||||
|
startAge: 20,
|
||||||
|
turnTime: new Date(startTime),
|
||||||
|
recentWarTime: null,
|
||||||
|
lastTurn: { command: '휴식' },
|
||||||
|
penalty: {},
|
||||||
|
inheritancePoints: {},
|
||||||
|
meta: {
|
||||||
|
...domainGeneral.meta,
|
||||||
|
...seedGeneral.meta,
|
||||||
|
killturn,
|
||||||
|
npcType: seedGeneral.npcType,
|
||||||
|
crewTypeId: seedGeneral.crewTypeId,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const cloneNpcGeneral = (source: TurnGeneral, id: number): TurnGeneral => {
|
||||||
|
const cloned = structuredClone(source);
|
||||||
|
return {
|
||||||
|
...cloned,
|
||||||
|
id,
|
||||||
|
name: `${source.name}#M${id}`,
|
||||||
|
userId: null,
|
||||||
|
npcState: Math.max(2, source.npcState),
|
||||||
|
nationId: 0,
|
||||||
|
cityId: 0,
|
||||||
|
troopId: 0,
|
||||||
|
officerLevel: 0,
|
||||||
|
meta: {
|
||||||
|
...cloned.meta,
|
||||||
|
lifecycleMemoryFixture: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const createProfileWorld = async (initialGeneralCount: number) => {
|
||||||
|
const scenario = await loadScenarioDefinitionById(SCENARIO_ID);
|
||||||
|
const map = await loadMapDefinitionByName(scenario.config.environment.mapName);
|
||||||
|
const unitSet = await loadUnitSetDefinitionByName(scenario.config.environment.unitSet);
|
||||||
|
const startYear = scenario.startYear ?? 180;
|
||||||
|
const startMonth = 1;
|
||||||
|
const startTime = createGameDate(startYear, startMonth);
|
||||||
|
const bootstrap = buildScenarioBootstrap({
|
||||||
|
scenario,
|
||||||
|
map,
|
||||||
|
unitSet,
|
||||||
|
options: {
|
||||||
|
hiddenSeed: HIDDEN_SEED,
|
||||||
|
initialYear: startYear,
|
||||||
|
initialMonth: startMonth,
|
||||||
|
turnTermMinutes: 10,
|
||||||
|
includeNeutralNationInSeed: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (bootstrap.warnings.length > 0) {
|
||||||
|
throw new Error(`scenario bootstrap warnings: ${bootstrap.warnings.join(', ')}`);
|
||||||
|
}
|
||||||
|
const domainGeneralById = new Map(bootstrap.snapshot.generals.map((general) => [general.id, general]));
|
||||||
|
const scenarioGenerals = bootstrap.seed.generals.map((seedGeneral) => {
|
||||||
|
const domainGeneral = domainGeneralById.get(seedGeneral.id);
|
||||||
|
if (!domainGeneral) {
|
||||||
|
throw new Error(`missing scenario general: ${seedGeneral.id}`);
|
||||||
|
}
|
||||||
|
return buildTurnGeneral(domainGeneral, seedGeneral, startTime, startYear, startMonth);
|
||||||
|
});
|
||||||
|
const generals = Array.from({ length: initialGeneralCount }, (_, index) =>
|
||||||
|
cloneNpcGeneral(scenarioGenerals[index % scenarioGenerals.length]!, index + 1)
|
||||||
|
);
|
||||||
|
const snapshot: TurnWorldSnapshot = {
|
||||||
|
scenarioConfig: bootstrap.snapshot.scenarioConfig,
|
||||||
|
scenarioMeta: bootstrap.snapshot.scenarioMeta,
|
||||||
|
worldConfig: {
|
||||||
|
fiction: scenario.fiction,
|
||||||
|
npcMode: 2,
|
||||||
|
turnTermMinutes: 10,
|
||||||
|
tournamentTrig: false,
|
||||||
|
},
|
||||||
|
map,
|
||||||
|
unitSet,
|
||||||
|
generals,
|
||||||
|
cities: bootstrap.snapshot.cities,
|
||||||
|
nations: bootstrap.snapshot.nations,
|
||||||
|
troops: bootstrap.snapshot.troops,
|
||||||
|
diplomacy: bootstrap.snapshot.diplomacy.map((entry) => ({
|
||||||
|
fromNationId: entry.fromNationId,
|
||||||
|
toNationId: entry.toNationId,
|
||||||
|
state: entry.state,
|
||||||
|
term: entry.durationMonths,
|
||||||
|
dead: 0,
|
||||||
|
meta: {},
|
||||||
|
})),
|
||||||
|
events: [],
|
||||||
|
initialEvents: [],
|
||||||
|
};
|
||||||
|
const state: TurnWorldState = {
|
||||||
|
id: 1,
|
||||||
|
currentYear: startYear,
|
||||||
|
currentMonth: startMonth,
|
||||||
|
tickSeconds: 600,
|
||||||
|
lastTurnTime: startTime,
|
||||||
|
clockBaseTime: startTime,
|
||||||
|
clockTick: 0,
|
||||||
|
clockMode: 'manual',
|
||||||
|
clockWallAnchor: startTime,
|
||||||
|
lastTurnTick: 0,
|
||||||
|
meta: {
|
||||||
|
scenarioId: SCENARIO_ID,
|
||||||
|
hiddenSeed: HIDDEN_SEED,
|
||||||
|
killturn: 480,
|
||||||
|
lastGeneralId: initialGeneralCount,
|
||||||
|
serverId: 'npc-lifecycle-memory-profile',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const schedule: TurnSchedule = { entries: [{ startMinute: 0, tickMinutes: 10 }] };
|
||||||
|
const world = new InMemoryTurnWorld(state, snapshot, { schedule });
|
||||||
|
const reservedTurns = new InMemoryReservedTurnStore({} as never, {
|
||||||
|
maxGeneralTurns: 30,
|
||||||
|
maxNationTurns: 12,
|
||||||
|
leaseOwner: 'npc-lifecycle-memory-profile',
|
||||||
|
});
|
||||||
|
for (const general of generals) {
|
||||||
|
reservedTurns.getGeneralTurns(general.id);
|
||||||
|
}
|
||||||
|
return { world, reservedTurns, templateGenerals: scenarioGenerals };
|
||||||
|
};
|
||||||
|
|
||||||
|
profileDescribe('NPC 생성·사망 장기 구동 메모리 프로파일', () => {
|
||||||
|
it('격리 시나리오의 GC 안정 heap, rollback snapshot과 예약 큐 보유량을 기록한다', async () => {
|
||||||
|
expect(typeof globalThis.gc).toBe('function');
|
||||||
|
const rawScenario = process.env.NPC_LIFECYCLE_MEMORY_SCENARIO ?? 'balanced-churn';
|
||||||
|
if (!VALID_SCENARIOS.has(rawScenario as NpcLifecycleMemoryScenario)) {
|
||||||
|
throw new Error(`unknown NPC lifecycle memory scenario: ${rawScenario}`);
|
||||||
|
}
|
||||||
|
const scenario = rawScenario as NpcLifecycleMemoryScenario;
|
||||||
|
const cycles = readPositiveInteger('NPC_LIFECYCLE_MEMORY_CYCLES', 80);
|
||||||
|
const batchSize = readPositiveInteger('NPC_LIFECYCLE_MEMORY_BATCH_SIZE', 100);
|
||||||
|
const sampleEvery = readPositiveInteger('NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY', 5);
|
||||||
|
const baseGeneralCount = readPositiveInteger('NPC_LIFECYCLE_MEMORY_BASE_GENERALS', 1_200);
|
||||||
|
const pruneDeletedQueues = process.env.NPC_LIFECYCLE_MEMORY_PRUNE_DELETED === '1';
|
||||||
|
const initialGeneralCount =
|
||||||
|
scenario === 'death-drain' ? baseGeneralCount + cycles * batchSize : baseGeneralCount;
|
||||||
|
const { world, reservedTurns, templateGenerals } = await createProfileWorld(initialGeneralCount);
|
||||||
|
const stateManager = new EngineStateManager();
|
||||||
|
stateManager.register('world', {
|
||||||
|
capture: () => world.captureState(),
|
||||||
|
restore: (snapshot) => world.restoreState(snapshot),
|
||||||
|
});
|
||||||
|
stateManager.register('reservedTurns', {
|
||||||
|
capture: () => reservedTurns.captureTransactionState(),
|
||||||
|
restore: (snapshot) => reservedTurns.restoreState(snapshot),
|
||||||
|
});
|
||||||
|
|
||||||
|
const startedAtMs = performance.now();
|
||||||
|
const samples: NpcLifecycleMemorySample[] = [
|
||||||
|
captureLifecycleMemorySample({
|
||||||
|
world,
|
||||||
|
reservedTurns,
|
||||||
|
startedAtMs,
|
||||||
|
cycle: 0,
|
||||||
|
phase: 'initial',
|
||||||
|
includePending: false,
|
||||||
|
includeSnapshot: true,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
const activeGeneralIds = world
|
||||||
|
.listGenerals()
|
||||||
|
.map((general) => general.id)
|
||||||
|
.sort((left, right) => left - right);
|
||||||
|
let nextGeneralId = Math.max(...activeGeneralIds) + 1;
|
||||||
|
let createdTotal = 0;
|
||||||
|
let deletedTotal = 0;
|
||||||
|
let rolledBackCycles = 0;
|
||||||
|
|
||||||
|
const addGenerals = (count: number): void => {
|
||||||
|
for (let index = 0; index < count; index += 1) {
|
||||||
|
const generalId = nextGeneralId++;
|
||||||
|
const source = templateGenerals[(generalId - 1) % templateGenerals.length]!;
|
||||||
|
if (!world.addGeneral(cloneNpcGeneral(source, generalId))) {
|
||||||
|
throw new Error(`failed to add profile general ${generalId}`);
|
||||||
|
}
|
||||||
|
reservedTurns.ensureGeneralTurns(generalId);
|
||||||
|
activeGeneralIds.push(generalId);
|
||||||
|
createdTotal += 1;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const deleteGenerals = (count: number): void => {
|
||||||
|
const targetIds = activeGeneralIds.splice(0, count);
|
||||||
|
for (const generalId of targetIds) {
|
||||||
|
if (!world.deleteGeneralWithLifecycle(generalId, 180, 1)) {
|
||||||
|
throw new Error(`failed to delete profile general ${generalId}`);
|
||||||
|
}
|
||||||
|
deletedTotal += 1;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
for (let cycle = 1; cycle <= cycles; cycle += 1) {
|
||||||
|
const sampledCycle = cycle % sampleEvery === 0 || cycle === cycles;
|
||||||
|
try {
|
||||||
|
await stateManager.transaction(() => {
|
||||||
|
if (scenario === 'steady-state') {
|
||||||
|
for (let index = 0; index < batchSize; index += 1) {
|
||||||
|
const generalId = activeGeneralIds[((cycle - 1) * batchSize + index) % activeGeneralIds.length]!;
|
||||||
|
const current = world.getGeneralById(generalId);
|
||||||
|
if (!current) {
|
||||||
|
throw new Error(`missing steady-state general ${generalId}`);
|
||||||
|
}
|
||||||
|
world.updateGeneral(generalId, { experience: current.experience + 1 });
|
||||||
|
reservedTurns.shiftGeneralTurns(generalId, -1);
|
||||||
|
}
|
||||||
|
} else if (scenario === 'growth') {
|
||||||
|
addGenerals(batchSize);
|
||||||
|
} else if (scenario === 'death-drain') {
|
||||||
|
deleteGenerals(batchSize);
|
||||||
|
} else if (scenario === 'balanced-churn') {
|
||||||
|
deleteGenerals(batchSize);
|
||||||
|
addGenerals(batchSize);
|
||||||
|
} else {
|
||||||
|
addGenerals(batchSize);
|
||||||
|
deleteGenerals(batchSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sampledCycle) {
|
||||||
|
samples.push(
|
||||||
|
captureLifecycleMemorySample({
|
||||||
|
world,
|
||||||
|
reservedTurns,
|
||||||
|
startedAtMs,
|
||||||
|
cycle,
|
||||||
|
phase: 'in-transaction',
|
||||||
|
includePending: true,
|
||||||
|
includeSnapshot: false,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (scenario === 'rollback-churn') {
|
||||||
|
throw ROLLBACK_SENTINEL;
|
||||||
|
}
|
||||||
|
|
||||||
|
const worldChanges = world.peekDirtyState();
|
||||||
|
const reservedChanges = reservedTurns.peekDirtyState();
|
||||||
|
world.acknowledgeDirtyState(worldChanges);
|
||||||
|
reservedTurns.acknowledgeDirtyState(reservedChanges);
|
||||||
|
if (pruneDeletedQueues) {
|
||||||
|
reservedTurns.pruneDeletedEntityQueues(
|
||||||
|
worldChanges.deletedGenerals,
|
||||||
|
worldChanges.deletedNations
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (scenario !== 'rollback-churn' || error !== ROLLBACK_SENTINEL) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
rolledBackCycles += 1;
|
||||||
|
activeGeneralIds.splice(0, activeGeneralIds.length, ...world.listGenerals().map((general) => general.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sampledCycle) {
|
||||||
|
samples.push(
|
||||||
|
captureLifecycleMemorySample({
|
||||||
|
world,
|
||||||
|
reservedTurns,
|
||||||
|
startedAtMs,
|
||||||
|
cycle,
|
||||||
|
phase: 'post-flush',
|
||||||
|
includePending: true,
|
||||||
|
includeSnapshot: true,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const report = buildNpcLifecycleMemoryReport({
|
||||||
|
scenario,
|
||||||
|
pruneDeletedQueues,
|
||||||
|
initialGeneralCount,
|
||||||
|
cycles,
|
||||||
|
batchSize,
|
||||||
|
sampleEvery,
|
||||||
|
createdTotal,
|
||||||
|
deletedTotal,
|
||||||
|
rolledBackCycles,
|
||||||
|
startedAtMs,
|
||||||
|
samples,
|
||||||
|
});
|
||||||
|
const reportPath = resolve(
|
||||||
|
process.env.NPC_LIFECYCLE_MEMORY_CHILD_REPORT_PATH ??
|
||||||
|
`test-results/npc-lifecycle-memory-${scenario}.json`
|
||||||
|
);
|
||||||
|
mkdirSync(dirname(reportPath), { recursive: true });
|
||||||
|
writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||||
|
console.log(
|
||||||
|
`[NPC_LIFECYCLE_MEMORY_REPORT]${JSON.stringify({
|
||||||
|
reportPath,
|
||||||
|
scenario: report.scenario,
|
||||||
|
result: report.result,
|
||||||
|
memory: report.memory,
|
||||||
|
})}`
|
||||||
|
);
|
||||||
|
|
||||||
|
const expectedFinalGeneralCount =
|
||||||
|
scenario === 'growth'
|
||||||
|
? initialGeneralCount + cycles * batchSize
|
||||||
|
: scenario === 'death-drain'
|
||||||
|
? baseGeneralCount
|
||||||
|
: initialGeneralCount;
|
||||||
|
expect(report.result.finalGeneralCount).toBe(expectedFinalGeneralCount);
|
||||||
|
expect(report.result.rolledBackCycles).toBe(scenario === 'rollback-churn' ? cycles : 0);
|
||||||
|
if (pruneDeletedQueues || !['death-drain', 'balanced-churn'].includes(scenario)) {
|
||||||
|
expect(report.result.deadQueueRetentionCount).toBe(0);
|
||||||
|
} else {
|
||||||
|
expect(report.result.deadQueueRetentionCount).toBe(cycles * batchSize);
|
||||||
|
}
|
||||||
|
expect(world.peekDirtyState().lifecycleEvents).toHaveLength(0);
|
||||||
|
expect(reservedTurns.peekDirtyState().generalIds).toHaveLength(0);
|
||||||
|
}, 600_000);
|
||||||
|
});
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { linearRegressionSlope } from './helpers/npcLifecycleMemoryProfiler.js';
|
||||||
|
|
||||||
|
describe('NPC lifecycle memory profiler metrics', () => {
|
||||||
|
it('calculates the retained-byte slope from unevenly spaced samples', () => {
|
||||||
|
expect(
|
||||||
|
linearRegressionSlope([
|
||||||
|
{ x: 0, y: 100 },
|
||||||
|
{ x: 2, y: 140 },
|
||||||
|
{ x: 5, y: 200 },
|
||||||
|
])
|
||||||
|
).toBeCloseTo(20, 8);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns zero when a trend cannot be established', () => {
|
||||||
|
expect(linearRegressionSlope([])).toBe(0);
|
||||||
|
expect(linearRegressionSlope([{ x: 1, y: 10 }])).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -169,6 +169,51 @@ const buildHarness = (initialRevision: RevisionRow | null = null) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
describe('reserved turn daemon lease', () => {
|
describe('reserved turn daemon lease', () => {
|
||||||
|
it('prunes deleted general and nation queues together with their journal state', () => {
|
||||||
|
const harness = buildHarness();
|
||||||
|
harness.store.ensureGeneralTurns(7);
|
||||||
|
harness.store.ensureGeneralTurns(8);
|
||||||
|
harness.store.ensureNationTurns(3, 12);
|
||||||
|
harness.store.ensureNationTurns(4, 12);
|
||||||
|
|
||||||
|
expect(harness.store.getQueueCounts()).toMatchObject({
|
||||||
|
generalQueues: 2,
|
||||||
|
nationQueues: 2,
|
||||||
|
pendingGeneralInitializations: 2,
|
||||||
|
pendingNationInitializations: 2,
|
||||||
|
});
|
||||||
|
expect(harness.store.pruneDeletedEntityQueues([7], [3])).toEqual({
|
||||||
|
generalQueues: 1,
|
||||||
|
nationQueues: 1,
|
||||||
|
});
|
||||||
|
expect(harness.store.getQueueCounts()).toMatchObject({
|
||||||
|
generalQueues: 1,
|
||||||
|
nationQueues: 1,
|
||||||
|
pendingGeneralInitializations: 1,
|
||||||
|
pendingNationInitializations: 1,
|
||||||
|
});
|
||||||
|
expect(harness.store.getGeneralTurns(8)).toHaveLength(2);
|
||||||
|
expect(harness.store.getNationTurns(4, 12)).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('restores pruned queues from the transaction savepoint', () => {
|
||||||
|
const harness = buildHarness();
|
||||||
|
harness.store.ensureGeneralTurns(7);
|
||||||
|
harness.store.ensureNationTurns(3, 12);
|
||||||
|
const savepoint = harness.store.captureTransactionState();
|
||||||
|
|
||||||
|
harness.store.pruneDeletedEntityQueues([7], [3]);
|
||||||
|
expect(harness.store.getQueueCounts()).toMatchObject({ generalQueues: 0, nationQueues: 0 });
|
||||||
|
|
||||||
|
harness.store.restoreState(savepoint);
|
||||||
|
expect(harness.store.getQueueCounts()).toMatchObject({
|
||||||
|
generalQueues: 1,
|
||||||
|
nationQueues: 1,
|
||||||
|
pendingGeneralInitializations: 1,
|
||||||
|
pendingNationInitializations: 1,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('holds the queue lease from refresh through shift and releases it with the revision increment', async () => {
|
it('holds the queue lease from refresh through shift and releases it with the revision increment', async () => {
|
||||||
const harness = buildHarness();
|
const harness = buildHarness();
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ const context = {
|
|||||||
nations: 3,
|
nations: 3,
|
||||||
troops: 15,
|
troops: 15,
|
||||||
events: 4,
|
events: 4,
|
||||||
|
generalTurnQueues: 2461,
|
||||||
|
nationTurnQueues: 18,
|
||||||
lifecycleState: 'paused',
|
lifecycleState: 'paused',
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -33,6 +35,7 @@ describe('turn daemon memory reporting', () => {
|
|||||||
expect(result.message).toContain('profile=hwe reason=interval');
|
expect(result.message).toContain('profile=hwe reason=interval');
|
||||||
expect(result.message).toContain('heapLimitMiB=3072');
|
expect(result.message).toContain('heapLimitMiB=3072');
|
||||||
expect(result.message).toContain('year=214 month=12 generals=2461');
|
expect(result.message).toContain('year=214 month=12 generals=2461');
|
||||||
|
expect(result.message).toContain('generalTurnQueues=2461 nationTurnQueues=18');
|
||||||
expect(result.message).toContain('lifecycle=paused');
|
expect(result.message).toContain('lifecycle=paused');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
# NPC 생명주기 메모리 프로파일
|
||||||
|
|
||||||
|
## 목적과 해석 경계
|
||||||
|
|
||||||
|
턴 데몬이 오래 실행될 때의 안정 상태와 NPC 생성·사망 churn을 서로 분리해
|
||||||
|
측정한다. 각 case는 별도 Node/Vitest worker에서 실행하며 명시적 GC 뒤의
|
||||||
|
retained heap, RSS, rollback participant의 V8 직렬화 크기, 실제 live 장수 수와
|
||||||
|
예약 턴 큐 수를 기록한다.
|
||||||
|
|
||||||
|
이 프로파일은 `scenario_2601`의 실제 장수 shape, `InMemoryTurnWorld`,
|
||||||
|
`InMemoryReservedTurnStore`, `EngineStateManager.transaction()`과 flush 뒤
|
||||||
|
acknowledgement를 사용한다. PostgreSQL/Redis I/O, 실제 NPC AI·전투·월간 event,
|
||||||
|
다른 PM2 process와 allocator/cgroup 경합은 포함하지 않는다. 따라서 heap과
|
||||||
|
snapshot은 엔진 상태 보유량 비교에 사용하고 RSS는 동일 host의 보조 high-water로만
|
||||||
|
해석한다.
|
||||||
|
|
||||||
|
## 시나리오
|
||||||
|
|
||||||
|
| 이름 | 고정 조건 | 확인하는 위험 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `steady-state` | 1,200명, 매 cycle 100명 update/예약 턴 shift, 생성·사망 0 | 장기 transaction/flush 자체의 retained heap drift |
|
||||||
|
| `growth` | 1,200명에서 cycle마다 100명 생성 | live NPC 1명당 world+예약 큐 증가량 |
|
||||||
|
| `death-drain` | 종료 시 1,200명이 되도록 큰 roster에서 cycle마다 100명 사망 | 사망 뒤 world와 예약 큐가 실제로 줄어드는지 |
|
||||||
|
| `balanced-churn` | 1,200명을 유지하면서 cycle마다 100명 사망+100명 생성 | live 수가 일정해도 과거 ID/queue가 남는 누수 |
|
||||||
|
| `rollback-churn` | cycle마다 100명 생성+사망 후 강제 실패 | rollback snapshot 복원 뒤 retained state drift |
|
||||||
|
|
||||||
|
`death-drain`과 `balanced-churn`은 `retain`과 `prune`을 같은 입력으로 A/B
|
||||||
|
실행할 수 있다. `retain`은 수정 전처럼 삭제된 예약 큐를 남기는 비교 모드이고,
|
||||||
|
`prune`은 PostgreSQL world flush 성공 뒤 삭제된 장수·국가 큐를 제거하는 현재
|
||||||
|
제품 경계다. 큐 제거는 `EngineStateManager.transaction()` 안에서 수행되므로 이후
|
||||||
|
오류가 발생하면 transaction savepoint가 큐와 journal set까지 복원한다.
|
||||||
|
|
||||||
|
## 실행
|
||||||
|
|
||||||
|
기본 행렬은 1,200명, 80 cycle, cycle당 100명, 5 cycle 간격 sample이다.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
NPC_LIFECYCLE_MEMORY_REPORT_PATH=/dev/shm/npc-lifecycle-memory.json \
|
||||||
|
pnpm --filter @sammo-ts/game-engine profile:npc-lifecycle-memory
|
||||||
|
```
|
||||||
|
|
||||||
|
시나리오와 규모를 좁힐 수 있다.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
NPC_LIFECYCLE_MEMORY_SCENARIOS=balanced-churn@retain,balanced-churn@prune \
|
||||||
|
NPC_LIFECYCLE_MEMORY_CYCLES=240 \
|
||||||
|
NPC_LIFECYCLE_MEMORY_BATCH_SIZE=100 \
|
||||||
|
NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY=15 \
|
||||||
|
NPC_LIFECYCLE_MEMORY_BASE_GENERALS=1200 \
|
||||||
|
NPC_LIFECYCLE_MEMORY_REPORT_PATH=/dev/shm/npc-lifecycle-memory-churn-240.json \
|
||||||
|
pnpm --filter @sammo-ts/game-engine profile:npc-lifecycle-memory
|
||||||
|
```
|
||||||
|
|
||||||
|
지원 변수:
|
||||||
|
|
||||||
|
- `NPC_LIFECYCLE_MEMORY_SCENARIOS`: 쉼표 구분 `name@prune|retain`
|
||||||
|
- `NPC_LIFECYCLE_MEMORY_CYCLES`: transaction/flush 반복 횟수
|
||||||
|
- `NPC_LIFECYCLE_MEMORY_BATCH_SIZE`: cycle당 update/create/delete 수
|
||||||
|
- `NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY`: 명시적 GC와 snapshot sample 간격
|
||||||
|
- `NPC_LIFECYCLE_MEMORY_BASE_GENERALS`: steady/growth/churn의 live 기준 수
|
||||||
|
- `NPC_LIFECYCLE_MEMORY_REPETITIONS`: 각 독립 case 반복 횟수
|
||||||
|
- `NPC_LIFECYCLE_MEMORY_REPORT_PATH`: aggregate JSON 경로
|
||||||
|
|
||||||
|
## 2026-08-24 기준 결과
|
||||||
|
|
||||||
|
Node v24.18.0, Linux x64 shared 개발 host에서 80×100 행렬을 독립적으로 두 번
|
||||||
|
실행했다. 게임 결과에 영향을 주지 않는 profile 수치 중 retained heap delta는
|
||||||
|
두 번의 차이가 14 KiB 이내였고 snapshot delta와 최종 queue 수는 정확히
|
||||||
|
일치했다.
|
||||||
|
|
||||||
|
| case | 최종 live/queue | retained heap delta (run 1/2) | snapshot delta | 해석 |
|
||||||
|
| --- | ---: | ---: | ---: | --- |
|
||||||
|
| steady 80 | 1,200 / 1,200 | +121,664 / +126,720 B | 0 B | transaction 반복 자체는 안정 |
|
||||||
|
| growth 8,000명 | 9,200 / 9,200 | +37,487,264 / +37,494,272 B | +13,807,868 B | live NPC 약 4.69 KiB heap, 1.73 KiB snapshot/명 |
|
||||||
|
| death retain | 1,200 / 9,200 | -11,407,296 / -11,393,936 B | -7,450,860 B | world는 줄지만 죽은 큐 8,000개 잔존 |
|
||||||
|
| death prune | 1,200 / 1,200 | -37,058,496 / -37,054,712 B | -13,802,736 B | live roster와 queue가 함께 감소 |
|
||||||
|
| balanced retain | 1,200 / 9,200 | +25,970,064 / +25,969,504 B | +6,357,008 B | live 수가 같아도 과거 큐가 선형 증가 |
|
||||||
|
| balanced prune | 1,200 / 1,200 | +314,464 / +305,984 B | +5,132 B | 8,000명 churn 뒤 dead queue 0 |
|
||||||
|
| rollback churn | 1,200 / 1,200 | +1,982,920 / +1,985,664 B | +72,000 B | 80회 강제 rollback 후 live/queue 복원 |
|
||||||
|
|
||||||
|
장기 soak도 별도로 실행했다.
|
||||||
|
|
||||||
|
- steady 720 cycle: live/queue `1,200/1,200`, heap `+217,192 B`, snapshot `0 B`,
|
||||||
|
후반 slope 약 `182 B/cycle`
|
||||||
|
- balanced 24,000명 churn, retain: queue 25,200, heap `+77,121,104 B`,
|
||||||
|
snapshot `+19,063,275 B`
|
||||||
|
- 같은 churn, prune: queue 1,200, heap `+321,256 B`, snapshot `+7,400 B`
|
||||||
|
|
||||||
|
수정 전 예약 큐 보유량은 churn 1명당 retained heap 약 3.2 KiB, 직렬화된
|
||||||
|
transaction participant 약 794 B로 선형 증가했다. 정리 후 두 장기 case에서
|
||||||
|
dead queue는 0이며 live 수가 일정한 snapshot은 ID 문자열 길이 차이 외에는
|
||||||
|
거의 일정했다.
|
||||||
|
|
||||||
|
`growth` 9,200명 sample에서 in-transaction heap은 같은 cycle의 flush 후보다
|
||||||
|
최대 약 14.1 MiB 높았다. 현재 world rollback은 live state 전체를 deep clone하므로
|
||||||
|
다음 절감 후보는 mutation별 copy-on-write/delta journal이다. 다만 이 변경은
|
||||||
|
오류 rollback, RNG와 persistence 재시도 순서에 직접 닿으므로 이번 작업에서는
|
||||||
|
예약 큐처럼 소유권이 명확한 항목만 정리하고 world snapshot 구조는 바꾸지 않는다.
|
||||||
|
|
||||||
|
## 운영 관찰
|
||||||
|
|
||||||
|
턴 데몬 5분 telemetry는 world entity 수에 더해 `generalTurnQueues`와
|
||||||
|
`nationTurnQueues`를 출력한다. 정상 flush가 이어지는 장기 서버에서는
|
||||||
|
`generalTurnQueues - generals`가 사망 누적으로 계속 증가하지 않아야 한다.
|
||||||
|
heap/RSS만으로 queue leak을 추정하지 말고 두 count와 OOM/restart, 월 진행을 함께
|
||||||
|
관찰한다.
|
||||||
@@ -245,6 +245,13 @@ Transaction 실패 시 in-memory snapshot을 복원하고 event는 재시도 가
|
|||||||
Checkpoint의 단일 소유자는 `InMemoryTurnWorld`이며 state store는 이를
|
Checkpoint의 단일 소유자는 `InMemoryTurnWorld`이며 state store는 이를
|
||||||
위임 조회합니다. 별도 checkpoint 복사본이 snapshot보다 앞서 나가지 않습니다.
|
위임 조회합니다. 별도 checkpoint 복사본이 snapshot보다 앞서 나가지 않습니다.
|
||||||
|
|
||||||
|
DB flush가 성공해 장수·국가 행과 예약 턴 행을 삭제한 뒤에는 같은
|
||||||
|
`EngineStateManager` transaction 안에서 해당 in-memory 예약 큐도 제거합니다.
|
||||||
|
실패하면 savepoint가 큐를 복원합니다. 5분 memory telemetry의
|
||||||
|
`generalTurnQueues`/`nationTurnQueues`와 world entity count로 장기 churn의 queue
|
||||||
|
잔존 여부를 확인할 수 있으며, 재현 행렬과 기준 수치는
|
||||||
|
[`npc-lifecycle-memory-profile.md`](./npc-lifecycle-memory-profile.md)에 기록합니다.
|
||||||
|
|
||||||
예약 턴은 revision/CAS와 lease를 사용합니다. API의 편집과 daemon의 실행이
|
예약 턴은 revision/CAS와 lease를 사용합니다. API의 편집과 daemon의 실행이
|
||||||
경합해도 오래된 revision이 새 queue를 덮어쓰지 않게 합니다.
|
경합해도 오래된 revision이 새 queue를 덮어쓰지 않게 합니다.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user