perf: 300명 실시간 부하 측정 도구를 추가
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import { assertRuntimeMetadataFinalized, loadConfig, loadTokens } from './config.js';
|
||||
import { describeDryRun, runLoadTest } from './runner.js';
|
||||
|
||||
type Command = 'run' | 'dry-run' | 'validate';
|
||||
|
||||
const usage = (): never => {
|
||||
process.stderr.write('usage: cli.ts <validate|dry-run|run> --config <file> [--tokens <0600-gitignored-file>] [--output <new-json-file>]\n');
|
||||
process.exit(64);
|
||||
};
|
||||
|
||||
const parseArguments = (argv: readonly string[]): { command: Command; config: string; tokens?: string; output?: string } => {
|
||||
const command = argv[0];
|
||||
if (!['run', 'dry-run', 'validate'].includes(command ?? '')) usage();
|
||||
const values = new Map<string, string>();
|
||||
for (let index = 1; index < argv.length; index += 2) {
|
||||
const flag = argv[index];
|
||||
const value = argv[index + 1];
|
||||
if (!flag || !['--config', '--tokens', '--output'].includes(flag) || !value) usage();
|
||||
values.set(flag, value);
|
||||
}
|
||||
const config = values.get('--config');
|
||||
if (!config) usage();
|
||||
if (command === 'run' && (!values.get('--tokens') || !values.get('--output'))) usage();
|
||||
if (command === 'validate' && (values.has('--tokens') || values.has('--output'))) usage();
|
||||
if (command === 'dry-run' && values.has('--output')) usage();
|
||||
return {
|
||||
command: command as Command,
|
||||
config: config!,
|
||||
...(values.get('--tokens') ? { tokens: values.get('--tokens')! } : {}),
|
||||
...(values.get('--output') ? { output: values.get('--output')! } : {}),
|
||||
};
|
||||
};
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const args = parseArguments(process.argv.slice(2));
|
||||
const workspaceRoot = path.resolve(import.meta.dirname, '../../..');
|
||||
const { config, sha256 } = await loadConfig(args.config);
|
||||
if (args.command === 'validate') {
|
||||
process.stdout.write(`${JSON.stringify({ valid: true, name: config.name, configSha256: sha256 })}\n`);
|
||||
return;
|
||||
}
|
||||
const tokens = args.tokens ? await loadTokens(args.tokens, workspaceRoot, config.capacity.authenticatedViewers) : null;
|
||||
if (args.command === 'dry-run') {
|
||||
process.stdout.write(`${JSON.stringify({ valid: true, tokenFileValidated: tokens !== null, configSha256: sha256, plan: describeDryRun(config) }, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
assertRuntimeMetadataFinalized(config);
|
||||
const output = path.resolve(args.output!);
|
||||
await mkdir(path.dirname(output), { recursive: true });
|
||||
const result = await runLoadTest({ config, configSha256: sha256, tokens: tokens!, workspaceRoot });
|
||||
await writeFile(output, `${JSON.stringify(result, null, 2)}\n`, { encoding: 'utf8', flag: 'wx', mode: 0o600 });
|
||||
const failedRequests = result.phases.reduce((total, phase) => total + Object.values(phase.metrics.http.errors).reduce((sum, count) => sum + count, 0), 0);
|
||||
process.stdout.write(`${JSON.stringify({ completed: true, phases: result.phases.length, failedRequests, outputWritten: true })}\n`);
|
||||
};
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : 'unknown load-test error';
|
||||
process.stderr.write(`${message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,251 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { lstat, readFile, realpath, stat } from 'node:fs/promises';
|
||||
import { execFile } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export type PhaseKind = 'idle' | 'own' | 'global' | 'mixed';
|
||||
|
||||
export interface LoadOperation {
|
||||
name: string;
|
||||
procedure: string;
|
||||
type: 'query';
|
||||
weight: number;
|
||||
input?: unknown;
|
||||
}
|
||||
|
||||
export interface LoadPhase {
|
||||
name: string;
|
||||
kind: PhaseKind;
|
||||
durationMs: number;
|
||||
sseConnections: number;
|
||||
requestIntervalMs: number | null;
|
||||
operations: LoadOperation[];
|
||||
}
|
||||
|
||||
export interface LoadConfig {
|
||||
$schema?: string;
|
||||
version: 1;
|
||||
name: string;
|
||||
target: {
|
||||
baseUrl: string;
|
||||
trpcPath: string;
|
||||
ssePath: string;
|
||||
publicProfile: false;
|
||||
allowedHosts: string[];
|
||||
};
|
||||
isolation: {
|
||||
postgresSchema: string;
|
||||
redisPrefix: string;
|
||||
};
|
||||
capacity: {
|
||||
authenticatedViewers: number;
|
||||
npcGenerals: number;
|
||||
humanGenerals: number;
|
||||
turnIntervalMs: number;
|
||||
};
|
||||
runtimeMetadata: {
|
||||
fixtureSha256: string;
|
||||
imageDigest: string;
|
||||
postgresVersion: string;
|
||||
redisVersion: string;
|
||||
};
|
||||
phases: LoadPhase[];
|
||||
}
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
|
||||
const integerAtLeast = (value: unknown, minimum: number): boolean =>
|
||||
typeof value === 'number' && Number.isInteger(value) && value >= minimum;
|
||||
|
||||
const hasOnlyKeys = (value: Record<string, unknown>, allowed: readonly string[]): boolean =>
|
||||
Object.keys(value).every((key) => allowed.includes(key));
|
||||
|
||||
const isPrivateTargetHost = (hostname: string): boolean => {
|
||||
const normalized = hostname.toLowerCase().replace(/^\[|\]$/gu, '');
|
||||
if (normalized === 'localhost' || normalized === '::1' || normalized.endsWith('.localhost')) return true;
|
||||
if (normalized.endsWith('.internal') || normalized.endsWith('.local')) return true;
|
||||
const parts = normalized.split('.').map(Number);
|
||||
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
|
||||
return normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe8') || normalized.startsWith('fe9') || normalized.startsWith('fea') || normalized.startsWith('feb');
|
||||
}
|
||||
return parts[0] === 10 || parts[0] === 127 || (parts[0] === 192 && parts[1] === 168) || (parts[0] === 172 && parts[1]! >= 16 && parts[1]! <= 31);
|
||||
};
|
||||
|
||||
export const validateLoadConfig = (raw: unknown): LoadConfig => {
|
||||
const issues: string[] = [];
|
||||
if (!isRecord(raw)) throw new Error('config must be a JSON object');
|
||||
if (!hasOnlyKeys(raw, ['$schema', 'version', 'name', 'target', 'isolation', 'capacity', 'runtimeMetadata', 'phases'])) {
|
||||
issues.push('config contains unknown fields');
|
||||
}
|
||||
if (raw.version !== 1) issues.push('version must be 1');
|
||||
if (typeof raw.name !== 'string' || !/^[a-z0-9][a-z0-9-]{0,63}$/u.test(raw.name)) issues.push('name is invalid');
|
||||
|
||||
const target = raw.target;
|
||||
if (!isRecord(target)) {
|
||||
issues.push('target must be an object');
|
||||
} else {
|
||||
if (!hasOnlyKeys(target, ['baseUrl', 'trpcPath', 'ssePath', 'publicProfile', 'allowedHosts'])) {
|
||||
issues.push('target contains unknown fields');
|
||||
}
|
||||
let parsedUrl: URL | null = null;
|
||||
try {
|
||||
parsedUrl = new URL(typeof target.baseUrl === 'string' ? target.baseUrl : 'invalid:');
|
||||
} catch {
|
||||
issues.push('target.baseUrl must be a URL');
|
||||
}
|
||||
if (parsedUrl && !['http:', 'https:'].includes(parsedUrl.protocol)) issues.push('target.baseUrl must use HTTP(S)');
|
||||
if (parsedUrl && !isPrivateTargetHost(parsedUrl.hostname)) issues.push('target.baseUrl must use a loopback or private/internal hostname');
|
||||
if (parsedUrl && (parsedUrl.username || parsedUrl.password || parsedUrl.search || parsedUrl.hash)) {
|
||||
issues.push('target.baseUrl must not contain credentials, query, or fragment');
|
||||
}
|
||||
if (target.publicProfile !== false) issues.push('target.publicProfile must be false');
|
||||
if (!Array.isArray(target.allowedHosts) || target.allowedHosts.length === 0 || !target.allowedHosts.every((item) => typeof item === 'string' && item.length > 0)) {
|
||||
issues.push('target.allowedHosts must contain explicit hostnames');
|
||||
} else if (parsedUrl && !target.allowedHosts.includes(parsedUrl.hostname)) {
|
||||
issues.push('target hostname is not explicitly allowlisted');
|
||||
}
|
||||
if (typeof target.trpcPath !== 'string' || !target.trpcPath.startsWith('/')) issues.push('target.trpcPath must be absolute');
|
||||
if (typeof target.ssePath !== 'string' || !target.ssePath.startsWith('/')) issues.push('target.ssePath must be absolute');
|
||||
}
|
||||
|
||||
const isolation = raw.isolation;
|
||||
if (!isRecord(isolation)) {
|
||||
issues.push('isolation must be an object');
|
||||
} else {
|
||||
if (!hasOnlyKeys(isolation, ['postgresSchema', 'redisPrefix'])) issues.push('isolation contains unknown fields');
|
||||
if (typeof isolation.postgresSchema !== 'string' || !/^load_[a-z0-9_]+$/u.test(isolation.postgresSchema)) {
|
||||
issues.push('isolation.postgresSchema must start with load_');
|
||||
}
|
||||
if (typeof isolation.redisPrefix !== 'string' || !/^load-tests:[a-z0-9:_-]+:$/u.test(isolation.redisPrefix)) {
|
||||
issues.push('isolation.redisPrefix must be load-tests scoped and end with a colon');
|
||||
}
|
||||
}
|
||||
|
||||
const capacity = raw.capacity;
|
||||
if (!isRecord(capacity)) {
|
||||
issues.push('capacity must be an object');
|
||||
} else {
|
||||
if (!hasOnlyKeys(capacity, ['authenticatedViewers', 'npcGenerals', 'humanGenerals', 'turnIntervalMs'])) issues.push('capacity contains unknown fields');
|
||||
if (!integerAtLeast(capacity.authenticatedViewers, 1)) issues.push('capacity.authenticatedViewers must be positive');
|
||||
if (!integerAtLeast(capacity.npcGenerals, 0)) issues.push('capacity.npcGenerals must be non-negative');
|
||||
if (!integerAtLeast(capacity.humanGenerals, 0)) issues.push('capacity.humanGenerals must be non-negative');
|
||||
if (!integerAtLeast(capacity.turnIntervalMs, 1000)) issues.push('capacity.turnIntervalMs must be at least 1000');
|
||||
}
|
||||
|
||||
const metadata = raw.runtimeMetadata;
|
||||
if (!isRecord(metadata)) {
|
||||
issues.push('runtimeMetadata must be an object');
|
||||
} else {
|
||||
if (!hasOnlyKeys(metadata, ['fixtureSha256', 'imageDigest', 'postgresVersion', 'redisVersion'])) issues.push('runtimeMetadata contains unknown fields');
|
||||
if (typeof metadata.fixtureSha256 !== 'string' || !/^sha256:[a-f0-9]{64}$/u.test(metadata.fixtureSha256)) issues.push('runtimeMetadata.fixtureSha256 must be a sha256 digest');
|
||||
for (const field of ['imageDigest', 'postgresVersion', 'redisVersion'] as const) {
|
||||
if (typeof metadata[field] !== 'string' || metadata[field].length === 0) issues.push(`runtimeMetadata.${field} is required`);
|
||||
}
|
||||
}
|
||||
|
||||
const phases = raw.phases;
|
||||
if (!Array.isArray(phases)) {
|
||||
issues.push('phases must be an array');
|
||||
} else {
|
||||
const kinds = new Set<string>();
|
||||
for (const [index, phase] of phases.entries()) {
|
||||
if (!isRecord(phase)) {
|
||||
issues.push(`phases[${index}] must be an object`);
|
||||
continue;
|
||||
}
|
||||
if (!hasOnlyKeys(phase, ['name', 'kind', 'durationMs', 'sseConnections', 'requestIntervalMs', 'operations'])) issues.push(`phases[${index}] contains unknown fields`);
|
||||
if (typeof phase.name !== 'string' || !/^[a-z0-9][a-z0-9-]{0,31}$/u.test(phase.name)) issues.push(`phases[${index}].name is invalid`);
|
||||
if (!['idle', 'own', 'global', 'mixed'].includes(String(phase.kind))) issues.push(`phases[${index}].kind is invalid`);
|
||||
else kinds.add(String(phase.kind));
|
||||
if (!integerAtLeast(phase.durationMs, 1000)) issues.push(`phases[${index}].durationMs must be at least 1000`);
|
||||
if (!integerAtLeast(phase.sseConnections, 0)) issues.push(`phases[${index}].sseConnections must be non-negative`);
|
||||
const operations = phase.operations;
|
||||
if (!Array.isArray(operations)) {
|
||||
issues.push(`phases[${index}].operations must be an array`);
|
||||
continue;
|
||||
}
|
||||
if (phase.kind === 'idle') {
|
||||
if (phase.requestIntervalMs !== null || operations.length !== 0) issues.push(`phases[${index}] idle phase must not issue HTTP requests`);
|
||||
} else if (!integerAtLeast(phase.requestIntervalMs, 50) || operations.length === 0) {
|
||||
issues.push(`phases[${index}] active phase requires an interval and operations`);
|
||||
}
|
||||
for (const [operationIndex, operation] of operations.entries()) {
|
||||
if (!isRecord(operation)) {
|
||||
issues.push(`phases[${index}].operations[${operationIndex}] must be an object`);
|
||||
continue;
|
||||
}
|
||||
if (!hasOnlyKeys(operation, ['name', 'procedure', 'type', 'weight', 'input'])) issues.push(`phases[${index}].operations[${operationIndex}] contains unknown fields`);
|
||||
if (typeof operation.name !== 'string' || !/^[a-z0-9][a-z0-9-]{0,31}$/u.test(operation.name)) issues.push(`phases[${index}].operations[${operationIndex}].name is invalid`);
|
||||
if (typeof operation.procedure !== 'string' || !/^[A-Za-z][A-Za-z0-9_.]+$/u.test(operation.procedure)) issues.push(`phases[${index}].operations[${operationIndex}].procedure is invalid`);
|
||||
if (operation.type !== 'query') issues.push(`phases[${index}].operations[${operationIndex}] must be a read-only query`);
|
||||
if (!integerAtLeast(operation.weight, 1) || Number(operation.weight) > 100) issues.push(`phases[${index}].operations[${operationIndex}].weight must be 1..100`);
|
||||
}
|
||||
}
|
||||
for (const required of ['idle', 'own', 'global', 'mixed']) {
|
||||
if (!kinds.has(required)) issues.push(`phases must include ${required}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (isRecord(capacity) && Array.isArray(phases)) {
|
||||
for (const [index, phase] of phases.entries()) {
|
||||
if (isRecord(phase) && typeof phase.sseConnections === 'number' && typeof capacity.authenticatedViewers === 'number' && phase.sseConnections > capacity.authenticatedViewers) {
|
||||
issues.push(`phases[${index}].sseConnections exceeds authenticatedViewers`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (issues.length > 0) throw new Error(`invalid load configuration:\n- ${issues.join('\n- ')}`);
|
||||
return raw as unknown as LoadConfig;
|
||||
};
|
||||
|
||||
export const canonicalJson = (value: unknown): string => {
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
|
||||
if (isRecord(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(',')}}`;
|
||||
return JSON.stringify(value);
|
||||
};
|
||||
|
||||
export const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex');
|
||||
|
||||
export const loadConfig = async (configPath: string): Promise<{ config: LoadConfig; sha256: string }> => {
|
||||
const text = await readFile(configPath, 'utf8');
|
||||
const parsed: unknown = JSON.parse(text);
|
||||
const config = validateLoadConfig(parsed);
|
||||
return { config, sha256: sha256(canonicalJson(config)) };
|
||||
};
|
||||
|
||||
export const assertRuntimeMetadataFinalized = (config: LoadConfig): void => {
|
||||
const placeholderFields = Object.entries(config.runtimeMetadata)
|
||||
.filter(([, value]) => value.includes('replace-before-measurement') || /^sha256:0{64}$/u.test(value))
|
||||
.map(([key]) => key);
|
||||
if (placeholderFields.length > 0) {
|
||||
throw new Error(`runtime metadata placeholders must be replaced before run: ${placeholderFields.join(', ')}`);
|
||||
}
|
||||
};
|
||||
|
||||
export const loadTokens = async (tokenPath: string, workspaceRoot: string, requiredCount: number): Promise<string[]> => {
|
||||
const absolute = path.resolve(tokenPath);
|
||||
const root = await realpath(path.resolve(workspaceRoot));
|
||||
const linkStat = await lstat(absolute);
|
||||
if (linkStat.isSymbolicLink()) throw new Error('token file must not be a symbolic link');
|
||||
const resolved = await realpath(absolute);
|
||||
if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) throw new Error('token file must be inside the workspace and gitignored');
|
||||
const fileStat = await stat(absolute);
|
||||
if ((fileStat.mode & 0o777) !== 0o600) throw new Error('token file mode must be exactly 0600');
|
||||
try {
|
||||
await execFileAsync('git', ['check-ignore', '--quiet', '--', resolved], { cwd: root });
|
||||
} catch {
|
||||
throw new Error('token file must be covered by .gitignore');
|
||||
}
|
||||
const parsed: unknown = JSON.parse(await readFile(absolute, 'utf8'));
|
||||
if (!isRecord(parsed) || !hasOnlyKeys(parsed, ['tokens']) || !Array.isArray(parsed.tokens)) throw new Error('token file must contain only a tokens array');
|
||||
if (!parsed.tokens.every((token) => typeof token === 'string' && token.length >= 16)) throw new Error('each bearer token must be a non-empty string of at least 16 characters');
|
||||
if (new Set(parsed.tokens).size !== parsed.tokens.length) throw new Error('token file contains duplicate tokens');
|
||||
if (parsed.tokens.length < requiredCount) throw new Error(`token file has fewer than ${requiredCount} entries`);
|
||||
return parsed.tokens.slice(0, requiredCount);
|
||||
};
|
||||
|
||||
export const expandWeightedOperations = (operations: readonly LoadOperation[]): LoadOperation[] =>
|
||||
operations.flatMap((operation) => Array.from({ length: operation.weight }, () => operation));
|
||||
@@ -0,0 +1,168 @@
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
||||
|
||||
export interface DistributionSummary {
|
||||
count: number;
|
||||
min: number | null;
|
||||
max: number | null;
|
||||
mean: number | null;
|
||||
p50: number | null;
|
||||
p95: number | null;
|
||||
p99: number | null;
|
||||
}
|
||||
|
||||
const rounded = (value: number): number => Math.round(value * 1000) / 1000;
|
||||
|
||||
export const percentile = (sorted: readonly number[], percentileValue: number): number | null => {
|
||||
if (sorted.length === 0) return null;
|
||||
if (percentileValue <= 0) return sorted[0] ?? null;
|
||||
if (percentileValue >= 100) return sorted.at(-1) ?? null;
|
||||
const rank = Math.ceil((percentileValue / 100) * sorted.length) - 1;
|
||||
return sorted[Math.max(0, rank)] ?? null;
|
||||
};
|
||||
|
||||
export const summarizeDistribution = (values: readonly number[]): DistributionSummary => {
|
||||
if (values.length === 0) return { count: 0, min: null, max: null, mean: null, p50: null, p95: null, p99: null };
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length;
|
||||
return {
|
||||
count: sorted.length,
|
||||
min: rounded(sorted[0]!),
|
||||
max: rounded(sorted.at(-1)!),
|
||||
mean: rounded(mean),
|
||||
p50: rounded(percentile(sorted, 50)!),
|
||||
p95: rounded(percentile(sorted, 95)!),
|
||||
p99: rounded(percentile(sorted, 99)!),
|
||||
};
|
||||
};
|
||||
|
||||
export class PhaseMetrics {
|
||||
readonly httpLatencyMs = new Map<string, number[]>();
|
||||
readonly httpSuccess = new Map<string, number>();
|
||||
readonly httpErrors = new Map<string, number>();
|
||||
readonly httpResults = new Map<string, number>();
|
||||
readonly sseEvents = new Map<string, number>();
|
||||
readonly processRssBytes: number[] = [];
|
||||
readonly sseActiveConnections: number[] = [];
|
||||
sseActiveCurrent = 0;
|
||||
sseAttempts = 0;
|
||||
sseOpened = 0;
|
||||
sseClosed = 0;
|
||||
sseReconnects = 0;
|
||||
sseFailures = 0;
|
||||
ssePrivacyViolations = 0;
|
||||
|
||||
recordHttp(name: string, latencyMs: number, outcome: string | null): void {
|
||||
const values = this.httpLatencyMs.get(name) ?? [];
|
||||
values.push(latencyMs);
|
||||
this.httpLatencyMs.set(name, values);
|
||||
const target = outcome === null ? this.httpSuccess : this.httpErrors;
|
||||
const key = outcome === null ? name : `${name}:${outcome}`;
|
||||
target.set(key, (target.get(key) ?? 0) + 1);
|
||||
}
|
||||
|
||||
recordSseEvent(name: string): void {
|
||||
const safeName = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/u.test(name) ? name : 'invalid-name';
|
||||
this.sseEvents.set(safeName, (this.sseEvents.get(safeName) ?? 0) + 1);
|
||||
}
|
||||
|
||||
recordHttpResult(name: string, result: string): void {
|
||||
const safeResult = /^[a-z][a-z0-9-]{0,31}$/u.test(result) ? result : 'other';
|
||||
const key = `${name}:${safeResult}`;
|
||||
this.httpResults.set(key, (this.httpResults.get(key) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const mapToObject = (value: ReadonlyMap<string, number>): Record<string, number> =>
|
||||
Object.fromEntries([...value.entries()].sort(([left], [right]) => left.localeCompare(right)));
|
||||
|
||||
export interface PhaseMetricSummary {
|
||||
http: {
|
||||
success: Record<string, number>;
|
||||
errors: Record<string, number>;
|
||||
results: Record<string, number>;
|
||||
latencyMs: Record<string, DistributionSummary>;
|
||||
};
|
||||
sse: {
|
||||
attempts: number;
|
||||
opened: number;
|
||||
closed: number;
|
||||
reconnects: number;
|
||||
failures: number;
|
||||
events: Record<string, number>;
|
||||
privacyViolations: number;
|
||||
activeConnections: DistributionSummary;
|
||||
};
|
||||
process: {
|
||||
cpuPercentOfOneCore: number;
|
||||
rssBytes: DistributionSummary;
|
||||
eventLoopLagMs: Omit<DistributionSummary, 'count' | 'mean'> & { mean: number };
|
||||
};
|
||||
}
|
||||
|
||||
export class ProcessSampler {
|
||||
private readonly histogram = monitorEventLoopDelay({ resolution: 20 });
|
||||
private readonly startCpu = process.cpuUsage();
|
||||
private readonly startNs = process.hrtime.bigint();
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(private readonly metrics: PhaseMetrics) {}
|
||||
|
||||
start(): void {
|
||||
this.histogram.enable();
|
||||
this.sample();
|
||||
this.timer = setInterval(() => this.sample(), 1000);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
private sample(): void {
|
||||
this.metrics.processRssBytes.push(process.memoryUsage().rss);
|
||||
this.metrics.sseActiveConnections.push(this.metrics.sseActiveCurrent);
|
||||
}
|
||||
|
||||
stop(activeConnections: number): PhaseMetricSummary['process'] {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
this.sample();
|
||||
this.metrics.sseActiveConnections.push(activeConnections);
|
||||
this.histogram.disable();
|
||||
const elapsedMs = Number(process.hrtime.bigint() - this.startNs) / 1_000_000;
|
||||
const cpu = process.cpuUsage(this.startCpu);
|
||||
const cpuMs = (cpu.user + cpu.system) / 1000;
|
||||
const fromNs = (value: number): number => (Number.isFinite(value) ? rounded(value / 1_000_000) : 0);
|
||||
return {
|
||||
cpuPercentOfOneCore: rounded((cpuMs / Math.max(elapsedMs, 1)) * 100),
|
||||
rssBytes: summarizeDistribution(this.metrics.processRssBytes),
|
||||
eventLoopLagMs: {
|
||||
min: fromNs(this.histogram.min),
|
||||
max: fromNs(this.histogram.max),
|
||||
mean: fromNs(this.histogram.mean),
|
||||
p50: fromNs(this.histogram.percentile(50)),
|
||||
p95: fromNs(this.histogram.percentile(95)),
|
||||
p99: fromNs(this.histogram.percentile(99)),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const summarizePhaseMetrics = (metrics: PhaseMetrics, processSummary: PhaseMetricSummary['process']): PhaseMetricSummary => ({
|
||||
http: {
|
||||
success: mapToObject(metrics.httpSuccess),
|
||||
errors: mapToObject(metrics.httpErrors),
|
||||
results: mapToObject(metrics.httpResults),
|
||||
latencyMs: Object.fromEntries(
|
||||
[...metrics.httpLatencyMs.entries()]
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([name, values]) => [name, summarizeDistribution(values)])
|
||||
),
|
||||
},
|
||||
sse: {
|
||||
attempts: metrics.sseAttempts,
|
||||
opened: metrics.sseOpened,
|
||||
closed: metrics.sseClosed,
|
||||
reconnects: metrics.sseReconnects,
|
||||
failures: metrics.sseFailures,
|
||||
events: mapToObject(metrics.sseEvents),
|
||||
privacyViolations: metrics.ssePrivacyViolations,
|
||||
activeConnections: summarizeDistribution(metrics.sseActiveConnections),
|
||||
},
|
||||
process: processSummary,
|
||||
});
|
||||
@@ -0,0 +1,201 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { setMaxListeners } from 'node:events';
|
||||
import { promisify } from 'node:util';
|
||||
import os from 'node:os';
|
||||
import { readFile as readTextFile } from 'node:fs/promises';
|
||||
|
||||
import { canonicalJson, expandWeightedOperations, sha256, type LoadConfig, type LoadPhase } from './config.js';
|
||||
import { PhaseMetrics, ProcessSampler, summarizePhaseMetrics, type PhaseMetricSummary } from './metrics.js';
|
||||
import { runSseConnection } from './sse.js';
|
||||
import { executeTrpcQuery } from './trpc.js';
|
||||
import type { DashboardRevisions } from './trpc.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const wait = (milliseconds: number, signal: AbortSignal): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
if (signal.aborted) return resolve();
|
||||
const done = () => {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener('abort', done);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(done, milliseconds);
|
||||
signal.addEventListener('abort', done, { once: true });
|
||||
});
|
||||
|
||||
const loadText = async (file: string): Promise<string | null> => {
|
||||
try {
|
||||
return (await readTextFile(file, 'utf8')).trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const runtimeAndHost = async () => {
|
||||
const cpuQuota = await loadText('/sys/fs/cgroup/cpu.max');
|
||||
const memoryLimit = await loadText('/sys/fs/cgroup/memory.max');
|
||||
const runtime = { node: process.version, v8: process.versions.v8 };
|
||||
const cpus = os.cpus();
|
||||
const host = {
|
||||
platform: os.platform(),
|
||||
release: os.release(),
|
||||
arch: os.arch(),
|
||||
logicalCpuCount: cpus.length,
|
||||
cpuModel: cpus[0]?.model ?? 'unknown',
|
||||
totalMemoryBytes: os.totalmem(),
|
||||
cgroupCpuMax: cpuQuota,
|
||||
cgroupMemoryMax: memoryLimit,
|
||||
};
|
||||
return {
|
||||
runtime,
|
||||
host,
|
||||
runtimeSha256: sha256(canonicalJson(runtime)),
|
||||
hostSha256: sha256(canonicalJson(host)),
|
||||
};
|
||||
};
|
||||
|
||||
const gitMetadata = async (workspaceRoot: string) => {
|
||||
const run = async (args: string[]): Promise<string> => (await execFileAsync('git', args, { cwd: workspaceRoot })).stdout.trim();
|
||||
const [commit, tree, status] = await Promise.all([run(['rev-parse', 'HEAD']), run(['rev-parse', 'HEAD^{tree}']), run(['status', '--porcelain=v1'])]);
|
||||
return { commit, tree, dirty: status.length > 0 };
|
||||
};
|
||||
|
||||
const runHttpViewers = async (options: {
|
||||
config: LoadConfig;
|
||||
phase: LoadPhase;
|
||||
tokens: readonly string[];
|
||||
signal: AbortSignal;
|
||||
metrics: PhaseMetrics;
|
||||
}): Promise<void> => {
|
||||
if (options.phase.requestIntervalMs === null || options.phase.operations.length === 0) return;
|
||||
const schedule = expandWeightedOperations(options.phase.operations);
|
||||
const interval = options.phase.requestIntervalMs;
|
||||
await Promise.all(
|
||||
options.tokens.map(async (token, viewerIndex) => {
|
||||
const stagger = Math.floor((viewerIndex / options.tokens.length) * interval);
|
||||
await wait(stagger, options.signal);
|
||||
let iteration = 0;
|
||||
let dashboardRevisions: DashboardRevisions = {};
|
||||
while (!options.signal.aborted) {
|
||||
const configuredOperation = schedule[(viewerIndex + iteration) % schedule.length]!;
|
||||
const operation =
|
||||
configuredOperation.procedure === 'dashboard.getContextBundleDelta' && Object.keys(dashboardRevisions).length > 0
|
||||
? {
|
||||
...configuredOperation,
|
||||
input: {
|
||||
...(typeof configuredOperation.input === 'object' && configuredOperation.input !== null
|
||||
? configuredOperation.input
|
||||
: {}),
|
||||
known: dashboardRevisions,
|
||||
forceSnapshot: false,
|
||||
},
|
||||
}
|
||||
: configuredOperation;
|
||||
const observedRevisions = await executeTrpcQuery({
|
||||
baseUrl: options.config.target.baseUrl,
|
||||
trpcPath: options.config.target.trpcPath,
|
||||
operation,
|
||||
token,
|
||||
signal: options.signal,
|
||||
metrics: options.metrics,
|
||||
});
|
||||
if (observedRevisions) dashboardRevisions = { ...dashboardRevisions, ...observedRevisions };
|
||||
iteration += 1;
|
||||
await wait(interval, options.signal);
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
export interface PhaseResult {
|
||||
name: string;
|
||||
kind: LoadPhase['kind'];
|
||||
configuredDurationMs: number;
|
||||
elapsedMs: number;
|
||||
metrics: PhaseMetricSummary;
|
||||
}
|
||||
|
||||
const runPhase = async (config: LoadConfig, phase: LoadPhase, tokens: readonly string[]): Promise<PhaseResult> => {
|
||||
const metrics = new PhaseMetrics();
|
||||
const sampler = new ProcessSampler(metrics);
|
||||
const controller = new AbortController();
|
||||
setMaxListeners(0, controller.signal);
|
||||
let activeConnections = 0;
|
||||
const started = performance.now();
|
||||
sampler.start();
|
||||
const timer = setTimeout(() => controller.abort(), phase.durationMs);
|
||||
const sseUrl = new URL(config.target.ssePath, config.target.baseUrl).toString();
|
||||
const sseTasks = tokens.slice(0, phase.sseConnections).map((token) =>
|
||||
runSseConnection({
|
||||
url: sseUrl,
|
||||
token,
|
||||
signal: controller.signal,
|
||||
metrics,
|
||||
onActiveChange: (delta) => {
|
||||
activeConnections += delta;
|
||||
metrics.sseActiveCurrent = activeConnections;
|
||||
},
|
||||
})
|
||||
);
|
||||
const httpTask = runHttpViewers({ config, phase, tokens, signal: controller.signal, metrics });
|
||||
const settled = await Promise.allSettled([...sseTasks, httpTask]);
|
||||
clearTimeout(timer);
|
||||
const rejected = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected');
|
||||
if (rejected) {
|
||||
sampler.stop(activeConnections);
|
||||
throw rejected.reason;
|
||||
}
|
||||
const processSummary = sampler.stop(activeConnections);
|
||||
return {
|
||||
name: phase.name,
|
||||
kind: phase.kind,
|
||||
configuredDurationMs: phase.durationMs,
|
||||
elapsedMs: Math.round(performance.now() - started),
|
||||
metrics: summarizePhaseMetrics(metrics, processSummary),
|
||||
};
|
||||
};
|
||||
|
||||
export const describeDryRun = (config: LoadConfig) => ({
|
||||
name: config.name,
|
||||
targetHost: new URL(config.target.baseUrl).hostname,
|
||||
isolation: config.isolation,
|
||||
capacity: config.capacity,
|
||||
phases: config.phases.map((phase) => ({
|
||||
name: phase.name,
|
||||
kind: phase.kind,
|
||||
durationMs: phase.durationMs,
|
||||
sseConnections: phase.sseConnections,
|
||||
requestIntervalMs: phase.requestIntervalMs,
|
||||
operations: phase.operations.map((operation) => ({ name: operation.name, weight: operation.weight })),
|
||||
})),
|
||||
});
|
||||
|
||||
export const runLoadTest = async (options: {
|
||||
config: LoadConfig;
|
||||
configSha256: string;
|
||||
tokens: readonly string[];
|
||||
workspaceRoot: string;
|
||||
}) => {
|
||||
const startedAt = new Date().toISOString();
|
||||
const [git, environment] = await Promise.all([gitMetadata(options.workspaceRoot), runtimeAndHost()]);
|
||||
const phases: PhaseResult[] = [];
|
||||
for (const phase of options.config.phases) phases.push(await runPhase(options.config, phase, options.tokens));
|
||||
return {
|
||||
formatVersion: 1,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
config: {
|
||||
name: options.config.name,
|
||||
sha256: options.configSha256,
|
||||
capacity: options.config.capacity,
|
||||
isolation: options.config.isolation,
|
||||
},
|
||||
git,
|
||||
runtime: environment.runtime,
|
||||
host: environment.host,
|
||||
hashes: { configSha256: options.configSha256, runtimeSha256: environment.runtimeSha256, hostSha256: environment.hostSha256 },
|
||||
targetRuntime: options.config.runtimeMetadata,
|
||||
phases,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,134 @@
|
||||
import type { PhaseMetrics } from './metrics.js';
|
||||
|
||||
const forbiddenKeys = new Set([
|
||||
'at',
|
||||
'lastTurnTime',
|
||||
'revision',
|
||||
'generalId',
|
||||
'cityId',
|
||||
'nationId',
|
||||
'entityId',
|
||||
'mailboxId',
|
||||
'messageId',
|
||||
'senderId',
|
||||
]);
|
||||
|
||||
export const containsForbiddenPublicField = (value: unknown): boolean => {
|
||||
if (Array.isArray(value)) return value.some(containsForbiddenPublicField);
|
||||
if (typeof value !== 'object' || value === null) return false;
|
||||
return Object.entries(value).some(([key, item]) => forbiddenKeys.has(key) || containsForbiddenPublicField(item));
|
||||
};
|
||||
|
||||
export interface ParsedSseEvent {
|
||||
event: string;
|
||||
data: string;
|
||||
}
|
||||
|
||||
export class SseParser {
|
||||
private buffer = '';
|
||||
private eventName = 'message';
|
||||
private data: string[] = [];
|
||||
|
||||
constructor(private readonly onEvent: (event: ParsedSseEvent) => void) {}
|
||||
|
||||
push(chunk: string): void {
|
||||
this.buffer += chunk;
|
||||
let newline = this.buffer.indexOf('\n');
|
||||
while (newline >= 0) {
|
||||
const rawLine = this.buffer.slice(0, newline);
|
||||
this.buffer = this.buffer.slice(newline + 1);
|
||||
this.consumeLine(rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine);
|
||||
newline = this.buffer.indexOf('\n');
|
||||
}
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (this.buffer.length > 0) this.consumeLine(this.buffer.endsWith('\r') ? this.buffer.slice(0, -1) : this.buffer);
|
||||
this.buffer = '';
|
||||
this.dispatch();
|
||||
}
|
||||
|
||||
private consumeLine(line: string): void {
|
||||
if (line === '') {
|
||||
this.dispatch();
|
||||
return;
|
||||
}
|
||||
if (line.startsWith(':')) return;
|
||||
const separator = line.indexOf(':');
|
||||
const field = separator < 0 ? line : line.slice(0, separator);
|
||||
let value = separator < 0 ? '' : line.slice(separator + 1);
|
||||
if (value.startsWith(' ')) value = value.slice(1);
|
||||
if (field === 'event') this.eventName = value;
|
||||
if (field === 'data') this.data.push(value);
|
||||
}
|
||||
|
||||
private dispatch(): void {
|
||||
if (this.data.length > 0) this.onEvent({ event: this.eventName, data: this.data.join('\n') });
|
||||
this.eventName = 'message';
|
||||
this.data = [];
|
||||
}
|
||||
}
|
||||
|
||||
const wait = (milliseconds: number, signal: AbortSignal): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
if (signal.aborted) return resolve();
|
||||
const done = () => {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener('abort', done);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(done, milliseconds);
|
||||
signal.addEventListener('abort', done, { once: true });
|
||||
});
|
||||
|
||||
export const runSseConnection = async (options: {
|
||||
url: string;
|
||||
token: string;
|
||||
signal: AbortSignal;
|
||||
metrics: PhaseMetrics;
|
||||
onActiveChange: (delta: number) => void;
|
||||
reconnectDelayMs?: number;
|
||||
}): Promise<void> => {
|
||||
let priorAttempt = false;
|
||||
while (!options.signal.aborted) {
|
||||
if (priorAttempt) options.metrics.sseReconnects += 1;
|
||||
priorAttempt = true;
|
||||
options.metrics.sseAttempts += 1;
|
||||
let active = false;
|
||||
try {
|
||||
const response = await fetch(options.url, {
|
||||
headers: { accept: 'text/event-stream', authorization: `Bearer ${options.token}` },
|
||||
signal: options.signal,
|
||||
});
|
||||
if (!response.ok || !response.body) {
|
||||
options.metrics.sseFailures += 1;
|
||||
await response.body?.cancel();
|
||||
} else {
|
||||
options.metrics.sseOpened += 1;
|
||||
active = true;
|
||||
options.onActiveChange(1);
|
||||
const parser = new SseParser(({ event, data }) => {
|
||||
options.metrics.recordSseEvent(event);
|
||||
try {
|
||||
if (containsForbiddenPublicField(JSON.parse(data))) options.metrics.ssePrivacyViolations += 1;
|
||||
} catch {
|
||||
options.metrics.sseFailures += 1;
|
||||
}
|
||||
});
|
||||
const decoder = new TextDecoder();
|
||||
for await (const chunk of response.body) parser.push(decoder.decode(chunk, { stream: true }));
|
||||
parser.push(decoder.decode());
|
||||
parser.finish();
|
||||
if (!options.signal.aborted) options.metrics.sseFailures += 1;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!options.signal.aborted) options.metrics.sseFailures += 1;
|
||||
} finally {
|
||||
if (active) {
|
||||
options.metrics.sseClosed += 1;
|
||||
options.onActiveChange(-1);
|
||||
}
|
||||
}
|
||||
if (!options.signal.aborted) await wait(options.reconnectDelayMs ?? 1000, options.signal);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,104 @@
|
||||
import type { LoadOperation } from './config.js';
|
||||
import type { PhaseMetrics } from './metrics.js';
|
||||
|
||||
export interface TrpcRequest {
|
||||
url: string;
|
||||
init: RequestInit;
|
||||
}
|
||||
|
||||
export const buildTrpcQuery = (baseUrl: string, trpcPath: string, operation: LoadOperation, token: string): TrpcRequest => {
|
||||
const normalizedPath = trpcPath.endsWith('/') ? trpcPath.slice(0, -1) : trpcPath;
|
||||
const url = new URL(`${normalizedPath}/${operation.procedure}`, baseUrl);
|
||||
if (operation.input !== undefined) url.searchParams.set('input', JSON.stringify({ json: operation.input }));
|
||||
return {
|
||||
url: url.toString(),
|
||||
init: {
|
||||
method: 'GET',
|
||||
headers: { accept: 'application/json', authorization: `Bearer ${token}` },
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const classifyTrpcPayload = (payload: unknown): string | null => {
|
||||
if (typeof payload !== 'object' || payload === null) return 'invalid-payload';
|
||||
if ('error' in payload) {
|
||||
const error = (payload as { error?: unknown }).error;
|
||||
if (typeof error === 'object' && error !== null && 'data' in error) {
|
||||
const data = (error as { data?: unknown }).data;
|
||||
if (typeof data === 'object' && data !== null && 'code' in data && typeof (data as { code?: unknown }).code === 'string') {
|
||||
const code = (data as { code: string }).code;
|
||||
return `trpc-${/^[A-Z_]+$/u.test(code) ? code.toLowerCase() : 'error'}`;
|
||||
}
|
||||
}
|
||||
return 'trpc-error';
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const asRecord = (value: unknown): Record<string, unknown> | null =>
|
||||
typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record<string, unknown>) : null;
|
||||
|
||||
const unwrapTrpcData = (payload: unknown): unknown => {
|
||||
const result = asRecord(asRecord(payload)?.result);
|
||||
const data = asRecord(result?.data);
|
||||
return data && 'json' in data ? data.json : result?.data;
|
||||
};
|
||||
|
||||
export interface DashboardRevisions {
|
||||
context?: string;
|
||||
commandTable?: string;
|
||||
boardAccess?: string;
|
||||
}
|
||||
|
||||
export const extractDashboardRevisions = (payload: unknown): { revisions: DashboardRevisions; resultKinds: string[] } | null => {
|
||||
const data = asRecord(unwrapTrpcData(payload));
|
||||
if (!data) return null;
|
||||
const revisions: DashboardRevisions = {};
|
||||
const resultKinds: string[] = [];
|
||||
for (const [wireName, outputName] of [
|
||||
['context', 'context'],
|
||||
['commandTable', 'commandTable'],
|
||||
['boardAccess', 'boardAccess'],
|
||||
] as const) {
|
||||
const slice = asRecord(data[wireName]);
|
||||
if (!slice) continue;
|
||||
if (typeof slice.revision === 'string' && /^[A-Za-z0-9_-]{22}$/u.test(slice.revision)) revisions[outputName] = slice.revision;
|
||||
if (typeof slice.kind === 'string' && ['unchanged', 'snapshot', 'patch'].includes(slice.kind)) resultKinds.push(slice.kind);
|
||||
else resultKinds.push('other');
|
||||
}
|
||||
return { revisions, resultKinds };
|
||||
};
|
||||
|
||||
export const executeTrpcQuery = async (options: {
|
||||
baseUrl: string;
|
||||
trpcPath: string;
|
||||
operation: LoadOperation;
|
||||
token: string;
|
||||
signal: AbortSignal;
|
||||
metrics: PhaseMetrics;
|
||||
}): Promise<DashboardRevisions | undefined> => {
|
||||
const started = performance.now();
|
||||
let outcome: string | null;
|
||||
try {
|
||||
const request = buildTrpcQuery(options.baseUrl, options.trpcPath, options.operation, options.token);
|
||||
const response = await fetch(request.url, { ...request.init, signal: options.signal });
|
||||
if (!response.ok) {
|
||||
outcome = `http-${response.status}`;
|
||||
await response.body?.cancel();
|
||||
} else {
|
||||
const payload: unknown = await response.json();
|
||||
outcome = classifyTrpcPayload(payload);
|
||||
if (outcome === null && options.operation.procedure === 'dashboard.getContextBundleDelta') {
|
||||
const dashboard = extractDashboardRevisions(payload);
|
||||
for (const kind of dashboard?.resultKinds ?? []) options.metrics.recordHttpResult(options.operation.name, kind);
|
||||
options.metrics.recordHttp(options.operation.name, performance.now() - started, outcome);
|
||||
return dashboard?.revisions;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (options.signal.aborted) return;
|
||||
outcome = error instanceof TypeError ? 'network' : 'client';
|
||||
}
|
||||
options.metrics.recordHttp(options.operation.name, performance.now() - started, outcome);
|
||||
return undefined;
|
||||
};
|
||||
Reference in New Issue
Block a user