merge: PM2 runtime safety hardening
This commit is contained in:
@@ -13,7 +13,7 @@ import {
|
|||||||
import { isRecord } from '@sammo-ts/common';
|
import { isRecord } from '@sammo-ts/common';
|
||||||
|
|
||||||
import type { BuildCommand, BuildRunner } from './buildRunner.js';
|
import type { BuildCommand, BuildRunner } from './buildRunner.js';
|
||||||
import type { ProcessManager } from './processManager.js';
|
import { sanitizeManagedProcessEnv, type ProcessManager } from './processManager.js';
|
||||||
import type {
|
import type {
|
||||||
GatewayClaimedProfileUpdate,
|
GatewayClaimedProfileUpdate,
|
||||||
GatewayOperationRecord,
|
GatewayOperationRecord,
|
||||||
@@ -368,7 +368,7 @@ export const buildProcessDefinitions = (
|
|||||||
battleSim: { name: string; script: string; cwd: string; env: Record<string, string> };
|
battleSim: { name: string; script: string; cwd: string; env: Record<string, string> };
|
||||||
tournament: { name: string; script: string; cwd: string; env: Record<string, string> };
|
tournament: { name: string; script: string; cwd: string; env: Record<string, string> };
|
||||||
} => {
|
} => {
|
||||||
const baseEnv = { ...(config.baseEnv ?? {}) };
|
const baseEnv = sanitizeManagedProcessEnv(config.baseEnv ?? {});
|
||||||
const frontendName = buildProcessName(profile.profileName, 'frontend');
|
const frontendName = buildProcessName(profile.profileName, 'frontend');
|
||||||
const apiName = buildProcessName(profile.profileName, 'api');
|
const apiName = buildProcessName(profile.profileName, 'api');
|
||||||
const daemonName = buildProcessName(profile.profileName, 'daemon');
|
const daemonName = buildProcessName(profile.profileName, 'daemon');
|
||||||
@@ -1453,11 +1453,18 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
buildWorkspace: workspace.root,
|
buildWorkspace: workspace.root,
|
||||||
};
|
};
|
||||||
const started = await this.startProfile(builtProfile, assertLease);
|
const started = await this.startProfile(builtProfile, assertLease);
|
||||||
if (!started) {
|
const ready = started && (await this.waitForProfileReadiness(builtProfile, assertLease));
|
||||||
await updateClaimedProfile({ status: 'STOPPED', lastError: 'Failed to start profile processes.' }, () =>
|
if (!ready) {
|
||||||
|
if (started) {
|
||||||
|
await this.stopProfile(builtProfile, assertLease);
|
||||||
|
}
|
||||||
|
const detail = started
|
||||||
|
? 'reset completed but profile processes failed readiness'
|
||||||
|
: 'reset completed but profile processes failed to start';
|
||||||
|
await updateClaimedProfile({ status: 'STOPPED', lastError: detail }, () =>
|
||||||
this.repository.updateStatus(profile.profileName, 'STOPPED')
|
this.repository.updateStatus(profile.profileName, 'STOPPED')
|
||||||
);
|
);
|
||||||
return { status: 'FAILED', detail: 'reset completed but profile processes failed to start' };
|
return { status: 'FAILED', detail };
|
||||||
}
|
}
|
||||||
await updateClaimedProfile({ lastError: null }, async () => {
|
await updateClaimedProfile({ lastError: null }, async () => {
|
||||||
await this.repository.updateLastError(profile.profileName, null);
|
await this.repository.updateLastError(profile.profileName, null);
|
||||||
@@ -1723,6 +1730,17 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
];
|
];
|
||||||
const attemptedNames: string[] = [];
|
const attemptedNames: string[] = [];
|
||||||
try {
|
try {
|
||||||
|
const expectedNames = new Set(orderedDefinitions.map((definition) => definition.name));
|
||||||
|
const existingNames = new Set(
|
||||||
|
(await this.processManager.list())
|
||||||
|
.filter((process) => expectedNames.has(process.name))
|
||||||
|
.map((process) => process.name)
|
||||||
|
);
|
||||||
|
for (const name of existingNames) {
|
||||||
|
await assertLease?.();
|
||||||
|
await this.processManager.delete(name);
|
||||||
|
await assertLease?.();
|
||||||
|
}
|
||||||
for (const definition of orderedDefinitions) {
|
for (const definition of orderedDefinitions) {
|
||||||
await assertLease?.();
|
await assertLease?.();
|
||||||
attemptedNames.push(definition.name);
|
attemptedNames.push(definition.name);
|
||||||
@@ -1774,12 +1792,17 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
this.fetchImpl(frontendUrl),
|
this.fetchImpl(frontendUrl),
|
||||||
this.processManager.list(),
|
this.processManager.list(),
|
||||||
]);
|
]);
|
||||||
const online = new Set(
|
const expectedProcesses = processes.filter((process) => expectedNames.includes(process.name));
|
||||||
processes
|
const safeProcesses = expectedProcesses.filter(
|
||||||
.filter((process) => process.status.toLowerCase() === 'online')
|
(process) => process.status.toLowerCase() === 'online' && (process.restartCount ?? 0) === 0
|
||||||
.map((process) => process.name)
|
|
||||||
);
|
);
|
||||||
if (api.ok && frontend.ok && expectedNames.every((name) => online.has(name))) {
|
if (
|
||||||
|
api.ok &&
|
||||||
|
frontend.ok &&
|
||||||
|
expectedProcesses.length === expectedNames.length &&
|
||||||
|
safeProcesses.length === expectedNames.length &&
|
||||||
|
new Set(safeProcesses.map((process) => process.name)).size === expectedNames.length
|
||||||
|
) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
|
|||||||
@@ -4,13 +4,13 @@ import type { GatewayOrchestratorConfig } from '../config.js';
|
|||||||
import { createGatewayProfileRepository } from './profileRepository.js';
|
import { createGatewayProfileRepository } from './profileRepository.js';
|
||||||
import { GatewayOrchestrator } from './gatewayOrchestrator.js';
|
import { GatewayOrchestrator } from './gatewayOrchestrator.js';
|
||||||
import { Pm2ProcessManager } from './pm2ProcessManager.js';
|
import { Pm2ProcessManager } from './pm2ProcessManager.js';
|
||||||
|
import { sanitizeManagedProcessEnv } from './processManager.js';
|
||||||
import { PnpmBuildRunner } from './buildRunner.js';
|
import { PnpmBuildRunner } from './buildRunner.js';
|
||||||
import { resolveWorkspaceRoot } from './workspaceRoot.js';
|
import { resolveWorkspaceRoot } from './workspaceRoot.js';
|
||||||
import { GitWorkspaceManager } from './workspaceManager.js';
|
import { GitWorkspaceManager } from './workspaceManager.js';
|
||||||
|
|
||||||
export const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
|
export const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
|
||||||
const entries = Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === 'string');
|
return sanitizeManagedProcessEnv(env);
|
||||||
return Object.fromEntries(entries);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export const createGatewayOrchestrator = (
|
export const createGatewayOrchestrator = (
|
||||||
|
|||||||
@@ -1,7 +1,12 @@
|
|||||||
import { createRequire } from 'node:module';
|
import { createRequire } from 'node:module';
|
||||||
|
|
||||||
import type * as Pm2 from 'pm2';
|
import type * as Pm2 from 'pm2';
|
||||||
import type { ProcessManager, ManagedProcessInfo, ProcessDefinition } from './processManager.js';
|
import {
|
||||||
|
sanitizePm2IdentityEnv,
|
||||||
|
type ProcessManager,
|
||||||
|
type ManagedProcessInfo,
|
||||||
|
type ProcessDefinition,
|
||||||
|
} from './processManager.js';
|
||||||
|
|
||||||
type Pm2Module = typeof Pm2;
|
type Pm2Module = typeof Pm2;
|
||||||
|
|
||||||
@@ -27,6 +32,20 @@ const withPm2 = async <T>(handler: (pm2: Pm2Module) => Promise<T>): Promise<T> =
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const buildPm2StartOptions = (definition: ProcessDefinition) => ({
|
||||||
|
name: definition.name,
|
||||||
|
script: definition.script,
|
||||||
|
cwd: definition.cwd,
|
||||||
|
args: definition.args,
|
||||||
|
env: sanitizePm2IdentityEnv(definition.env ?? {}),
|
||||||
|
autorestart: true,
|
||||||
|
max_restarts: 5,
|
||||||
|
min_uptime: 10_000,
|
||||||
|
restart_delay: 2_000,
|
||||||
|
kill_timeout: 15_000,
|
||||||
|
time: true,
|
||||||
|
});
|
||||||
|
|
||||||
export class Pm2ProcessManager implements ProcessManager {
|
export class Pm2ProcessManager implements ProcessManager {
|
||||||
async list(): Promise<ManagedProcessInfo[]> {
|
async list(): Promise<ManagedProcessInfo[]> {
|
||||||
return withPm2(
|
return withPm2(
|
||||||
@@ -44,6 +63,7 @@ export class Pm2ProcessManager implements ProcessManager {
|
|||||||
pid: item.pid ?? undefined,
|
pid: item.pid ?? undefined,
|
||||||
cwd: item.pm2_env?.pm_cwd ?? undefined,
|
cwd: item.pm2_env?.pm_cwd ?? undefined,
|
||||||
script: item.pm2_env?.pm_exec_path ?? undefined,
|
script: item.pm2_env?.pm_exec_path ?? undefined,
|
||||||
|
restartCount: item.pm2_env?.restart_time ?? 0,
|
||||||
})) ?? [];
|
})) ?? [];
|
||||||
resolve(normalized);
|
resolve(normalized);
|
||||||
});
|
});
|
||||||
@@ -55,24 +75,26 @@ export class Pm2ProcessManager implements ProcessManager {
|
|||||||
await withPm2(
|
await withPm2(
|
||||||
(pm2) =>
|
(pm2) =>
|
||||||
new Promise<void>((resolve, reject) => {
|
new Promise<void>((resolve, reject) => {
|
||||||
pm2.start(
|
pm2.list((listError, list) => {
|
||||||
{
|
if (listError) {
|
||||||
name: definition.name,
|
reject(listError);
|
||||||
script: definition.script,
|
return;
|
||||||
cwd: definition.cwd,
|
|
||||||
args: definition.args,
|
|
||||||
env: definition.env,
|
|
||||||
autorestart: true,
|
|
||||||
time: true,
|
|
||||||
},
|
|
||||||
(error) => {
|
|
||||||
if (error) {
|
|
||||||
reject(error);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
resolve();
|
|
||||||
}
|
}
|
||||||
);
|
if (list?.some((item) => item.name === definition.name)) {
|
||||||
|
reject(new Error(`PM2 process name already exists: ${definition.name}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pm2.start(
|
||||||
|
buildPm2StartOptions(definition),
|
||||||
|
(error) => {
|
||||||
|
if (error) {
|
||||||
|
reject(error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
);
|
||||||
|
});
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ export interface ManagedProcessInfo {
|
|||||||
pid?: number;
|
pid?: number;
|
||||||
cwd?: string;
|
cwd?: string;
|
||||||
script?: string;
|
script?: string;
|
||||||
|
restartCount?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ProcessDefinition {
|
export interface ProcessDefinition {
|
||||||
@@ -20,3 +21,52 @@ export interface ProcessManager {
|
|||||||
stop(name: string): Promise<void>;
|
stop(name: string): Promise<void>;
|
||||||
delete(name: string): Promise<void>;
|
delete(name: string): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PM2_INTERNAL_ENV_KEYS = new Set([
|
||||||
|
'NODE_APP_INSTANCE',
|
||||||
|
'autorestart',
|
||||||
|
'autostart',
|
||||||
|
'created_at',
|
||||||
|
'exec_interpreter',
|
||||||
|
'exec_mode',
|
||||||
|
'exit_code',
|
||||||
|
'instance_var',
|
||||||
|
'instances',
|
||||||
|
'merge_logs',
|
||||||
|
'name',
|
||||||
|
'namespace',
|
||||||
|
'node_args',
|
||||||
|
'node_version',
|
||||||
|
'pm_cwd',
|
||||||
|
'pm_err_log_path',
|
||||||
|
'pm_exec_path',
|
||||||
|
'pm_id',
|
||||||
|
'pm_out_log_path',
|
||||||
|
'pm_pid_path',
|
||||||
|
'pm_uptime',
|
||||||
|
'restart_time',
|
||||||
|
'status',
|
||||||
|
'unstable_restarts',
|
||||||
|
'version',
|
||||||
|
'vizion',
|
||||||
|
'watch',
|
||||||
|
]);
|
||||||
|
|
||||||
|
export const sanitizePm2IdentityEnv = (env: NodeJS.ProcessEnv | Record<string, string>): Record<string, string> =>
|
||||||
|
Object.fromEntries(
|
||||||
|
Object.entries(env).filter(
|
||||||
|
(entry): entry is [string, string] =>
|
||||||
|
typeof entry[1] === 'string' &&
|
||||||
|
!entry[0].startsWith('axm_') &&
|
||||||
|
!entry[0].startsWith('pm_') &&
|
||||||
|
!PM2_INTERNAL_ENV_KEYS.has(entry[0])
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
export const sanitizeManagedProcessEnv = (env: NodeJS.ProcessEnv | Record<string, string>): Record<string, string> => {
|
||||||
|
const sanitized = sanitizePm2IdentityEnv(env);
|
||||||
|
delete sanitized.GATEWAY_ROLE;
|
||||||
|
delete sanitized.GAME_API_ROLE;
|
||||||
|
delete sanitized.GAME_ENGINE_ROLE;
|
||||||
|
return sanitized;
|
||||||
|
};
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { buildEnvMap } from '../src/orchestrator/orchestratorFactory.js';
|
||||||
|
|
||||||
|
describe('buildEnvMap', () => {
|
||||||
|
it('removes PM2 child identity before creating Git and profile process environments', () => {
|
||||||
|
expect(
|
||||||
|
buildEnvMap({
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
GATEWAY_ROLE: 'orchestrator',
|
||||||
|
pm_id: '2',
|
||||||
|
pm_exec_path: '/workspace/core2026/app/gateway-api/dist/index.js',
|
||||||
|
name: 'sammo:gateway-orchestrator',
|
||||||
|
axm_dynamic: '{}',
|
||||||
|
})
|
||||||
|
).toEqual({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -174,6 +174,14 @@ describe('GatewayOrchestrator first-class operations', () => {
|
|||||||
'sammo:che:2:battle-sim-worker',
|
'sammo:che:2:battle-sim-worker',
|
||||||
'sammo:che:2:tournament-worker',
|
'sammo:che:2:tournament-worker',
|
||||||
]);
|
]);
|
||||||
|
expect(harness.deleted).toEqual([
|
||||||
|
'sammo:che:2:game-frontend',
|
||||||
|
'sammo:che:2:game-api',
|
||||||
|
'sammo:che:2:turn-daemon',
|
||||||
|
'sammo:che:2:auction-worker',
|
||||||
|
'sammo:che:2:battle-sim-worker',
|
||||||
|
'sammo:che:2:tournament-worker',
|
||||||
|
]);
|
||||||
expect(harness.completions).toEqual(['SUCCEEDED']);
|
expect(harness.completions).toEqual(['SUCCEEDED']);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -236,6 +244,12 @@ describe('GatewayOrchestrator first-class operations', () => {
|
|||||||
|
|
||||||
expect(harness.completions).toEqual(['FAILED']);
|
expect(harness.completions).toEqual(['FAILED']);
|
||||||
expect(harness.deleted).toEqual([
|
expect(harness.deleted).toEqual([
|
||||||
|
'sammo:che:2:game-frontend',
|
||||||
|
'sammo:che:2:game-api',
|
||||||
|
'sammo:che:2:turn-daemon',
|
||||||
|
'sammo:che:2:auction-worker',
|
||||||
|
'sammo:che:2:battle-sim-worker',
|
||||||
|
'sammo:che:2:tournament-worker',
|
||||||
'sammo:che:2:turn-daemon',
|
'sammo:che:2:turn-daemon',
|
||||||
'sammo:che:2:game-api',
|
'sammo:che:2:game-api',
|
||||||
'sammo:che:2:game-frontend',
|
'sammo:che:2:game-frontend',
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
buildWorkspaceCommands,
|
buildWorkspaceCommands,
|
||||||
planProfileReconcile,
|
planProfileReconcile,
|
||||||
} from '../src/orchestrator/gatewayOrchestrator.js';
|
} from '../src/orchestrator/gatewayOrchestrator.js';
|
||||||
|
import { sanitizeManagedProcessEnv } from '../src/orchestrator/processManager.js';
|
||||||
import type { GatewayProfileRecord } from '../src/orchestrator/profileRepository.js';
|
import type { GatewayProfileRecord } from '../src/orchestrator/profileRepository.js';
|
||||||
|
|
||||||
const buildProfile = (buildWorkspace?: string): GatewayProfileRecord => ({
|
const buildProfile = (buildWorkspace?: string): GatewayProfileRecord => ({
|
||||||
@@ -116,6 +117,9 @@ describe('buildProcessDefinitions', () => {
|
|||||||
const buildWorkspace = '/srv/sammo/worktrees/0123456789abcdef';
|
const buildWorkspace = '/srv/sammo/worktrees/0123456789abcdef';
|
||||||
const definitions = buildProcessDefinitions(buildProfile(buildWorkspace), processConfig);
|
const definitions = buildProcessDefinitions(buildProfile(buildWorkspace), processConfig);
|
||||||
|
|
||||||
|
expect(Object.values(definitions)).toHaveLength(6);
|
||||||
|
expect(new Set(Object.values(definitions).map((definition) => definition.name)).size).toBe(6);
|
||||||
|
|
||||||
expect(definitions.frontend).toMatchObject({
|
expect(definitions.frontend).toMatchObject({
|
||||||
cwd: path.join(buildWorkspace, 'app', 'game-frontend'),
|
cwd: path.join(buildWorkspace, 'app', 'game-frontend'),
|
||||||
script: path.join(buildWorkspace, 'app', 'game-frontend', 'node_modules', 'vite', 'bin', 'vite.js'),
|
script: path.join(buildWorkspace, 'app', 'game-frontend', 'node_modules', 'vite', 'bin', 'vite.js'),
|
||||||
@@ -167,6 +171,50 @@ describe('buildProcessDefinitions', () => {
|
|||||||
expect(definitions.battleSim.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-api'));
|
expect(definitions.battleSim.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-api'));
|
||||||
expect(definitions.tournament.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-api'));
|
expect(definitions.tournament.cwd).toBe(path.join(processConfig.workspaceRoot, 'app', 'game-api'));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not forward PM2 identity or parent runtime roles to profile processes', () => {
|
||||||
|
const definitions = buildProcessDefinitions(buildProfile(), {
|
||||||
|
...processConfig,
|
||||||
|
baseEnv: {
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
GATEWAY_ROLE: 'orchestrator',
|
||||||
|
NODE_APP_INSTANCE: '2',
|
||||||
|
name: 'sammo:gateway-orchestrator',
|
||||||
|
pm_id: '2',
|
||||||
|
pm_exec_path: '/srv/controller.js',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const definition of Object.values(definitions)) {
|
||||||
|
expect(definition.env).toMatchObject({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
|
||||||
|
expect(definition.env).not.toHaveProperty('pm_id');
|
||||||
|
expect(definition.env).not.toHaveProperty('pm_exec_path');
|
||||||
|
expect(definition.env).not.toHaveProperty('name');
|
||||||
|
expect(definition.env).not.toHaveProperty('NODE_APP_INSTANCE');
|
||||||
|
expect(definition.env).not.toHaveProperty('GATEWAY_ROLE');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('sanitizeManagedProcessEnv', () => {
|
||||||
|
it('keeps application configuration while removing PM2 metadata and role selectors', () => {
|
||||||
|
expect(
|
||||||
|
sanitizeManagedProcessEnv({
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
PATH: '/usr/local/bin:/usr/bin',
|
||||||
|
GATEWAY_ROLE: 'orchestrator',
|
||||||
|
GAME_API_ROLE: 'server',
|
||||||
|
NODE_APP_INSTANCE: '2',
|
||||||
|
name: 'sammo:gateway-orchestrator',
|
||||||
|
pm_id: '2',
|
||||||
|
pm_cwd: '/srv/controller',
|
||||||
|
axm_monitor: '{}',
|
||||||
|
})
|
||||||
|
).toEqual({
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
PATH: '/usr/local/bin:/usr/bin',
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('buildWorkspaceCommands', () => {
|
describe('buildWorkspaceCommands', () => {
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { buildPm2StartOptions } from '../src/orchestrator/pm2ProcessManager.js';
|
||||||
|
|
||||||
|
describe('buildPm2StartOptions', () => {
|
||||||
|
it('enforces bounded restart policy and strips inherited PM2 identity at the PM2 boundary', () => {
|
||||||
|
const options = buildPm2StartOptions({
|
||||||
|
name: 'sammo:che:2:game-api',
|
||||||
|
script: '/srv/sammo/app/game-api/dist/index.js',
|
||||||
|
cwd: '/srv/sammo/app/game-api',
|
||||||
|
env: {
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
GAME_API_ROLE: 'server',
|
||||||
|
pm_id: '2',
|
||||||
|
pm_exec_path: '/srv/sammo/app/gateway-api/dist/index.js',
|
||||||
|
name: 'sammo:gateway-orchestrator',
|
||||||
|
NODE_APP_INSTANCE: '2',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(options).toMatchObject({
|
||||||
|
name: 'sammo:che:2:game-api',
|
||||||
|
autorestart: true,
|
||||||
|
max_restarts: 5,
|
||||||
|
min_uptime: 10_000,
|
||||||
|
restart_delay: 2_000,
|
||||||
|
kill_timeout: 15_000,
|
||||||
|
env: {
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
GAME_API_ROLE: 'server',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(options.env).not.toHaveProperty('pm_id');
|
||||||
|
expect(options.env).not.toHaveProperty('pm_exec_path');
|
||||||
|
expect(options.env).not.toHaveProperty('name');
|
||||||
|
expect(options.env).not.toHaveProperty('NODE_APP_INSTANCE');
|
||||||
|
expect(options.env).toHaveProperty('GAME_API_ROLE', 'server');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
|
|
||||||
|
import { sanitizeManagedProcessEnv } from '@sammo-ts/gateway-api';
|
||||||
|
|
||||||
const parsePositiveInt = (value: string | undefined, fallback: number, name: string): number => {
|
const parsePositiveInt = (value: string | undefined, fallback: number, name: string): number => {
|
||||||
if (!value) return fallback;
|
if (!value) return fallback;
|
||||||
const parsed = Number(value);
|
const parsed = Number(value);
|
||||||
@@ -49,8 +51,6 @@ export const resolveReleaseControllerConfig = (env: NodeJS.ProcessEnv = process.
|
|||||||
60000,
|
60000,
|
||||||
'RELEASE_CONTROLLER_READINESS_TIMEOUT_MS'
|
'RELEASE_CONTROLLER_READINESS_TIMEOUT_MS'
|
||||||
),
|
),
|
||||||
baseEnv: Object.fromEntries(
|
baseEnv: sanitizeManagedProcessEnv(env),
|
||||||
Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === 'string')
|
|
||||||
),
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import {
|
|||||||
type ProcessDefinition,
|
type ProcessDefinition,
|
||||||
type ProcessManager,
|
type ProcessManager,
|
||||||
readReleaseManifest,
|
readReleaseManifest,
|
||||||
|
sanitizeManagedProcessEnv,
|
||||||
} from '@sammo-ts/gateway-api';
|
} from '@sammo-ts/gateway-api';
|
||||||
|
|
||||||
import type { ReleaseControllerConfig } from './config.js';
|
import type { ReleaseControllerConfig } from './config.js';
|
||||||
@@ -26,7 +27,7 @@ export const buildGatewayReleaseCommands = (
|
|||||||
config: ReleaseControllerConfig
|
config: ReleaseControllerConfig
|
||||||
): BuildCommand[] => {
|
): BuildCommand[] => {
|
||||||
const env = {
|
const env = {
|
||||||
...config.baseEnv,
|
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||||
VITE_APP_BASE_PATH: config.gatewayBasePath,
|
VITE_APP_BASE_PATH: config.gatewayBasePath,
|
||||||
VITE_GATEWAY_API_URL: `${config.gatewayBasePath}/api/trpc`,
|
VITE_GATEWAY_API_URL: `${config.gatewayBasePath}/api/trpc`,
|
||||||
VITE_GAME_API_URL_TEMPLATE: '/{profile}/api/trpc',
|
VITE_GAME_API_URL_TEMPLATE: '/{profile}/api/trpc',
|
||||||
@@ -49,7 +50,7 @@ export const buildGatewayMigrationCommand = (workspaceRoot: string, config: Rele
|
|||||||
args: ['--filter', '@sammo-ts/infra', 'prisma:migrate:deploy:gateway'],
|
args: ['--filter', '@sammo-ts/infra', 'prisma:migrate:deploy:gateway'],
|
||||||
cwd: workspaceRoot,
|
cwd: workspaceRoot,
|
||||||
env: {
|
env: {
|
||||||
...config.baseEnv,
|
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -63,7 +64,7 @@ export const buildGatewayProcessDefinitions = (
|
|||||||
const apiScript = path.join(apiCwd, 'dist', 'index.js');
|
const apiScript = path.join(apiCwd, 'dist', 'index.js');
|
||||||
const frontendScript = path.join(frontendCwd, 'node_modules', 'vite', 'bin', 'vite.js');
|
const frontendScript = path.join(frontendCwd, 'node_modules', 'vite', 'bin', 'vite.js');
|
||||||
const env = {
|
const env = {
|
||||||
...config.baseEnv,
|
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||||
GATEWAY_API_HOST: '0.0.0.0',
|
GATEWAY_API_HOST: '0.0.0.0',
|
||||||
GATEWAY_API_PORT: String(config.gatewayApiPort),
|
GATEWAY_API_PORT: String(config.gatewayApiPort),
|
||||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||||
@@ -231,12 +232,19 @@ export class GatewayReleaseController {
|
|||||||
try {
|
try {
|
||||||
const [api, frontend] = await Promise.all([this.fetchImpl(apiUrl), this.fetchImpl(frontendUrl)]);
|
const [api, frontend] = await Promise.all([this.fetchImpl(apiUrl), this.fetchImpl(frontendUrl)]);
|
||||||
const processes = await this.processManager.list();
|
const processes = await this.processManager.list();
|
||||||
const online = new Set(
|
const expected = processes.filter((process) => PROCESS_NAMES.includes(process.name as (typeof PROCESS_NAMES)[number]));
|
||||||
processes
|
const safe = expected.filter(
|
||||||
.filter((process) => process.status.toLowerCase() === 'online')
|
(process) => process.status.toLowerCase() === 'online' && (process.restartCount ?? 0) === 0
|
||||||
.map((process) => process.name)
|
|
||||||
);
|
);
|
||||||
if (api.ok && frontend.ok && PROCESS_NAMES.every((name) => online.has(name))) return;
|
if (
|
||||||
|
api.ok &&
|
||||||
|
frontend.ok &&
|
||||||
|
expected.length === PROCESS_NAMES.length &&
|
||||||
|
safe.length === PROCESS_NAMES.length &&
|
||||||
|
new Set(safe.map((process) => process.name)).size === PROCESS_NAMES.length
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Retry until the bounded deadline.
|
// Retry until the bounded deadline.
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
type ProcessDefinition,
|
type ProcessDefinition,
|
||||||
type ProcessManager,
|
type ProcessManager,
|
||||||
readReleaseManifest,
|
readReleaseManifest,
|
||||||
|
sanitizeManagedProcessEnv,
|
||||||
} from '@sammo-ts/gateway-api';
|
} from '@sammo-ts/gateway-api';
|
||||||
|
|
||||||
import type { ReleaseControllerConfig } from './config.js';
|
import type { ReleaseControllerConfig } from './config.js';
|
||||||
@@ -20,7 +21,7 @@ export const buildReleaseControllerCommands = (
|
|||||||
needsInstall: boolean,
|
needsInstall: boolean,
|
||||||
config: ReleaseControllerConfig
|
config: ReleaseControllerConfig
|
||||||
): BuildCommand[] => {
|
): BuildCommand[] => {
|
||||||
const env = config.baseEnv;
|
const env = sanitizeManagedProcessEnv(config.baseEnv);
|
||||||
return [
|
return [
|
||||||
...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []),
|
...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []),
|
||||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env },
|
{ command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env },
|
||||||
@@ -42,7 +43,7 @@ export const buildReleaseControllerDefinition = (
|
|||||||
cwd: path.join(workspaceRoot, 'app', 'release-controller'),
|
cwd: path.join(workspaceRoot, 'app', 'release-controller'),
|
||||||
args: ['daemon'],
|
args: ['daemon'],
|
||||||
env: {
|
env: {
|
||||||
...config.baseEnv,
|
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||||
GATEWAY_DB_SCHEMA: config.gatewayDbSchema,
|
GATEWAY_DB_SCHEMA: config.gatewayDbSchema,
|
||||||
RELEASE_CONTROLLER_WORKSPACE_ROOT: workspaceRoot,
|
RELEASE_CONTROLLER_WORKSPACE_ROOT: workspaceRoot,
|
||||||
@@ -89,10 +90,16 @@ export const upgradeReleaseController = async (options: {
|
|||||||
await options.processManager.start(buildReleaseControllerDefinition(workspace.root, options.config));
|
await options.processManager.start(buildReleaseControllerDefinition(workspace.root, options.config));
|
||||||
const deadline = Date.now() + (options.readinessTimeoutMs ?? options.config.readinessTimeoutMs);
|
const deadline = Date.now() + (options.readinessTimeoutMs ?? options.config.readinessTimeoutMs);
|
||||||
while (Date.now() < deadline) {
|
while (Date.now() < deadline) {
|
||||||
const active = (await options.processManager.list()).find(
|
const matching = (await options.processManager.list()).filter(
|
||||||
(process) => process.name === CONTROLLER_PROCESS_NAME && process.status.toLowerCase() === 'online'
|
(process) => process.name === CONTROLLER_PROCESS_NAME
|
||||||
);
|
);
|
||||||
if (active) return { commitSha, workspace: workspace.root };
|
if (
|
||||||
|
matching.length === 1 &&
|
||||||
|
matching[0]?.status.toLowerCase() === 'online' &&
|
||||||
|
(matching[0]?.restartCount ?? 0) === 0
|
||||||
|
) {
|
||||||
|
return { commitSha, workspace: workspace.root };
|
||||||
|
}
|
||||||
await new Promise<void>((resolve) => setTimeout(resolve, 250));
|
await new Promise<void>((resolve) => setTimeout(resolve, 250));
|
||||||
}
|
}
|
||||||
throw new Error('Release controller did not become online before the timeout.');
|
throw new Error('Release controller did not become online before the timeout.');
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import { afterEach, describe, expect, it } from 'vitest';
|
|||||||
|
|
||||||
import { resolveReleaseControllerConfig, type ReleaseControllerConfig } from '../src/config.js';
|
import { resolveReleaseControllerConfig, type ReleaseControllerConfig } from '../src/config.js';
|
||||||
import { buildGatewayProcessDefinitions, GatewayReleaseController } from '../src/releaseController.js';
|
import { buildGatewayProcessDefinitions, GatewayReleaseController } from '../src/releaseController.js';
|
||||||
import { upgradeReleaseController } from '../src/selfUpgrade.js';
|
import { buildReleaseControllerDefinition, upgradeReleaseController } from '../src/selfUpgrade.js';
|
||||||
|
|
||||||
const SHA = '1111111111111111111111111111111111111111';
|
const SHA = '1111111111111111111111111111111111111111';
|
||||||
const OLD_SHA = '2222222222222222222222222222222222222222';
|
const OLD_SHA = '2222222222222222222222222222222222222222';
|
||||||
@@ -124,6 +124,26 @@ it('runs Gateway preview from the frontend workspace dependency', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not forward release-controller PM2 identity to Gateway processes', () => {
|
||||||
|
const definitions = buildGatewayProcessDefinitions('/srv/sammo/release', {
|
||||||
|
...config,
|
||||||
|
baseEnv: {
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
GATEWAY_ROLE: 'orchestrator',
|
||||||
|
name: 'sammo:release-controller',
|
||||||
|
pm_id: '3',
|
||||||
|
pm_exec_path: '/srv/release-controller.js',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const definition of definitions) {
|
||||||
|
expect(definition.env).toMatchObject({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
|
||||||
|
expect(definition.env).not.toHaveProperty('pm_id');
|
||||||
|
expect(definition.env).not.toHaveProperty('pm_exec_path');
|
||||||
|
expect(definition.env).not.toHaveProperty('name');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
describe('GatewayReleaseController', () => {
|
describe('GatewayReleaseController', () => {
|
||||||
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
||||||
const workspace = await createReleaseWorkspace();
|
const workspace = await createReleaseWorkspace();
|
||||||
@@ -225,9 +245,44 @@ describe('resolveReleaseControllerConfig', () => {
|
|||||||
|
|
||||||
expect(new URL(resolved.gatewayDatabaseUrl).searchParams.get('schema')).toBe('gateway_release');
|
expect(new URL(resolved.gatewayDatabaseUrl).searchParams.get('schema')).toBe('gateway_release');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('removes PM2 metadata from the inherited controller environment', () => {
|
||||||
|
const resolved = resolveReleaseControllerConfig({
|
||||||
|
GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo',
|
||||||
|
RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller',
|
||||||
|
pm_id: '3',
|
||||||
|
name: 'sammo:release-controller',
|
||||||
|
axm_monitor: '{}',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(resolved.baseEnv).toMatchObject({
|
||||||
|
GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo',
|
||||||
|
RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller',
|
||||||
|
});
|
||||||
|
expect(resolved.baseEnv).not.toHaveProperty('pm_id');
|
||||||
|
expect(resolved.baseEnv).not.toHaveProperty('name');
|
||||||
|
expect(resolved.baseEnv).not.toHaveProperty('axm_monitor');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('upgradeReleaseController', () => {
|
describe('upgradeReleaseController', () => {
|
||||||
|
it('does not forward the old controller PM2 identity to its replacement', () => {
|
||||||
|
const definition = buildReleaseControllerDefinition('/srv/sammo/release', {
|
||||||
|
...config,
|
||||||
|
baseEnv: {
|
||||||
|
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||||
|
name: 'sammo:release-controller',
|
||||||
|
pm_id: '3',
|
||||||
|
pm_exec_path: '/srv/old-controller.js',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(definition.env).toMatchObject({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
|
||||||
|
expect(definition.env).not.toHaveProperty('pm_id');
|
||||||
|
expect(definition.env).not.toHaveProperty('pm_exec_path');
|
||||||
|
expect(definition.env).not.toHaveProperty('name');
|
||||||
|
});
|
||||||
|
|
||||||
it('switches the controller daemon from a separately invoked CLI process', async () => {
|
it('switches the controller daemon from a separately invoked CLI process', async () => {
|
||||||
const workspace = await createReleaseWorkspace();
|
const workspace = await createReleaseWorkspace();
|
||||||
const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]);
|
const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]);
|
||||||
|
|||||||
@@ -32,6 +32,12 @@ profile 범위 권한과 별개인 전역 `admin.releases.manage` 권한이 필
|
|||||||
`GAME_API_ROLE=server|*-worker`, turn daemon에
|
`GAME_API_ROLE=server|*-worker`, turn daemon에
|
||||||
`GAME_ENGINE_ROLE=turn-daemon`을 명시합니다. library import나 PM2 wrapper의
|
`GAME_ENGINE_ROLE=turn-daemon`을 명시합니다. library import나 PM2 wrapper의
|
||||||
argv만으로 실행 역할을 추론하지 않습니다.
|
argv만으로 실행 역할을 추론하지 않습니다.
|
||||||
|
- PM2 child에서 상속된 `pm_id`, `name`, `pm_exec_path`, `NODE_APP_INSTANCE`와
|
||||||
|
`axm_*` 메타데이터는 새 definition에 전달하지 않습니다. 동일 process 이름은
|
||||||
|
시작 전에 제거하며 PM2 start는 남은 동일 이름을 거부합니다.
|
||||||
|
- Runtime process는 10초 이전의 불안정 종료에 대해 최대 5회, 2초 간격으로만
|
||||||
|
자동 재시작합니다. Readiness는 예상 process 수가 정확하고 모든 restart count가
|
||||||
|
0일 때만 성공합니다.
|
||||||
- migration 이후 이전 애플리케이션으로 돌아갈 때 schema 하위 호환성이
|
- migration 이후 이전 애플리케이션으로 돌아갈 때 schema 하위 호환성이
|
||||||
유지됩니다.
|
유지됩니다.
|
||||||
|
|
||||||
@@ -61,7 +67,9 @@ profile 범위 권한과 별개인 전역 `admin.releases.manage` 권한이 필
|
|||||||
사용합니다. Source와 scenario를 먼저 불러온 뒤 turn 간격, 가오픈·정식 오픈,
|
사용합니다. Source와 scenario를 먼저 불러온 뒤 turn 간격, 가오픈·정식 오픈,
|
||||||
NPC와 자동 진행 설정을 확인하고 요청해 주세요.
|
NPC와 자동 진행 설정을 확인하고 요청해 주세요.
|
||||||
|
|
||||||
이 모드는 build와 migration 후 scenario seeder를 실행합니다. 현 시즌의 장수,
|
이 모드는 build와 migration 후 기존 season/tick metadata를 읽고 scenario seeder를 실행합니다.
|
||||||
|
빈 profile schema도 migration을 먼저 적용하므로 최초 `world_state` 조회가
|
||||||
|
table 부재로 실패하지 않습니다. 현 시즌의 장수,
|
||||||
국가, 도시, command queue와 시장·경매 등은 새 scenario 기준으로 교체됩니다.
|
국가, 도시, command queue와 시장·경매 등은 새 scenario 기준으로 교체됩니다.
|
||||||
다음 장기보존 자료는 reset 범위 밖에 있으므로 기수를 넘어 유지됩니다.
|
다음 장기보존 자료는 reset 범위 밖에 있으므로 기수를 넘어 유지됩니다.
|
||||||
|
|
||||||
@@ -76,7 +84,7 @@ scenario 설정을 확인한 뒤 실행해 주세요.
|
|||||||
### Profile 실패와 재시도
|
### Profile 실패와 재시도
|
||||||
|
|
||||||
Build는 현재 runtime을 멈추기 전에 수행합니다. Migration 또는 새 process
|
Build는 현재 runtime을 멈추기 전에 수행합니다. Migration 또는 새 process
|
||||||
readiness가 실패하면 작업은 `FAILED`가 되며 orchestrator는 이전 worktree의
|
readiness가 실패하면 작업은 `FAILED`가 됩니다. DB 유지 배포는 이전 worktree의
|
||||||
process 복구를 시도합니다. 관리자 화면의 오류와 PM2 process 상태를 확인한
|
process 복구를 시도합니다. 관리자 화면의 오류와 PM2 process 상태를 확인한
|
||||||
뒤 원인을 해결하고 실패한 작업을 재시도해 주세요. 재시도는 처음 고정된 commit을
|
뒤 원인을 해결하고 실패한 작업을 재시도해 주세요. 재시도는 처음 고정된 commit을
|
||||||
사용합니다.
|
사용합니다.
|
||||||
@@ -159,7 +167,8 @@ pnpm --filter @sammo-ts/release-controller self-upgrade -- COMMIT <full-sha>
|
|||||||
배포 후:
|
배포 후:
|
||||||
|
|
||||||
- 작업이 `SUCCEEDED`이고 고정 commit이 요청한 commit과 같은지 확인합니다.
|
- 작업이 `SUCCEEDED`이고 고정 commit이 요청한 commit과 같은지 확인합니다.
|
||||||
- PM2의 cwd와 script가 게시된 worktree를 가리키는지 확인합니다.
|
- PM2 process 이름별 항목이 정확히 하나이고 restart count가 0이며, cwd와
|
||||||
|
script가 게시된 worktree를 가리키는지 확인합니다.
|
||||||
- `/gateway/` 또는 대상 profile prefix에 직접 접속하고 새로고침합니다.
|
- `/gateway/` 또는 대상 profile prefix에 직접 접속하고 새로고침합니다.
|
||||||
- API health, tRPC, SSE와 정적 자산 경로를 확인합니다.
|
- API health, tRPC, SSE와 정적 자산 경로를 확인합니다.
|
||||||
- DB 유지 배포에서는 현재 season/scenario와 핵심 게임 상태가 유지됐는지
|
- DB 유지 배포에서는 현재 season/scenario와 핵심 게임 상태가 유지됐는지
|
||||||
|
|||||||
Reference in New Issue
Block a user