fix(runtime): bound PM2 process restarts and identity

This commit is contained in:
2026-08-04 16:01:43 +00:00
parent 1e66c69d60
commit b589f2edb2
13 changed files with 343 additions and 50 deletions
+3 -3
View File
@@ -1,5 +1,7 @@
import path from 'node:path';
import { sanitizeManagedProcessEnv } from '@sammo-ts/gateway-api';
const parsePositiveInt = (value: string | undefined, fallback: number, name: string): number => {
if (!value) return fallback;
const parsed = Number(value);
@@ -49,8 +51,6 @@ export const resolveReleaseControllerConfig = (env: NodeJS.ProcessEnv = process.
60000,
'RELEASE_CONTROLLER_READINESS_TIMEOUT_MS'
),
baseEnv: Object.fromEntries(
Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === 'string')
),
baseEnv: sanitizeManagedProcessEnv(env),
};
};
@@ -12,6 +12,7 @@ import {
type ProcessDefinition,
type ProcessManager,
readReleaseManifest,
sanitizeManagedProcessEnv,
} from '@sammo-ts/gateway-api';
import type { ReleaseControllerConfig } from './config.js';
@@ -26,7 +27,7 @@ export const buildGatewayReleaseCommands = (
config: ReleaseControllerConfig
): BuildCommand[] => {
const env = {
...config.baseEnv,
...sanitizeManagedProcessEnv(config.baseEnv),
VITE_APP_BASE_PATH: config.gatewayBasePath,
VITE_GATEWAY_API_URL: `${config.gatewayBasePath}/api/trpc`,
VITE_GAME_API_URL_TEMPLATE: '/{profile}/api/trpc',
@@ -49,7 +50,7 @@ export const buildGatewayMigrationCommand = (workspaceRoot: string, config: Rele
args: ['--filter', '@sammo-ts/infra', 'prisma:migrate:deploy:gateway'],
cwd: workspaceRoot,
env: {
...config.baseEnv,
...sanitizeManagedProcessEnv(config.baseEnv),
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
},
});
@@ -63,7 +64,7 @@ export const buildGatewayProcessDefinitions = (
const apiScript = path.join(apiCwd, 'dist', 'index.js');
const frontendScript = path.join(frontendCwd, 'node_modules', 'vite', 'bin', 'vite.js');
const env = {
...config.baseEnv,
...sanitizeManagedProcessEnv(config.baseEnv),
GATEWAY_API_HOST: '0.0.0.0',
GATEWAY_API_PORT: String(config.gatewayApiPort),
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
@@ -231,12 +232,19 @@ export class GatewayReleaseController {
try {
const [api, frontend] = await Promise.all([this.fetchImpl(apiUrl), this.fetchImpl(frontendUrl)]);
const processes = await this.processManager.list();
const online = new Set(
processes
.filter((process) => process.status.toLowerCase() === 'online')
.map((process) => process.name)
const expected = processes.filter((process) => PROCESS_NAMES.includes(process.name as (typeof PROCESS_NAMES)[number]));
const safe = expected.filter(
(process) => process.status.toLowerCase() === 'online' && (process.restartCount ?? 0) === 0
);
if (api.ok && frontend.ok && PROCESS_NAMES.every((name) => online.has(name))) return;
if (
api.ok &&
frontend.ok &&
expected.length === PROCESS_NAMES.length &&
safe.length === PROCESS_NAMES.length &&
new Set(safe.map((process) => process.name)).size === PROCESS_NAMES.length
) {
return;
}
} catch {
// Retry until the bounded deadline.
}
+12 -5
View File
@@ -8,6 +8,7 @@ import {
type ProcessDefinition,
type ProcessManager,
readReleaseManifest,
sanitizeManagedProcessEnv,
} from '@sammo-ts/gateway-api';
import type { ReleaseControllerConfig } from './config.js';
@@ -20,7 +21,7 @@ export const buildReleaseControllerCommands = (
needsInstall: boolean,
config: ReleaseControllerConfig
): BuildCommand[] => {
const env = config.baseEnv;
const env = sanitizeManagedProcessEnv(config.baseEnv);
return [
...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []),
{ command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env },
@@ -42,7 +43,7 @@ export const buildReleaseControllerDefinition = (
cwd: path.join(workspaceRoot, 'app', 'release-controller'),
args: ['daemon'],
env: {
...config.baseEnv,
...sanitizeManagedProcessEnv(config.baseEnv),
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
GATEWAY_DB_SCHEMA: config.gatewayDbSchema,
RELEASE_CONTROLLER_WORKSPACE_ROOT: workspaceRoot,
@@ -89,10 +90,16 @@ export const upgradeReleaseController = async (options: {
await options.processManager.start(buildReleaseControllerDefinition(workspace.root, options.config));
const deadline = Date.now() + (options.readinessTimeoutMs ?? options.config.readinessTimeoutMs);
while (Date.now() < deadline) {
const active = (await options.processManager.list()).find(
(process) => process.name === CONTROLLER_PROCESS_NAME && process.status.toLowerCase() === 'online'
const matching = (await options.processManager.list()).filter(
(process) => process.name === CONTROLLER_PROCESS_NAME
);
if (active) return { commitSha, workspace: workspace.root };
if (
matching.length === 1 &&
matching[0]?.status.toLowerCase() === 'online' &&
(matching[0]?.restartCount ?? 0) === 0
) {
return { commitSha, workspace: workspace.root };
}
await new Promise<void>((resolve) => setTimeout(resolve, 250));
}
throw new Error('Release controller did not become online before the timeout.');
@@ -15,7 +15,7 @@ import { afterEach, describe, expect, it } from 'vitest';
import { resolveReleaseControllerConfig, type ReleaseControllerConfig } from '../src/config.js';
import { buildGatewayProcessDefinitions, GatewayReleaseController } from '../src/releaseController.js';
import { upgradeReleaseController } from '../src/selfUpgrade.js';
import { buildReleaseControllerDefinition, upgradeReleaseController } from '../src/selfUpgrade.js';
const SHA = '1111111111111111111111111111111111111111';
const OLD_SHA = '2222222222222222222222222222222222222222';
@@ -124,6 +124,26 @@ it('runs Gateway preview from the frontend workspace dependency', () => {
);
});
it('does not forward release-controller PM2 identity to Gateway processes', () => {
const definitions = buildGatewayProcessDefinitions('/srv/sammo/release', {
...config,
baseEnv: {
DATABASE_URL: 'postgresql://integration.invalid/sammo',
GATEWAY_ROLE: 'orchestrator',
name: 'sammo:release-controller',
pm_id: '3',
pm_exec_path: '/srv/release-controller.js',
},
});
for (const definition of definitions) {
expect(definition.env).toMatchObject({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
expect(definition.env).not.toHaveProperty('pm_id');
expect(definition.env).not.toHaveProperty('pm_exec_path');
expect(definition.env).not.toHaveProperty('name');
}
});
describe('GatewayReleaseController', () => {
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
const workspace = await createReleaseWorkspace();
@@ -225,9 +245,44 @@ describe('resolveReleaseControllerConfig', () => {
expect(new URL(resolved.gatewayDatabaseUrl).searchParams.get('schema')).toBe('gateway_release');
});
it('removes PM2 metadata from the inherited controller environment', () => {
const resolved = resolveReleaseControllerConfig({
GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo',
RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller',
pm_id: '3',
name: 'sammo:release-controller',
axm_monitor: '{}',
});
expect(resolved.baseEnv).toMatchObject({
GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo',
RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller',
});
expect(resolved.baseEnv).not.toHaveProperty('pm_id');
expect(resolved.baseEnv).not.toHaveProperty('name');
expect(resolved.baseEnv).not.toHaveProperty('axm_monitor');
});
});
describe('upgradeReleaseController', () => {
it('does not forward the old controller PM2 identity to its replacement', () => {
const definition = buildReleaseControllerDefinition('/srv/sammo/release', {
...config,
baseEnv: {
DATABASE_URL: 'postgresql://integration.invalid/sammo',
name: 'sammo:release-controller',
pm_id: '3',
pm_exec_path: '/srv/old-controller.js',
},
});
expect(definition.env).toMatchObject({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
expect(definition.env).not.toHaveProperty('pm_id');
expect(definition.env).not.toHaveProperty('pm_exec_path');
expect(definition.env).not.toHaveProperty('name');
});
it('switches the controller daemon from a separately invoked CLI process', async () => {
const workspace = await createReleaseWorkspace();
const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]);