fix(runtime): bound PM2 process restarts and identity
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
import path from 'node:path';
|
||||
|
||||
import { sanitizeManagedProcessEnv } from '@sammo-ts/gateway-api';
|
||||
|
||||
const parsePositiveInt = (value: string | undefined, fallback: number, name: string): number => {
|
||||
if (!value) return fallback;
|
||||
const parsed = Number(value);
|
||||
@@ -49,8 +51,6 @@ export const resolveReleaseControllerConfig = (env: NodeJS.ProcessEnv = process.
|
||||
60000,
|
||||
'RELEASE_CONTROLLER_READINESS_TIMEOUT_MS'
|
||||
),
|
||||
baseEnv: Object.fromEntries(
|
||||
Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === 'string')
|
||||
),
|
||||
baseEnv: sanitizeManagedProcessEnv(env),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
type ProcessDefinition,
|
||||
type ProcessManager,
|
||||
readReleaseManifest,
|
||||
sanitizeManagedProcessEnv,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
|
||||
import type { ReleaseControllerConfig } from './config.js';
|
||||
@@ -26,7 +27,7 @@ export const buildGatewayReleaseCommands = (
|
||||
config: ReleaseControllerConfig
|
||||
): BuildCommand[] => {
|
||||
const env = {
|
||||
...config.baseEnv,
|
||||
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||
VITE_APP_BASE_PATH: config.gatewayBasePath,
|
||||
VITE_GATEWAY_API_URL: `${config.gatewayBasePath}/api/trpc`,
|
||||
VITE_GAME_API_URL_TEMPLATE: '/{profile}/api/trpc',
|
||||
@@ -49,7 +50,7 @@ export const buildGatewayMigrationCommand = (workspaceRoot: string, config: Rele
|
||||
args: ['--filter', '@sammo-ts/infra', 'prisma:migrate:deploy:gateway'],
|
||||
cwd: workspaceRoot,
|
||||
env: {
|
||||
...config.baseEnv,
|
||||
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||
},
|
||||
});
|
||||
@@ -63,7 +64,7 @@ export const buildGatewayProcessDefinitions = (
|
||||
const apiScript = path.join(apiCwd, 'dist', 'index.js');
|
||||
const frontendScript = path.join(frontendCwd, 'node_modules', 'vite', 'bin', 'vite.js');
|
||||
const env = {
|
||||
...config.baseEnv,
|
||||
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||
GATEWAY_API_HOST: '0.0.0.0',
|
||||
GATEWAY_API_PORT: String(config.gatewayApiPort),
|
||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||
@@ -231,12 +232,19 @@ export class GatewayReleaseController {
|
||||
try {
|
||||
const [api, frontend] = await Promise.all([this.fetchImpl(apiUrl), this.fetchImpl(frontendUrl)]);
|
||||
const processes = await this.processManager.list();
|
||||
const online = new Set(
|
||||
processes
|
||||
.filter((process) => process.status.toLowerCase() === 'online')
|
||||
.map((process) => process.name)
|
||||
const expected = processes.filter((process) => PROCESS_NAMES.includes(process.name as (typeof PROCESS_NAMES)[number]));
|
||||
const safe = expected.filter(
|
||||
(process) => process.status.toLowerCase() === 'online' && (process.restartCount ?? 0) === 0
|
||||
);
|
||||
if (api.ok && frontend.ok && PROCESS_NAMES.every((name) => online.has(name))) return;
|
||||
if (
|
||||
api.ok &&
|
||||
frontend.ok &&
|
||||
expected.length === PROCESS_NAMES.length &&
|
||||
safe.length === PROCESS_NAMES.length &&
|
||||
new Set(safe.map((process) => process.name)).size === PROCESS_NAMES.length
|
||||
) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// Retry until the bounded deadline.
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type ProcessDefinition,
|
||||
type ProcessManager,
|
||||
readReleaseManifest,
|
||||
sanitizeManagedProcessEnv,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
|
||||
import type { ReleaseControllerConfig } from './config.js';
|
||||
@@ -20,7 +21,7 @@ export const buildReleaseControllerCommands = (
|
||||
needsInstall: boolean,
|
||||
config: ReleaseControllerConfig
|
||||
): BuildCommand[] => {
|
||||
const env = config.baseEnv;
|
||||
const env = sanitizeManagedProcessEnv(config.baseEnv);
|
||||
return [
|
||||
...(needsInstall ? [{ command: 'pnpm', args: ['install', '--frozen-lockfile'], cwd: workspaceRoot, env }] : []),
|
||||
{ command: 'pnpm', args: ['--filter', '@sammo-ts/common', 'build'], cwd: workspaceRoot, env },
|
||||
@@ -42,7 +43,7 @@ export const buildReleaseControllerDefinition = (
|
||||
cwd: path.join(workspaceRoot, 'app', 'release-controller'),
|
||||
args: ['daemon'],
|
||||
env: {
|
||||
...config.baseEnv,
|
||||
...sanitizeManagedProcessEnv(config.baseEnv),
|
||||
GATEWAY_DATABASE_URL: config.gatewayDatabaseUrl,
|
||||
GATEWAY_DB_SCHEMA: config.gatewayDbSchema,
|
||||
RELEASE_CONTROLLER_WORKSPACE_ROOT: workspaceRoot,
|
||||
@@ -89,10 +90,16 @@ export const upgradeReleaseController = async (options: {
|
||||
await options.processManager.start(buildReleaseControllerDefinition(workspace.root, options.config));
|
||||
const deadline = Date.now() + (options.readinessTimeoutMs ?? options.config.readinessTimeoutMs);
|
||||
while (Date.now() < deadline) {
|
||||
const active = (await options.processManager.list()).find(
|
||||
(process) => process.name === CONTROLLER_PROCESS_NAME && process.status.toLowerCase() === 'online'
|
||||
const matching = (await options.processManager.list()).filter(
|
||||
(process) => process.name === CONTROLLER_PROCESS_NAME
|
||||
);
|
||||
if (active) return { commitSha, workspace: workspace.root };
|
||||
if (
|
||||
matching.length === 1 &&
|
||||
matching[0]?.status.toLowerCase() === 'online' &&
|
||||
(matching[0]?.restartCount ?? 0) === 0
|
||||
) {
|
||||
return { commitSha, workspace: workspace.root };
|
||||
}
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
throw new Error('Release controller did not become online before the timeout.');
|
||||
|
||||
@@ -15,7 +15,7 @@ import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { resolveReleaseControllerConfig, type ReleaseControllerConfig } from '../src/config.js';
|
||||
import { buildGatewayProcessDefinitions, GatewayReleaseController } from '../src/releaseController.js';
|
||||
import { upgradeReleaseController } from '../src/selfUpgrade.js';
|
||||
import { buildReleaseControllerDefinition, upgradeReleaseController } from '../src/selfUpgrade.js';
|
||||
|
||||
const SHA = '1111111111111111111111111111111111111111';
|
||||
const OLD_SHA = '2222222222222222222222222222222222222222';
|
||||
@@ -124,6 +124,26 @@ it('runs Gateway preview from the frontend workspace dependency', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('does not forward release-controller PM2 identity to Gateway processes', () => {
|
||||
const definitions = buildGatewayProcessDefinitions('/srv/sammo/release', {
|
||||
...config,
|
||||
baseEnv: {
|
||||
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||
GATEWAY_ROLE: 'orchestrator',
|
||||
name: 'sammo:release-controller',
|
||||
pm_id: '3',
|
||||
pm_exec_path: '/srv/release-controller.js',
|
||||
},
|
||||
});
|
||||
|
||||
for (const definition of definitions) {
|
||||
expect(definition.env).toMatchObject({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
|
||||
expect(definition.env).not.toHaveProperty('pm_id');
|
||||
expect(definition.env).not.toHaveProperty('pm_exec_path');
|
||||
expect(definition.env).not.toHaveProperty('name');
|
||||
}
|
||||
});
|
||||
|
||||
describe('GatewayReleaseController', () => {
|
||||
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
@@ -225,9 +245,44 @@ describe('resolveReleaseControllerConfig', () => {
|
||||
|
||||
expect(new URL(resolved.gatewayDatabaseUrl).searchParams.get('schema')).toBe('gateway_release');
|
||||
});
|
||||
|
||||
it('removes PM2 metadata from the inherited controller environment', () => {
|
||||
const resolved = resolveReleaseControllerConfig({
|
||||
GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo',
|
||||
RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller',
|
||||
pm_id: '3',
|
||||
name: 'sammo:release-controller',
|
||||
axm_monitor: '{}',
|
||||
});
|
||||
|
||||
expect(resolved.baseEnv).toMatchObject({
|
||||
GATEWAY_DATABASE_URL: 'postgresql://user:pass@127.0.0.1:5432/sammo',
|
||||
RELEASE_CONTROLLER_WORKSPACE_ROOT: '/srv/sammo/controller',
|
||||
});
|
||||
expect(resolved.baseEnv).not.toHaveProperty('pm_id');
|
||||
expect(resolved.baseEnv).not.toHaveProperty('name');
|
||||
expect(resolved.baseEnv).not.toHaveProperty('axm_monitor');
|
||||
});
|
||||
});
|
||||
|
||||
describe('upgradeReleaseController', () => {
|
||||
it('does not forward the old controller PM2 identity to its replacement', () => {
|
||||
const definition = buildReleaseControllerDefinition('/srv/sammo/release', {
|
||||
...config,
|
||||
baseEnv: {
|
||||
DATABASE_URL: 'postgresql://integration.invalid/sammo',
|
||||
name: 'sammo:release-controller',
|
||||
pm_id: '3',
|
||||
pm_exec_path: '/srv/old-controller.js',
|
||||
},
|
||||
});
|
||||
|
||||
expect(definition.env).toMatchObject({ DATABASE_URL: 'postgresql://integration.invalid/sammo' });
|
||||
expect(definition.env).not.toHaveProperty('pm_id');
|
||||
expect(definition.env).not.toHaveProperty('pm_exec_path');
|
||||
expect(definition.env).not.toHaveProperty('name');
|
||||
});
|
||||
|
||||
it('switches the controller daemon from a separately invoked CLI process', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
const running = new Map([['sammo:release-controller', '/srv/sammo/old/app/release-controller']]);
|
||||
|
||||
Reference in New Issue
Block a user