fix(gateway): protect active profile workspaces
This commit is contained in:
@@ -305,6 +305,19 @@ const buildProcessName = (
|
|||||||
const isMissingProcessError = (error: unknown): boolean =>
|
const isMissingProcessError = (error: unknown): boolean =>
|
||||||
error instanceof Error && /process or namespace not found/i.test(error.message);
|
error instanceof Error && /process or namespace not found/i.test(error.message);
|
||||||
|
|
||||||
|
const isRuntimeProcessActive = (status: string): boolean => {
|
||||||
|
const normalized = status.toLowerCase();
|
||||||
|
return normalized === 'online' || normalized === 'launching' || normalized === 'stopping';
|
||||||
|
};
|
||||||
|
|
||||||
|
const isPathInside = (candidate: string | undefined, root: string): boolean => {
|
||||||
|
if (!candidate) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const relative = path.relative(path.resolve(root), path.resolve(candidate));
|
||||||
|
return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
|
||||||
|
};
|
||||||
|
|
||||||
export const buildProcessDefinitions = (
|
export const buildProcessDefinitions = (
|
||||||
profile: GatewayProfileRecord,
|
profile: GatewayProfileRecord,
|
||||||
config: GatewayProcessConfig
|
config: GatewayProcessConfig
|
||||||
@@ -1003,10 +1016,36 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
workspaceMap.set(workspace, entry);
|
workspaceMap.set(workspace, entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const activeProcesses = (await this.processManager.list()).filter((process) =>
|
||||||
|
isRuntimeProcessActive(process.status)
|
||||||
|
);
|
||||||
|
const referencedWorkspaces = new Set<string>();
|
||||||
|
for (const [workspace, entry] of workspaceMap.entries()) {
|
||||||
|
const profileProcessNames = new Set(
|
||||||
|
entry.profileNames.flatMap((profileName) => [
|
||||||
|
buildProcessName(profileName, 'api'),
|
||||||
|
buildProcessName(profileName, 'daemon'),
|
||||||
|
buildProcessName(profileName, 'auction'),
|
||||||
|
buildProcessName(profileName, 'battle-sim'),
|
||||||
|
buildProcessName(profileName, 'tournament'),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
activeProcesses.some(
|
||||||
|
(process) =>
|
||||||
|
profileProcessNames.has(process.name) ||
|
||||||
|
isPathInside(process.cwd, workspace) ||
|
||||||
|
isPathInside(process.script, workspace)
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
referencedWorkspaces.add(workspace);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const removed: string[] = [];
|
const removed: string[] = [];
|
||||||
const skipped: string[] = [];
|
const skipped: string[] = [];
|
||||||
for (const [workspace, entry] of workspaceMap.entries()) {
|
for (const [workspace, entry] of workspaceMap.entries()) {
|
||||||
if (!entry.lastUsedAt || entry.hasActiveBuild) {
|
if (!entry.lastUsedAt || entry.hasActiveBuild || referencedWorkspaces.has(workspace)) {
|
||||||
skipped.push(workspace);
|
skipped.push(workspace);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,6 +42,8 @@ export class Pm2ProcessManager implements ProcessManager {
|
|||||||
name: item.name ?? 'unknown',
|
name: item.name ?? 'unknown',
|
||||||
status: item.pm2_env?.status ?? 'unknown',
|
status: item.pm2_env?.status ?? 'unknown',
|
||||||
pid: item.pid ?? undefined,
|
pid: item.pid ?? undefined,
|
||||||
|
cwd: item.pm2_env?.pm_cwd ?? undefined,
|
||||||
|
script: item.pm2_env?.pm_exec_path ?? undefined,
|
||||||
})) ?? [];
|
})) ?? [];
|
||||||
resolve(normalized);
|
resolve(normalized);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ export interface ManagedProcessInfo {
|
|||||||
name: string;
|
name: string;
|
||||||
status: string;
|
status: string;
|
||||||
pid?: number;
|
pid?: number;
|
||||||
|
cwd?: string;
|
||||||
|
script?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ProcessDefinition {
|
export interface ProcessDefinition {
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js';
|
||||||
|
import type { ProcessManager } from '../src/orchestrator/processManager.js';
|
||||||
|
import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
|
||||||
|
import type { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js';
|
||||||
|
|
||||||
|
const oldUsage = '2025-01-01T00:00:00.000Z';
|
||||||
|
|
||||||
|
const makeProfile = (
|
||||||
|
profileName: string,
|
||||||
|
workspace: string,
|
||||||
|
overrides: Partial<GatewayProfileRecord> = {}
|
||||||
|
): GatewayProfileRecord => ({
|
||||||
|
profileName,
|
||||||
|
profile: profileName.split(':')[0] ?? 'che',
|
||||||
|
scenario: profileName.split(':')[1] ?? 'default',
|
||||||
|
apiPort: 15_003,
|
||||||
|
status: 'RUNNING',
|
||||||
|
buildStatus: 'SUCCEEDED',
|
||||||
|
buildCommitSha: '0123456789abcdef0123456789abcdef01234567',
|
||||||
|
buildWorkspace: workspace,
|
||||||
|
buildLastUsedAt: oldUsage,
|
||||||
|
meta: {},
|
||||||
|
createdAt: oldUsage,
|
||||||
|
updatedAt: oldUsage,
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
|
||||||
|
const createHarness = (
|
||||||
|
profiles: GatewayProfileRecord[],
|
||||||
|
processes: Awaited<ReturnType<ProcessManager['list']>>,
|
||||||
|
workspaceExists = true
|
||||||
|
) => {
|
||||||
|
const removeCalls: string[] = [];
|
||||||
|
const clearedProfiles: string[][] = [];
|
||||||
|
|
||||||
|
const repository = {
|
||||||
|
listProfiles: async () => profiles,
|
||||||
|
clearWorkspaceUsage: async (profileNames: string[]) => {
|
||||||
|
clearedProfiles.push(profileNames);
|
||||||
|
},
|
||||||
|
} as unknown as GatewayProfileRepository;
|
||||||
|
const processManager: ProcessManager = {
|
||||||
|
list: async () => processes,
|
||||||
|
start: async () => {},
|
||||||
|
stop: async () => {},
|
||||||
|
delete: async () => {},
|
||||||
|
};
|
||||||
|
const workspaceManager = {
|
||||||
|
remove: async (workspace: string) => {
|
||||||
|
removeCalls.push(workspace);
|
||||||
|
return workspaceExists;
|
||||||
|
},
|
||||||
|
} as unknown as GitWorkspaceManager;
|
||||||
|
const orchestrator = new GatewayOrchestrator({
|
||||||
|
repository,
|
||||||
|
processManager,
|
||||||
|
buildRunner: { run: async () => ({ ok: true, exitCode: 0, output: '' }) },
|
||||||
|
workspaceManager,
|
||||||
|
processConfig: {
|
||||||
|
workspaceRoot: '/srv/sammo',
|
||||||
|
redisKeyPrefix: 'sammo:test',
|
||||||
|
gameTokenSecret: 'test-secret',
|
||||||
|
},
|
||||||
|
reconcileIntervalMs: 60_000,
|
||||||
|
scheduleIntervalMs: 60_000,
|
||||||
|
buildIntervalMs: 60_000,
|
||||||
|
adminActionIntervalMs: 60_000,
|
||||||
|
now: () => new Date('2026-07-30T00:00:00.000Z'),
|
||||||
|
});
|
||||||
|
|
||||||
|
return { orchestrator, removeCalls, clearedProfiles };
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('GatewayOrchestrator workspace cleanup', () => {
|
||||||
|
it('skips a workspace referenced by any active process cwd', async () => {
|
||||||
|
const workspace = '/srv/sammo/worktrees/active';
|
||||||
|
const harness = createHarness(
|
||||||
|
[makeProfile('che:default', workspace)],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
name: 'sammo:che:default:frontend',
|
||||||
|
status: 'online',
|
||||||
|
cwd: `${workspace}/app/game-frontend`,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [],
|
||||||
|
skipped: [workspace],
|
||||||
|
});
|
||||||
|
expect(harness.removeCalls).toEqual([]);
|
||||||
|
expect(harness.clearedProfiles).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips a workspace when only one profile process is active and cwd metadata is absent', async () => {
|
||||||
|
const workspace = '/srv/sammo/worktrees/partial';
|
||||||
|
const harness = createHarness(
|
||||||
|
[makeProfile('che:default', workspace)],
|
||||||
|
[{ name: 'sammo:che:default:tournament-worker', status: 'launching' }]
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [],
|
||||||
|
skipped: [workspace],
|
||||||
|
});
|
||||||
|
expect(harness.removeCalls).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips a workspace referenced only by an active process script', async () => {
|
||||||
|
const workspace = '/srv/sammo/worktrees/script-reference';
|
||||||
|
const harness = createHarness(
|
||||||
|
[makeProfile('che:default', workspace)],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
name: 'unregistered-worker-name',
|
||||||
|
status: 'online',
|
||||||
|
script: `${workspace}/app/game-api/dist/index.js`,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [],
|
||||||
|
skipped: [workspace],
|
||||||
|
});
|
||||||
|
expect(harness.removeCalls).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('protects a shared workspace when a process for either profile is active', async () => {
|
||||||
|
const workspace = '/srv/sammo/worktrees/shared';
|
||||||
|
const harness = createHarness(
|
||||||
|
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
|
||||||
|
[{ name: 'sammo:hwe:default:game-api', status: 'stopping' }]
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [],
|
||||||
|
skipped: [workspace],
|
||||||
|
});
|
||||||
|
expect(harness.removeCalls).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('removes an old unreferenced workspace and clears every profile reference', async () => {
|
||||||
|
const workspace = '/srv/sammo/worktrees/stale';
|
||||||
|
const harness = createHarness(
|
||||||
|
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
|
||||||
|
[{ name: 'sammo:che:default:game-api', status: 'stopped', cwd: `${workspace}/app/game-api` }]
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [workspace],
|
||||||
|
skipped: [],
|
||||||
|
});
|
||||||
|
expect(harness.removeCalls).toEqual([workspace]);
|
||||||
|
expect(harness.clearedProfiles).toEqual([['che:default', 'hwe:default']]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not treat a sibling path with the same prefix as a workspace reference', async () => {
|
||||||
|
const workspace = '/srv/sammo/worktrees/commit-a';
|
||||||
|
const harness = createHarness(
|
||||||
|
[makeProfile('che:default', workspace)],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
name: 'unregistered-worker-name',
|
||||||
|
status: 'online',
|
||||||
|
cwd: '/srv/sammo/worktrees/commit-a-old/app/game-api',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [workspace],
|
||||||
|
skipped: [],
|
||||||
|
});
|
||||||
|
expect(harness.removeCalls).toEqual([workspace]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears a stale database reference when the workspace is already missing', async () => {
|
||||||
|
const workspace = '/srv/sammo/worktrees/missing';
|
||||||
|
const harness = createHarness([makeProfile('che:default', workspace)], [], false);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [workspace],
|
||||||
|
skipped: [],
|
||||||
|
});
|
||||||
|
expect(harness.removeCalls).toEqual([workspace]);
|
||||||
|
expect(harness.clearedProfiles).toEqual([['che:default']]);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user