devsam/core2026
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gateway): 배포 worktree 자동 정리 추가

Browse Source 
Profile과 Gateway release의 현재 실행 및 rollback 경계를 보호하면서 오래된 commit worktree를 주기적으로 정리한다.
This commit is contained in:
Hide_D
2026-08-20 16:25:40 +00:00
parent ce7807bb4f
commit 1279a68f1c
9 changed files with 437 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/gateway-api/src/orchestrator/gatewayOrchestrator.ts
+60 -72
View File
@@ -44,7 +44,11 @@ import {
writeProfileReleaseSource,
type ProfileReleaseSource,
} from './profileReleaseSource.js';
import type { GitWorkspaceManager } from './workspaceManager.js';
import {
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
type GitWorkspaceManager,
} from './workspaceManager.js';
import type { AdminSeedUser } from './seedProfileDatabase.js';
import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js';
@@ -73,6 +77,8 @@ export interface GatewayOrchestratorOptions {
cancelGame?: typeof defaultCancelGame;
}
const WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
export interface ProfileRuntimeState {
frontendRunning: boolean;
apiRunning: boolean;
@@ -629,11 +635,13 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
private scheduleTimer?: NodeJS.Timeout;
private buildTimer?: NodeJS.Timeout;
private adminActionTimer?: NodeJS.Timeout;
private workspaceCleanupTimer?: NodeJS.Timeout;
private reconcileInFlight = false;
private scheduleInFlight = false;
private buildInFlight = false;
private adminActionInFlight = false;
private operationInFlight = false;
private workspaceCleanupInFlight = false;
private activeOperationAbortSignal?: AbortSignal;
private readonly resetInFlight = new Set<string>();
private readonly operationLeaseOwner = randomUUID();
@@ -714,7 +722,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
start(): void {
this.stopping = false;
this.trackTask(this.reconcileNow());
this.trackTask(this.runOperationsNow());
this.trackTask(this.runOperationsNow().then(() => this.cleanupWorkspacesScheduled()));
this.trackTask(this.runAdminActionsNow());
this.reconcileTimer = setInterval(() => this.trackTask(this.reconcileNow()), this.reconcileIntervalMs);
this.scheduleTimer = setInterval(() => this.trackTask(this.runScheduleNow()), this.scheduleIntervalMs);
@@ -723,6 +731,10 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
this.trackTask(this.runOperationsNow());
this.trackTask(this.runAdminActionsNow());
}, this.adminActionIntervalMs);
this.workspaceCleanupTimer = setInterval(
() => this.trackTask(this.cleanupWorkspacesScheduled()),
WORKSPACE_CLEANUP_INTERVAL_MS
);
}
async stop(): Promise<void> {
@@ -747,6 +759,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
if (this.adminActionTimer) {
clearInterval(this.adminActionTimer);
}
if (this.workspaceCleanupTimer) {
clearInterval(this.workspaceCleanupTimer);
}
await Promise.allSettled([...this.inFlightTasks]);
}
@@ -903,7 +918,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}
async runBuildQueueNow(): Promise<void> {
if (this.stopping || this.buildInFlight) {
if (this.stopping || this.buildInFlight || this.workspaceCleanupInFlight) {
return;
}
this.buildInFlight = true;
@@ -965,7 +980,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}
async runOperationsNow(): Promise<void> {
if (this.stopping || this.operationInFlight || this.buildInFlight) {
if (this.stopping || this.operationInFlight || this.buildInFlight || this.workspaceCleanupInFlight) {
return;
}
this.operationInFlight = true;
@@ -2160,83 +2175,56 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
const profiles = await this.repository.listProfiles();
const cutoff = this.computeCutoffDate(6);
const workspaceMap = new Map<string, { profileNames: string[]; lastUsedAt?: Date; hasActiveBuild: boolean }>();
for (const profile of profiles) {
const workspace = profile.buildWorkspace;
if (!workspace) {
continue;
}
const entry = workspaceMap.get(workspace) ?? {
profileNames: [],
lastUsedAt: undefined,
hasActiveBuild: false,
};
entry.profileNames.push(profile.profileName);
if (profile.buildLastUsedAt) {
const usedAt = new Date(profile.buildLastUsedAt);
if (!entry.lastUsedAt || usedAt > entry.lastUsedAt) {
entry.lastUsedAt = usedAt;
if (this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) {
const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces();
return { removed: [], skipped: managedWorkspaces.map((workspace) => workspace.root) };
}
this.workspaceCleanupInFlight = true;
try {
const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces();
const profiles = await this.repository.listProfiles();
const protectedWorkspaces = new Set<string>();
for (const profile of profiles) {
if (profile.buildWorkspace) {
protectedWorkspaces.add(path.resolve(profile.buildWorkspace));
}
if (profile.buildCommitSha && (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED')) {
protectedWorkspaces.add(
path.resolve(this.workspaceManager.workspacePathForCommit(profile.buildCommitSha))
);
}
}
if (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED') {
entry.hasActiveBuild = true;
}
workspaceMap.set(workspace, entry);
}
const activeProcesses = (await this.processManager.list()).filter((process) =>
isRuntimeProcessActive(process.status)
);
const referencedWorkspaces = new Set<string>();
for (const [workspace, entry] of workspaceMap.entries()) {
const profileProcessNames = new Set(
entry.profileNames.flatMap((profileName) => [
buildProcessName(profileName, 'frontend'),
buildProcessName(profileName, 'api'),
buildProcessName(profileName, 'daemon'),
buildProcessName(profileName, 'auction'),
buildProcessName(profileName, 'battle-sim'),
buildProcessName(profileName, 'tournament'),
])
const activeProcesses = (await this.processManager.list()).filter((process) =>
isRuntimeProcessActive(process.status)
);
if (
activeProcesses.some(
(process) =>
profileProcessNames.has(process.name) ||
isPathInside(process.cwd, workspace) ||
isPathInside(process.script, workspace)
)
) {
referencedWorkspaces.add(workspace);
for (const workspace of managedWorkspaces) {
if (
activeProcesses.some(
(process) =>
isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root)
)
) {
protectedWorkspaces.add(workspace.root);
}
}
}
const removed: string[] = [];
const skipped: string[] = [];
for (const [workspace, entry] of workspaceMap.entries()) {
if (!entry.lastUsedAt || entry.hasActiveBuild || referencedWorkspaces.has(workspace)) {
skipped.push(workspace);
continue;
}
if (entry.lastUsedAt > cutoff) {
skipped.push(workspace);
continue;
}
await this.workspaceManager.remove(workspace);
await this.repository.clearWorkspaceUsage(entry.profileNames);
removed.push(workspace);
return await this.workspaceManager.cleanup({
protectedPaths: [...protectedWorkspaces],
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
});
} finally {
this.workspaceCleanupInFlight = false;
}
return { removed, skipped };
}
private computeCutoffDate(months: number): Date {
const date = this.now();
const cutoff = new Date(date);
cutoff.setMonth(cutoff.getMonth() - months);
return cutoff;
private async cleanupWorkspacesScheduled(): Promise<void> {
if (this.stopping || this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) return;
const result = await this.cleanupStaleWorkspaces();
if (result.removed.length > 0) {
console.info(`[gateway-orchestrator] removed ${result.removed.length} stale profile worktrees`);
}
}
private async startProfile(profile: GatewayProfileRecord, assertLease?: () => Promise<void>): Promise<boolean> {
app/gateway-api/src/orchestrator/workspaceManager.ts
+113 -1
View File
@@ -6,6 +6,7 @@ export interface WorkspaceManagerOptions {
repoRoot: string;
worktreeRoot: string;
baseEnv?: Record<string, string>;
now?: () => Date;
}
export interface WorkspaceInfo {
@@ -14,6 +15,26 @@ export interface WorkspaceInfo {
needsInstall: boolean;
}
export interface ManagedWorkspaceInfo {
root: string;
commitSha: string;
lastUsedAt: Date;
}
export interface ManagedWorkspaceCleanupOptions {
protectedPaths?: readonly string[];
retentionMs: number;
keepNewest: number;
}
export interface ManagedWorkspaceCleanupResult {
removed: string[];
skipped: string[];
}
export const DEFAULT_MANAGED_WORKSPACE_RETENTION_MS = 24 * 60 * 60 * 1_000;
export const DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST = 2;
const runGit = (args: string[], cwd: string, env?: Record<string, string>): Promise<{ ok: boolean; output: string }> =>
new Promise((resolve) => {
const child = spawn('git', args, {
@@ -58,11 +79,13 @@ export class GitWorkspaceManager {
private readonly repoRoot: string;
private readonly worktreeRoot: string;
private readonly baseEnv?: Record<string, string>;
private readonly now: () => Date;
constructor(options: WorkspaceManagerOptions) {
this.repoRoot = options.repoRoot;
this.worktreeRoot = options.worktreeRoot;
this.baseEnv = options.baseEnv;
this.now = options.now ?? (() => new Date());
}
async resolveCommit(sourceMode: 'BRANCH' | 'COMMIT', sourceRef: string): Promise<string> {
@@ -124,6 +147,8 @@ export class GitWorkspaceManager {
} else {
await this.assertReusableWorkspace(workspacePath, commitSha);
}
const usedAt = this.now();
fs.utimesSync(workspacePath, usedAt, usedAt);
return {
root: workspacePath,
@@ -138,13 +163,100 @@ export class GitWorkspaceManager {
return false;
}
await this.assertRegisteredWorkspace(resolved);
const status = await runGit(['status', '--porcelain'], resolved, this.baseEnv);
if (!status.ok) {
throw new Error(status.output || 'Failed to inspect managed workspace.');
}
if (status.output.trim()) {
throw new Error('Managed workspace has uncommitted changes.');
}
const result = await runGit(['worktree', 'remove', '--force', resolved], this.repoRoot, this.baseEnv);
if (!result.ok) {
fs.rmSync(resolved, { recursive: true, force: true });
throw new Error(result.output || 'Failed to remove git worktree.');
}
return true;
}
workspacePathForCommit(commitSha: string): string {
if (!COMMIT_SHA_PATTERN.test(commitSha)) {
throw new Error('Invalid commit SHA.');
}
return path.join(this.worktreeRoot, commitSha);
}
async listManagedWorkspaces(): Promise<ManagedWorkspaceInfo[]> {
const listed = await runGit(['worktree', 'list', '--porcelain'], this.repoRoot, this.baseEnv);
if (!listed.ok) {
throw new Error(listed.output || 'Failed to inspect git worktrees.');
}
const workspaces: ManagedWorkspaceInfo[] = [];
for (const block of listed.output.split(/\n\n+/)) {
const lines = block.split('\n');
const worktreeLine = lines.find((line) => line.startsWith('worktree '));
const headLine = lines.find((line) => line.startsWith('HEAD '));
if (!worktreeLine || !headLine) continue;
const workspacePath = path.resolve(worktreeLine.slice('worktree '.length));
const commitSha = headLine.slice('HEAD '.length);
try {
this.assertManagedWorkspacePath(workspacePath);
} catch {
continue;
}
if (!COMMIT_SHA_PATTERN.test(commitSha) || !fs.existsSync(workspacePath)) continue;
workspaces.push({
root: workspacePath,
commitSha,
lastUsedAt: fs.statSync(workspacePath).mtime,
});
}
return workspaces;
}
async cleanup(options: ManagedWorkspaceCleanupOptions): Promise<ManagedWorkspaceCleanupResult> {
if (!Number.isFinite(options.retentionMs) || options.retentionMs < 0) {
throw new Error('Workspace retention must be a non-negative duration.');
}
if (!Number.isInteger(options.keepNewest) || options.keepNewest < 0) {
throw new Error('Workspace keepNewest must be a non-negative integer.');
}
const protectedPaths = new Set((options.protectedPaths ?? []).map((item) => path.resolve(item)));
const workspaces = await this.listManagedWorkspaces();
const unprotectedNewest = [...workspaces]
.filter((workspace) => !protectedPaths.has(workspace.root))
.sort((left, right) => right.lastUsedAt.getTime() - left.lastUsedAt.getTime())
.slice(0, options.keepNewest);
const retainedNewestPaths = new Set(unprotectedNewest.map((workspace) => workspace.root));
const cutoff = this.now().getTime() - options.retentionMs;
const removed: string[] = [];
const skipped: string[] = [];
for (const workspace of workspaces) {
if (
protectedPaths.has(workspace.root) ||
retainedNewestPaths.has(workspace.root) ||
workspace.lastUsedAt.getTime() > cutoff
) {
skipped.push(workspace.root);
continue;
}
try {
if (await this.remove(workspace.root)) {
removed.push(workspace.root);
} else {
skipped.push(workspace.root);
}
} catch {
skipped.push(workspace.root);
}
}
const pruned = await runGit(['worktree', 'prune', '--expire', 'now'], this.repoRoot, this.baseEnv);
if (!pruned.ok) {
throw new Error(pruned.output || 'Failed to prune git worktree metadata.');
}
return { removed, skipped };
}
private assertManagedWorkspacePath(workspacePath: string): string {
const resolved = path.resolve(workspacePath);
const root = path.resolve(this.worktreeRoot);
app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts
+66 -117
View File
@@ -1,15 +1,23 @@
import path from 'node:path';
import { describe, expect, it } from 'vitest';
import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js';
import type { ProcessManager } from '../src/orchestrator/processManager.js';
import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
import type { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js';
import {
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
type GitWorkspaceManager,
type ManagedWorkspaceCleanupOptions,
} from '../src/orchestrator/workspaceManager.js';
const COMMIT_SHA = '0123456789abcdef0123456789abcdef01234567';
const oldUsage = '2025-01-01T00:00:00.000Z';
const makeProfile = (
profileName: string,
workspace: string,
workspace: string | undefined,
overrides: Partial<GatewayProfileRecord> = {}
): GatewayProfileRecord => ({
profileName,
@@ -20,7 +28,7 @@ const makeProfile = (
apiPort: 15_003,
status: 'RUNNING',
buildStatus: 'SUCCEEDED',
buildCommitSha: '0123456789abcdef0123456789abcdef01234567',
buildCommitSha: COMMIT_SHA,
buildWorkspace: workspace,
buildLastUsedAt: oldUsage,
meta: {},
@@ -32,17 +40,10 @@ const makeProfile = (
const createHarness = (
profiles: GatewayProfileRecord[],
processes: Awaited<ReturnType<ProcessManager['list']>>,
workspaceExists = true
managedPaths: string[]
) => {
const removeCalls: string[] = [];
const clearedProfiles: string[][] = [];
const repository = {
listProfiles: async () => profiles,
clearWorkspaceUsage: async (profileNames: string[]) => {
clearedProfiles.push(profileNames);
},
} as unknown as GatewayProfileRepository;
const cleanupCalls: ManagedWorkspaceCleanupOptions[] = [];
const repository = { listProfiles: async () => profiles } as unknown as GatewayProfileRepository;
const processManager: ProcessManager = {
list: async () => processes,
start: async () => {},
@@ -50,9 +51,16 @@ const createHarness = (
delete: async () => {},
};
const workspaceManager = {
remove: async (workspace: string) => {
removeCalls.push(workspace);
return workspaceExists;
listManagedWorkspaces: async () =>
managedPaths.map((root) => ({ root, commitSha: path.basename(root), lastUsedAt: new Date(oldUsage) })),
workspacePathForCommit: (commitSha: string) => `/srv/sammo/worktrees/${commitSha}`,
cleanup: async (options: ManagedWorkspaceCleanupOptions) => {
cleanupCalls.push(options);
const protectedPaths = new Set(options.protectedPaths);
return {
removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)),
skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)),
};
},
} as unknown as GitWorkspaceManager;
const orchestrator = new GatewayOrchestrator({
@@ -70,126 +78,67 @@ const createHarness = (
scheduleIntervalMs: 60_000,
buildIntervalMs: 60_000,
adminActionIntervalMs: 60_000,
now: () => new Date('2026-07-30T00:00:00.000Z'),
});
return { orchestrator, removeCalls, clearedProfiles };
return { orchestrator, cleanupCalls };
};
describe('GatewayOrchestrator workspace cleanup', () => {
it('skips a workspace referenced by any active process cwd', async () => {
const workspace = '/srv/sammo/worktrees/active';
it('always protects every workspace currently selected by a profile', async () => {
const current = '/srv/sammo/worktrees/current';
const stale = '/srv/sammo/worktrees/stale';
const harness = createHarness([makeProfile('che:default', current)], [], [current, stale]);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [stale],
skipped: [current],
});
expect(harness.cleanupCalls[0]).toMatchObject({
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
});
});
it('protects the commit target of queued and running builds before the profile reference changes', async () => {
const target = `/srv/sammo/worktrees/${COMMIT_SHA}`;
const harness = createHarness([makeProfile('che:default', undefined, { buildStatus: 'QUEUED' })], [], [target]);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [],
skipped: [target],
});
});
it('protects an otherwise orphaned workspace referenced by any active process cwd or script', async () => {
const cwdWorkspace = '/srv/sammo/worktrees/cwd-orphan';
const scriptWorkspace = '/srv/sammo/worktrees/script-orphan';
const stale = '/srv/sammo/worktrees/stale';
const harness = createHarness(
[makeProfile('che:default', workspace)],
[],
[
{
name: 'sammo:che:default:frontend',
status: 'online',
cwd: `${workspace}/app/game-frontend`,
},
]
{ name: 'custom-build', status: 'online', cwd: `${cwdWorkspace}/app/game-api` },
{ name: 'custom-worker', status: 'launching', script: `${scriptWorkspace}/dist/index.js` },
{ name: 'stopped-worker', status: 'stopped', cwd: `${stale}/app/game-api` },
],
[cwdWorkspace, scriptWorkspace, stale]
);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [],
skipped: [workspace],
removed: [stale],
skipped: [cwdWorkspace, scriptWorkspace],
});
expect(harness.removeCalls).toEqual([]);
expect(harness.clearedProfiles).toEqual([]);
});
it('skips a workspace when only one profile process is active and cwd metadata is absent', async () => {
const workspace = '/srv/sammo/worktrees/partial';
const harness = createHarness(
[makeProfile('che:default', workspace)],
[{ name: 'sammo:che:default:tournament-worker', status: 'launching' }]
);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [],
skipped: [workspace],
});
expect(harness.removeCalls).toEqual([]);
});
it('skips a workspace referenced only by an active process script', async () => {
const workspace = '/srv/sammo/worktrees/script-reference';
const harness = createHarness(
[makeProfile('che:default', workspace)],
[
{
name: 'unregistered-worker-name',
status: 'online',
script: `${workspace}/app/game-api/dist/index.js`,
},
]
);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [],
skipped: [workspace],
});
expect(harness.removeCalls).toEqual([]);
});
it('protects a shared workspace when a process for either profile is active', async () => {
const workspace = '/srv/sammo/worktrees/shared';
const harness = createHarness(
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
[{ name: 'sammo:hwe:default:game-api', status: 'stopping' }]
);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [],
skipped: [workspace],
});
expect(harness.removeCalls).toEqual([]);
});
it('removes an old unreferenced workspace and clears every profile reference', async () => {
const workspace = '/srv/sammo/worktrees/stale';
const harness = createHarness(
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
[{ name: 'sammo:che:default:game-api', status: 'stopped', cwd: `${workspace}/app/game-api` }]
);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [workspace],
skipped: [],
});
expect(harness.removeCalls).toEqual([workspace]);
expect(harness.clearedProfiles).toEqual([['che:default', 'hwe:default']]);
});
it('does not treat a sibling path with the same prefix as a workspace reference', async () => {
it('does not confuse sibling path prefixes with an active workspace reference', async () => {
const workspace = '/srv/sammo/worktrees/commit-a';
const harness = createHarness(
[makeProfile('che:default', workspace)],
[
{
name: 'unregistered-worker-name',
status: 'online',
cwd: '/srv/sammo/worktrees/commit-a-old/app/game-api',
},
]
[],
[{ name: 'custom-worker', status: 'online', cwd: `${workspace}-old/app/game-api` }],
[workspace]
);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [workspace],
skipped: [],
});
expect(harness.removeCalls).toEqual([workspace]);
});
it('clears a stale database reference when the workspace is already missing', async () => {
const workspace = '/srv/sammo/worktrees/missing';
const harness = createHarness([makeProfile('che:default', workspace)], [], false);
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [workspace],
skipped: [],
});
expect(harness.removeCalls).toEqual([workspace]);
expect(harness.clearedProfiles).toEqual([['che:default']]);
});
});
app/gateway-api/test/workspaceManager.test.ts
+42
View File
@@ -163,4 +163,46 @@ describe('GitWorkspaceManager source resolution', () => {
);
expect(fs.existsSync(unregistered)).toBe(true);
});
it('cleans only expired unprotected worktrees beyond the newest cache and preserves dirty work', async () => {
const fixture = createRepositoryFixture();
const now = new Date('2026-08-20T12:00:00.000Z');
const manager = new GitWorkspaceManager({
repoRoot: fixture.checkout,
worktreeRoot: fixture.worktrees,
now: () => now,
});
const workspaces = [await manager.prepare(fixture.firstCommit)];
for (let index = 2; index <= 5; index += 1) {
fs.writeFileSync(path.join(fixture.source, 'version.txt'), `version ${index}\n`);
git(fixture.source, 'add', 'version.txt');
git(fixture.source, 'commit', '-m', `version ${index}`);
git(fixture.source, 'push', 'origin', 'main');
const commit = await manager.resolveCommit('BRANCH', 'main');
workspaces.push(await manager.prepare(commit));
}
const expired = new Date('2026-08-01T00:00:00.000Z');
for (const workspace of workspaces) fs.utimesSync(workspace.root, expired, expired);
fs.writeFileSync(path.join(workspaces[1]!.root, 'preserve-me.txt'), 'uncommitted\n');
fs.utimesSync(workspaces[1]!.root, expired, expired);
const recent = new Date('2026-08-20T11:00:00.000Z');
fs.utimesSync(workspaces[4]!.root, recent, recent);
const result = await manager.cleanup({
protectedPaths: [workspaces[0]!.root],
retentionMs: 24 * 60 * 60 * 1_000,
keepNewest: 1,
});
expect(result.removed).toHaveLength(2);
expect(result.removed).toEqual(expect.arrayContaining([workspaces[2]!.root, workspaces[3]!.root]));
expect(result.skipped).toHaveLength(3);
expect(result.skipped).toEqual(
expect.arrayContaining([workspaces[0]!.root, workspaces[1]!.root, workspaces[4]!.root])
);
expect(fs.existsSync(workspaces[0]!.root)).toBe(true);
expect(fs.existsSync(workspaces[1]!.root)).toBe(true);
expect(fs.existsSync(workspaces[2]!.root)).toBe(false);
expect(fs.existsSync(workspaces[3]!.root)).toBe(false);
expect(fs.existsSync(workspaces[4]!.root)).toBe(true);
});
});
app/release-controller/README.md
+7
View File
@@ -70,6 +70,13 @@ pnpm --filter @sammo-ts/release-controller status
pnpm --filter @sammo-ts/release-controller run-once
```
Daemon은 시작 시와 이후 24시간마다 commit worktree를 자동 정리합니다. 현재·이전
Gateway release와 활성 PM2 process가 사용하는 경로는 항상 보호하고, 나머지는
마지막 사용 후 24시간과 최신 2개 cache를 보장한 뒤 제거합니다. 변경이 있거나 Git
제거가 실패한 worktree는 raw directory 삭제로 우회하지 않고 다음 주기까지
보존합니다. Profile worktree는 Gateway orchestrator가 같은 정책으로 별도
관리합니다.
## Controller self-upgrade
이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를
app/release-controller/src/index.ts
+14 -1
View File
@@ -7,7 +7,7 @@ import {
} from '@sammo-ts/gateway-api';
import { resolveReleaseControllerConfig } from './config.js';
import { GatewayReleaseController } from './releaseController.js';
import { GatewayReleaseController, RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS } from './releaseController.js';
import { upgradeReleaseController } from './selfUpgrade.js';
export * from './config.js';
@@ -67,6 +67,7 @@ const main = async (): Promise<void> => {
}
if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`);
let stopping = false;
let nextWorkspaceCleanupAt = 0;
const stop = async (): Promise<void> => {
if (stopping) return;
stopping = true;
@@ -75,6 +76,18 @@ const main = async (): Promise<void> => {
process.once('SIGINT', () => void stop());
process.once('SIGTERM', () => void stop());
while (!stopping) {
const now = Date.now();
if (now >= nextWorkspaceCleanupAt) {
nextWorkspaceCleanupAt = now + RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS;
try {
const result = await controller.cleanupStaleWorkspaces();
if (result.removed.length > 0) {
console.info(`[release-controller] removed ${result.removed.length} stale Gateway worktrees`);
}
} catch (error) {
console.error('[release-controller] workspace cleanup failed', error);
}
}
await controller.runOnce();
await new Promise<void>((resolve) => setTimeout(resolve, config.pollIntervalMs));
}
app/release-controller/src/releaseController.ts
+39
View File
@@ -6,6 +6,8 @@ import {
assertReleaseComponents,
buildTurboReleaseCommand,
buildTurboReleaseTaskCommand,
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
type BuildCommand,
type BuildProgressEvent,
type BuildRunner,
@@ -27,6 +29,16 @@ const HEARTBEAT_INTERVAL_MS = 60_000;
const CANCELLATION_POLL_INTERVAL_MS = 500;
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
export const RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
const isRuntimeProcessActive = (status: string): boolean =>
['online', 'launching', 'stopping'].includes(status.toLowerCase());
const isPathInside = (candidate: string | undefined, root: string): boolean => {
if (!candidate) return false;
const relative = path.relative(path.resolve(root), path.resolve(candidate));
return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
};
const managedPostgresPoolMax = (env: Record<string, string>, roleVariable: string, fallback: number): string =>
String(resolvePostgresPoolMax(env[roleVariable] ?? env.POSTGRES_POOL_MAX, fallback));
@@ -132,6 +144,33 @@ export class GatewayReleaseController {
private readonly fetchImpl: typeof fetch = fetch
) {}
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
const [state, processes, workspaces] = await Promise.all([
this.repository.getState(),
this.processManager.list(),
this.workspaceManager.listManagedWorkspaces(),
]);
const protectedWorkspaces = new Set<string>();
if (state.activeWorkspace) protectedWorkspaces.add(path.resolve(state.activeWorkspace));
if (state.previousWorkspace) protectedWorkspaces.add(path.resolve(state.previousWorkspace));
const activeProcesses = processes.filter((process) => isRuntimeProcessActive(process.status));
for (const workspace of workspaces) {
if (
activeProcesses.some(
(process) =>
isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root)
)
) {
protectedWorkspaces.add(workspace.root);
}
}
return this.workspaceManager.cleanup({
protectedPaths: [...protectedWorkspaces],
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
});
}
private sanitizeLogMessage(message: string): string {
let sanitized = stripVTControlCharacters(message);
const sensitiveValues = new Set([
app/release-controller/test/releaseController.test.ts
+73 -8
View File
@@ -2,14 +2,17 @@ import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import type {
BuildRunner,
GatewayReleaseOperationRecord,
GatewayReleaseRepository,
GatewayReleaseStateRecord,
GitWorkspaceManager,
ProcessDefinition,
ProcessManager,
import {
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
type BuildRunner,
type GatewayReleaseOperationRecord,
type GatewayReleaseRepository,
type GatewayReleaseStateRecord,
type GitWorkspaceManager,
type ManagedWorkspaceCleanupOptions,
type ProcessDefinition,
type ProcessManager,
} from '@sammo-ts/gateway-api';
import { afterEach, describe, expect, it } from 'vitest';
@@ -178,6 +181,68 @@ it('rejects Gateway definitions before switching processes when Redis connection
});
describe('GatewayReleaseController', () => {
it('protects active, rollback, and running-process worktrees while delegating bounded cleanup', async () => {
const active = '/srv/sammo/releases/active';
const previous = '/srv/sammo/releases/previous';
const controllerWorkspace = '/srv/sammo/releases/controller';
const stale = '/srv/sammo/releases/stale';
const managedPaths = [active, previous, controllerWorkspace, stale];
const cleanupCalls: ManagedWorkspaceCleanupOptions[] = [];
const harness = createRepository();
const workspaceManager = {
listManagedWorkspaces: async () =>
managedPaths.map((root) => ({
root,
commitSha: SHA,
lastUsedAt: new Date('2025-01-01T00:00:00.000Z'),
})),
cleanup: async (options: ManagedWorkspaceCleanupOptions) => {
cleanupCalls.push(options);
const protectedPaths = new Set(options.protectedPaths);
return {
removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)),
skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)),
};
},
} as unknown as GitWorkspaceManager;
const controller = new GatewayReleaseController(
{
...harness.repository,
getState: async () => ({
...state,
activeWorkspace: active,
previousCommitSha: SHA,
previousWorkspace: previous,
}),
},
workspaceManager,
{ run: async () => ({ ok: true, exitCode: 0, output: '' }) },
{
list: async () => [
{
name: 'sammo:release-controller',
status: 'online',
cwd: `${controllerWorkspace}/app/release-controller`,
},
{ name: 'old-build', status: 'stopped', cwd: `${stale}/app/gateway-api` },
],
start: async () => {},
stop: async () => {},
delete: async () => {},
},
config
);
await expect(controller.cleanupStaleWorkspaces()).resolves.toEqual({
removed: [stale],
skipped: [active, previous, controllerWorkspace],
});
expect(cleanupCalls[0]).toMatchObject({
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
});
});
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
const workspace = await createReleaseWorkspace();
const harness = createRepository();