fix(gateway): 배포 worktree 자동 정리 추가
Profile과 Gateway release의 현재 실행 및 rollback 경계를 보호하면서 오래된 commit worktree를 주기적으로 정리한다.
This commit is contained in:
@@ -70,6 +70,13 @@ pnpm --filter @sammo-ts/release-controller status
|
||||
pnpm --filter @sammo-ts/release-controller run-once
|
||||
```
|
||||
|
||||
Daemon은 시작 시와 이후 24시간마다 commit worktree를 자동 정리합니다. 현재·이전
|
||||
Gateway release와 활성 PM2 process가 사용하는 경로는 항상 보호하고, 나머지는
|
||||
마지막 사용 후 24시간과 최신 2개 cache를 보장한 뒤 제거합니다. 변경이 있거나 Git
|
||||
제거가 실패한 worktree는 raw directory 삭제로 우회하지 않고 다음 주기까지
|
||||
보존합니다. Profile worktree는 Gateway orchestrator가 같은 정책으로 별도
|
||||
관리합니다.
|
||||
|
||||
## Controller self-upgrade
|
||||
|
||||
이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
} from '@sammo-ts/gateway-api';
|
||||
|
||||
import { resolveReleaseControllerConfig } from './config.js';
|
||||
import { GatewayReleaseController } from './releaseController.js';
|
||||
import { GatewayReleaseController, RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS } from './releaseController.js';
|
||||
import { upgradeReleaseController } from './selfUpgrade.js';
|
||||
|
||||
export * from './config.js';
|
||||
@@ -67,6 +67,7 @@ const main = async (): Promise<void> => {
|
||||
}
|
||||
if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`);
|
||||
let stopping = false;
|
||||
let nextWorkspaceCleanupAt = 0;
|
||||
const stop = async (): Promise<void> => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
@@ -75,6 +76,18 @@ const main = async (): Promise<void> => {
|
||||
process.once('SIGINT', () => void stop());
|
||||
process.once('SIGTERM', () => void stop());
|
||||
while (!stopping) {
|
||||
const now = Date.now();
|
||||
if (now >= nextWorkspaceCleanupAt) {
|
||||
nextWorkspaceCleanupAt = now + RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS;
|
||||
try {
|
||||
const result = await controller.cleanupStaleWorkspaces();
|
||||
if (result.removed.length > 0) {
|
||||
console.info(`[release-controller] removed ${result.removed.length} stale Gateway worktrees`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[release-controller] workspace cleanup failed', error);
|
||||
}
|
||||
}
|
||||
await controller.runOnce();
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, config.pollIntervalMs));
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
assertReleaseComponents,
|
||||
buildTurboReleaseCommand,
|
||||
buildTurboReleaseTaskCommand,
|
||||
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
type BuildCommand,
|
||||
type BuildProgressEvent,
|
||||
type BuildRunner,
|
||||
@@ -27,6 +29,16 @@ const HEARTBEAT_INTERVAL_MS = 60_000;
|
||||
const CANCELLATION_POLL_INTERVAL_MS = 500;
|
||||
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
|
||||
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
|
||||
export const RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
|
||||
const isRuntimeProcessActive = (status: string): boolean =>
|
||||
['online', 'launching', 'stopping'].includes(status.toLowerCase());
|
||||
|
||||
const isPathInside = (candidate: string | undefined, root: string): boolean => {
|
||||
if (!candidate) return false;
|
||||
const relative = path.relative(path.resolve(root), path.resolve(candidate));
|
||||
return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
|
||||
};
|
||||
|
||||
const managedPostgresPoolMax = (env: Record<string, string>, roleVariable: string, fallback: number): string =>
|
||||
String(resolvePostgresPoolMax(env[roleVariable] ?? env.POSTGRES_POOL_MAX, fallback));
|
||||
@@ -132,6 +144,33 @@ export class GatewayReleaseController {
|
||||
private readonly fetchImpl: typeof fetch = fetch
|
||||
) {}
|
||||
|
||||
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
|
||||
const [state, processes, workspaces] = await Promise.all([
|
||||
this.repository.getState(),
|
||||
this.processManager.list(),
|
||||
this.workspaceManager.listManagedWorkspaces(),
|
||||
]);
|
||||
const protectedWorkspaces = new Set<string>();
|
||||
if (state.activeWorkspace) protectedWorkspaces.add(path.resolve(state.activeWorkspace));
|
||||
if (state.previousWorkspace) protectedWorkspaces.add(path.resolve(state.previousWorkspace));
|
||||
const activeProcesses = processes.filter((process) => isRuntimeProcessActive(process.status));
|
||||
for (const workspace of workspaces) {
|
||||
if (
|
||||
activeProcesses.some(
|
||||
(process) =>
|
||||
isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root)
|
||||
)
|
||||
) {
|
||||
protectedWorkspaces.add(workspace.root);
|
||||
}
|
||||
}
|
||||
return this.workspaceManager.cleanup({
|
||||
protectedPaths: [...protectedWorkspaces],
|
||||
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
});
|
||||
}
|
||||
|
||||
private sanitizeLogMessage(message: string): string {
|
||||
let sanitized = stripVTControlCharacters(message);
|
||||
const sensitiveValues = new Set([
|
||||
|
||||
@@ -2,14 +2,17 @@ import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import type {
|
||||
BuildRunner,
|
||||
GatewayReleaseOperationRecord,
|
||||
GatewayReleaseRepository,
|
||||
GatewayReleaseStateRecord,
|
||||
GitWorkspaceManager,
|
||||
ProcessDefinition,
|
||||
ProcessManager,
|
||||
import {
|
||||
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
type BuildRunner,
|
||||
type GatewayReleaseOperationRecord,
|
||||
type GatewayReleaseRepository,
|
||||
type GatewayReleaseStateRecord,
|
||||
type GitWorkspaceManager,
|
||||
type ManagedWorkspaceCleanupOptions,
|
||||
type ProcessDefinition,
|
||||
type ProcessManager,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
@@ -178,6 +181,68 @@ it('rejects Gateway definitions before switching processes when Redis connection
|
||||
});
|
||||
|
||||
describe('GatewayReleaseController', () => {
|
||||
it('protects active, rollback, and running-process worktrees while delegating bounded cleanup', async () => {
|
||||
const active = '/srv/sammo/releases/active';
|
||||
const previous = '/srv/sammo/releases/previous';
|
||||
const controllerWorkspace = '/srv/sammo/releases/controller';
|
||||
const stale = '/srv/sammo/releases/stale';
|
||||
const managedPaths = [active, previous, controllerWorkspace, stale];
|
||||
const cleanupCalls: ManagedWorkspaceCleanupOptions[] = [];
|
||||
const harness = createRepository();
|
||||
const workspaceManager = {
|
||||
listManagedWorkspaces: async () =>
|
||||
managedPaths.map((root) => ({
|
||||
root,
|
||||
commitSha: SHA,
|
||||
lastUsedAt: new Date('2025-01-01T00:00:00.000Z'),
|
||||
})),
|
||||
cleanup: async (options: ManagedWorkspaceCleanupOptions) => {
|
||||
cleanupCalls.push(options);
|
||||
const protectedPaths = new Set(options.protectedPaths);
|
||||
return {
|
||||
removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)),
|
||||
skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)),
|
||||
};
|
||||
},
|
||||
} as unknown as GitWorkspaceManager;
|
||||
const controller = new GatewayReleaseController(
|
||||
{
|
||||
...harness.repository,
|
||||
getState: async () => ({
|
||||
...state,
|
||||
activeWorkspace: active,
|
||||
previousCommitSha: SHA,
|
||||
previousWorkspace: previous,
|
||||
}),
|
||||
},
|
||||
workspaceManager,
|
||||
{ run: async () => ({ ok: true, exitCode: 0, output: '' }) },
|
||||
{
|
||||
list: async () => [
|
||||
{
|
||||
name: 'sammo:release-controller',
|
||||
status: 'online',
|
||||
cwd: `${controllerWorkspace}/app/release-controller`,
|
||||
},
|
||||
{ name: 'old-build', status: 'stopped', cwd: `${stale}/app/gateway-api` },
|
||||
],
|
||||
start: async () => {},
|
||||
stop: async () => {},
|
||||
delete: async () => {},
|
||||
},
|
||||
config
|
||||
);
|
||||
|
||||
await expect(controller.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [stale],
|
||||
skipped: [active, previous, controllerWorkspace],
|
||||
});
|
||||
expect(cleanupCalls[0]).toMatchObject({
|
||||
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
});
|
||||
});
|
||||
|
||||
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
const harness = createRepository();
|
||||
|
||||
Reference in New Issue
Block a user