fix(gateway): 배포 worktree 자동 정리 추가
Profile과 Gateway release의 현재 실행 및 rollback 경계를 보호하면서 오래된 commit worktree를 주기적으로 정리한다.
This commit is contained in:
@@ -44,7 +44,11 @@ import {
|
|||||||
writeProfileReleaseSource,
|
writeProfileReleaseSource,
|
||||||
type ProfileReleaseSource,
|
type ProfileReleaseSource,
|
||||||
} from './profileReleaseSource.js';
|
} from './profileReleaseSource.js';
|
||||||
import type { GitWorkspaceManager } from './workspaceManager.js';
|
import {
|
||||||
|
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
|
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
|
type GitWorkspaceManager,
|
||||||
|
} from './workspaceManager.js';
|
||||||
import type { AdminSeedUser } from './seedProfileDatabase.js';
|
import type { AdminSeedUser } from './seedProfileDatabase.js';
|
||||||
import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js';
|
import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js';
|
||||||
|
|
||||||
@@ -73,6 +77,8 @@ export interface GatewayOrchestratorOptions {
|
|||||||
cancelGame?: typeof defaultCancelGame;
|
cancelGame?: typeof defaultCancelGame;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||||
|
|
||||||
export interface ProfileRuntimeState {
|
export interface ProfileRuntimeState {
|
||||||
frontendRunning: boolean;
|
frontendRunning: boolean;
|
||||||
apiRunning: boolean;
|
apiRunning: boolean;
|
||||||
@@ -629,11 +635,13 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
private scheduleTimer?: NodeJS.Timeout;
|
private scheduleTimer?: NodeJS.Timeout;
|
||||||
private buildTimer?: NodeJS.Timeout;
|
private buildTimer?: NodeJS.Timeout;
|
||||||
private adminActionTimer?: NodeJS.Timeout;
|
private adminActionTimer?: NodeJS.Timeout;
|
||||||
|
private workspaceCleanupTimer?: NodeJS.Timeout;
|
||||||
private reconcileInFlight = false;
|
private reconcileInFlight = false;
|
||||||
private scheduleInFlight = false;
|
private scheduleInFlight = false;
|
||||||
private buildInFlight = false;
|
private buildInFlight = false;
|
||||||
private adminActionInFlight = false;
|
private adminActionInFlight = false;
|
||||||
private operationInFlight = false;
|
private operationInFlight = false;
|
||||||
|
private workspaceCleanupInFlight = false;
|
||||||
private activeOperationAbortSignal?: AbortSignal;
|
private activeOperationAbortSignal?: AbortSignal;
|
||||||
private readonly resetInFlight = new Set<string>();
|
private readonly resetInFlight = new Set<string>();
|
||||||
private readonly operationLeaseOwner = randomUUID();
|
private readonly operationLeaseOwner = randomUUID();
|
||||||
@@ -714,7 +722,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
start(): void {
|
start(): void {
|
||||||
this.stopping = false;
|
this.stopping = false;
|
||||||
this.trackTask(this.reconcileNow());
|
this.trackTask(this.reconcileNow());
|
||||||
this.trackTask(this.runOperationsNow());
|
this.trackTask(this.runOperationsNow().then(() => this.cleanupWorkspacesScheduled()));
|
||||||
this.trackTask(this.runAdminActionsNow());
|
this.trackTask(this.runAdminActionsNow());
|
||||||
this.reconcileTimer = setInterval(() => this.trackTask(this.reconcileNow()), this.reconcileIntervalMs);
|
this.reconcileTimer = setInterval(() => this.trackTask(this.reconcileNow()), this.reconcileIntervalMs);
|
||||||
this.scheduleTimer = setInterval(() => this.trackTask(this.runScheduleNow()), this.scheduleIntervalMs);
|
this.scheduleTimer = setInterval(() => this.trackTask(this.runScheduleNow()), this.scheduleIntervalMs);
|
||||||
@@ -723,6 +731,10 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
this.trackTask(this.runOperationsNow());
|
this.trackTask(this.runOperationsNow());
|
||||||
this.trackTask(this.runAdminActionsNow());
|
this.trackTask(this.runAdminActionsNow());
|
||||||
}, this.adminActionIntervalMs);
|
}, this.adminActionIntervalMs);
|
||||||
|
this.workspaceCleanupTimer = setInterval(
|
||||||
|
() => this.trackTask(this.cleanupWorkspacesScheduled()),
|
||||||
|
WORKSPACE_CLEANUP_INTERVAL_MS
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async stop(): Promise<void> {
|
async stop(): Promise<void> {
|
||||||
@@ -747,6 +759,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
if (this.adminActionTimer) {
|
if (this.adminActionTimer) {
|
||||||
clearInterval(this.adminActionTimer);
|
clearInterval(this.adminActionTimer);
|
||||||
}
|
}
|
||||||
|
if (this.workspaceCleanupTimer) {
|
||||||
|
clearInterval(this.workspaceCleanupTimer);
|
||||||
|
}
|
||||||
await Promise.allSettled([...this.inFlightTasks]);
|
await Promise.allSettled([...this.inFlightTasks]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -903,7 +918,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async runBuildQueueNow(): Promise<void> {
|
async runBuildQueueNow(): Promise<void> {
|
||||||
if (this.stopping || this.buildInFlight) {
|
if (this.stopping || this.buildInFlight || this.workspaceCleanupInFlight) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.buildInFlight = true;
|
this.buildInFlight = true;
|
||||||
@@ -965,7 +980,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async runOperationsNow(): Promise<void> {
|
async runOperationsNow(): Promise<void> {
|
||||||
if (this.stopping || this.operationInFlight || this.buildInFlight) {
|
if (this.stopping || this.operationInFlight || this.buildInFlight || this.workspaceCleanupInFlight) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.operationInFlight = true;
|
this.operationInFlight = true;
|
||||||
@@ -2160,83 +2175,56 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
|
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
|
||||||
|
if (this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) {
|
||||||
|
const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces();
|
||||||
|
return { removed: [], skipped: managedWorkspaces.map((workspace) => workspace.root) };
|
||||||
|
}
|
||||||
|
this.workspaceCleanupInFlight = true;
|
||||||
|
try {
|
||||||
|
const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces();
|
||||||
const profiles = await this.repository.listProfiles();
|
const profiles = await this.repository.listProfiles();
|
||||||
const cutoff = this.computeCutoffDate(6);
|
const protectedWorkspaces = new Set<string>();
|
||||||
const workspaceMap = new Map<string, { profileNames: string[]; lastUsedAt?: Date; hasActiveBuild: boolean }>();
|
|
||||||
for (const profile of profiles) {
|
for (const profile of profiles) {
|
||||||
const workspace = profile.buildWorkspace;
|
if (profile.buildWorkspace) {
|
||||||
if (!workspace) {
|
protectedWorkspaces.add(path.resolve(profile.buildWorkspace));
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
const entry = workspaceMap.get(workspace) ?? {
|
if (profile.buildCommitSha && (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED')) {
|
||||||
profileNames: [],
|
protectedWorkspaces.add(
|
||||||
lastUsedAt: undefined,
|
path.resolve(this.workspaceManager.workspacePathForCommit(profile.buildCommitSha))
|
||||||
hasActiveBuild: false,
|
);
|
||||||
};
|
|
||||||
entry.profileNames.push(profile.profileName);
|
|
||||||
if (profile.buildLastUsedAt) {
|
|
||||||
const usedAt = new Date(profile.buildLastUsedAt);
|
|
||||||
if (!entry.lastUsedAt || usedAt > entry.lastUsedAt) {
|
|
||||||
entry.lastUsedAt = usedAt;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED') {
|
|
||||||
entry.hasActiveBuild = true;
|
|
||||||
}
|
|
||||||
workspaceMap.set(workspace, entry);
|
|
||||||
}
|
|
||||||
|
|
||||||
const activeProcesses = (await this.processManager.list()).filter((process) =>
|
const activeProcesses = (await this.processManager.list()).filter((process) =>
|
||||||
isRuntimeProcessActive(process.status)
|
isRuntimeProcessActive(process.status)
|
||||||
);
|
);
|
||||||
const referencedWorkspaces = new Set<string>();
|
for (const workspace of managedWorkspaces) {
|
||||||
for (const [workspace, entry] of workspaceMap.entries()) {
|
|
||||||
const profileProcessNames = new Set(
|
|
||||||
entry.profileNames.flatMap((profileName) => [
|
|
||||||
buildProcessName(profileName, 'frontend'),
|
|
||||||
buildProcessName(profileName, 'api'),
|
|
||||||
buildProcessName(profileName, 'daemon'),
|
|
||||||
buildProcessName(profileName, 'auction'),
|
|
||||||
buildProcessName(profileName, 'battle-sim'),
|
|
||||||
buildProcessName(profileName, 'tournament'),
|
|
||||||
])
|
|
||||||
);
|
|
||||||
if (
|
if (
|
||||||
activeProcesses.some(
|
activeProcesses.some(
|
||||||
(process) =>
|
(process) =>
|
||||||
profileProcessNames.has(process.name) ||
|
isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root)
|
||||||
isPathInside(process.cwd, workspace) ||
|
|
||||||
isPathInside(process.script, workspace)
|
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
referencedWorkspaces.add(workspace);
|
protectedWorkspaces.add(workspace.root);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const removed: string[] = [];
|
return await this.workspaceManager.cleanup({
|
||||||
const skipped: string[] = [];
|
protectedPaths: [...protectedWorkspaces],
|
||||||
for (const [workspace, entry] of workspaceMap.entries()) {
|
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
if (!entry.lastUsedAt || entry.hasActiveBuild || referencedWorkspaces.has(workspace)) {
|
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
skipped.push(workspace);
|
});
|
||||||
continue;
|
} finally {
|
||||||
|
this.workspaceCleanupInFlight = false;
|
||||||
}
|
}
|
||||||
if (entry.lastUsedAt > cutoff) {
|
|
||||||
skipped.push(workspace);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
await this.workspaceManager.remove(workspace);
|
|
||||||
await this.repository.clearWorkspaceUsage(entry.profileNames);
|
|
||||||
removed.push(workspace);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return { removed, skipped };
|
private async cleanupWorkspacesScheduled(): Promise<void> {
|
||||||
|
if (this.stopping || this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) return;
|
||||||
|
const result = await this.cleanupStaleWorkspaces();
|
||||||
|
if (result.removed.length > 0) {
|
||||||
|
console.info(`[gateway-orchestrator] removed ${result.removed.length} stale profile worktrees`);
|
||||||
}
|
}
|
||||||
|
|
||||||
private computeCutoffDate(months: number): Date {
|
|
||||||
const date = this.now();
|
|
||||||
const cutoff = new Date(date);
|
|
||||||
cutoff.setMonth(cutoff.getMonth() - months);
|
|
||||||
return cutoff;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async startProfile(profile: GatewayProfileRecord, assertLease?: () => Promise<void>): Promise<boolean> {
|
private async startProfile(profile: GatewayProfileRecord, assertLease?: () => Promise<void>): Promise<boolean> {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ export interface WorkspaceManagerOptions {
|
|||||||
repoRoot: string;
|
repoRoot: string;
|
||||||
worktreeRoot: string;
|
worktreeRoot: string;
|
||||||
baseEnv?: Record<string, string>;
|
baseEnv?: Record<string, string>;
|
||||||
|
now?: () => Date;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface WorkspaceInfo {
|
export interface WorkspaceInfo {
|
||||||
@@ -14,6 +15,26 @@ export interface WorkspaceInfo {
|
|||||||
needsInstall: boolean;
|
needsInstall: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ManagedWorkspaceInfo {
|
||||||
|
root: string;
|
||||||
|
commitSha: string;
|
||||||
|
lastUsedAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ManagedWorkspaceCleanupOptions {
|
||||||
|
protectedPaths?: readonly string[];
|
||||||
|
retentionMs: number;
|
||||||
|
keepNewest: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ManagedWorkspaceCleanupResult {
|
||||||
|
removed: string[];
|
||||||
|
skipped: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_MANAGED_WORKSPACE_RETENTION_MS = 24 * 60 * 60 * 1_000;
|
||||||
|
export const DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST = 2;
|
||||||
|
|
||||||
const runGit = (args: string[], cwd: string, env?: Record<string, string>): Promise<{ ok: boolean; output: string }> =>
|
const runGit = (args: string[], cwd: string, env?: Record<string, string>): Promise<{ ok: boolean; output: string }> =>
|
||||||
new Promise((resolve) => {
|
new Promise((resolve) => {
|
||||||
const child = spawn('git', args, {
|
const child = spawn('git', args, {
|
||||||
@@ -58,11 +79,13 @@ export class GitWorkspaceManager {
|
|||||||
private readonly repoRoot: string;
|
private readonly repoRoot: string;
|
||||||
private readonly worktreeRoot: string;
|
private readonly worktreeRoot: string;
|
||||||
private readonly baseEnv?: Record<string, string>;
|
private readonly baseEnv?: Record<string, string>;
|
||||||
|
private readonly now: () => Date;
|
||||||
|
|
||||||
constructor(options: WorkspaceManagerOptions) {
|
constructor(options: WorkspaceManagerOptions) {
|
||||||
this.repoRoot = options.repoRoot;
|
this.repoRoot = options.repoRoot;
|
||||||
this.worktreeRoot = options.worktreeRoot;
|
this.worktreeRoot = options.worktreeRoot;
|
||||||
this.baseEnv = options.baseEnv;
|
this.baseEnv = options.baseEnv;
|
||||||
|
this.now = options.now ?? (() => new Date());
|
||||||
}
|
}
|
||||||
|
|
||||||
async resolveCommit(sourceMode: 'BRANCH' | 'COMMIT', sourceRef: string): Promise<string> {
|
async resolveCommit(sourceMode: 'BRANCH' | 'COMMIT', sourceRef: string): Promise<string> {
|
||||||
@@ -124,6 +147,8 @@ export class GitWorkspaceManager {
|
|||||||
} else {
|
} else {
|
||||||
await this.assertReusableWorkspace(workspacePath, commitSha);
|
await this.assertReusableWorkspace(workspacePath, commitSha);
|
||||||
}
|
}
|
||||||
|
const usedAt = this.now();
|
||||||
|
fs.utimesSync(workspacePath, usedAt, usedAt);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
root: workspacePath,
|
root: workspacePath,
|
||||||
@@ -138,13 +163,100 @@ export class GitWorkspaceManager {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
await this.assertRegisteredWorkspace(resolved);
|
await this.assertRegisteredWorkspace(resolved);
|
||||||
|
const status = await runGit(['status', '--porcelain'], resolved, this.baseEnv);
|
||||||
|
if (!status.ok) {
|
||||||
|
throw new Error(status.output || 'Failed to inspect managed workspace.');
|
||||||
|
}
|
||||||
|
if (status.output.trim()) {
|
||||||
|
throw new Error('Managed workspace has uncommitted changes.');
|
||||||
|
}
|
||||||
const result = await runGit(['worktree', 'remove', '--force', resolved], this.repoRoot, this.baseEnv);
|
const result = await runGit(['worktree', 'remove', '--force', resolved], this.repoRoot, this.baseEnv);
|
||||||
if (!result.ok) {
|
if (!result.ok) {
|
||||||
fs.rmSync(resolved, { recursive: true, force: true });
|
throw new Error(result.output || 'Failed to remove git worktree.');
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
workspacePathForCommit(commitSha: string): string {
|
||||||
|
if (!COMMIT_SHA_PATTERN.test(commitSha)) {
|
||||||
|
throw new Error('Invalid commit SHA.');
|
||||||
|
}
|
||||||
|
return path.join(this.worktreeRoot, commitSha);
|
||||||
|
}
|
||||||
|
|
||||||
|
async listManagedWorkspaces(): Promise<ManagedWorkspaceInfo[]> {
|
||||||
|
const listed = await runGit(['worktree', 'list', '--porcelain'], this.repoRoot, this.baseEnv);
|
||||||
|
if (!listed.ok) {
|
||||||
|
throw new Error(listed.output || 'Failed to inspect git worktrees.');
|
||||||
|
}
|
||||||
|
const workspaces: ManagedWorkspaceInfo[] = [];
|
||||||
|
for (const block of listed.output.split(/\n\n+/)) {
|
||||||
|
const lines = block.split('\n');
|
||||||
|
const worktreeLine = lines.find((line) => line.startsWith('worktree '));
|
||||||
|
const headLine = lines.find((line) => line.startsWith('HEAD '));
|
||||||
|
if (!worktreeLine || !headLine) continue;
|
||||||
|
const workspacePath = path.resolve(worktreeLine.slice('worktree '.length));
|
||||||
|
const commitSha = headLine.slice('HEAD '.length);
|
||||||
|
try {
|
||||||
|
this.assertManagedWorkspacePath(workspacePath);
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!COMMIT_SHA_PATTERN.test(commitSha) || !fs.existsSync(workspacePath)) continue;
|
||||||
|
workspaces.push({
|
||||||
|
root: workspacePath,
|
||||||
|
commitSha,
|
||||||
|
lastUsedAt: fs.statSync(workspacePath).mtime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return workspaces;
|
||||||
|
}
|
||||||
|
|
||||||
|
async cleanup(options: ManagedWorkspaceCleanupOptions): Promise<ManagedWorkspaceCleanupResult> {
|
||||||
|
if (!Number.isFinite(options.retentionMs) || options.retentionMs < 0) {
|
||||||
|
throw new Error('Workspace retention must be a non-negative duration.');
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(options.keepNewest) || options.keepNewest < 0) {
|
||||||
|
throw new Error('Workspace keepNewest must be a non-negative integer.');
|
||||||
|
}
|
||||||
|
const protectedPaths = new Set((options.protectedPaths ?? []).map((item) => path.resolve(item)));
|
||||||
|
const workspaces = await this.listManagedWorkspaces();
|
||||||
|
const unprotectedNewest = [...workspaces]
|
||||||
|
.filter((workspace) => !protectedPaths.has(workspace.root))
|
||||||
|
.sort((left, right) => right.lastUsedAt.getTime() - left.lastUsedAt.getTime())
|
||||||
|
.slice(0, options.keepNewest);
|
||||||
|
const retainedNewestPaths = new Set(unprotectedNewest.map((workspace) => workspace.root));
|
||||||
|
const cutoff = this.now().getTime() - options.retentionMs;
|
||||||
|
const removed: string[] = [];
|
||||||
|
const skipped: string[] = [];
|
||||||
|
|
||||||
|
for (const workspace of workspaces) {
|
||||||
|
if (
|
||||||
|
protectedPaths.has(workspace.root) ||
|
||||||
|
retainedNewestPaths.has(workspace.root) ||
|
||||||
|
workspace.lastUsedAt.getTime() > cutoff
|
||||||
|
) {
|
||||||
|
skipped.push(workspace.root);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (await this.remove(workspace.root)) {
|
||||||
|
removed.push(workspace.root);
|
||||||
|
} else {
|
||||||
|
skipped.push(workspace.root);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
skipped.push(workspace.root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const pruned = await runGit(['worktree', 'prune', '--expire', 'now'], this.repoRoot, this.baseEnv);
|
||||||
|
if (!pruned.ok) {
|
||||||
|
throw new Error(pruned.output || 'Failed to prune git worktree metadata.');
|
||||||
|
}
|
||||||
|
return { removed, skipped };
|
||||||
|
}
|
||||||
|
|
||||||
private assertManagedWorkspacePath(workspacePath: string): string {
|
private assertManagedWorkspacePath(workspacePath: string): string {
|
||||||
const resolved = path.resolve(workspacePath);
|
const resolved = path.resolve(workspacePath);
|
||||||
const root = path.resolve(this.worktreeRoot);
|
const root = path.resolve(this.worktreeRoot);
|
||||||
|
|||||||
@@ -1,15 +1,23 @@
|
|||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js';
|
import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js';
|
||||||
import type { ProcessManager } from '../src/orchestrator/processManager.js';
|
import type { ProcessManager } from '../src/orchestrator/processManager.js';
|
||||||
import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
|
import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
|
||||||
import type { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js';
|
import {
|
||||||
|
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
|
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
|
type GitWorkspaceManager,
|
||||||
|
type ManagedWorkspaceCleanupOptions,
|
||||||
|
} from '../src/orchestrator/workspaceManager.js';
|
||||||
|
|
||||||
|
const COMMIT_SHA = '0123456789abcdef0123456789abcdef01234567';
|
||||||
const oldUsage = '2025-01-01T00:00:00.000Z';
|
const oldUsage = '2025-01-01T00:00:00.000Z';
|
||||||
|
|
||||||
const makeProfile = (
|
const makeProfile = (
|
||||||
profileName: string,
|
profileName: string,
|
||||||
workspace: string,
|
workspace: string | undefined,
|
||||||
overrides: Partial<GatewayProfileRecord> = {}
|
overrides: Partial<GatewayProfileRecord> = {}
|
||||||
): GatewayProfileRecord => ({
|
): GatewayProfileRecord => ({
|
||||||
profileName,
|
profileName,
|
||||||
@@ -20,7 +28,7 @@ const makeProfile = (
|
|||||||
apiPort: 15_003,
|
apiPort: 15_003,
|
||||||
status: 'RUNNING',
|
status: 'RUNNING',
|
||||||
buildStatus: 'SUCCEEDED',
|
buildStatus: 'SUCCEEDED',
|
||||||
buildCommitSha: '0123456789abcdef0123456789abcdef01234567',
|
buildCommitSha: COMMIT_SHA,
|
||||||
buildWorkspace: workspace,
|
buildWorkspace: workspace,
|
||||||
buildLastUsedAt: oldUsage,
|
buildLastUsedAt: oldUsage,
|
||||||
meta: {},
|
meta: {},
|
||||||
@@ -32,17 +40,10 @@ const makeProfile = (
|
|||||||
const createHarness = (
|
const createHarness = (
|
||||||
profiles: GatewayProfileRecord[],
|
profiles: GatewayProfileRecord[],
|
||||||
processes: Awaited<ReturnType<ProcessManager['list']>>,
|
processes: Awaited<ReturnType<ProcessManager['list']>>,
|
||||||
workspaceExists = true
|
managedPaths: string[]
|
||||||
) => {
|
) => {
|
||||||
const removeCalls: string[] = [];
|
const cleanupCalls: ManagedWorkspaceCleanupOptions[] = [];
|
||||||
const clearedProfiles: string[][] = [];
|
const repository = { listProfiles: async () => profiles } as unknown as GatewayProfileRepository;
|
||||||
|
|
||||||
const repository = {
|
|
||||||
listProfiles: async () => profiles,
|
|
||||||
clearWorkspaceUsage: async (profileNames: string[]) => {
|
|
||||||
clearedProfiles.push(profileNames);
|
|
||||||
},
|
|
||||||
} as unknown as GatewayProfileRepository;
|
|
||||||
const processManager: ProcessManager = {
|
const processManager: ProcessManager = {
|
||||||
list: async () => processes,
|
list: async () => processes,
|
||||||
start: async () => {},
|
start: async () => {},
|
||||||
@@ -50,9 +51,16 @@ const createHarness = (
|
|||||||
delete: async () => {},
|
delete: async () => {},
|
||||||
};
|
};
|
||||||
const workspaceManager = {
|
const workspaceManager = {
|
||||||
remove: async (workspace: string) => {
|
listManagedWorkspaces: async () =>
|
||||||
removeCalls.push(workspace);
|
managedPaths.map((root) => ({ root, commitSha: path.basename(root), lastUsedAt: new Date(oldUsage) })),
|
||||||
return workspaceExists;
|
workspacePathForCommit: (commitSha: string) => `/srv/sammo/worktrees/${commitSha}`,
|
||||||
|
cleanup: async (options: ManagedWorkspaceCleanupOptions) => {
|
||||||
|
cleanupCalls.push(options);
|
||||||
|
const protectedPaths = new Set(options.protectedPaths);
|
||||||
|
return {
|
||||||
|
removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)),
|
||||||
|
skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)),
|
||||||
|
};
|
||||||
},
|
},
|
||||||
} as unknown as GitWorkspaceManager;
|
} as unknown as GitWorkspaceManager;
|
||||||
const orchestrator = new GatewayOrchestrator({
|
const orchestrator = new GatewayOrchestrator({
|
||||||
@@ -70,126 +78,67 @@ const createHarness = (
|
|||||||
scheduleIntervalMs: 60_000,
|
scheduleIntervalMs: 60_000,
|
||||||
buildIntervalMs: 60_000,
|
buildIntervalMs: 60_000,
|
||||||
adminActionIntervalMs: 60_000,
|
adminActionIntervalMs: 60_000,
|
||||||
now: () => new Date('2026-07-30T00:00:00.000Z'),
|
|
||||||
});
|
});
|
||||||
|
return { orchestrator, cleanupCalls };
|
||||||
return { orchestrator, removeCalls, clearedProfiles };
|
|
||||||
};
|
};
|
||||||
|
|
||||||
describe('GatewayOrchestrator workspace cleanup', () => {
|
describe('GatewayOrchestrator workspace cleanup', () => {
|
||||||
it('skips a workspace referenced by any active process cwd', async () => {
|
it('always protects every workspace currently selected by a profile', async () => {
|
||||||
const workspace = '/srv/sammo/worktrees/active';
|
const current = '/srv/sammo/worktrees/current';
|
||||||
|
const stale = '/srv/sammo/worktrees/stale';
|
||||||
|
const harness = createHarness([makeProfile('che:default', current)], [], [current, stale]);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [stale],
|
||||||
|
skipped: [current],
|
||||||
|
});
|
||||||
|
expect(harness.cleanupCalls[0]).toMatchObject({
|
||||||
|
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
|
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('protects the commit target of queued and running builds before the profile reference changes', async () => {
|
||||||
|
const target = `/srv/sammo/worktrees/${COMMIT_SHA}`;
|
||||||
|
const harness = createHarness([makeProfile('che:default', undefined, { buildStatus: 'QUEUED' })], [], [target]);
|
||||||
|
|
||||||
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [],
|
||||||
|
skipped: [target],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('protects an otherwise orphaned workspace referenced by any active process cwd or script', async () => {
|
||||||
|
const cwdWorkspace = '/srv/sammo/worktrees/cwd-orphan';
|
||||||
|
const scriptWorkspace = '/srv/sammo/worktrees/script-orphan';
|
||||||
|
const stale = '/srv/sammo/worktrees/stale';
|
||||||
const harness = createHarness(
|
const harness = createHarness(
|
||||||
[makeProfile('che:default', workspace)],
|
[],
|
||||||
[
|
[
|
||||||
{
|
{ name: 'custom-build', status: 'online', cwd: `${cwdWorkspace}/app/game-api` },
|
||||||
name: 'sammo:che:default:frontend',
|
{ name: 'custom-worker', status: 'launching', script: `${scriptWorkspace}/dist/index.js` },
|
||||||
status: 'online',
|
{ name: 'stopped-worker', status: 'stopped', cwd: `${stale}/app/game-api` },
|
||||||
cwd: `${workspace}/app/game-frontend`,
|
],
|
||||||
},
|
[cwdWorkspace, scriptWorkspace, stale]
|
||||||
]
|
|
||||||
);
|
);
|
||||||
|
|
||||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
removed: [],
|
removed: [stale],
|
||||||
skipped: [workspace],
|
skipped: [cwdWorkspace, scriptWorkspace],
|
||||||
});
|
});
|
||||||
expect(harness.removeCalls).toEqual([]);
|
|
||||||
expect(harness.clearedProfiles).toEqual([]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('skips a workspace when only one profile process is active and cwd metadata is absent', async () => {
|
it('does not confuse sibling path prefixes with an active workspace reference', async () => {
|
||||||
const workspace = '/srv/sammo/worktrees/partial';
|
|
||||||
const harness = createHarness(
|
|
||||||
[makeProfile('che:default', workspace)],
|
|
||||||
[{ name: 'sammo:che:default:tournament-worker', status: 'launching' }]
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
|
||||||
removed: [],
|
|
||||||
skipped: [workspace],
|
|
||||||
});
|
|
||||||
expect(harness.removeCalls).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('skips a workspace referenced only by an active process script', async () => {
|
|
||||||
const workspace = '/srv/sammo/worktrees/script-reference';
|
|
||||||
const harness = createHarness(
|
|
||||||
[makeProfile('che:default', workspace)],
|
|
||||||
[
|
|
||||||
{
|
|
||||||
name: 'unregistered-worker-name',
|
|
||||||
status: 'online',
|
|
||||||
script: `${workspace}/app/game-api/dist/index.js`,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
|
||||||
removed: [],
|
|
||||||
skipped: [workspace],
|
|
||||||
});
|
|
||||||
expect(harness.removeCalls).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('protects a shared workspace when a process for either profile is active', async () => {
|
|
||||||
const workspace = '/srv/sammo/worktrees/shared';
|
|
||||||
const harness = createHarness(
|
|
||||||
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
|
|
||||||
[{ name: 'sammo:hwe:default:game-api', status: 'stopping' }]
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
|
||||||
removed: [],
|
|
||||||
skipped: [workspace],
|
|
||||||
});
|
|
||||||
expect(harness.removeCalls).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('removes an old unreferenced workspace and clears every profile reference', async () => {
|
|
||||||
const workspace = '/srv/sammo/worktrees/stale';
|
|
||||||
const harness = createHarness(
|
|
||||||
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
|
|
||||||
[{ name: 'sammo:che:default:game-api', status: 'stopped', cwd: `${workspace}/app/game-api` }]
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
|
||||||
removed: [workspace],
|
|
||||||
skipped: [],
|
|
||||||
});
|
|
||||||
expect(harness.removeCalls).toEqual([workspace]);
|
|
||||||
expect(harness.clearedProfiles).toEqual([['che:default', 'hwe:default']]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('does not treat a sibling path with the same prefix as a workspace reference', async () => {
|
|
||||||
const workspace = '/srv/sammo/worktrees/commit-a';
|
const workspace = '/srv/sammo/worktrees/commit-a';
|
||||||
const harness = createHarness(
|
const harness = createHarness(
|
||||||
[makeProfile('che:default', workspace)],
|
[],
|
||||||
[
|
[{ name: 'custom-worker', status: 'online', cwd: `${workspace}-old/app/game-api` }],
|
||||||
{
|
[workspace]
|
||||||
name: 'unregistered-worker-name',
|
|
||||||
status: 'online',
|
|
||||||
cwd: '/srv/sammo/worktrees/commit-a-old/app/game-api',
|
|
||||||
},
|
|
||||||
]
|
|
||||||
);
|
);
|
||||||
|
|
||||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
removed: [workspace],
|
removed: [workspace],
|
||||||
skipped: [],
|
skipped: [],
|
||||||
});
|
});
|
||||||
expect(harness.removeCalls).toEqual([workspace]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('clears a stale database reference when the workspace is already missing', async () => {
|
|
||||||
const workspace = '/srv/sammo/worktrees/missing';
|
|
||||||
const harness = createHarness([makeProfile('che:default', workspace)], [], false);
|
|
||||||
|
|
||||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
|
||||||
removed: [workspace],
|
|
||||||
skipped: [],
|
|
||||||
});
|
|
||||||
expect(harness.removeCalls).toEqual([workspace]);
|
|
||||||
expect(harness.clearedProfiles).toEqual([['che:default']]);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -163,4 +163,46 @@ describe('GitWorkspaceManager source resolution', () => {
|
|||||||
);
|
);
|
||||||
expect(fs.existsSync(unregistered)).toBe(true);
|
expect(fs.existsSync(unregistered)).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('cleans only expired unprotected worktrees beyond the newest cache and preserves dirty work', async () => {
|
||||||
|
const fixture = createRepositoryFixture();
|
||||||
|
const now = new Date('2026-08-20T12:00:00.000Z');
|
||||||
|
const manager = new GitWorkspaceManager({
|
||||||
|
repoRoot: fixture.checkout,
|
||||||
|
worktreeRoot: fixture.worktrees,
|
||||||
|
now: () => now,
|
||||||
|
});
|
||||||
|
const workspaces = [await manager.prepare(fixture.firstCommit)];
|
||||||
|
for (let index = 2; index <= 5; index += 1) {
|
||||||
|
fs.writeFileSync(path.join(fixture.source, 'version.txt'), `version ${index}\n`);
|
||||||
|
git(fixture.source, 'add', 'version.txt');
|
||||||
|
git(fixture.source, 'commit', '-m', `version ${index}`);
|
||||||
|
git(fixture.source, 'push', 'origin', 'main');
|
||||||
|
const commit = await manager.resolveCommit('BRANCH', 'main');
|
||||||
|
workspaces.push(await manager.prepare(commit));
|
||||||
|
}
|
||||||
|
const expired = new Date('2026-08-01T00:00:00.000Z');
|
||||||
|
for (const workspace of workspaces) fs.utimesSync(workspace.root, expired, expired);
|
||||||
|
fs.writeFileSync(path.join(workspaces[1]!.root, 'preserve-me.txt'), 'uncommitted\n');
|
||||||
|
fs.utimesSync(workspaces[1]!.root, expired, expired);
|
||||||
|
const recent = new Date('2026-08-20T11:00:00.000Z');
|
||||||
|
fs.utimesSync(workspaces[4]!.root, recent, recent);
|
||||||
|
|
||||||
|
const result = await manager.cleanup({
|
||||||
|
protectedPaths: [workspaces[0]!.root],
|
||||||
|
retentionMs: 24 * 60 * 60 * 1_000,
|
||||||
|
keepNewest: 1,
|
||||||
|
});
|
||||||
|
expect(result.removed).toHaveLength(2);
|
||||||
|
expect(result.removed).toEqual(expect.arrayContaining([workspaces[2]!.root, workspaces[3]!.root]));
|
||||||
|
expect(result.skipped).toHaveLength(3);
|
||||||
|
expect(result.skipped).toEqual(
|
||||||
|
expect.arrayContaining([workspaces[0]!.root, workspaces[1]!.root, workspaces[4]!.root])
|
||||||
|
);
|
||||||
|
expect(fs.existsSync(workspaces[0]!.root)).toBe(true);
|
||||||
|
expect(fs.existsSync(workspaces[1]!.root)).toBe(true);
|
||||||
|
expect(fs.existsSync(workspaces[2]!.root)).toBe(false);
|
||||||
|
expect(fs.existsSync(workspaces[3]!.root)).toBe(false);
|
||||||
|
expect(fs.existsSync(workspaces[4]!.root)).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -70,6 +70,13 @@ pnpm --filter @sammo-ts/release-controller status
|
|||||||
pnpm --filter @sammo-ts/release-controller run-once
|
pnpm --filter @sammo-ts/release-controller run-once
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Daemon은 시작 시와 이후 24시간마다 commit worktree를 자동 정리합니다. 현재·이전
|
||||||
|
Gateway release와 활성 PM2 process가 사용하는 경로는 항상 보호하고, 나머지는
|
||||||
|
마지막 사용 후 24시간과 최신 2개 cache를 보장한 뒤 제거합니다. 변경이 있거나 Git
|
||||||
|
제거가 실패한 worktree는 raw directory 삭제로 우회하지 않고 다음 주기까지
|
||||||
|
보존합니다. Profile worktree는 Gateway orchestrator가 같은 정책으로 별도
|
||||||
|
관리합니다.
|
||||||
|
|
||||||
## Controller self-upgrade
|
## Controller self-upgrade
|
||||||
|
|
||||||
이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를
|
이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import {
|
|||||||
} from '@sammo-ts/gateway-api';
|
} from '@sammo-ts/gateway-api';
|
||||||
|
|
||||||
import { resolveReleaseControllerConfig } from './config.js';
|
import { resolveReleaseControllerConfig } from './config.js';
|
||||||
import { GatewayReleaseController } from './releaseController.js';
|
import { GatewayReleaseController, RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS } from './releaseController.js';
|
||||||
import { upgradeReleaseController } from './selfUpgrade.js';
|
import { upgradeReleaseController } from './selfUpgrade.js';
|
||||||
|
|
||||||
export * from './config.js';
|
export * from './config.js';
|
||||||
@@ -67,6 +67,7 @@ const main = async (): Promise<void> => {
|
|||||||
}
|
}
|
||||||
if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`);
|
if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`);
|
||||||
let stopping = false;
|
let stopping = false;
|
||||||
|
let nextWorkspaceCleanupAt = 0;
|
||||||
const stop = async (): Promise<void> => {
|
const stop = async (): Promise<void> => {
|
||||||
if (stopping) return;
|
if (stopping) return;
|
||||||
stopping = true;
|
stopping = true;
|
||||||
@@ -75,6 +76,18 @@ const main = async (): Promise<void> => {
|
|||||||
process.once('SIGINT', () => void stop());
|
process.once('SIGINT', () => void stop());
|
||||||
process.once('SIGTERM', () => void stop());
|
process.once('SIGTERM', () => void stop());
|
||||||
while (!stopping) {
|
while (!stopping) {
|
||||||
|
const now = Date.now();
|
||||||
|
if (now >= nextWorkspaceCleanupAt) {
|
||||||
|
nextWorkspaceCleanupAt = now + RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS;
|
||||||
|
try {
|
||||||
|
const result = await controller.cleanupStaleWorkspaces();
|
||||||
|
if (result.removed.length > 0) {
|
||||||
|
console.info(`[release-controller] removed ${result.removed.length} stale Gateway worktrees`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[release-controller] workspace cleanup failed', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
await controller.runOnce();
|
await controller.runOnce();
|
||||||
await new Promise<void>((resolve) => setTimeout(resolve, config.pollIntervalMs));
|
await new Promise<void>((resolve) => setTimeout(resolve, config.pollIntervalMs));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import {
|
|||||||
assertReleaseComponents,
|
assertReleaseComponents,
|
||||||
buildTurboReleaseCommand,
|
buildTurboReleaseCommand,
|
||||||
buildTurboReleaseTaskCommand,
|
buildTurboReleaseTaskCommand,
|
||||||
|
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
|
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
type BuildCommand,
|
type BuildCommand,
|
||||||
type BuildProgressEvent,
|
type BuildProgressEvent,
|
||||||
type BuildRunner,
|
type BuildRunner,
|
||||||
@@ -27,6 +29,16 @@ const HEARTBEAT_INTERVAL_MS = 60_000;
|
|||||||
const CANCELLATION_POLL_INTERVAL_MS = 500;
|
const CANCELLATION_POLL_INTERVAL_MS = 500;
|
||||||
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
|
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
|
||||||
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
|
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
|
||||||
|
export const RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||||
|
|
||||||
|
const isRuntimeProcessActive = (status: string): boolean =>
|
||||||
|
['online', 'launching', 'stopping'].includes(status.toLowerCase());
|
||||||
|
|
||||||
|
const isPathInside = (candidate: string | undefined, root: string): boolean => {
|
||||||
|
if (!candidate) return false;
|
||||||
|
const relative = path.relative(path.resolve(root), path.resolve(candidate));
|
||||||
|
return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
|
||||||
|
};
|
||||||
|
|
||||||
const managedPostgresPoolMax = (env: Record<string, string>, roleVariable: string, fallback: number): string =>
|
const managedPostgresPoolMax = (env: Record<string, string>, roleVariable: string, fallback: number): string =>
|
||||||
String(resolvePostgresPoolMax(env[roleVariable] ?? env.POSTGRES_POOL_MAX, fallback));
|
String(resolvePostgresPoolMax(env[roleVariable] ?? env.POSTGRES_POOL_MAX, fallback));
|
||||||
@@ -132,6 +144,33 @@ export class GatewayReleaseController {
|
|||||||
private readonly fetchImpl: typeof fetch = fetch
|
private readonly fetchImpl: typeof fetch = fetch
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
|
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
|
||||||
|
const [state, processes, workspaces] = await Promise.all([
|
||||||
|
this.repository.getState(),
|
||||||
|
this.processManager.list(),
|
||||||
|
this.workspaceManager.listManagedWorkspaces(),
|
||||||
|
]);
|
||||||
|
const protectedWorkspaces = new Set<string>();
|
||||||
|
if (state.activeWorkspace) protectedWorkspaces.add(path.resolve(state.activeWorkspace));
|
||||||
|
if (state.previousWorkspace) protectedWorkspaces.add(path.resolve(state.previousWorkspace));
|
||||||
|
const activeProcesses = processes.filter((process) => isRuntimeProcessActive(process.status));
|
||||||
|
for (const workspace of workspaces) {
|
||||||
|
if (
|
||||||
|
activeProcesses.some(
|
||||||
|
(process) =>
|
||||||
|
isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root)
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
protectedWorkspaces.add(workspace.root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return this.workspaceManager.cleanup({
|
||||||
|
protectedPaths: [...protectedWorkspaces],
|
||||||
|
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
|
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private sanitizeLogMessage(message: string): string {
|
private sanitizeLogMessage(message: string): string {
|
||||||
let sanitized = stripVTControlCharacters(message);
|
let sanitized = stripVTControlCharacters(message);
|
||||||
const sensitiveValues = new Set([
|
const sensitiveValues = new Set([
|
||||||
|
|||||||
@@ -2,14 +2,17 @@ import fs from 'node:fs/promises';
|
|||||||
import os from 'node:os';
|
import os from 'node:os';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
|
|
||||||
import type {
|
import {
|
||||||
BuildRunner,
|
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
GatewayReleaseOperationRecord,
|
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
GatewayReleaseRepository,
|
type BuildRunner,
|
||||||
GatewayReleaseStateRecord,
|
type GatewayReleaseOperationRecord,
|
||||||
GitWorkspaceManager,
|
type GatewayReleaseRepository,
|
||||||
ProcessDefinition,
|
type GatewayReleaseStateRecord,
|
||||||
ProcessManager,
|
type GitWorkspaceManager,
|
||||||
|
type ManagedWorkspaceCleanupOptions,
|
||||||
|
type ProcessDefinition,
|
||||||
|
type ProcessManager,
|
||||||
} from '@sammo-ts/gateway-api';
|
} from '@sammo-ts/gateway-api';
|
||||||
import { afterEach, describe, expect, it } from 'vitest';
|
import { afterEach, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
@@ -178,6 +181,68 @@ it('rejects Gateway definitions before switching processes when Redis connection
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('GatewayReleaseController', () => {
|
describe('GatewayReleaseController', () => {
|
||||||
|
it('protects active, rollback, and running-process worktrees while delegating bounded cleanup', async () => {
|
||||||
|
const active = '/srv/sammo/releases/active';
|
||||||
|
const previous = '/srv/sammo/releases/previous';
|
||||||
|
const controllerWorkspace = '/srv/sammo/releases/controller';
|
||||||
|
const stale = '/srv/sammo/releases/stale';
|
||||||
|
const managedPaths = [active, previous, controllerWorkspace, stale];
|
||||||
|
const cleanupCalls: ManagedWorkspaceCleanupOptions[] = [];
|
||||||
|
const harness = createRepository();
|
||||||
|
const workspaceManager = {
|
||||||
|
listManagedWorkspaces: async () =>
|
||||||
|
managedPaths.map((root) => ({
|
||||||
|
root,
|
||||||
|
commitSha: SHA,
|
||||||
|
lastUsedAt: new Date('2025-01-01T00:00:00.000Z'),
|
||||||
|
})),
|
||||||
|
cleanup: async (options: ManagedWorkspaceCleanupOptions) => {
|
||||||
|
cleanupCalls.push(options);
|
||||||
|
const protectedPaths = new Set(options.protectedPaths);
|
||||||
|
return {
|
||||||
|
removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)),
|
||||||
|
skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
} as unknown as GitWorkspaceManager;
|
||||||
|
const controller = new GatewayReleaseController(
|
||||||
|
{
|
||||||
|
...harness.repository,
|
||||||
|
getState: async () => ({
|
||||||
|
...state,
|
||||||
|
activeWorkspace: active,
|
||||||
|
previousCommitSha: SHA,
|
||||||
|
previousWorkspace: previous,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
workspaceManager,
|
||||||
|
{ run: async () => ({ ok: true, exitCode: 0, output: '' }) },
|
||||||
|
{
|
||||||
|
list: async () => [
|
||||||
|
{
|
||||||
|
name: 'sammo:release-controller',
|
||||||
|
status: 'online',
|
||||||
|
cwd: `${controllerWorkspace}/app/release-controller`,
|
||||||
|
},
|
||||||
|
{ name: 'old-build', status: 'stopped', cwd: `${stale}/app/gateway-api` },
|
||||||
|
],
|
||||||
|
start: async () => {},
|
||||||
|
stop: async () => {},
|
||||||
|
delete: async () => {},
|
||||||
|
},
|
||||||
|
config
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(controller.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||||
|
removed: [stale],
|
||||||
|
skipped: [active, previous, controllerWorkspace],
|
||||||
|
});
|
||||||
|
expect(cleanupCalls[0]).toMatchObject({
|
||||||
|
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||||
|
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
||||||
const workspace = await createReleaseWorkspace();
|
const workspace = await createReleaseWorkspace();
|
||||||
const harness = createRepository();
|
const harness = createRepository();
|
||||||
|
|||||||
@@ -69,6 +69,29 @@ profile 범위 권한과 별개인 전역 `admin.releases.manage` 권한이 필
|
|||||||
- migration 이후 이전 애플리케이션으로 돌아갈 때 schema 하위 호환성이
|
- migration 이후 이전 애플리케이션으로 돌아갈 때 schema 하위 호환성이
|
||||||
유지됩니다.
|
유지됩니다.
|
||||||
|
|
||||||
|
## Commit worktree 자동 정리
|
||||||
|
|
||||||
|
Profile orchestrator와 Gateway release-controller는 서로 다른 worktree root를
|
||||||
|
사용하지만 같은 보존 정책을 적용합니다. 각 daemon은 시작 시 한 번, 이후 24시간마다
|
||||||
|
자신이 소유한 commit worktree를 점검합니다.
|
||||||
|
|
||||||
|
- `GatewayProfile.buildWorkspace`, `RUNNING`/`QUEUED` profile 빌드 대상,
|
||||||
|
`GatewayReleaseState`의 active/previous workspace는 기간과 무관하게 보호합니다.
|
||||||
|
- 활성 PM2 process의 cwd 또는 script 아래에 있는 worktree도 보호합니다. 여기에는
|
||||||
|
self-upgrade된 release-controller worktree도 포함됩니다.
|
||||||
|
- 보호 대상이 아닌 worktree는 마지막 prepare 이후 최소 24시간을 유예하고, 그중
|
||||||
|
최신 2개는 재시도 cache로 더 남깁니다. 나머지는 Git worktree로 제거하고
|
||||||
|
`git worktree prune --expire now`로 사라진 metadata를 정리합니다.
|
||||||
|
- tracked 또는 untracked 변경이 있으면 자동 삭제하지 않습니다. Git 제거 실패를
|
||||||
|
raw directory 삭제로 우회하지 않으며 다음 주기까지 보존합니다.
|
||||||
|
- 정리는 commit checkout과 재생성 가능한 build artifact만 대상으로 합니다.
|
||||||
|
Gateway/profile PostgreSQL, Redis, image, runtime data volume에는 접근하지 않습니다.
|
||||||
|
|
||||||
|
따라서 하루 안에 매우 많은 commit을 연속 배포하면 유예 구간만큼 일시적으로 늘 수
|
||||||
|
있지만, active/rollback/current profile 경로 외의 장기 누적은 다음 정리 주기에
|
||||||
|
제거됩니다. Profile 관리자 API의 `admin.profiles.cleanupWorkspaces`는 같은 보호
|
||||||
|
규칙을 사용하므로 진행 중인 build/operation이 있으면 전체 정리를 보류합니다.
|
||||||
|
|
||||||
## Profile 배포
|
## Profile 배포
|
||||||
|
|
||||||
버전 업데이트 화면에서 profile의 branch 또는 commit을 선택합니다. Branch는 worker가
|
버전 업데이트 화면에서 profile의 branch 또는 commit을 선택합니다. Branch는 worker가
|
||||||
|
|||||||
Reference in New Issue
Block a user