merge: 릴리스 빌드 안전 중단 통합

This commit is contained in:
2026-08-20 02:28:28 +00:00
11 changed files with 554 additions and 48 deletions
+5 -2
View File
@@ -1453,7 +1453,7 @@ export const adminRouter = router({
if (!cancelled) { if (!cancelled) {
throw new TRPCError({ throw new TRPCError({
code: 'CONFLICT', code: 'CONFLICT',
message: 'Only queued operations can be cancelled.', message: 'Only queued operations or a DEPLOY that is still building can be cancelled.',
}); });
} }
return { ok: true }; return { ok: true };
@@ -1626,7 +1626,10 @@ export const adminRouter = router({
}), }),
cancel: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => { cancel: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => {
if (!(await ctx.releases.cancelOperation(input.id))) { if (!(await ctx.releases.cancelOperation(input.id))) {
throw new TRPCError({ code: 'CONFLICT', message: 'Only queued releases can be cancelled.' }); throw new TRPCError({
code: 'CONFLICT',
message: 'Only queued releases or a release that is still building can be cancelled.',
});
} }
return { ok: true }; return { ok: true };
}), }),
@@ -12,6 +12,7 @@ export interface BuildResult {
ok: boolean; ok: boolean;
exitCode: number | null; exitCode: number | null;
output: string; output: string;
aborted?: boolean;
} }
export type BuildProgressEvent = export type BuildProgressEvent =
@@ -21,8 +22,13 @@ export type BuildProgressEvent =
export type BuildProgressObserver = (event: BuildProgressEvent) => void | Promise<void>; export type BuildProgressObserver = (event: BuildProgressEvent) => void | Promise<void>;
export interface BuildRunOptions {
signal?: AbortSignal;
terminateGraceMs?: number;
}
export interface BuildRunner { export interface BuildRunner {
run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult>; run(commands: BuildCommand[], onProgress?: BuildProgressObserver, options?: BuildRunOptions): Promise<BuildResult>;
} }
export const MAX_BUILD_OUTPUT_CHARS = 64 * 1024; export const MAX_BUILD_OUTPUT_CHARS = 64 * 1024;
@@ -77,7 +83,28 @@ export const buildTurboReleaseTaskCommand = (
const appendOutputTail = (current: string, chunk: unknown): string => const appendOutputTail = (current: string, chunk: unknown): string =>
`${current}${String(chunk)}`.slice(-MAX_BUILD_OUTPUT_CHARS); `${current}${String(chunk)}`.slice(-MAX_BUILD_OUTPUT_CHARS);
const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver): Promise<BuildResult> => const terminateChildProcess = (pid: number | undefined, signal: NodeJS.Signals): void => {
if (!pid) return;
try {
if (process.platform !== 'win32') {
process.kill(-pid, signal);
return;
}
} catch {
// Fall back to the direct child below when the process group already exited.
}
try {
process.kill(pid, signal);
} catch {
// The child already exited.
}
};
const runCommand = (
command: BuildCommand,
onProgress?: BuildProgressObserver,
options?: BuildRunOptions
): Promise<BuildResult> =>
new Promise((resolve) => { new Promise((resolve) => {
let progressQueue = Promise.resolve(); let progressQueue = Promise.resolve();
const emit = (event: BuildProgressEvent) => { const emit = (event: BuildProgressEvent) => {
@@ -89,9 +116,25 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
cwd: command.cwd, cwd: command.cwd,
env: command.env, env: command.env,
stdio: ['ignore', 'pipe', 'pipe'], stdio: ['ignore', 'pipe', 'pipe'],
detached: process.platform !== 'win32',
}); });
let output = ''; let output = '';
let spawnFailed = false; let spawnFailed = false;
let aborted = false;
let killTimer: ReturnType<typeof setTimeout> | undefined;
const abort = () => {
if (aborted) return;
aborted = true;
output = appendOutputTail(output, '\nBuild cancelled by operator.');
terminateChildProcess(child.pid, 'SIGTERM');
killTimer = setTimeout(
() => terminateChildProcess(child.pid, 'SIGKILL'),
options?.terminateGraceMs ?? 5_000
);
killTimer.unref?.();
};
options?.signal?.addEventListener('abort', abort, { once: true });
if (options?.signal?.aborted) abort();
const lineBuffers = { stdout: '', stderr: '' }; const lineBuffers = { stdout: '', stderr: '' };
const emitOutput = (stream: 'stdout' | 'stderr', chunk: unknown, flush = false) => { const emitOutput = (stream: 'stdout' | 'stderr', chunk: unknown, flush = false) => {
if (flush && !lineBuffers[stream]) return; if (flush && !lineBuffers[stream]) return;
@@ -119,31 +162,47 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
output = appendOutputTail(output, error.message); output = appendOutputTail(output, error.message);
}); });
child.on('close', (code) => { child.on('close', (code) => {
options?.signal?.removeEventListener('abort', abort);
if (killTimer) clearTimeout(killTimer);
emitOutput('stdout', '', true); emitOutput('stdout', '', true);
emitOutput('stderr', '', true); emitOutput('stderr', '', true);
const exitCode = spawnFailed ? null : code; const exitCode = spawnFailed ? null : code;
emit({ type: 'COMMAND_END', command, exitCode }); emit({ type: 'COMMAND_END', command, exitCode });
void progressQueue.then(() => { void progressQueue.then(() => {
resolve({ resolve({
ok: !spawnFailed && code === 0, ok: !aborted && !spawnFailed && code === 0,
exitCode, exitCode,
output, output,
...(aborted ? { aborted: true } : {}),
}); });
}); });
}); });
}); });
export class PnpmBuildRunner implements BuildRunner { export class PnpmBuildRunner implements BuildRunner {
async run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult> { async run(
commands: BuildCommand[],
onProgress?: BuildProgressObserver,
options?: BuildRunOptions
): Promise<BuildResult> {
let mergedOutput = ''; let mergedOutput = '';
for (const command of commands) { for (const command of commands) {
const result = await runCommand(command, onProgress); if (options?.signal?.aborted) {
return {
ok: false,
exitCode: null,
output: appendOutputTail(mergedOutput, 'Build cancelled by operator.'),
aborted: true,
};
}
const result = await runCommand(command, onProgress, options);
mergedOutput = appendOutputTail(mergedOutput, result.output); mergedOutput = appendOutputTail(mergedOutput, result.output);
if (!result.ok) { if (!result.ok) {
return { return {
ok: false, ok: false,
exitCode: result.exitCode, exitCode: result.exitCode,
output: mergedOutput, output: mergedOutput,
...(result.aborted ? { aborted: true } : {}),
}; };
} }
} }
@@ -198,6 +198,7 @@ interface GatewayAdminActionResult {
const OPERATION_LEASE_DURATION_MS = 10 * 60_000; const OPERATION_LEASE_DURATION_MS = 10 * 60_000;
const OPERATION_HEARTBEAT_INTERVAL_MS = 60_000; const OPERATION_HEARTBEAT_INTERVAL_MS = 60_000;
const OPERATION_CANCELLATION_POLL_INTERVAL_MS = 500;
class OperationLeaseLostError extends Error {} class OperationLeaseLostError extends Error {}
@@ -633,6 +634,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
private buildInFlight = false; private buildInFlight = false;
private adminActionInFlight = false; private adminActionInFlight = false;
private operationInFlight = false; private operationInFlight = false;
private activeOperationAbortSignal?: AbortSignal;
private readonly resetInFlight = new Set<string>(); private readonly resetInFlight = new Set<string>();
private readonly operationLeaseOwner = randomUUID(); private readonly operationLeaseOwner = randomUUID();
private readonly inFlightTasks = new Set<Promise<unknown>>(); private readonly inFlightTasks = new Set<Promise<unknown>>();
@@ -975,6 +977,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
if (!operation) { if (!operation) {
return; return;
} }
const abortController = new AbortController();
this.activeOperationAbortSignal = abortController.signal;
const heartbeatTimer = this.repository.renewOperationLease const heartbeatTimer = this.repository.renewOperationLease
? setInterval(() => { ? setInterval(() => {
void this.repository void this.repository
@@ -984,17 +988,36 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
this.now(), this.now(),
OPERATION_LEASE_DURATION_MS OPERATION_LEASE_DURATION_MS
) )
.then((renewed) => {
if (!renewed) abortController.abort();
})
.catch((error) => { .catch((error) => {
console.error('[gateway-orchestrator] operation heartbeat failed', error); console.error('[gateway-orchestrator] operation heartbeat failed', error);
}); });
}, OPERATION_HEARTBEAT_INTERVAL_MS) }, OPERATION_HEARTBEAT_INTERVAL_MS)
: undefined; : undefined;
const cancellationTimer = setInterval(() => {
void this.repository
.getOperation(operation.id)
.then((current) => {
if (
!current ||
current.status !== 'RUNNING' ||
current.leaseOwner !== this.operationLeaseOwner
) {
abortController.abort();
}
})
.catch(() => undefined);
}, OPERATION_CANCELLATION_POLL_INTERVAL_MS);
try { try {
await this.handleOperation(operation); await this.handleOperation(operation);
} finally { } finally {
clearInterval(cancellationTimer);
if (heartbeatTimer) { if (heartbeatTimer) {
clearInterval(heartbeatTimer); clearInterval(heartbeatTimer);
} }
this.activeOperationAbortSignal = undefined;
} }
} finally { } finally {
this.operationInFlight = false; this.operationInFlight = false;
@@ -1442,9 +1465,11 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
), ),
]; ];
await this.appendOperationLog(operationId, 'build', `${profile.profileName} 구성 요소를 빌드합니다.`); await this.appendOperationLog(operationId, 'build', `${profile.profileName} 구성 요소를 빌드합니다.`);
const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build')); const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build'), {
await assertLease(); signal: this.activeOperationAbortSignal,
});
if (!result.ok) { if (!result.ok) {
await assertLease();
const detail = result.output.slice(-4000) || 'selected workspace build failed'; const detail = result.output.slice(-4000) || 'selected workspace build failed';
await updateClaimedProfile({ await updateClaimedProfile({
buildStatus: 'FAILED', buildStatus: 'FAILED',
@@ -1455,6 +1480,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
} }
await this.appendOperationLog(operationId, 'switch', '기존 profile process를 정지합니다.'); await this.appendOperationLog(operationId, 'switch', '기존 profile process를 정지합니다.');
await assertLease();
await this.stopProfile(profile, assertLease); await this.stopProfile(profile, assertLease);
oldRuntimeStopped = true; oldRuntimeStopped = true;
const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile); const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile);
@@ -2030,7 +2056,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
return { return {
result: await this.buildRunner.run( result: await this.buildRunner.run(
commands, commands,
operationId ? this.buildProgress(operationId, 'build') : undefined operationId ? this.buildProgress(operationId, 'build') : undefined,
{ signal: this.activeOperationAbortSignal }
), ),
workspace, workspace,
}; };
@@ -2052,7 +2079,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
): Promise<Awaited<ReturnType<BuildRunner['run']>>> { ): Promise<Awaited<ReturnType<BuildRunner['run']>>> {
return this.buildRunner.run( return this.buildRunner.run(
[buildProfileMigrationCommand(workspaceRoot, profileDatabaseUrl, this.processConfig.baseEnv)], [buildProfileMigrationCommand(workspaceRoot, profileDatabaseUrl, this.processConfig.baseEnv)],
onProgress onProgress,
{ signal: this.activeOperationAbortSignal }
); );
} }
@@ -2106,7 +2134,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}, },
}, },
], ],
onProgress onProgress,
{ signal: this.activeOperationAbortSignal }
); );
} finally { } finally {
await fs.rm(tempDirectory, { recursive: true, force: true }); await fs.rm(tempDirectory, { recursive: true, force: true });
@@ -208,13 +208,18 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
return rows.map(mapLog); return rows.map(mapLog);
}, },
async appendOperationLog(id, input) { async appendOperationLog(id, input) {
const row = await prisma.gatewayReleaseLog.create({ const row = await prisma.$transaction(async (tx) => {
data: { await tx.$queryRaw<Array<{ id: string }>>`
operationId: id, SELECT "id" FROM "gateway_release_operation" WHERE "id" = ${id} FOR UPDATE
level: input.level, `;
phase: input.phase.slice(0, 64), return tx.gatewayReleaseLog.create({
message: input.message.slice(0, 4_000), data: {
}, operationId: id,
level: input.level,
phase: input.phase.slice(0, 64),
message: input.message.slice(0, 4_000),
},
});
}); });
return mapLog(row); return mapLog(row);
}, },
@@ -367,11 +372,48 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
}); });
}, },
async cancelOperation(id) { async cancelOperation(id) {
const updated = await prisma.gatewayReleaseOperation.updateMany({ const count = await prisma.$transaction(async (tx) => {
where: { id, status: 'QUEUED' }, const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus }>>`
data: { status: 'CANCELLED', completedAt: new Date() }, SELECT "status"
FROM "gateway_release_operation"
WHERE "id" = ${id}
FOR UPDATE
`;
const operation = rows[0];
if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
if (operation.status === 'RUNNING') {
const latestLog = await tx.gatewayReleaseLog.findFirst({
where: { operationId: id },
orderBy: { id: 'desc' },
select: { phase: true },
});
if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
}
const updated = await tx.gatewayReleaseOperation.updateMany({
where: { id, status: operation.status },
data: {
status: 'CANCELLED',
completedAt: new Date(),
leaseOwner: null,
leaseUntil: null,
heartbeatAt: null,
},
});
if (updated.count !== 1) return 0;
await tx.gatewayReleaseLog.create({
data: {
operationId: id,
level: 'INFO',
phase: 'cancel',
message:
operation.status === 'RUNNING'
? '실행 중인 Gateway 빌드를 중단했습니다. 현재 active release는 유지됩니다.'
: '대기 중인 Gateway 릴리스를 취소했습니다.',
},
});
return 1;
}); });
return updated.count === 1; return count === 1;
}, },
async retryOperation(id, requestedBy) { async retryOperation(id, requestedBy) {
const row = await prisma.$transaction(async (tx) => { const row = await prisma.$transaction(async (tx) => {
@@ -586,13 +586,18 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
return rows.map(mapOperationLog); return rows.map(mapOperationLog);
}, },
async appendOperationLog(id, input) { async appendOperationLog(id, input) {
const row = await prisma.gatewayOperationLog.create({ const row = await prisma.$transaction(async (tx) => {
data: { await tx.$queryRaw<Array<{ id: string }>>`
operationId: id, SELECT "id" FROM "gateway_operation" WHERE "id" = ${id} FOR UPDATE
level: input.level, `;
phase: input.phase.slice(0, 64), return tx.gatewayOperationLog.create({
message: input.message.slice(0, 4_000), data: {
}, operationId: id,
level: input.level,
phase: input.phase.slice(0, 64),
message: input.message.slice(0, 4_000),
},
});
}); });
return mapOperationLog(row); return mapOperationLog(row);
}, },
@@ -805,21 +810,61 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
}, },
async cancelOperation(id: string): Promise<boolean> { async cancelOperation(id: string): Promise<boolean> {
const count = await prisma.$transaction(async (tx) => { const count = await prisma.$transaction(async (tx) => {
const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus; type: GatewayOperationType }>>`
SELECT "status", "type"
FROM "gateway_operation"
WHERE "id" = ${id}
FOR UPDATE
`;
const operation = rows[0];
if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
let runningBuildCancelled = false;
if (operation.status === 'RUNNING') {
if (operation.type !== 'DEPLOY') return 0;
const latestLog = await tx.gatewayOperationLog.findFirst({
where: { operationId: id },
orderBy: { id: 'desc' },
select: { phase: true },
});
if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
runningBuildCancelled = true;
}
const result = await tx.gatewayOperation.updateMany({ const result = await tx.gatewayOperation.updateMany({
where: { id, status: 'QUEUED' }, where: { id, status: operation.status },
data: { status: 'CANCELLED', completedAt: new Date() }, data: {
status: 'CANCELLED',
completedAt: new Date(),
leaseOwner: null,
leaseUntil: null,
heartbeatAt: null,
},
}); });
if (result.count === 1) { if (result.count !== 1) return 0;
await tx.gatewayOperationLog.create({ if (runningBuildCancelled) {
await tx.gatewayProfile.updateMany({
where: { operations: { some: { id } } },
data: { data: {
operationId: id, buildStatus: 'SUCCEEDED',
level: 'INFO', buildRequestedAt: null,
phase: 'cancel', buildStartedAt: null,
message: '대기 중인 작업이 취소되었습니다.', buildCompletedAt: null,
buildError: null,
}, },
}); });
} }
return result.count; await tx.gatewayOperationLog.create({
data: {
operationId: id,
level: 'INFO',
phase: 'cancel',
message: runningBuildCancelled
? '실행 중인 빌드를 중단했습니다. 기존 runtime과 DB는 유지됩니다.'
: '대기 중인 작업이 취소되었습니다.',
},
});
return 1;
}); });
return count === 1; return count === 1;
}, },
+24
View File
@@ -145,4 +145,28 @@ describe('PnpmBuildRunner', () => {
{ type: 'COMMAND_END' }, { type: 'COMMAND_END' },
]); ]);
}); });
it('terminates a running build process group when the operation is cancelled', async () => {
const runner = new PnpmBuildRunner();
const abortController = new AbortController();
const startedAt = Date.now();
const timer = setTimeout(() => abortController.abort(), 50);
const result = await runner.run(
[
{
command: process.execPath,
args: ['-e', "setInterval(() => process.stdout.write('still-running\\n'), 25);"],
cwd: process.cwd(),
},
],
undefined,
{ signal: abortController.signal, terminateGraceMs: 100 }
);
clearTimeout(timer);
expect(result).toMatchObject({ ok: false, aborted: true });
expect(result.output).toContain('Build cancelled by operator.');
expect(Date.now() - startedAt).toBeLessThan(2_000);
});
}); });
@@ -249,6 +249,72 @@ describeDatabase('gateway operation lease and profile serialization', () => {
).resolves.toMatchObject({ status: 'SUCCEEDED' }); ).resolves.toMatchObject({ status: 'SUCCEEDED' });
}); });
it('cancels only a running profile DEPLOY build and fences its worker lease', async () => {
const operation = await repository.createOperation({
profileName,
type: 'DEPLOY',
sourceMode: 'BRANCH',
sourceRef: 'main',
requestedBy: 'admin',
});
const now = new Date('2030-01-01T00:00:00.000Z');
await repository.claimNextOperation(now, { ownerId: 'worker-a', durationMs: 10_000 });
await repository.updateProfileForOperation?.(operation.id, 'worker-a', profileName, {
buildStatus: 'RUNNING',
buildError: 'temporary build output',
});
await repository.appendOperationLog(operation.id, {
level: 'INFO',
phase: 'build',
message: 'building profile',
});
await expect(repository.cancelOperation(operation.id)).resolves.toBe(true);
await expect(repository.getOperation(operation.id)).resolves.toMatchObject({
status: 'CANCELLED',
leaseOwner: undefined,
});
await expect(repository.getProfile(profileName)).resolves.toMatchObject({
buildStatus: 'SUCCEEDED',
buildError: undefined,
});
await expect(repository.renewOperationLease?.(operation.id, 'worker-a', now, 10_000)).resolves.toBe(false);
});
it('cancels a running Gateway build but rejects cancellation after migration starts', async () => {
const buildOperation = await releaseRepository.createOperation({
type: 'DEPLOY',
sourceMode: 'BRANCH',
sourceRef: 'main',
requestedBy: 'admin',
});
const now = new Date('2030-01-01T00:00:00.000Z');
await releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 });
await releaseRepository.appendOperationLog(buildOperation.id, {
level: 'INFO',
phase: 'build',
message: 'building Gateway',
});
await expect(releaseRepository.cancelOperation(buildOperation.id)).resolves.toBe(true);
await expect(
releaseRepository.renewOperationLease(buildOperation.id, 'release-worker', now, 10_000)
).resolves.toBe(false);
const migrationOperation = await releaseRepository.createOperation({
type: 'DEPLOY',
sourceMode: 'BRANCH',
sourceRef: 'main',
requestedBy: 'admin',
});
await releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 });
await releaseRepository.appendOperationLog(migrationOperation.id, {
level: 'INFO',
phase: 'migration',
message: 'migrating Gateway',
});
await expect(releaseRepository.cancelOperation(migrationOperation.id)).resolves.toBe(false);
});
it('pins retry to the first resolved commit and preserves its install generation', async () => { it('pins retry to the first resolved commit and preserves its install generation', async () => {
const operation = await repository.createOperation({ const operation = await repository.createOperation({
profileName, profileName,
@@ -457,6 +457,33 @@ const installFixture = async (page: Page, state: FixtureState) => {
state.operations = [operation, ...state.operations]; state.operations = [operation, ...state.operations];
return response(operation); return response(operation);
} }
if (name === 'admin.operations.cancel') {
const id = JSON.stringify(body).match(/[0-9a-f]{8}-[0-9a-f-]{27,}/u)?.[0];
state.operations = state.operations.map((operation) =>
operation.id === id ? { ...operation, status: 'CANCELLED' as const } : operation
);
return response({ ok: true });
}
if (name === 'admin.releases.cancel') {
const id = JSON.stringify(body).match(/[0-9a-f]{8}-[0-9a-f-]{27,}/u)?.[0];
state.gatewayOperations = state.gatewayOperations.map((operation) =>
operation.id === id ? { ...operation, status: 'CANCELLED' as const } : operation
);
return response({ ok: true });
}
if (name === 'admin.releases.retry') {
const previous = state.gatewayOperations[0];
if (!previous) throw new Error('Release retry fixture is missing');
const retried = {
...previous,
id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
status: 'QUEUED' as const,
sourceMode: 'COMMIT' as const,
sourceRef: previous.resolvedCommitSha ?? previous.sourceRef,
};
state.gatewayOperations = [retried, ...state.gatewayOperations];
return response(retried);
}
throw new Error(`Unhandled tRPC operation: ${name}`); throw new Error(`Unhandled tRPC operation: ${name}`);
}); });
await route.fulfill({ await route.fulfill({
@@ -1203,6 +1230,77 @@ test('scenario-only operator resets the server-selected version without Git or G
expect(JSON.stringify(request?.body)).not.toContain('"sourceRef"'); expect(JSON.stringify(request?.body)).not.toContain('"sourceRef"');
}); });
test('stops a running profile build while keeping the existing runtime available', async ({ page }) => {
const operation: Operation = {
id: '12121212-1212-4212-8212-121212121212',
profileName: 'che:default',
type: 'DEPLOY',
status: 'RUNNING',
sourceMode: 'COMMIT',
sourceRef: '0123456789abcdef0123456789abcdef01234567',
payload: {},
requestedBy: 'admin',
createdAt: '2026-08-20T01:00:00.000Z',
updatedAt: '2026-08-20T01:00:01.000Z',
};
const state: FixtureState = {
operations: [operation],
gatewayOperations: [],
runtimeRunning: true,
requestBodies: [],
profileLogsEmpty: true,
};
await installFixture(page, state);
page.on('dialog', (dialog) => dialog.accept());
await page.goto('admin/servers/che%3Adefault/version');
await expect(page.getByTestId('profile-build-recovery-guide')).toContainText('기존 profile runtime과 게임 DB는');
await page.getByRole('button', { name: '빌드 중단' }).click();
await expect(page.getByText('프로필 빌드를 중단했습니다.').first()).toBeVisible();
await expect(page.getByTestId('operations-table').getByText('CANCELLED', { exact: true })).toBeVisible();
expect(state.requestBodies.some((entry) => entry.operation === 'admin.operations.cancel')).toBe(true);
});
test('stops and retries a running Gateway build from the release GUI', async ({ page }) => {
const state: FixtureState = {
operations: [],
gatewayOperations: [
{
id: '34343434-3434-4434-8434-343434343434',
type: 'DEPLOY',
status: 'RUNNING',
sourceMode: 'COMMIT',
sourceRef: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
resolvedCommitSha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
payload: {},
requestedBy: 'admin',
createdAt: '2026-08-20T01:00:00.000Z',
updatedAt: '2026-08-20T01:00:01.000Z',
},
],
runtimeRunning: true,
requestBodies: [],
gatewayLogsEmpty: true,
};
await installFixture(page, state);
page.on('dialog', (dialog) => dialog.accept());
await page.goto('admin/releases');
await expect(page.getByTestId('gateway-build-recovery-guide')).toContainText(
'migration 또는 process 전환이 시작된 뒤에는'
);
await page.getByRole('button', { name: '빌드 중단' }).click();
await expect(page.getByText('Gateway 빌드를 중단했습니다.').first()).toBeVisible();
await expect(page.getByTestId('gateway-release-table').getByText('CANCELLED', { exact: true })).toBeVisible();
await page.getByRole('button', { name: '재시도' }).click();
await expect(page.getByText('Gateway 릴리스 재시도 작업을 등록했습니다.').first()).toBeVisible();
await expect(page.getByTestId('gateway-release-table').getByText('QUEUED', { exact: true })).toBeVisible();
expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.cancel')).toBe(true);
expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.retry')).toBe(true);
});
test('controls gateway deployment and rollback through the external controller queue', async ({ page }, testInfo) => { test('controls gateway deployment and rollback through the external controller queue', async ({ page }, testInfo) => {
const state: FixtureState = { const state: FixtureState = {
operations: [], operations: [],
@@ -546,6 +546,37 @@ const requestGatewayRollback = async () => {
} }
}; };
const cancelGatewayRelease = async (operation: GatewayReleaseOperation) => {
clearStatus();
const prompt =
operation.status === 'RUNNING'
? '실행 중인 Gateway 빌드를 중단하시겠습니까? process 전환 또는 migration이 시작된 뒤에는 중단할 수 없습니다.'
: '대기 중인 Gateway 릴리스를 취소하시겠습니까?';
if (!window.confirm(prompt)) return;
try {
await adminClient.releases.cancel.mutate({ id: operation.id });
selectedGatewayOperationId.value = operation.id;
message.value =
operation.status === 'RUNNING' ? 'Gateway 빌드를 중단했습니다.' : 'Gateway 릴리스를 취소했습니다.';
await loadState(true);
} catch (error) {
errorMessage.value = error instanceof Error ? error.message : 'Gateway 릴리스 중단에 실패했습니다.';
}
};
const retryGatewayRelease = async (operation: GatewayReleaseOperation) => {
clearStatus();
if (!window.confirm('같은 고정 커밋으로 Gateway 릴리스를 다시 실행하시겠습니까?')) return;
try {
const retried = await adminClient.releases.retry.mutate({ id: operation.id });
selectedGatewayOperationId.value = retried.id;
message.value = 'Gateway 릴리스 재시도 작업을 등록했습니다.';
await loadState(true);
} catch (error) {
errorMessage.value = error instanceof Error ? error.message : 'Gateway 릴리스 재시도에 실패했습니다.';
}
};
const loadScenarios = async () => { const loadScenarios = async () => {
clearStatus(); clearStatus();
if (form.sourceMode !== 'CURRENT' && !form.sourceRef.trim()) { if (form.sourceMode !== 'CURRENT' && !form.sourceRef.trim()) {
@@ -688,13 +719,17 @@ const requestGameCancellation = async () => {
const cancelOperation = async (operation: Operation) => { const cancelOperation = async (operation: Operation) => {
clearStatus(); clearStatus();
if (!window.confirm('대기 중인 작업을 취소하시겠습니까?')) { const prompt =
operation.status === 'RUNNING'
? '실행 중인 프로필 빌드를 중단하시겠습니까? 기존 runtime과 게임 DB는 유지됩니다.'
: '대기 중인 작업을 취소하시겠습니까?';
if (!window.confirm(prompt)) {
return; return;
} }
try { try {
await adminClient.operations.cancel.mutate({ id: operation.id }); await adminClient.operations.cancel.mutate({ id: operation.id });
selectedProfileOperationId.value = operation.id; selectedProfileOperationId.value = operation.id;
message.value = '작업을 취소했습니다.'; message.value = operation.status === 'RUNNING' ? '프로필 빌드를 중단했습니다.' : '작업을 취소했습니다.';
await loadState(true); await loadState(true);
} catch (error) { } catch (error) {
errorMessage.value = error instanceof Error ? error.message : '작업 취소에 실패했습니다.'; errorMessage.value = error instanceof Error ? error.message : '작업 취소에 실패했습니다.';
@@ -1191,6 +1226,14 @@ onBeforeUnmount(() => {
class="rounded-lg border border-violet-800/70 bg-zinc-900 p-5 space-y-4" class="rounded-lg border border-violet-800/70 bg-zinc-900 p-5 space-y-4"
data-testid="gateway-release-panel" data-testid="gateway-release-panel"
> >
<div
class="rounded border border-amber-800/80 bg-amber-950/30 px-4 py-3 text-sm text-amber-100"
data-testid="gateway-build-recovery-guide"
>
<strong>빌드가 멈춘 경우:</strong> 로그의 마지막 단계가 build일 때만
<strong>빌드 중단</strong> 누르고 CANCELLED를 확인한 <strong>재시도</strong>하세요.
migration 또는 process 전환이 시작된 뒤에는 DB와 runtime 보호를 위해 중단할 없습니다.
</div>
<div class="flex flex-col gap-2 md:flex-row md:items-center md:justify-between"> <div class="flex flex-col gap-2 md:flex-row md:items-center md:justify-between">
<div> <div>
<h3 class="text-lg font-semibold">Gateway 릴리스</h3> <h3 class="text-lg font-semibold">Gateway 릴리스</h3>
@@ -1388,6 +1431,24 @@ onBeforeUnmount(() => {
: '오류 보기' : '오류 보기'
}} }}
</button> </button>
<button
v-if="operation.status === 'QUEUED' || operation.status === 'RUNNING'"
type="button"
class="rounded border border-red-800 px-2 py-1 text-red-300 hover:bg-red-950 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-red-400"
@click="cancelGatewayRelease(operation)"
>
{{ operation.status === 'RUNNING' ? '빌드 중단' : '취소' }}
</button>
<button
v-else-if="
operation.status === 'FAILED' || operation.status === 'CANCELLED'
"
type="button"
class="rounded border border-amber-700 px-2 py-1 text-amber-300 hover:bg-amber-950 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-400"
@click="retryGatewayRelease(operation)"
>
재시도
</button>
</div> </div>
</td> </td>
</tr> </tr>
@@ -1476,6 +1537,14 @@ onBeforeUnmount(() => {
</section> </section>
<section v-if="mode !== 'gateway'" class="rounded-lg border border-zinc-800 bg-zinc-900 p-5"> <section v-if="mode !== 'gateway'" class="rounded-lg border border-zinc-800 bg-zinc-900 p-5">
<div
class="mb-4 rounded border border-amber-800/80 bg-amber-950/30 px-4 py-3 text-sm text-amber-100"
data-testid="profile-build-recovery-guide"
>
<strong>DB 보존 업데이트 빌드가 멈춘 경우:</strong> <strong>빌드 중단</strong>을 누르고
CANCELLED를 확인한 뒤 <strong>재시도</strong>하세요. 기존 profile runtime과 게임 DB는
유지됩니다. RESET·migration·process 전환 단계는 이 화면에서 강제 중단하지 않습니다.
</div>
<div class="mb-4 flex items-center justify-between"> <div class="mb-4 flex items-center justify-between">
<h3 class="text-lg font-semibold">작업 이력</h3> <h3 class="text-lg font-semibold">작업 이력</h3>
<span class="text-xs text-zinc-500">3초마다 상태 갱신</span> <span class="text-xs text-zinc-500">3초마다 상태 갱신</span>
@@ -1565,11 +1634,14 @@ onBeforeUnmount(() => {
로그 로그
</button> </button>
<button <button
v-if="operation.status === 'QUEUED'" v-if="
operation.status === 'QUEUED' ||
(operation.status === 'RUNNING' && operation.type === 'DEPLOY')
"
class="rounded border border-red-800 px-2 py-1 text-xs text-red-300 hover:bg-red-950" class="rounded border border-red-800 px-2 py-1 text-xs text-red-300 hover:bg-red-950"
@click="cancelOperation(operation)" @click="cancelOperation(operation)"
> >
취소 {{ operation.status === 'RUNNING' ? '빌드 중단' : '취소' }}
</button> </button>
<button <button
v-else-if=" v-else-if="
+28
View File
@@ -87,6 +87,34 @@ pnpm --filter @sammo-ts/release-controller self-upgrade COMMIT <full-sha>
Database migration은 일반적으로 되돌리지 않습니다. 이전 애플리케이션으로 Database migration은 일반적으로 되돌리지 않습니다. 이전 애플리케이션으로
rollback하려면 새 schema와의 하위 호환성을 릴리스 전에 확인해 주세요. rollback하려면 새 schema와의 하위 호환성을 릴리스 전에 확인해 주세요.
## 멈춘 빌드 복구
운영 container나 PM2 process를 먼저 종료하지 마세요. 관리자 화면의
`Gateway 릴리스` 또는 profile `버전 업데이트` 작업 이력에서 로그의 마지막 단계와
작업 상태를 확인합니다.
1. `RUNNING`이고 마지막 단계가 `claim`, `resolve`, `workspace`, `build` 중 하나이면
`빌드 중단`을 누릅니다.
2. 작업이 `CANCELLED`가 되고 로그에 빌드 종료가 기록될 때까지 기다립니다. Controller와
orchestrator는 해당 process group에 SIGTERM을 보내고 제한 시간 뒤 SIGKILL로
정리하며, 기존 active Gateway/profile runtime과 profile DB는 유지합니다.
3. 같은 행의 `재시도`를 누르면 최초 작업이 고정한 commit으로 새 작업을 등록합니다.
branch의 최신 commit을 새로 선택하려면 새 배포 작업을 등록합니다.
마지막 단계가 `migration`, `switch`, `readiness`이면 중단 요청을 거부합니다. 이 구간에서
container restart, PM2 delete 또는 DB row 직접 변경으로 lease를 무효화하지 말고 작업 로그와
controller/orchestrator 상태를 조사합니다. Profile 상태가 `PAUSED`이면 runtime 장애가 아니라
turn gate가 닫힌 상태이므로 배포 완료 후 서버 관리 화면에서 `턴 재개`를 사용합니다.
호스트에서는 stack wrapper로 container와 로그를 읽기 전용 확인합니다. 운영 stack의 가까운
README에 정의된 경로에서 다음 순서로 확인하며, `down --volumes``RESET`은 빌드 복구에
사용하지 않습니다.
```sh
./scripts/stack.sh ps
./scripts/stack.sh logs runtime
```
`release-manifest.json``controllerProtocol`이 올라간 릴리스는 controller를 `release-manifest.json``controllerProtocol`이 올라간 릴리스는 controller를
먼저 self-upgrade해야 합니다. Protocol 2는 `GatewayReleaseLog` 진행 로그 저장을 먼저 self-upgrade해야 합니다. Protocol 2는 `GatewayReleaseLog` 진행 로그 저장을
요구합니다. 구형 controller로 새 Gateway만 배포하면 관리자 화면과 controller의 요구합니다. 구형 controller로 새 Gateway만 배포하면 관리자 화면과 controller의
@@ -24,6 +24,7 @@ import type { ReleaseControllerConfig } from './config.js';
const LEASE_DURATION_MS = 10 * 60_000; const LEASE_DURATION_MS = 10 * 60_000;
const HEARTBEAT_INTERVAL_MS = 60_000; const HEARTBEAT_INTERVAL_MS = 60_000;
const CANCELLATION_POLL_INTERVAL_MS = 500;
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const; const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu; const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
@@ -187,9 +188,25 @@ export class GatewayReleaseController {
}); });
if (!operation) return null; if (!operation) return null;
await this.appendLog(operation.id, 'claim', `릴리스 작업을 시작합니다. 시도 ${operation.attempts}회차.`); await this.appendLog(operation.id, 'claim', `릴리스 작업을 시작합니다. 시도 ${operation.attempts}회차.`);
const abortController = new AbortController();
const heartbeat = setInterval(() => { const heartbeat = setInterval(() => {
void this.repository.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS); void this.repository
.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS)
.then((renewed) => {
if (!renewed) abortController.abort();
})
.catch(() => undefined);
}, HEARTBEAT_INTERVAL_MS); }, HEARTBEAT_INTERVAL_MS);
const cancellationWatcher = setInterval(() => {
void this.repository
.getOperation(operation.id)
.then((current) => {
if (!current || current.status !== 'RUNNING' || current.leaseOwner !== this.ownerId) {
abortController.abort();
}
})
.catch(() => undefined);
}, CANCELLATION_POLL_INTERVAL_MS);
let resolvedCommitSha: string | undefined; let resolvedCommitSha: string | undefined;
try { try {
await this.appendLog(operation.id, 'resolve', '현재 Gateway 릴리스 상태를 확인합니다.'); await this.appendLog(operation.id, 'resolve', '현재 Gateway 릴리스 상태를 확인합니다.');
@@ -208,7 +225,7 @@ export class GatewayReleaseController {
throw new Error('Gateway release lease was lost while pinning the commit.'); throw new Error('Gateway release lease was lost while pinning the commit.');
} }
await this.appendLog(operation.id, 'resolve', `대상 커밋을 ${resolvedCommitSha}로 고정했습니다.`); await this.appendLog(operation.id, 'resolve', `대상 커밋을 ${resolvedCommitSha}로 고정했습니다.`);
await this.deploy(operation, deploymentState, resolvedCommitSha); await this.deploy(operation, deploymentState, resolvedCommitSha, abortController.signal);
await this.appendLog(operation.id, 'complete', 'Gateway 릴리스가 완료되었습니다.'); await this.appendLog(operation.id, 'complete', 'Gateway 릴리스가 완료되었습니다.');
return await this.repository.completeOperation( return await this.repository.completeOperation(
operation.id, operation.id,
@@ -217,6 +234,17 @@ export class GatewayReleaseController {
this.ownerId this.ownerId
); );
} catch (error) { } catch (error) {
const current = await this.repository.getOperation(operation.id);
if (
!current ||
current.status !== 'RUNNING' ||
(abortController.signal.aborted && current.leaseOwner !== this.ownerId)
) {
if (current?.status === 'CANCELLED') {
await this.appendLog(operation.id, 'cancel', '실행 중인 Gateway 빌드가 종료되었습니다.');
}
return current;
}
const detail = error instanceof Error ? error.message : String(error); const detail = error instanceof Error ? error.message : String(error);
await this.appendLog(operation.id, 'failed', detail, 'ERROR'); await this.appendLog(operation.id, 'failed', detail, 'ERROR');
await this.repository.recordStateError(detail); await this.repository.recordStateError(detail);
@@ -228,13 +256,21 @@ export class GatewayReleaseController {
); );
} finally { } finally {
clearInterval(heartbeat); clearInterval(heartbeat);
clearInterval(cancellationWatcher);
}
}
private async assertOperationLease(operationId: string): Promise<void> {
if (!(await this.repository.renewOperationLease(operationId, this.ownerId, this.now(), LEASE_DURATION_MS))) {
throw new Error(`Gateway release lease lost: ${operationId}`);
} }
} }
private async deploy( private async deploy(
operation: GatewayReleaseOperationRecord, operation: GatewayReleaseOperationRecord,
state: GatewayReleaseStateRecord, state: GatewayReleaseStateRecord,
commitSha: string commitSha: string,
signal: AbortSignal
): Promise<void> { ): Promise<void> {
await this.appendLog(operation.id, 'workspace', `커밋 ${commitSha}의 worktree를 준비합니다.`); await this.appendLog(operation.id, 'workspace', `커밋 ${commitSha}의 worktree를 준비합니다.`);
const workspace = await this.workspaceManager.prepare(commitSha); const workspace = await this.workspaceManager.prepare(commitSha);
@@ -244,13 +280,16 @@ export class GatewayReleaseController {
await this.appendLog(operation.id, 'build', 'Gateway 구성 요소를 빌드합니다.'); await this.appendLog(operation.id, 'build', 'Gateway 구성 요소를 빌드합니다.');
const build = await this.buildRunner.run( const build = await this.buildRunner.run(
buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config), buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config),
this.buildProgress(operation.id, 'build') this.buildProgress(operation.id, 'build'),
{ signal }
); );
if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`); if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`);
await this.appendLog(operation.id, 'migration', 'Gateway database migration을 적용합니다.'); await this.appendLog(operation.id, 'migration', 'Gateway database migration을 적용합니다.');
await this.assertOperationLease(operation.id);
const migration = await this.buildRunner.run( const migration = await this.buildRunner.run(
[buildGatewayMigrationCommand(workspace.root, this.config)], [buildGatewayMigrationCommand(workspace.root, this.config)],
this.buildProgress(operation.id, 'migration') this.buildProgress(operation.id, 'migration'),
{ signal }
); );
if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`); if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`);
await this.appendLog(operation.id, 'migration', 'Gateway database migration이 완료되었습니다.'); await this.appendLog(operation.id, 'migration', 'Gateway database migration이 완료되었습니다.');
@@ -259,6 +298,7 @@ export class GatewayReleaseController {
? buildGatewayProcessDefinitions(state.activeWorkspace, this.config) ? buildGatewayProcessDefinitions(state.activeWorkspace, this.config)
: []; : [];
await this.appendLog(operation.id, 'switch', '기존 Gateway process를 정지합니다.'); await this.appendLog(operation.id, 'switch', '기존 Gateway process를 정지합니다.');
await this.assertOperationLease(operation.id);
await this.stopManagedProcesses(operation.id); await this.stopManagedProcesses(operation.id);
try { try {
await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config), operation.id); await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config), operation.id);