fix: 실행 중인 릴리스 빌드를 안전하게 중단

Gateway와 프로필 DEPLOY의 빌드 단계만 취소하고 lease와 phase 전환을 직렬화한다. 관리자 화면에 중단·재시도 절차와 회귀 검증을 추가한다.
This commit is contained in:
2026-08-20 02:27:52 +00:00
parent 1bacdaef5f
commit 466f030889
11 changed files with 554 additions and 48 deletions
+5 -2
View File
@@ -1453,7 +1453,7 @@ export const adminRouter = router({
if (!cancelled) {
throw new TRPCError({
code: 'CONFLICT',
message: 'Only queued operations can be cancelled.',
message: 'Only queued operations or a DEPLOY that is still building can be cancelled.',
});
}
return { ok: true };
@@ -1626,7 +1626,10 @@ export const adminRouter = router({
}),
cancel: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => {
if (!(await ctx.releases.cancelOperation(input.id))) {
throw new TRPCError({ code: 'CONFLICT', message: 'Only queued releases can be cancelled.' });
throw new TRPCError({
code: 'CONFLICT',
message: 'Only queued releases or a release that is still building can be cancelled.',
});
}
return { ok: true };
}),
@@ -12,6 +12,7 @@ export interface BuildResult {
ok: boolean;
exitCode: number | null;
output: string;
aborted?: boolean;
}
export type BuildProgressEvent =
@@ -21,8 +22,13 @@ export type BuildProgressEvent =
export type BuildProgressObserver = (event: BuildProgressEvent) => void | Promise<void>;
export interface BuildRunOptions {
signal?: AbortSignal;
terminateGraceMs?: number;
}
export interface BuildRunner {
run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult>;
run(commands: BuildCommand[], onProgress?: BuildProgressObserver, options?: BuildRunOptions): Promise<BuildResult>;
}
export const MAX_BUILD_OUTPUT_CHARS = 64 * 1024;
@@ -77,7 +83,28 @@ export const buildTurboReleaseTaskCommand = (
const appendOutputTail = (current: string, chunk: unknown): string =>
`${current}${String(chunk)}`.slice(-MAX_BUILD_OUTPUT_CHARS);
const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver): Promise<BuildResult> =>
const terminateChildProcess = (pid: number | undefined, signal: NodeJS.Signals): void => {
if (!pid) return;
try {
if (process.platform !== 'win32') {
process.kill(-pid, signal);
return;
}
} catch {
// Fall back to the direct child below when the process group already exited.
}
try {
process.kill(pid, signal);
} catch {
// The child already exited.
}
};
const runCommand = (
command: BuildCommand,
onProgress?: BuildProgressObserver,
options?: BuildRunOptions
): Promise<BuildResult> =>
new Promise((resolve) => {
let progressQueue = Promise.resolve();
const emit = (event: BuildProgressEvent) => {
@@ -89,9 +116,25 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
cwd: command.cwd,
env: command.env,
stdio: ['ignore', 'pipe', 'pipe'],
detached: process.platform !== 'win32',
});
let output = '';
let spawnFailed = false;
let aborted = false;
let killTimer: ReturnType<typeof setTimeout> | undefined;
const abort = () => {
if (aborted) return;
aborted = true;
output = appendOutputTail(output, '\nBuild cancelled by operator.');
terminateChildProcess(child.pid, 'SIGTERM');
killTimer = setTimeout(
() => terminateChildProcess(child.pid, 'SIGKILL'),
options?.terminateGraceMs ?? 5_000
);
killTimer.unref?.();
};
options?.signal?.addEventListener('abort', abort, { once: true });
if (options?.signal?.aborted) abort();
const lineBuffers = { stdout: '', stderr: '' };
const emitOutput = (stream: 'stdout' | 'stderr', chunk: unknown, flush = false) => {
if (flush && !lineBuffers[stream]) return;
@@ -119,31 +162,47 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
output = appendOutputTail(output, error.message);
});
child.on('close', (code) => {
options?.signal?.removeEventListener('abort', abort);
if (killTimer) clearTimeout(killTimer);
emitOutput('stdout', '', true);
emitOutput('stderr', '', true);
const exitCode = spawnFailed ? null : code;
emit({ type: 'COMMAND_END', command, exitCode });
void progressQueue.then(() => {
resolve({
ok: !spawnFailed && code === 0,
ok: !aborted && !spawnFailed && code === 0,
exitCode,
output,
...(aborted ? { aborted: true } : {}),
});
});
});
});
export class PnpmBuildRunner implements BuildRunner {
async run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult> {
async run(
commands: BuildCommand[],
onProgress?: BuildProgressObserver,
options?: BuildRunOptions
): Promise<BuildResult> {
let mergedOutput = '';
for (const command of commands) {
const result = await runCommand(command, onProgress);
if (options?.signal?.aborted) {
return {
ok: false,
exitCode: null,
output: appendOutputTail(mergedOutput, 'Build cancelled by operator.'),
aborted: true,
};
}
const result = await runCommand(command, onProgress, options);
mergedOutput = appendOutputTail(mergedOutput, result.output);
if (!result.ok) {
return {
ok: false,
exitCode: result.exitCode,
output: mergedOutput,
...(result.aborted ? { aborted: true } : {}),
};
}
}
@@ -198,6 +198,7 @@ interface GatewayAdminActionResult {
const OPERATION_LEASE_DURATION_MS = 10 * 60_000;
const OPERATION_HEARTBEAT_INTERVAL_MS = 60_000;
const OPERATION_CANCELLATION_POLL_INTERVAL_MS = 500;
class OperationLeaseLostError extends Error {}
@@ -633,6 +634,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
private buildInFlight = false;
private adminActionInFlight = false;
private operationInFlight = false;
private activeOperationAbortSignal?: AbortSignal;
private readonly resetInFlight = new Set<string>();
private readonly operationLeaseOwner = randomUUID();
private readonly inFlightTasks = new Set<Promise<unknown>>();
@@ -975,6 +977,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
if (!operation) {
return;
}
const abortController = new AbortController();
this.activeOperationAbortSignal = abortController.signal;
const heartbeatTimer = this.repository.renewOperationLease
? setInterval(() => {
void this.repository
@@ -984,17 +988,36 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
this.now(),
OPERATION_LEASE_DURATION_MS
)
.then((renewed) => {
if (!renewed) abortController.abort();
})
.catch((error) => {
console.error('[gateway-orchestrator] operation heartbeat failed', error);
});
}, OPERATION_HEARTBEAT_INTERVAL_MS)
: undefined;
const cancellationTimer = setInterval(() => {
void this.repository
.getOperation(operation.id)
.then((current) => {
if (
!current ||
current.status !== 'RUNNING' ||
current.leaseOwner !== this.operationLeaseOwner
) {
abortController.abort();
}
})
.catch(() => undefined);
}, OPERATION_CANCELLATION_POLL_INTERVAL_MS);
try {
await this.handleOperation(operation);
} finally {
clearInterval(cancellationTimer);
if (heartbeatTimer) {
clearInterval(heartbeatTimer);
}
this.activeOperationAbortSignal = undefined;
}
} finally {
this.operationInFlight = false;
@@ -1442,9 +1465,11 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
),
];
await this.appendOperationLog(operationId, 'build', `${profile.profileName} 구성 요소를 빌드합니다.`);
const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build'));
await assertLease();
const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build'), {
signal: this.activeOperationAbortSignal,
});
if (!result.ok) {
await assertLease();
const detail = result.output.slice(-4000) || 'selected workspace build failed';
await updateClaimedProfile({
buildStatus: 'FAILED',
@@ -1455,6 +1480,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}
await this.appendOperationLog(operationId, 'switch', '기존 profile process를 정지합니다.');
await assertLease();
await this.stopProfile(profile, assertLease);
oldRuntimeStopped = true;
const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile);
@@ -2030,7 +2056,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
return {
result: await this.buildRunner.run(
commands,
operationId ? this.buildProgress(operationId, 'build') : undefined
operationId ? this.buildProgress(operationId, 'build') : undefined,
{ signal: this.activeOperationAbortSignal }
),
workspace,
};
@@ -2052,7 +2079,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
): Promise<Awaited<ReturnType<BuildRunner['run']>>> {
return this.buildRunner.run(
[buildProfileMigrationCommand(workspaceRoot, profileDatabaseUrl, this.processConfig.baseEnv)],
onProgress
onProgress,
{ signal: this.activeOperationAbortSignal }
);
}
@@ -2106,7 +2134,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
},
},
],
onProgress
onProgress,
{ signal: this.activeOperationAbortSignal }
);
} finally {
await fs.rm(tempDirectory, { recursive: true, force: true });
@@ -208,13 +208,18 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
return rows.map(mapLog);
},
async appendOperationLog(id, input) {
const row = await prisma.gatewayReleaseLog.create({
data: {
operationId: id,
level: input.level,
phase: input.phase.slice(0, 64),
message: input.message.slice(0, 4_000),
},
const row = await prisma.$transaction(async (tx) => {
await tx.$queryRaw<Array<{ id: string }>>`
SELECT "id" FROM "gateway_release_operation" WHERE "id" = ${id} FOR UPDATE
`;
return tx.gatewayReleaseLog.create({
data: {
operationId: id,
level: input.level,
phase: input.phase.slice(0, 64),
message: input.message.slice(0, 4_000),
},
});
});
return mapLog(row);
},
@@ -367,11 +372,48 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
});
},
async cancelOperation(id) {
const updated = await prisma.gatewayReleaseOperation.updateMany({
where: { id, status: 'QUEUED' },
data: { status: 'CANCELLED', completedAt: new Date() },
const count = await prisma.$transaction(async (tx) => {
const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus }>>`
SELECT "status"
FROM "gateway_release_operation"
WHERE "id" = ${id}
FOR UPDATE
`;
const operation = rows[0];
if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
if (operation.status === 'RUNNING') {
const latestLog = await tx.gatewayReleaseLog.findFirst({
where: { operationId: id },
orderBy: { id: 'desc' },
select: { phase: true },
});
if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
}
const updated = await tx.gatewayReleaseOperation.updateMany({
where: { id, status: operation.status },
data: {
status: 'CANCELLED',
completedAt: new Date(),
leaseOwner: null,
leaseUntil: null,
heartbeatAt: null,
},
});
if (updated.count !== 1) return 0;
await tx.gatewayReleaseLog.create({
data: {
operationId: id,
level: 'INFO',
phase: 'cancel',
message:
operation.status === 'RUNNING'
? '실행 중인 Gateway 빌드를 중단했습니다. 현재 active release는 유지됩니다.'
: '대기 중인 Gateway 릴리스를 취소했습니다.',
},
});
return 1;
});
return updated.count === 1;
return count === 1;
},
async retryOperation(id, requestedBy) {
const row = await prisma.$transaction(async (tx) => {
@@ -586,13 +586,18 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
return rows.map(mapOperationLog);
},
async appendOperationLog(id, input) {
const row = await prisma.gatewayOperationLog.create({
data: {
operationId: id,
level: input.level,
phase: input.phase.slice(0, 64),
message: input.message.slice(0, 4_000),
},
const row = await prisma.$transaction(async (tx) => {
await tx.$queryRaw<Array<{ id: string }>>`
SELECT "id" FROM "gateway_operation" WHERE "id" = ${id} FOR UPDATE
`;
return tx.gatewayOperationLog.create({
data: {
operationId: id,
level: input.level,
phase: input.phase.slice(0, 64),
message: input.message.slice(0, 4_000),
},
});
});
return mapOperationLog(row);
},
@@ -805,21 +810,61 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
},
async cancelOperation(id: string): Promise<boolean> {
const count = await prisma.$transaction(async (tx) => {
const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus; type: GatewayOperationType }>>`
SELECT "status", "type"
FROM "gateway_operation"
WHERE "id" = ${id}
FOR UPDATE
`;
const operation = rows[0];
if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
let runningBuildCancelled = false;
if (operation.status === 'RUNNING') {
if (operation.type !== 'DEPLOY') return 0;
const latestLog = await tx.gatewayOperationLog.findFirst({
where: { operationId: id },
orderBy: { id: 'desc' },
select: { phase: true },
});
if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
runningBuildCancelled = true;
}
const result = await tx.gatewayOperation.updateMany({
where: { id, status: 'QUEUED' },
data: { status: 'CANCELLED', completedAt: new Date() },
where: { id, status: operation.status },
data: {
status: 'CANCELLED',
completedAt: new Date(),
leaseOwner: null,
leaseUntil: null,
heartbeatAt: null,
},
});
if (result.count === 1) {
await tx.gatewayOperationLog.create({
if (result.count !== 1) return 0;
if (runningBuildCancelled) {
await tx.gatewayProfile.updateMany({
where: { operations: { some: { id } } },
data: {
operationId: id,
level: 'INFO',
phase: 'cancel',
message: '대기 중인 작업이 취소되었습니다.',
buildStatus: 'SUCCEEDED',
buildRequestedAt: null,
buildStartedAt: null,
buildCompletedAt: null,
buildError: null,
},
});
}
return result.count;
await tx.gatewayOperationLog.create({
data: {
operationId: id,
level: 'INFO',
phase: 'cancel',
message: runningBuildCancelled
? '실행 중인 빌드를 중단했습니다. 기존 runtime과 DB는 유지됩니다.'
: '대기 중인 작업이 취소되었습니다.',
},
});
return 1;
});
return count === 1;
},
+24
View File
@@ -145,4 +145,28 @@ describe('PnpmBuildRunner', () => {
{ type: 'COMMAND_END' },
]);
});
it('terminates a running build process group when the operation is cancelled', async () => {
const runner = new PnpmBuildRunner();
const abortController = new AbortController();
const startedAt = Date.now();
const timer = setTimeout(() => abortController.abort(), 50);
const result = await runner.run(
[
{
command: process.execPath,
args: ['-e', "setInterval(() => process.stdout.write('still-running\\n'), 25);"],
cwd: process.cwd(),
},
],
undefined,
{ signal: abortController.signal, terminateGraceMs: 100 }
);
clearTimeout(timer);
expect(result).toMatchObject({ ok: false, aborted: true });
expect(result.output).toContain('Build cancelled by operator.');
expect(Date.now() - startedAt).toBeLessThan(2_000);
});
});
@@ -249,6 +249,72 @@ describeDatabase('gateway operation lease and profile serialization', () => {
).resolves.toMatchObject({ status: 'SUCCEEDED' });
});
it('cancels only a running profile DEPLOY build and fences its worker lease', async () => {
const operation = await repository.createOperation({
profileName,
type: 'DEPLOY',
sourceMode: 'BRANCH',
sourceRef: 'main',
requestedBy: 'admin',
});
const now = new Date('2030-01-01T00:00:00.000Z');
await repository.claimNextOperation(now, { ownerId: 'worker-a', durationMs: 10_000 });
await repository.updateProfileForOperation?.(operation.id, 'worker-a', profileName, {
buildStatus: 'RUNNING',
buildError: 'temporary build output',
});
await repository.appendOperationLog(operation.id, {
level: 'INFO',
phase: 'build',
message: 'building profile',
});
await expect(repository.cancelOperation(operation.id)).resolves.toBe(true);
await expect(repository.getOperation(operation.id)).resolves.toMatchObject({
status: 'CANCELLED',
leaseOwner: undefined,
});
await expect(repository.getProfile(profileName)).resolves.toMatchObject({
buildStatus: 'SUCCEEDED',
buildError: undefined,
});
await expect(repository.renewOperationLease?.(operation.id, 'worker-a', now, 10_000)).resolves.toBe(false);
});
it('cancels a running Gateway build but rejects cancellation after migration starts', async () => {
const buildOperation = await releaseRepository.createOperation({
type: 'DEPLOY',
sourceMode: 'BRANCH',
sourceRef: 'main',
requestedBy: 'admin',
});
const now = new Date('2030-01-01T00:00:00.000Z');
await releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 });
await releaseRepository.appendOperationLog(buildOperation.id, {
level: 'INFO',
phase: 'build',
message: 'building Gateway',
});
await expect(releaseRepository.cancelOperation(buildOperation.id)).resolves.toBe(true);
await expect(
releaseRepository.renewOperationLease(buildOperation.id, 'release-worker', now, 10_000)
).resolves.toBe(false);
const migrationOperation = await releaseRepository.createOperation({
type: 'DEPLOY',
sourceMode: 'BRANCH',
sourceRef: 'main',
requestedBy: 'admin',
});
await releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 });
await releaseRepository.appendOperationLog(migrationOperation.id, {
level: 'INFO',
phase: 'migration',
message: 'migrating Gateway',
});
await expect(releaseRepository.cancelOperation(migrationOperation.id)).resolves.toBe(false);
});
it('pins retry to the first resolved commit and preserves its install generation', async () => {
const operation = await repository.createOperation({
profileName,
@@ -457,6 +457,33 @@ const installFixture = async (page: Page, state: FixtureState) => {
state.operations = [operation, ...state.operations];
return response(operation);
}
if (name === 'admin.operations.cancel') {
const id = JSON.stringify(body).match(/[0-9a-f]{8}-[0-9a-f-]{27,}/u)?.[0];
state.operations = state.operations.map((operation) =>
operation.id === id ? { ...operation, status: 'CANCELLED' as const } : operation
);
return response({ ok: true });
}
if (name === 'admin.releases.cancel') {
const id = JSON.stringify(body).match(/[0-9a-f]{8}-[0-9a-f-]{27,}/u)?.[0];
state.gatewayOperations = state.gatewayOperations.map((operation) =>
operation.id === id ? { ...operation, status: 'CANCELLED' as const } : operation
);
return response({ ok: true });
}
if (name === 'admin.releases.retry') {
const previous = state.gatewayOperations[0];
if (!previous) throw new Error('Release retry fixture is missing');
const retried = {
...previous,
id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
status: 'QUEUED' as const,
sourceMode: 'COMMIT' as const,
sourceRef: previous.resolvedCommitSha ?? previous.sourceRef,
};
state.gatewayOperations = [retried, ...state.gatewayOperations];
return response(retried);
}
throw new Error(`Unhandled tRPC operation: ${name}`);
});
await route.fulfill({
@@ -1203,6 +1230,77 @@ test('scenario-only operator resets the server-selected version without Git or G
expect(JSON.stringify(request?.body)).not.toContain('"sourceRef"');
});
test('stops a running profile build while keeping the existing runtime available', async ({ page }) => {
const operation: Operation = {
id: '12121212-1212-4212-8212-121212121212',
profileName: 'che:default',
type: 'DEPLOY',
status: 'RUNNING',
sourceMode: 'COMMIT',
sourceRef: '0123456789abcdef0123456789abcdef01234567',
payload: {},
requestedBy: 'admin',
createdAt: '2026-08-20T01:00:00.000Z',
updatedAt: '2026-08-20T01:00:01.000Z',
};
const state: FixtureState = {
operations: [operation],
gatewayOperations: [],
runtimeRunning: true,
requestBodies: [],
profileLogsEmpty: true,
};
await installFixture(page, state);
page.on('dialog', (dialog) => dialog.accept());
await page.goto('admin/servers/che%3Adefault/version');
await expect(page.getByTestId('profile-build-recovery-guide')).toContainText('기존 profile runtime과 게임 DB는');
await page.getByRole('button', { name: '빌드 중단' }).click();
await expect(page.getByText('프로필 빌드를 중단했습니다.').first()).toBeVisible();
await expect(page.getByTestId('operations-table').getByText('CANCELLED', { exact: true })).toBeVisible();
expect(state.requestBodies.some((entry) => entry.operation === 'admin.operations.cancel')).toBe(true);
});
test('stops and retries a running Gateway build from the release GUI', async ({ page }) => {
const state: FixtureState = {
operations: [],
gatewayOperations: [
{
id: '34343434-3434-4434-8434-343434343434',
type: 'DEPLOY',
status: 'RUNNING',
sourceMode: 'COMMIT',
sourceRef: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
resolvedCommitSha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
payload: {},
requestedBy: 'admin',
createdAt: '2026-08-20T01:00:00.000Z',
updatedAt: '2026-08-20T01:00:01.000Z',
},
],
runtimeRunning: true,
requestBodies: [],
gatewayLogsEmpty: true,
};
await installFixture(page, state);
page.on('dialog', (dialog) => dialog.accept());
await page.goto('admin/releases');
await expect(page.getByTestId('gateway-build-recovery-guide')).toContainText(
'migration 또는 process 전환이 시작된 뒤에는'
);
await page.getByRole('button', { name: '빌드 중단' }).click();
await expect(page.getByText('Gateway 빌드를 중단했습니다.').first()).toBeVisible();
await expect(page.getByTestId('gateway-release-table').getByText('CANCELLED', { exact: true })).toBeVisible();
await page.getByRole('button', { name: '재시도' }).click();
await expect(page.getByText('Gateway 릴리스 재시도 작업을 등록했습니다.').first()).toBeVisible();
await expect(page.getByTestId('gateway-release-table').getByText('QUEUED', { exact: true })).toBeVisible();
expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.cancel')).toBe(true);
expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.retry')).toBe(true);
});
test('controls gateway deployment and rollback through the external controller queue', async ({ page }, testInfo) => {
const state: FixtureState = {
operations: [],
@@ -546,6 +546,37 @@ const requestGatewayRollback = async () => {
}
};
const cancelGatewayRelease = async (operation: GatewayReleaseOperation) => {
clearStatus();
const prompt =
operation.status === 'RUNNING'
? '실행 중인 Gateway 빌드를 중단하시겠습니까? process 전환 또는 migration이 시작된 뒤에는 중단할 수 없습니다.'
: '대기 중인 Gateway 릴리스를 취소하시겠습니까?';
if (!window.confirm(prompt)) return;
try {
await adminClient.releases.cancel.mutate({ id: operation.id });
selectedGatewayOperationId.value = operation.id;
message.value =
operation.status === 'RUNNING' ? 'Gateway 빌드를 중단했습니다.' : 'Gateway 릴리스를 취소했습니다.';
await loadState(true);
} catch (error) {
errorMessage.value = error instanceof Error ? error.message : 'Gateway 릴리스 중단에 실패했습니다.';
}
};
const retryGatewayRelease = async (operation: GatewayReleaseOperation) => {
clearStatus();
if (!window.confirm('같은 고정 커밋으로 Gateway 릴리스를 다시 실행하시겠습니까?')) return;
try {
const retried = await adminClient.releases.retry.mutate({ id: operation.id });
selectedGatewayOperationId.value = retried.id;
message.value = 'Gateway 릴리스 재시도 작업을 등록했습니다.';
await loadState(true);
} catch (error) {
errorMessage.value = error instanceof Error ? error.message : 'Gateway 릴리스 재시도에 실패했습니다.';
}
};
const loadScenarios = async () => {
clearStatus();
if (form.sourceMode !== 'CURRENT' && !form.sourceRef.trim()) {
@@ -688,13 +719,17 @@ const requestGameCancellation = async () => {
const cancelOperation = async (operation: Operation) => {
clearStatus();
if (!window.confirm('대기 중인 작업을 취소하시겠습니까?')) {
const prompt =
operation.status === 'RUNNING'
? '실행 중인 프로필 빌드를 중단하시겠습니까? 기존 runtime과 게임 DB는 유지됩니다.'
: '대기 중인 작업을 취소하시겠습니까?';
if (!window.confirm(prompt)) {
return;
}
try {
await adminClient.operations.cancel.mutate({ id: operation.id });
selectedProfileOperationId.value = operation.id;
message.value = '작업을 취소했습니다.';
message.value = operation.status === 'RUNNING' ? '프로필 빌드를 중단했습니다.' : '작업을 취소했습니다.';
await loadState(true);
} catch (error) {
errorMessage.value = error instanceof Error ? error.message : '작업 취소에 실패했습니다.';
@@ -1191,6 +1226,14 @@ onBeforeUnmount(() => {
class="rounded-lg border border-violet-800/70 bg-zinc-900 p-5 space-y-4"
data-testid="gateway-release-panel"
>
<div
class="rounded border border-amber-800/80 bg-amber-950/30 px-4 py-3 text-sm text-amber-100"
data-testid="gateway-build-recovery-guide"
>
<strong>빌드가 멈춘 경우:</strong> 로그의 마지막 단계가 build일 때만
<strong>빌드 중단</strong> 누르고 CANCELLED를 확인한 <strong>재시도</strong>하세요.
migration 또는 process 전환이 시작된 뒤에는 DB와 runtime 보호를 위해 중단할 없습니다.
</div>
<div class="flex flex-col gap-2 md:flex-row md:items-center md:justify-between">
<div>
<h3 class="text-lg font-semibold">Gateway 릴리스</h3>
@@ -1388,6 +1431,24 @@ onBeforeUnmount(() => {
: '오류 보기'
}}
</button>
<button
v-if="operation.status === 'QUEUED' || operation.status === 'RUNNING'"
type="button"
class="rounded border border-red-800 px-2 py-1 text-red-300 hover:bg-red-950 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-red-400"
@click="cancelGatewayRelease(operation)"
>
{{ operation.status === 'RUNNING' ? '빌드 중단' : '취소' }}
</button>
<button
v-else-if="
operation.status === 'FAILED' || operation.status === 'CANCELLED'
"
type="button"
class="rounded border border-amber-700 px-2 py-1 text-amber-300 hover:bg-amber-950 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-400"
@click="retryGatewayRelease(operation)"
>
재시도
</button>
</div>
</td>
</tr>
@@ -1476,6 +1537,14 @@ onBeforeUnmount(() => {
</section>
<section v-if="mode !== 'gateway'" class="rounded-lg border border-zinc-800 bg-zinc-900 p-5">
<div
class="mb-4 rounded border border-amber-800/80 bg-amber-950/30 px-4 py-3 text-sm text-amber-100"
data-testid="profile-build-recovery-guide"
>
<strong>DB 보존 업데이트 빌드가 멈춘 경우:</strong> <strong>빌드 중단</strong>을 누르고
CANCELLED를 확인한 뒤 <strong>재시도</strong>하세요. 기존 profile runtime과 게임 DB는
유지됩니다. RESET·migration·process 전환 단계는 이 화면에서 강제 중단하지 않습니다.
</div>
<div class="mb-4 flex items-center justify-between">
<h3 class="text-lg font-semibold">작업 이력</h3>
<span class="text-xs text-zinc-500">3초마다 상태 갱신</span>
@@ -1565,11 +1634,14 @@ onBeforeUnmount(() => {
로그
</button>
<button
v-if="operation.status === 'QUEUED'"
v-if="
operation.status === 'QUEUED' ||
(operation.status === 'RUNNING' && operation.type === 'DEPLOY')
"
class="rounded border border-red-800 px-2 py-1 text-xs text-red-300 hover:bg-red-950"
@click="cancelOperation(operation)"
>
취소
{{ operation.status === 'RUNNING' ? '빌드 중단' : '취소' }}
</button>
<button
v-else-if="
+28
View File
@@ -87,6 +87,34 @@ pnpm --filter @sammo-ts/release-controller self-upgrade COMMIT <full-sha>
Database migration은 일반적으로 되돌리지 않습니다. 이전 애플리케이션으로
rollback하려면 새 schema와의 하위 호환성을 릴리스 전에 확인해 주세요.
## 멈춘 빌드 복구
운영 container나 PM2 process를 먼저 종료하지 마세요. 관리자 화면의
`Gateway 릴리스` 또는 profile `버전 업데이트` 작업 이력에서 로그의 마지막 단계와
작업 상태를 확인합니다.
1. `RUNNING`이고 마지막 단계가 `claim`, `resolve`, `workspace`, `build` 중 하나이면
`빌드 중단`을 누릅니다.
2. 작업이 `CANCELLED`가 되고 로그에 빌드 종료가 기록될 때까지 기다립니다. Controller와
orchestrator는 해당 process group에 SIGTERM을 보내고 제한 시간 뒤 SIGKILL로
정리하며, 기존 active Gateway/profile runtime과 profile DB는 유지합니다.
3. 같은 행의 `재시도`를 누르면 최초 작업이 고정한 commit으로 새 작업을 등록합니다.
branch의 최신 commit을 새로 선택하려면 새 배포 작업을 등록합니다.
마지막 단계가 `migration`, `switch`, `readiness`이면 중단 요청을 거부합니다. 이 구간에서
container restart, PM2 delete 또는 DB row 직접 변경으로 lease를 무효화하지 말고 작업 로그와
controller/orchestrator 상태를 조사합니다. Profile 상태가 `PAUSED`이면 runtime 장애가 아니라
turn gate가 닫힌 상태이므로 배포 완료 후 서버 관리 화면에서 `턴 재개`를 사용합니다.
호스트에서는 stack wrapper로 container와 로그를 읽기 전용 확인합니다. 운영 stack의 가까운
README에 정의된 경로에서 다음 순서로 확인하며, `down --volumes``RESET`은 빌드 복구에
사용하지 않습니다.
```sh
./scripts/stack.sh ps
./scripts/stack.sh logs runtime
```
`release-manifest.json``controllerProtocol`이 올라간 릴리스는 controller를
먼저 self-upgrade해야 합니다. Protocol 2는 `GatewayReleaseLog` 진행 로그 저장을
요구합니다. 구형 controller로 새 Gateway만 배포하면 관리자 화면과 controller의
@@ -24,6 +24,7 @@ import type { ReleaseControllerConfig } from './config.js';
const LEASE_DURATION_MS = 10 * 60_000;
const HEARTBEAT_INTERVAL_MS = 60_000;
const CANCELLATION_POLL_INTERVAL_MS = 500;
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
@@ -187,9 +188,25 @@ export class GatewayReleaseController {
});
if (!operation) return null;
await this.appendLog(operation.id, 'claim', `릴리스 작업을 시작합니다. 시도 ${operation.attempts}회차.`);
const abortController = new AbortController();
const heartbeat = setInterval(() => {
void this.repository.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS);
void this.repository
.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS)
.then((renewed) => {
if (!renewed) abortController.abort();
})
.catch(() => undefined);
}, HEARTBEAT_INTERVAL_MS);
const cancellationWatcher = setInterval(() => {
void this.repository
.getOperation(operation.id)
.then((current) => {
if (!current || current.status !== 'RUNNING' || current.leaseOwner !== this.ownerId) {
abortController.abort();
}
})
.catch(() => undefined);
}, CANCELLATION_POLL_INTERVAL_MS);
let resolvedCommitSha: string | undefined;
try {
await this.appendLog(operation.id, 'resolve', '현재 Gateway 릴리스 상태를 확인합니다.');
@@ -208,7 +225,7 @@ export class GatewayReleaseController {
throw new Error('Gateway release lease was lost while pinning the commit.');
}
await this.appendLog(operation.id, 'resolve', `대상 커밋을 ${resolvedCommitSha}로 고정했습니다.`);
await this.deploy(operation, deploymentState, resolvedCommitSha);
await this.deploy(operation, deploymentState, resolvedCommitSha, abortController.signal);
await this.appendLog(operation.id, 'complete', 'Gateway 릴리스가 완료되었습니다.');
return await this.repository.completeOperation(
operation.id,
@@ -217,6 +234,17 @@ export class GatewayReleaseController {
this.ownerId
);
} catch (error) {
const current = await this.repository.getOperation(operation.id);
if (
!current ||
current.status !== 'RUNNING' ||
(abortController.signal.aborted && current.leaseOwner !== this.ownerId)
) {
if (current?.status === 'CANCELLED') {
await this.appendLog(operation.id, 'cancel', '실행 중인 Gateway 빌드가 종료되었습니다.');
}
return current;
}
const detail = error instanceof Error ? error.message : String(error);
await this.appendLog(operation.id, 'failed', detail, 'ERROR');
await this.repository.recordStateError(detail);
@@ -228,13 +256,21 @@ export class GatewayReleaseController {
);
} finally {
clearInterval(heartbeat);
clearInterval(cancellationWatcher);
}
}
private async assertOperationLease(operationId: string): Promise<void> {
if (!(await this.repository.renewOperationLease(operationId, this.ownerId, this.now(), LEASE_DURATION_MS))) {
throw new Error(`Gateway release lease lost: ${operationId}`);
}
}
private async deploy(
operation: GatewayReleaseOperationRecord,
state: GatewayReleaseStateRecord,
commitSha: string
commitSha: string,
signal: AbortSignal
): Promise<void> {
await this.appendLog(operation.id, 'workspace', `커밋 ${commitSha}의 worktree를 준비합니다.`);
const workspace = await this.workspaceManager.prepare(commitSha);
@@ -244,13 +280,16 @@ export class GatewayReleaseController {
await this.appendLog(operation.id, 'build', 'Gateway 구성 요소를 빌드합니다.');
const build = await this.buildRunner.run(
buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config),
this.buildProgress(operation.id, 'build')
this.buildProgress(operation.id, 'build'),
{ signal }
);
if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`);
await this.appendLog(operation.id, 'migration', 'Gateway database migration을 적용합니다.');
await this.assertOperationLease(operation.id);
const migration = await this.buildRunner.run(
[buildGatewayMigrationCommand(workspace.root, this.config)],
this.buildProgress(operation.id, 'migration')
this.buildProgress(operation.id, 'migration'),
{ signal }
);
if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`);
await this.appendLog(operation.id, 'migration', 'Gateway database migration이 완료되었습니다.');
@@ -259,6 +298,7 @@ export class GatewayReleaseController {
? buildGatewayProcessDefinitions(state.activeWorkspace, this.config)
: [];
await this.appendLog(operation.id, 'switch', '기존 Gateway process를 정지합니다.');
await this.assertOperationLease(operation.id);
await this.stopManagedProcesses(operation.id);
try {
await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config), operation.id);