fix(gateway): 프로필 초기화와 전체 배포를 직렬화

진행 중인 프로필 RESET 또는 DEPLOY와 Gateway 전체 릴리스가 서로의 실행자를 중단하지 않도록 공통 advisory lock과 교차 RUNNING 검사를 추가한다. PostgreSQL 통합 테스트로 동시 claim과 후속 실행을 검증한다.
This commit is contained in:
2026-08-17 14:41:18 +00:00
parent e8e062514c
commit fb6add83a5
4 changed files with 75 additions and 3 deletions
@@ -1,6 +1,10 @@
import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra'; import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra';
import type { GatewayOperationStatus, GatewaySourceMode } from './profileRepository.js'; import {
CONTROL_PLANE_OPERATION_CLAIM_LOCK,
type GatewayOperationStatus,
type GatewaySourceMode,
} from './profileRepository.js';
export type GatewayReleaseOperationType = 'DEPLOY' | 'ROLLBACK'; export type GatewayReleaseOperationType = 'DEPLOY' | 'ROLLBACK';
@@ -230,8 +234,17 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
async claimNextOperation(now, lease) { async claimNextOperation(now, lease) {
const row = await prisma.$transaction(async (tx) => { const row = await prisma.$transaction(async (tx) => {
await tx.$queryRaw<Array<{ lock_result: string }>>` await tx.$queryRaw<Array<{ lock_result: string }>>`
SELECT pg_advisory_xact_lock(hashtextextended('gateway_release_operation_claim', 0))::text AS lock_result SELECT pg_advisory_xact_lock(
hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0)
)::text AS lock_result
`; `;
const runningProfileOperation = await tx.gatewayOperation.findFirst({
where: { status: 'RUNNING' },
select: { id: true },
});
if (runningProfileOperation) {
return null;
}
const staleBefore = new Date(now.getTime() - lease.durationMs); const staleBefore = new Date(now.getTime() - lease.durationMs);
const running = await tx.gatewayReleaseOperation.findFirst({ const running = await tx.gatewayReleaseOperation.findFirst({
where: { status: 'RUNNING' }, where: { status: 'RUNNING' },
@@ -1,6 +1,8 @@
import { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus } from '@sammo-ts/common'; import { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus } from '@sammo-ts/common';
import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra'; import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra';
export const CONTROL_PLANE_OPERATION_CLAIM_LOCK = 'gateway_control_plane_operation_claim';
export { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus }; export { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus };
export const GATEWAY_BUILD_STATUSES = ['IDLE', 'QUEUED', 'RUNNING', 'FAILED', 'SUCCEEDED'] as const; export const GATEWAY_BUILD_STATUSES = ['IDLE', 'QUEUED', 'RUNNING', 'FAILED', 'SUCCEEDED'] as const;
@@ -625,8 +627,17 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
): Promise<GatewayOperationRecord | null> { ): Promise<GatewayOperationRecord | null> {
const row = await prisma.$transaction(async (tx) => { const row = await prisma.$transaction(async (tx) => {
await tx.$queryRaw<Array<{ lock_result: string }>>` await tx.$queryRaw<Array<{ lock_result: string }>>`
SELECT pg_advisory_xact_lock(hashtextextended('gateway_operation_claim', 0))::text AS lock_result SELECT pg_advisory_xact_lock(
hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0)
)::text AS lock_result
`; `;
const runningRelease = await tx.gatewayReleaseOperation.findFirst({
where: { status: 'RUNNING' },
select: { id: true },
});
if (runningRelease) {
return null;
}
const staleBefore = lease ? new Date(now.getTime() - lease.durationMs) : now; const staleBefore = lease ? new Date(now.getTime() - lease.durationMs) : now;
const running = await tx.gatewayOperation.findFirst({ const running = await tx.gatewayOperation.findFirst({
where: { status: 'RUNNING' }, where: { status: 'RUNNING' },
@@ -1,6 +1,7 @@
import { createGatewayPostgresConnector } from '@sammo-ts/infra'; import { createGatewayPostgresConnector } from '@sammo-ts/infra';
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest';
import { createGatewayReleaseRepository } from '../src/orchestrator/gatewayReleaseRepository.js';
import { createGatewayProfileRepository } from '../src/orchestrator/profileRepository.js'; import { createGatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
const databaseUrl = process.env.GATEWAY_OPERATION_DATABASE_URL; const databaseUrl = process.env.GATEWAY_OPERATION_DATABASE_URL;
@@ -11,6 +12,7 @@ const secondProfileName = 'lease-test-2:1010';
describeDatabase('gateway operation lease and profile serialization', () => { describeDatabase('gateway operation lease and profile serialization', () => {
const connector = createGatewayPostgresConnector({ url: databaseUrl ?? '' }); const connector = createGatewayPostgresConnector({ url: databaseUrl ?? '' });
const repository = createGatewayProfileRepository(connector.prisma); const repository = createGatewayProfileRepository(connector.prisma);
const releaseRepository = createGatewayReleaseRepository(connector.prisma);
beforeAll(async () => { beforeAll(async () => {
await connector.connect(); await connector.connect();
@@ -29,6 +31,8 @@ describeDatabase('gateway operation lease and profile serialization', () => {
}); });
afterEach(async () => { afterEach(async () => {
await connector.prisma.gatewayReleaseOperation.deleteMany();
await connector.prisma.gatewayReleaseState.deleteMany();
await connector.prisma.gatewayOperation.deleteMany({ await connector.prisma.gatewayOperation.deleteMany({
where: { profileName: { in: [profileName, secondProfileName] } }, where: { profileName: { in: [profileName, secondProfileName] } },
}); });
@@ -115,6 +119,45 @@ describeDatabase('gateway operation lease and profile serialization', () => {
).resolves.toMatchObject({ id: second.id, leaseOwner: 'worker-b' }); ).resolves.toMatchObject({ id: second.id, leaseOwner: 'worker-b' });
}); });
it('serializes profile and Gateway release claims under one control-plane lock', async () => {
const profileOperation = await repository.createOperation({
profileName,
type: 'RESET',
sourceMode: 'COMMIT',
sourceRef: 'a'.repeat(40),
requestedBy: 'profile-admin',
});
const releaseOperation = await releaseRepository.createOperation({
type: 'DEPLOY',
sourceMode: 'COMMIT',
sourceRef: 'b'.repeat(40),
requestedBy: 'release-admin',
});
const now = new Date('2030-01-01T00:00:00.000Z');
const [profileClaim, releaseClaim] = await Promise.all([
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 1_000 }),
releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 1_000 }),
]);
expect([profileClaim, releaseClaim].filter(Boolean)).toHaveLength(1);
if (profileClaim) {
expect(profileClaim.id).toBe(profileOperation.id);
expect(releaseClaim).toBeNull();
await repository.completeOperation(profileOperation.id, 'SUCCEEDED', { error: null }, 'profile-worker');
await expect(
releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 1_000 })
).resolves.toMatchObject({ id: releaseOperation.id });
return;
}
expect(releaseClaim?.id).toBe(releaseOperation.id);
await releaseRepository.completeOperation(releaseOperation.id, 'SUCCEEDED', { error: null }, 'release-worker');
await expect(
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 1_000 })
).resolves.toMatchObject({ id: profileOperation.id });
});
it('does not let a future queued operation suppress runtime reconciliation early', async () => { it('does not let a future queued operation suppress runtime reconciliation early', async () => {
const now = new Date('2030-01-01T00:00:00.000Z'); const now = new Date('2030-01-01T00:00:00.000Z');
await repository.createOperation({ await repository.createOperation({
+5
View File
@@ -67,6 +67,11 @@ profile 범위 권한과 별개인 전역 `admin.releases.manage` 권한이 필
버전 업데이트 화면에서 profile의 branch 또는 commit을 선택합니다. Branch는 worker가 버전 업데이트 화면에서 profile의 branch 또는 commit을 선택합니다. Branch는 worker가
작업을 claim할 때 commit으로 해석하며, commit 입력은 전체 SHA로 고정됩니다. 작업을 claim할 때 commit으로 해석하며, commit 입력은 전체 SHA로 고정됩니다.
같은 profile에는 `QUEUED` 또는 `RUNNING` 작업을 동시에 하나만 둘 수 있습니다. 같은 profile에는 `QUEUED` 또는 `RUNNING` 작업을 동시에 하나만 둘 수 있습니다.
Profile 작업과 Gateway 전체 릴리스 claim도 같은 PostgreSQL advisory lock으로
직렬화합니다. 따라서 profile `RESET`/`DEPLOY`가 실행 중이면 Gateway 릴리스는
queue에서 기다리고, Gateway 릴리스가 실행 중이면 새 profile 작업이 기다립니다.
Gateway process 전환이 진행 중인 profile migration·seed 실행자를 중단하지 않는
운영 계약입니다.
### DB 유지 배포 ### DB 유지 배포