From fb6add83a503889e77d5864a95691a9d1d38f0bb Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 17 Aug 2026 14:41:18 +0000 Subject: [PATCH] =?UTF-8?q?fix(gateway):=20=ED=94=84=EB=A1=9C=ED=95=84=20?= =?UTF-8?q?=EC=B4=88=EA=B8=B0=ED=99=94=EC=99=80=20=EC=A0=84=EC=B2=B4=20?= =?UTF-8?q?=EB=B0=B0=ED=8F=AC=EB=A5=BC=20=EC=A7=81=EB=A0=AC=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 진행 중인 프로필 RESET 또는 DEPLOY와 Gateway 전체 릴리스가 서로의 실행자를 중단하지 않도록 공통 advisory lock과 교차 RUNNING 검사를 추가한다. PostgreSQL 통합 테스트로 동시 claim과 후속 실행을 검증한다. --- .../orchestrator/gatewayReleaseRepository.ts | 17 +++++++- .../src/orchestrator/profileRepository.ts | 13 +++++- .../profileOperationLease.integration.test.ts | 43 +++++++++++++++++++ docs/release-operations.md | 5 +++ 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts b/app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts index a7421222..9b000742 100644 --- a/app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts +++ b/app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts @@ -1,6 +1,10 @@ import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra'; -import type { GatewayOperationStatus, GatewaySourceMode } from './profileRepository.js'; +import { + CONTROL_PLANE_OPERATION_CLAIM_LOCK, + type GatewayOperationStatus, + type GatewaySourceMode, +} from './profileRepository.js'; export type GatewayReleaseOperationType = 'DEPLOY' | 'ROLLBACK'; @@ -230,8 +234,17 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat async claimNextOperation(now, lease) { const row = await prisma.$transaction(async (tx) => { await tx.$queryRaw>` - SELECT pg_advisory_xact_lock(hashtextextended('gateway_release_operation_claim', 0))::text AS lock_result + SELECT pg_advisory_xact_lock( + hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0) + )::text AS lock_result `; + const runningProfileOperation = await tx.gatewayOperation.findFirst({ + where: { status: 'RUNNING' }, + select: { id: true }, + }); + if (runningProfileOperation) { + return null; + } const staleBefore = new Date(now.getTime() - lease.durationMs); const running = await tx.gatewayReleaseOperation.findFirst({ where: { status: 'RUNNING' }, diff --git a/app/gateway-api/src/orchestrator/profileRepository.ts b/app/gateway-api/src/orchestrator/profileRepository.ts index 68e3c604..994106e7 100644 --- a/app/gateway-api/src/orchestrator/profileRepository.ts +++ b/app/gateway-api/src/orchestrator/profileRepository.ts @@ -1,6 +1,8 @@ import { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus } from '@sammo-ts/common'; import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra'; +export const CONTROL_PLANE_OPERATION_CLAIM_LOCK = 'gateway_control_plane_operation_claim'; + export { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus }; export const GATEWAY_BUILD_STATUSES = ['IDLE', 'QUEUED', 'RUNNING', 'FAILED', 'SUCCEEDED'] as const; @@ -625,8 +627,17 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat ): Promise { const row = await prisma.$transaction(async (tx) => { await tx.$queryRaw>` - SELECT pg_advisory_xact_lock(hashtextextended('gateway_operation_claim', 0))::text AS lock_result + SELECT pg_advisory_xact_lock( + hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0) + )::text AS lock_result `; + const runningRelease = await tx.gatewayReleaseOperation.findFirst({ + where: { status: 'RUNNING' }, + select: { id: true }, + }); + if (runningRelease) { + return null; + } const staleBefore = lease ? new Date(now.getTime() - lease.durationMs) : now; const running = await tx.gatewayOperation.findFirst({ where: { status: 'RUNNING' }, diff --git a/app/gateway-api/test/profileOperationLease.integration.test.ts b/app/gateway-api/test/profileOperationLease.integration.test.ts index 35bb32de..8d4ff698 100644 --- a/app/gateway-api/test/profileOperationLease.integration.test.ts +++ b/app/gateway-api/test/profileOperationLease.integration.test.ts @@ -1,6 +1,7 @@ import { createGatewayPostgresConnector } from '@sammo-ts/infra'; import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { createGatewayReleaseRepository } from '../src/orchestrator/gatewayReleaseRepository.js'; import { createGatewayProfileRepository } from '../src/orchestrator/profileRepository.js'; const databaseUrl = process.env.GATEWAY_OPERATION_DATABASE_URL; @@ -11,6 +12,7 @@ const secondProfileName = 'lease-test-2:1010'; describeDatabase('gateway operation lease and profile serialization', () => { const connector = createGatewayPostgresConnector({ url: databaseUrl ?? '' }); const repository = createGatewayProfileRepository(connector.prisma); + const releaseRepository = createGatewayReleaseRepository(connector.prisma); beforeAll(async () => { await connector.connect(); @@ -29,6 +31,8 @@ describeDatabase('gateway operation lease and profile serialization', () => { }); afterEach(async () => { + await connector.prisma.gatewayReleaseOperation.deleteMany(); + await connector.prisma.gatewayReleaseState.deleteMany(); await connector.prisma.gatewayOperation.deleteMany({ where: { profileName: { in: [profileName, secondProfileName] } }, }); @@ -115,6 +119,45 @@ describeDatabase('gateway operation lease and profile serialization', () => { ).resolves.toMatchObject({ id: second.id, leaseOwner: 'worker-b' }); }); + it('serializes profile and Gateway release claims under one control-plane lock', async () => { + const profileOperation = await repository.createOperation({ + profileName, + type: 'RESET', + sourceMode: 'COMMIT', + sourceRef: 'a'.repeat(40), + requestedBy: 'profile-admin', + }); + const releaseOperation = await releaseRepository.createOperation({ + type: 'DEPLOY', + sourceMode: 'COMMIT', + sourceRef: 'b'.repeat(40), + requestedBy: 'release-admin', + }); + const now = new Date('2030-01-01T00:00:00.000Z'); + + const [profileClaim, releaseClaim] = await Promise.all([ + repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 1_000 }), + releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 1_000 }), + ]); + + expect([profileClaim, releaseClaim].filter(Boolean)).toHaveLength(1); + if (profileClaim) { + expect(profileClaim.id).toBe(profileOperation.id); + expect(releaseClaim).toBeNull(); + await repository.completeOperation(profileOperation.id, 'SUCCEEDED', { error: null }, 'profile-worker'); + await expect( + releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 1_000 }) + ).resolves.toMatchObject({ id: releaseOperation.id }); + return; + } + + expect(releaseClaim?.id).toBe(releaseOperation.id); + await releaseRepository.completeOperation(releaseOperation.id, 'SUCCEEDED', { error: null }, 'release-worker'); + await expect( + repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 1_000 }) + ).resolves.toMatchObject({ id: profileOperation.id }); + }); + it('does not let a future queued operation suppress runtime reconciliation early', async () => { const now = new Date('2030-01-01T00:00:00.000Z'); await repository.createOperation({ diff --git a/docs/release-operations.md b/docs/release-operations.md index 467a54bf..35f16966 100644 --- a/docs/release-operations.md +++ b/docs/release-operations.md @@ -67,6 +67,11 @@ profile 범위 권한과 별개인 전역 `admin.releases.manage` 권한이 필 버전 업데이트 화면에서 profile의 branch 또는 commit을 선택합니다. Branch는 worker가 작업을 claim할 때 commit으로 해석하며, commit 입력은 전체 SHA로 고정됩니다. 같은 profile에는 `QUEUED` 또는 `RUNNING` 작업을 동시에 하나만 둘 수 있습니다. +Profile 작업과 Gateway 전체 릴리스 claim도 같은 PostgreSQL advisory lock으로 +직렬화합니다. 따라서 profile `RESET`/`DEPLOY`가 실행 중이면 Gateway 릴리스는 +queue에서 기다리고, Gateway 릴리스가 실행 중이면 새 profile 작업이 기다립니다. +Gateway process 전환이 진행 중인 profile migration·seed 실행자를 중단하지 않는 +운영 계약입니다. ### DB 유지 배포