fix(gateway): 프로필 초기화와 전체 배포를 직렬화
진행 중인 프로필 RESET 또는 DEPLOY와 Gateway 전체 릴리스가 서로의 실행자를 중단하지 않도록 공통 advisory lock과 교차 RUNNING 검사를 추가한다. PostgreSQL 통합 테스트로 동시 claim과 후속 실행을 검증한다.
This commit is contained in:
@@ -1,6 +1,10 @@
|
||||
import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra';
|
||||
|
||||
import type { GatewayOperationStatus, GatewaySourceMode } from './profileRepository.js';
|
||||
import {
|
||||
CONTROL_PLANE_OPERATION_CLAIM_LOCK,
|
||||
type GatewayOperationStatus,
|
||||
type GatewaySourceMode,
|
||||
} from './profileRepository.js';
|
||||
|
||||
export type GatewayReleaseOperationType = 'DEPLOY' | 'ROLLBACK';
|
||||
|
||||
@@ -230,8 +234,17 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
|
||||
async claimNextOperation(now, lease) {
|
||||
const row = await prisma.$transaction(async (tx) => {
|
||||
await tx.$queryRaw<Array<{ lock_result: string }>>`
|
||||
SELECT pg_advisory_xact_lock(hashtextextended('gateway_release_operation_claim', 0))::text AS lock_result
|
||||
SELECT pg_advisory_xact_lock(
|
||||
hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0)
|
||||
)::text AS lock_result
|
||||
`;
|
||||
const runningProfileOperation = await tx.gatewayOperation.findFirst({
|
||||
where: { status: 'RUNNING' },
|
||||
select: { id: true },
|
||||
});
|
||||
if (runningProfileOperation) {
|
||||
return null;
|
||||
}
|
||||
const staleBefore = new Date(now.getTime() - lease.durationMs);
|
||||
const running = await tx.gatewayReleaseOperation.findFirst({
|
||||
where: { status: 'RUNNING' },
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus } from '@sammo-ts/common';
|
||||
import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra';
|
||||
|
||||
export const CONTROL_PLANE_OPERATION_CLAIM_LOCK = 'gateway_control_plane_operation_claim';
|
||||
|
||||
export { GATEWAY_PROFILE_STATUSES, type GatewayProfileStatus };
|
||||
|
||||
export const GATEWAY_BUILD_STATUSES = ['IDLE', 'QUEUED', 'RUNNING', 'FAILED', 'SUCCEEDED'] as const;
|
||||
@@ -625,8 +627,17 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
): Promise<GatewayOperationRecord | null> {
|
||||
const row = await prisma.$transaction(async (tx) => {
|
||||
await tx.$queryRaw<Array<{ lock_result: string }>>`
|
||||
SELECT pg_advisory_xact_lock(hashtextextended('gateway_operation_claim', 0))::text AS lock_result
|
||||
SELECT pg_advisory_xact_lock(
|
||||
hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0)
|
||||
)::text AS lock_result
|
||||
`;
|
||||
const runningRelease = await tx.gatewayReleaseOperation.findFirst({
|
||||
where: { status: 'RUNNING' },
|
||||
select: { id: true },
|
||||
});
|
||||
if (runningRelease) {
|
||||
return null;
|
||||
}
|
||||
const staleBefore = lease ? new Date(now.getTime() - lease.durationMs) : now;
|
||||
const running = await tx.gatewayOperation.findFirst({
|
||||
where: { status: 'RUNNING' },
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createGatewayPostgresConnector } from '@sammo-ts/infra';
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import { createGatewayReleaseRepository } from '../src/orchestrator/gatewayReleaseRepository.js';
|
||||
import { createGatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
|
||||
|
||||
const databaseUrl = process.env.GATEWAY_OPERATION_DATABASE_URL;
|
||||
@@ -11,6 +12,7 @@ const secondProfileName = 'lease-test-2:1010';
|
||||
describeDatabase('gateway operation lease and profile serialization', () => {
|
||||
const connector = createGatewayPostgresConnector({ url: databaseUrl ?? '' });
|
||||
const repository = createGatewayProfileRepository(connector.prisma);
|
||||
const releaseRepository = createGatewayReleaseRepository(connector.prisma);
|
||||
|
||||
beforeAll(async () => {
|
||||
await connector.connect();
|
||||
@@ -29,6 +31,8 @@ describeDatabase('gateway operation lease and profile serialization', () => {
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await connector.prisma.gatewayReleaseOperation.deleteMany();
|
||||
await connector.prisma.gatewayReleaseState.deleteMany();
|
||||
await connector.prisma.gatewayOperation.deleteMany({
|
||||
where: { profileName: { in: [profileName, secondProfileName] } },
|
||||
});
|
||||
@@ -115,6 +119,45 @@ describeDatabase('gateway operation lease and profile serialization', () => {
|
||||
).resolves.toMatchObject({ id: second.id, leaseOwner: 'worker-b' });
|
||||
});
|
||||
|
||||
it('serializes profile and Gateway release claims under one control-plane lock', async () => {
|
||||
const profileOperation = await repository.createOperation({
|
||||
profileName,
|
||||
type: 'RESET',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: 'a'.repeat(40),
|
||||
requestedBy: 'profile-admin',
|
||||
});
|
||||
const releaseOperation = await releaseRepository.createOperation({
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: 'b'.repeat(40),
|
||||
requestedBy: 'release-admin',
|
||||
});
|
||||
const now = new Date('2030-01-01T00:00:00.000Z');
|
||||
|
||||
const [profileClaim, releaseClaim] = await Promise.all([
|
||||
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 1_000 }),
|
||||
releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 1_000 }),
|
||||
]);
|
||||
|
||||
expect([profileClaim, releaseClaim].filter(Boolean)).toHaveLength(1);
|
||||
if (profileClaim) {
|
||||
expect(profileClaim.id).toBe(profileOperation.id);
|
||||
expect(releaseClaim).toBeNull();
|
||||
await repository.completeOperation(profileOperation.id, 'SUCCEEDED', { error: null }, 'profile-worker');
|
||||
await expect(
|
||||
releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 1_000 })
|
||||
).resolves.toMatchObject({ id: releaseOperation.id });
|
||||
return;
|
||||
}
|
||||
|
||||
expect(releaseClaim?.id).toBe(releaseOperation.id);
|
||||
await releaseRepository.completeOperation(releaseOperation.id, 'SUCCEEDED', { error: null }, 'release-worker');
|
||||
await expect(
|
||||
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 1_000 })
|
||||
).resolves.toMatchObject({ id: profileOperation.id });
|
||||
});
|
||||
|
||||
it('does not let a future queued operation suppress runtime reconciliation early', async () => {
|
||||
const now = new Date('2030-01-01T00:00:00.000Z');
|
||||
await repository.createOperation({
|
||||
|
||||
Reference in New Issue
Block a user