diff --git a/app/game-api/test/battleSimWorker.integration.test.ts b/app/game-api/test/battleSimWorker.integration.test.ts index 87469ba..0ab872a 100644 --- a/app/game-api/test/battleSimWorker.integration.test.ts +++ b/app/game-api/test/battleSimWorker.integration.test.ts @@ -20,7 +20,8 @@ afterEach(() => { liveDescribe('battle simulator worker with live Redis', () => { it('consumes an isolated queue, produces a result, and stops cleanly', { timeout: 30_000 }, async () => { - const scenario = `battle-sim-e2e-${randomUUID()}`; + const namespace = process.env.CONDITIONAL_INTEGRATION_RUN_ID ?? randomUUID(); + const scenario = `battle-sim-e2e-${namespace}-${randomUUID()}`; const profileName = `che:${scenario}`; const requesterUserId = 'worker-e2e-user'; vi.stubEnv('PROFILE', 'che'); diff --git a/docs/integration-tests.md b/docs/integration-tests.md index 30bbec3..f540d3a 100644 --- a/docs/integration-tests.md +++ b/docs/integration-tests.md @@ -22,6 +22,13 @@ schema 환경 변수를 지정한 경우에도 이미 존재하는 schema는 거 HTTP transport fixture의 Redis key도 실행 ID를 profile namespace에 포함하고, runner 종료 시 그 실행 ID에 속한 key만 삭제합니다. 공유 Redis 전체에 `FLUSHDB`나 `FLUSHALL`을 실행하지 않습니다. +Battle simulator fixture도 실행 ID가 포함된 queue/result/notify namespace를 +사용합니다. 실행 중단 시 테스트의 `finally`가 실행되지 않아도 runner가 같은 +실행 ID의 key만 찾아 삭제합니다. +Runner는 test process group에 종료 신호를 전달하고 기본 10초 안에 종료되지 +않으면 `SIGKILL`로 전환한 뒤 schema와 Redis cleanup을 계속합니다. 이 유예 +시간은 중단 경계 검증에서만 `CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS` +(1~60초)로 줄일 수 있습니다. `SIGKILL`은 cleanup trap을 실행할 수 없으므로 자동 정리를 보장하지 않습니다. ## 준비 @@ -71,6 +78,9 @@ runtime role을 삭제하고 PID와 명령행 및 daemon 종료를 확인한 뒤 환경 변수는 `tools/conditional-integration-registry.tsv`에서 명시적으로 관리합니다. 새 `*_DATABASE_URL` gate가 registry에 없거나 registry 항목이 더 이상 테스트에 존재하지 않으면 runner가 테스트 실행 전에 실패합니다. +Registry는 marker 존재 여부뿐 아니라 중복, 형식과 지원 execution mode도 +검사합니다. 지원하지 않는 mode로 인해 test가 실행 group에서 빠지는 경우에도 +runner는 test 시작 전에 실패합니다. 관리자 시간 조정의 PostgreSQL 경계는 `runtimeClockShiftPersistence.integration.test.ts`, gateway action diff --git a/tools/run-conditional-integration.sh b/tools/run-conditional-integration.sh index 6f53128..c3c639a 100755 --- a/tools/run-conditional-integration.sh +++ b/tools/run-conditional-integration.sh @@ -40,6 +40,19 @@ set +a node_tag=$(printf '%s' "${CI_NODE_INDEX:-local}" | tr -cd 'a-zA-Z0-9_' | tr 'A-Z' 'a-z' | cut -c1-8) run_id=$(date -u +%m%d%H%M%S)_$$_${node_tag} export CONDITIONAL_INTEGRATION_RUN_ID=$run_id +schema_ownership_token="sammo-conditional-integration:$run_id" +supported_registry_modes="core create_general gateway_runtime immediate_action npc_possession reference_live_sortie reference_npc_possession select_pool" +term_grace_seconds=${CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS:-10} +case "$term_grace_seconds" in + ''|*[!0-9]*) + echo "CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS must be an integer from 1 to 60" >&2 + exit 64 + ;; +esac +if [ "$term_grace_seconds" -lt 1 ] || [ "$term_grace_seconds" -gt 60 ]; then + echo "CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS must be an integer from 1 to 60" >&2 + exit 64 +fi integration_schema=${CONDITIONAL_INTEGRATION_SCHEMA:-ci_${run_id}_integration} scenario_schema=${SCENARIO_SEED_INTEGRATION_SCHEMA:-ci_${run_id}_scenario_seed} npc_possession_schema=${NPC_POSSESSION_INTEGRATION_SCHEMA:-ci_${run_id}_npc_possession_integration} @@ -74,8 +87,9 @@ done report_dir=$(mktemp -d) summary_file="$report_dir/summary.tsv" -owned_schemas= +validated_registry_file="$report_dir/validated-registry.tsv" active_process_group= +cleanup_resources_started=0 build_database_url() { SCHEMA_NAME=$1 node --input-type=module -e ' @@ -107,39 +121,56 @@ base_database_url=$(build_database_url public) create_owned_schema() { schema=$1 - SCHEMA_NAME=$schema DATABASE_URL=$base_database_url \ + SCHEMA_NAME=$schema SCHEMA_OWNERSHIP_TOKEN=$schema_ownership_token \ + DATABASE_URL=$base_database_url \ pnpm --filter @sammo-ts/infra exec node --input-type=module -e ' import pg from "pg"; const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); + const quoteIdentifier = (value) => `"${value.replaceAll("\"", "\"\"")}"`; + const apostrophe = String.fromCharCode(39); + const quoteLiteral = (value) => + `${apostrophe}${value.replaceAll(apostrophe, apostrophe.repeat(2))}${apostrophe}`; await client.connect(); try { - await client.query(`CREATE SCHEMA "${process.env.SCHEMA_NAME}"`); + await client.query("BEGIN"); + await client.query(`CREATE SCHEMA ${quoteIdentifier(process.env.SCHEMA_NAME)}`); + await client.query( + `COMMENT ON SCHEMA ${quoteIdentifier(process.env.SCHEMA_NAME)} IS ${quoteLiteral( + process.env.SCHEMA_OWNERSHIP_TOKEN + )}` + ); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; } finally { await client.end(); } ' - owned_schemas="${owned_schemas}${schema} -" } drop_owned_schemas() { - cleanup_failed=0 - for schema in $owned_schemas; do - if ! SCHEMA_NAME=$schema DATABASE_URL=$base_database_url \ - pnpm --filter @sammo-ts/infra exec node --input-type=module -e ' - import pg from "pg"; - const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); - await client.connect(); - try { - await client.query(`DROP SCHEMA IF EXISTS "${process.env.SCHEMA_NAME}" CASCADE`); - } finally { - await client.end(); + SCHEMA_OWNERSHIP_TOKEN=$schema_ownership_token DATABASE_URL=$base_database_url \ + pnpm --filter @sammo-ts/infra exec node --input-type=module -e ' + import pg from "pg"; + const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); + const quoteIdentifier = (value) => `"${value.replaceAll("\"", "\"\"")}"`; + await client.connect(); + try { + const result = await client.query( + `SELECT nspname + FROM pg_namespace + WHERE obj_description(oid, $$pg_namespace$$) = $1 + ORDER BY nspname`, + [process.env.SCHEMA_OWNERSHIP_TOKEN] + ); + for (const { nspname } of result.rows) { + await client.query(`DROP SCHEMA ${quoteIdentifier(nspname)} CASCADE`); } - ' >/dev/null; then - cleanup_failed=1 - fi - done - return "$cleanup_failed" + } finally { + await client.end(); + } + ' >/dev/null } delete_owned_redis_keys() { @@ -151,6 +182,7 @@ delete_owned_redis_keys() { const patterns = [ `sammo:game:*:che:security-http-${runId}:*`, `sammo:game:*:che:nation-html-${runId}:*`, + `sammo:che:battle-sim-e2e-${runId}-*:battle-sim:*`, ]; await client.connect(); try { @@ -169,8 +201,19 @@ delete_owned_redis_keys() { terminate_active_process_group() { [ -n "$active_process_group" ] || return 0 - kill -TERM "-$active_process_group" 2>/dev/null || true - wait "$active_process_group" 2>/dev/null || true + process_group=$active_process_group + kill -TERM "-$process_group" 2>/dev/null || true + waited_seconds=0 + while kill -0 "-$process_group" 2>/dev/null && + [ "$waited_seconds" -lt "$term_grace_seconds" ]; do + sleep 1 + waited_seconds=$((waited_seconds + 1)) + done + if kill -0 "-$process_group" 2>/dev/null; then + echo "conditional integration process group did not stop after ${term_grace_seconds}s; sending SIGKILL" >&2 + kill -KILL "-$process_group" 2>/dev/null || true + fi + wait "$process_group" 2>/dev/null || true active_process_group= } @@ -196,11 +239,13 @@ handle_exit() { exit_status=$? trap - EXIT HUP INT TERM terminate_active_process_group - if ! drop_owned_schemas && [ "$exit_status" -eq 0 ]; then - exit_status=1 - fi - if ! delete_owned_redis_keys && [ "$exit_status" -eq 0 ]; then - exit_status=1 + if [ "$cleanup_resources_started" -eq 1 ]; then + if ! drop_owned_schemas && [ "$exit_status" -eq 0 ]; then + exit_status=1 + fi + if ! delete_owned_redis_keys && [ "$exit_status" -eq 0 ]; then + exit_status=1 + fi fi print_summary rm -rf "$report_dir" @@ -215,6 +260,7 @@ trap 'exit 143' TERM validate_marker_registry() { discovered_file="$report_dir/discovered-markers.txt" registered_file="$report_dir/registered-markers.txt" + registry_errors_file="$report_dir/registry-errors.txt" missing_file="$report_dir/unregistered-markers.txt" stale_file="$report_dir/stale-markers.txt" @@ -229,7 +275,44 @@ validate_marker_registry() { sed 's/process\.env\.//' | sort -u ) >"$discovered_file" - awk -F '\t' '!/^#/ && NF == 2 { print $1 }' "$registry_file" | sort -u >"$registered_file" + : >"$validated_registry_file" + awk -F '\t' -v supported_modes="$supported_registry_modes" \ + -v validated_registry_file="$validated_registry_file" ' + BEGIN { + split(supported_modes, mode_names, " "); + for (mode_index in mode_names) { + allowed_modes[mode_names[mode_index]] = 1; + } + } + /^#/ || /^[[:space:]]*$/ { next } + NF != 2 { + printf "line %d must contain exactly one tab-separated marker and mode\n", NR; + failed = 1; + next; + } + $1 !~ /^[A-Z][A-Z0-9_]*_DATABASE_URL$/ { + printf "line %d has invalid marker: %s\n", NR, $1; + failed = 1; + } + !($2 in allowed_modes) { + printf "line %d has unsupported execution mode: %s\n", NR, $2; + failed = 1; + } + seen[$1]++ { + printf "line %d duplicates marker: %s\n", NR, $1; + failed = 1; + } + { + print $1 "\t" $2 > validated_registry_file; + } + END { exit failed } + ' "$registry_file" >"$registry_errors_file" || { + echo "invalid conditional integration marker registry:" >&2 + sed 's/^/ /' "$registry_errors_file" >&2 + exit 65 + } + sort -u "$validated_registry_file" -o "$validated_registry_file" + cut -f1 "$validated_registry_file" >"$registered_file" comm -23 "$discovered_file" "$registered_file" >"$missing_file" if [ -s "$missing_file" ]; then @@ -248,7 +331,7 @@ validate_marker_registry() { markers_for_mode() { mode=$1 - awk -F '\t' -v mode="$mode" '!/^#/ && $2 == mode { print $1 }' "$registry_file" | + awk -F '\t' -v mode="$mode" '$2 == mode { print $1 }' "$validated_registry_file" | paste -sd '|' - } @@ -348,13 +431,14 @@ run_redis_only_tests() { export PATH cd "$workspace_root" +validate_marker_registry + pnpm install --frozen-lockfile pnpm --filter @sammo-ts/infra prisma:generate pnpm --filter @sammo-ts/common build pnpm --filter @sammo-ts/infra build -validate_marker_registry - +cleanup_resources_started=1 create_owned_schema "$integration_schema" create_owned_schema "$npc_possession_schema" create_owned_schema "$scenario_schema"