test: harden conditional cleanup boundaries

app/game-api/test/battleSimWorker.integration.test.ts

@@ -20,7 +20,8 @@ afterEach(() => {
 
 liveDescribe('battle simulator worker with live Redis', () => {
     it('consumes an isolated queue, produces a result, and stops cleanly', { timeout: 30_000 }, async () => {
-        const scenario = `battle-sim-e2e-${randomUUID()}`;
+        const namespace = process.env.CONDITIONAL_INTEGRATION_RUN_ID ?? randomUUID();
+        const scenario = `battle-sim-e2e-${namespace}-${randomUUID()}`;
         const profileName = `che:${scenario}`;
         const requesterUserId = 'worker-e2e-user';
         vi.stubEnv('PROFILE', 'che');

docs/integration-tests.md

@@ -22,6 +22,13 @@ schema 환경 변수를 지정한 경우에도 이미 존재하는 schema는 거
 HTTP transport fixture의 Redis key도 실행 ID를 profile namespace에 포함하고,
 runner 종료 시 그 실행 ID에 속한 key만 삭제합니다. 공유 Redis 전체에
 `FLUSHDB`나 `FLUSHALL`을 실행하지 않습니다.
+Battle simulator fixture도 실행 ID가 포함된 queue/result/notify namespace를
+사용합니다. 실행 중단 시 테스트의 `finally`가 실행되지 않아도 runner가 같은
+실행 ID의 key만 찾아 삭제합니다.
+Runner는 test process group에 종료 신호를 전달하고 기본 10초 안에 종료되지
+않으면 `SIGKILL`로 전환한 뒤 schema와 Redis cleanup을 계속합니다. 이 유예
+시간은 중단 경계 검증에서만 `CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS`
+(1~60초)로 줄일 수 있습니다.
 `SIGKILL`은 cleanup trap을 실행할 수 없으므로 자동 정리를 보장하지 않습니다.
 
 ## 준비
@@ -71,6 +78,9 @@ runtime role을 삭제하고 PID와 명령행 및 daemon 종료를 확인한 뒤
 환경 변수는 `tools/conditional-integration-registry.tsv`에서 명시적으로
 관리합니다. 새 `*_DATABASE_URL` gate가 registry에 없거나 registry 항목이
 더 이상 테스트에 존재하지 않으면 runner가 테스트 실행 전에 실패합니다.
+Registry는 marker 존재 여부뿐 아니라 중복, 형식과 지원 execution mode도
+검사합니다. 지원하지 않는 mode로 인해 test가 실행 group에서 빠지는 경우에도
+runner는 test 시작 전에 실패합니다.
 
 관리자 시간 조정의 PostgreSQL 경계는
 `runtimeClockShiftPersistence.integration.test.ts`, gateway action

tools/run-conditional-integration.sh

@@ -40,6 +40,19 @@ set +a
 node_tag=$(printf '%s' "${CI_NODE_INDEX:-local}" | tr -cd 'a-zA-Z0-9_' | tr 'A-Z' 'a-z' | cut -c1-8)
 run_id=$(date -u +%m%d%H%M%S)_$$_${node_tag}
 export CONDITIONAL_INTEGRATION_RUN_ID=$run_id
+schema_ownership_token="sammo-conditional-integration:$run_id"
+supported_registry_modes="core create_general gateway_runtime immediate_action npc_possession reference_live_sortie reference_npc_possession select_pool"
+term_grace_seconds=${CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS:-10}
+case "$term_grace_seconds" in
+    ''|*[!0-9]*)
+        echo "CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS must be an integer from 1 to 60" >&2
+        exit 64
+        ;;
+esac
+if [ "$term_grace_seconds" -lt 1 ] || [ "$term_grace_seconds" -gt 60 ]; then
+    echo "CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS must be an integer from 1 to 60" >&2
+    exit 64
+fi
 integration_schema=${CONDITIONAL_INTEGRATION_SCHEMA:-ci_${run_id}_integration}
 scenario_schema=${SCENARIO_SEED_INTEGRATION_SCHEMA:-ci_${run_id}_scenario_seed}
 npc_possession_schema=${NPC_POSSESSION_INTEGRATION_SCHEMA:-ci_${run_id}_npc_possession_integration}
@@ -74,8 +87,9 @@ done
 
 report_dir=$(mktemp -d)
 summary_file="$report_dir/summary.tsv"
-owned_schemas=
+validated_registry_file="$report_dir/validated-registry.tsv"
 active_process_group=
+cleanup_resources_started=0
 
 build_database_url() {
     SCHEMA_NAME=$1 node --input-type=module -e '
@@ -107,39 +121,56 @@ base_database_url=$(build_database_url public)
 
 create_owned_schema() {
     schema=$1
-    SCHEMA_NAME=$schema DATABASE_URL=$base_database_url \
+    SCHEMA_NAME=$schema SCHEMA_OWNERSHIP_TOKEN=$schema_ownership_token \
+        DATABASE_URL=$base_database_url \
         pnpm --filter @sammo-ts/infra exec node --input-type=module -e '
             import pg from "pg";
             const client = new pg.Client({ connectionString: process.env.DATABASE_URL });
+            const quoteIdentifier = (value) => `"${value.replaceAll("\"", "\"\"")}"`;
+            const apostrophe = String.fromCharCode(39);
+            const quoteLiteral = (value) =>
+                `${apostrophe}${value.replaceAll(apostrophe, apostrophe.repeat(2))}${apostrophe}`;
             await client.connect();
             try {
-                await client.query(`CREATE SCHEMA "${process.env.SCHEMA_NAME}"`);
+                await client.query("BEGIN");
+                await client.query(`CREATE SCHEMA ${quoteIdentifier(process.env.SCHEMA_NAME)}`);
+                await client.query(
+                    `COMMENT ON SCHEMA ${quoteIdentifier(process.env.SCHEMA_NAME)} IS ${quoteLiteral(
+                        process.env.SCHEMA_OWNERSHIP_TOKEN
+                    )}`
+                );
+                await client.query("COMMIT");
+            } catch (error) {
+                await client.query("ROLLBACK");
+                throw error;
             } finally {
                 await client.end();
             }
         '
-    owned_schemas="${owned_schemas}${schema}
-"
 }
 
 drop_owned_schemas() {
-    cleanup_failed=0
+    SCHEMA_OWNERSHIP_TOKEN=$schema_ownership_token DATABASE_URL=$base_database_url \
-    for schema in $owned_schemas; do
+        pnpm --filter @sammo-ts/infra exec node --input-type=module -e '
-        if ! SCHEMA_NAME=$schema DATABASE_URL=$base_database_url \
+            import pg from "pg";
-            pnpm --filter @sammo-ts/infra exec node --input-type=module -e '
+            const client = new pg.Client({ connectionString: process.env.DATABASE_URL });
-                import pg from "pg";
+            const quoteIdentifier = (value) => `"${value.replaceAll("\"", "\"\"")}"`;
-                const client = new pg.Client({ connectionString: process.env.DATABASE_URL });
+            await client.connect();
-                await client.connect();
+            try {
-                try {
+                const result = await client.query(
-                    await client.query(`DROP SCHEMA IF EXISTS "${process.env.SCHEMA_NAME}" CASCADE`);
+                    `SELECT nspname
-                } finally {
+                     FROM pg_namespace
-                    await client.end();
+                     WHERE obj_description(oid, $$pg_namespace$$) = $1
+                     ORDER BY nspname`,
+                    [process.env.SCHEMA_OWNERSHIP_TOKEN]
+                );
+                for (const { nspname } of result.rows) {
+                    await client.query(`DROP SCHEMA ${quoteIdentifier(nspname)} CASCADE`);
                 }
-            ' >/dev/null; then
+            } finally {
-            cleanup_failed=1
+                await client.end();
-        fi
+            }
-    done
+        ' >/dev/null
-    return "$cleanup_failed"
 }
 
 delete_owned_redis_keys() {
@@ -151,6 +182,7 @@ delete_owned_redis_keys() {
             const patterns = [
                 `sammo:game:*:che:security-http-${runId}:*`,
                 `sammo:game:*:che:nation-html-${runId}:*`,
+                `sammo:che:battle-sim-e2e-${runId}-*:battle-sim:*`,
             ];
             await client.connect();
             try {
@@ -169,8 +201,19 @@ delete_owned_redis_keys() {
 
 terminate_active_process_group() {
     [ -n "$active_process_group" ] || return 0
-    kill -TERM "-$active_process_group" 2>/dev/null || true
+    process_group=$active_process_group
-    wait "$active_process_group" 2>/dev/null || true
+    kill -TERM "-$process_group" 2>/dev/null || true
+    waited_seconds=0
+    while kill -0 "-$process_group" 2>/dev/null &&
+        [ "$waited_seconds" -lt "$term_grace_seconds" ]; do
+        sleep 1
+        waited_seconds=$((waited_seconds + 1))
+    done
+    if kill -0 "-$process_group" 2>/dev/null; then
+        echo "conditional integration process group did not stop after ${term_grace_seconds}s; sending SIGKILL" >&2
+        kill -KILL "-$process_group" 2>/dev/null || true
+    fi
+    wait "$process_group" 2>/dev/null || true
     active_process_group=
 }
 
@@ -196,11 +239,13 @@ handle_exit() {
     exit_status=$?
     trap - EXIT HUP INT TERM
     terminate_active_process_group
-    if ! drop_owned_schemas && [ "$exit_status" -eq 0 ]; then
+    if [ "$cleanup_resources_started" -eq 1 ]; then
-        exit_status=1
+        if ! drop_owned_schemas && [ "$exit_status" -eq 0 ]; then
-    fi
+            exit_status=1
-    if ! delete_owned_redis_keys && [ "$exit_status" -eq 0 ]; then
+        fi
-        exit_status=1
+        if ! delete_owned_redis_keys && [ "$exit_status" -eq 0 ]; then
+            exit_status=1
+        fi
     fi
     print_summary
     rm -rf "$report_dir"
@@ -215,6 +260,7 @@ trap 'exit 143' TERM
 validate_marker_registry() {
     discovered_file="$report_dir/discovered-markers.txt"
     registered_file="$report_dir/registered-markers.txt"
+    registry_errors_file="$report_dir/registry-errors.txt"
     missing_file="$report_dir/unregistered-markers.txt"
     stale_file="$report_dir/stale-markers.txt"
 
@@ -229,7 +275,44 @@ validate_marker_registry() {
             sed 's/process\.env\.//' |
             sort -u
     ) >"$discovered_file"
-    awk -F '\t' '!/^#/ && NF == 2 { print $1 }' "$registry_file" | sort -u >"$registered_file"
+    : >"$validated_registry_file"
+    awk -F '\t' -v supported_modes="$supported_registry_modes" \
+        -v validated_registry_file="$validated_registry_file" '
+        BEGIN {
+            split(supported_modes, mode_names, " ");
+            for (mode_index in mode_names) {
+                allowed_modes[mode_names[mode_index]] = 1;
+            }
+        }
+        /^#/ || /^[[:space:]]*$/ { next }
+        NF != 2 {
+            printf "line %d must contain exactly one tab-separated marker and mode\n", NR;
+            failed = 1;
+            next;
+        }
+        $1 !~ /^[A-Z][A-Z0-9_]*_DATABASE_URL$/ {
+            printf "line %d has invalid marker: %s\n", NR, $1;
+            failed = 1;
+        }
+        !($2 in allowed_modes) {
+            printf "line %d has unsupported execution mode: %s\n", NR, $2;
+            failed = 1;
+        }
+        seen[$1]++ {
+            printf "line %d duplicates marker: %s\n", NR, $1;
+            failed = 1;
+        }
+        {
+            print $1 "\t" $2 > validated_registry_file;
+        }
+        END { exit failed }
+    ' "$registry_file" >"$registry_errors_file" || {
+        echo "invalid conditional integration marker registry:" >&2
+        sed 's/^/  /' "$registry_errors_file" >&2
+        exit 65
+    }
+    sort -u "$validated_registry_file" -o "$validated_registry_file"
+    cut -f1 "$validated_registry_file" >"$registered_file"
 
     comm -23 "$discovered_file" "$registered_file" >"$missing_file"
     if [ -s "$missing_file" ]; then
@@ -248,7 +331,7 @@ validate_marker_registry() {
 
 markers_for_mode() {
     mode=$1
-    awk -F '\t' -v mode="$mode" '!/^#/ && $2 == mode { print $1 }' "$registry_file" |
+    awk -F '\t' -v mode="$mode" '$2 == mode { print $1 }' "$validated_registry_file" |
         paste -sd '|' -
 }
 
@@ -348,13 +431,14 @@ run_redis_only_tests() {
 
 export PATH
 cd "$workspace_root"
+validate_marker_registry
+
 pnpm install --frozen-lockfile
 pnpm --filter @sammo-ts/infra prisma:generate
 pnpm --filter @sammo-ts/common build
 pnpm --filter @sammo-ts/infra build
 
-validate_marker_registry
+cleanup_resources_started=1
-
 create_owned_schema "$integration_schema"
 create_owned_schema "$npc_possession_schema"
 create_owned_schema "$scenario_schema"