test(infra): verify NPC selection migration recovery

2026-07-31 04:58:15 +00:00
parent 9c331844f5
commit 5b776a52fd
3 changed files with 318 additions and 0 deletions
@@ -18,6 +18,7 @@
        "prisma:migrate:deploy:game": "prisma migrate deploy --schema prisma/game.prisma",
        "prisma:migrate:deploy:gateway": "PRISMA_SCHEMA=prisma/gateway.prisma prisma migrate deploy --schema prisma/gateway.prisma --config prisma.gateway.config.ts",
        "prisma:migrate:status:game": "prisma migrate status --schema prisma/game.prisma",
+        "verify:migration:npc-selection": "sh scripts/verify-npc-selection-token-migration.sh",
        "prisma:db:push:game": "prisma db push --schema prisma/game.prisma",
        "prisma:db:push:gateway": "prisma db push --schema prisma/gateway.prisma"
    },
@@ -33,6 +33,67 @@ chain을 적용하고 두 번째 실행은 `No pending migrations to apply`여
 - `city.trade` nullable, `city.trust` REAL
 - `auction_bid.meta` JSONB NOT NULL
 - `traffic_period`, `traffic_period_general`과 unique key
+- `select_npc_token`, `select_npc_token_valid_until_idx`
+- `general_user_id_key`

 검증이 끝나면 이름을 직접 확인한 임시 database와 role만 제거합니다. 공유
 database나 Compose volume을 삭제하지 않습니다.
+
+## NPC selection 중복 owner preflight
+
+`20260731000000_add_npc_selection_token`은 `general.user_id` 중복을 발견하면
+token table과 unique index를 만들기 전에 실패합니다. 실제 Prisma 실패
+metadata와 운영자 정리 뒤 recovery를 전용 tmpfs PostgreSQL에서 검증합니다.
+
+```sh
+pnpm --filter @sammo-ts/infra verify:migration:npc-selection
+```
+
+검증기는 target 직전 migration 상태에 synthetic 중복 owner를 넣고 다음
+순서를 확인합니다.
+
+1. `migrate deploy`가 실패합니다. Prisma 7.2 CLI의 최상위 오류는 내부
+   owner 진단 대신 `current transaction is aborted`로 표시됩니다.
+2. migration transaction의 table/index DDL이 남지 않습니다.
+3. target `_prisma_migrations`에는 `finished_at`, `rolled_back_at`, `logs`가
+   모두 NULL이고 `applied_steps_count=0`인 미완료 행이 하나 남습니다.
+4. 재실행은 이 미완료 이력 때문에 P3009로 차단됩니다.
+5. 중복을 정리하고 target을 `--rolled-back`으로 resolve한 뒤 deploy가
+   성공합니다.
+6. 두 번째 deploy는 no-op이고 migration status가 clean입니다.
+
+검증기는 `postgres:18.4-bookworm` container의 데이터 경로를 tmpfs로
+mount하며 Docker volume을 만들지 않습니다. 임의 포트와 실행 고유 schema를
+사용하고 EXIT/HUP/INT/TERM에서 소유 label을 확인한 정확한 container만
+제거합니다.
+
+운영 복구에서는 중복 owner를 임의 삭제하지 말고 진단된 계정과 장수를
+확인해 주세요. CLI 오류가 일반 transaction 오류만 표시하면 다음 read-only
+query로 대상을 확인합니다.
+
+```sql
+SELECT
+    user_id,
+    count(*) AS owner_count,
+    string_agg(id::TEXT, ',' ORDER BY id) AS general_ids
+FROM general
+WHERE user_id IS NOT NULL
+GROUP BY user_id
+HAVING count(*) > 1
+ORDER BY user_id;
+```
+
+중복 원인을 정리한 뒤 다음처럼 실패한 target만 rolled-back으로 표시하고
+deploy를 다시 실행합니다.
+
+```sh
+pnpm --filter @sammo-ts/infra exec prisma migrate resolve \
+  --rolled-back 20260731000000_add_npc_selection_token \
+  --schema prisma/game.prisma
+pnpm --filter @sammo-ts/infra prisma:migrate:deploy:game
+```
+
+다른 migration을 resolve하거나 중복을 정리하기 전에 resolve하지 말아
+주세요. 이 migration에는 성공 적용을 되돌리는 down migration이 없습니다.
+성공 뒤 복구가 필요하면 사전 DB backup을 복원하거나 별도 forward
+migration을 작성해 주세요.
@@ -0,0 +1,256 @@
+#!/bin/sh
+set -eu
+
+script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
+package_dir="$(dirname "$script_dir")"
+prisma_dir="$package_dir/prisma"
+target_migration=20260731000000_add_npc_selection_token
+task_label=devsam.core2026.npc-selection-migration-preflight
+container_name="sammo-npc-migration-preflight-$(date +%s)-$$"
+schema_name="npc_selection_migration_preflight_$(date +%s)_$$"
+work_dir="$(mktemp -d /tmp/sammo-npc-migration-preflight.XXXXXX)"
+container_created=0
+
+case "$container_name" in
+    sammo-npc-migration-preflight-[0-9]*-[0-9]*) ;;
+    *) echo "unsafe migration preflight container name" >&2; exit 64 ;;
+esac
+case "$schema_name" in
+    npc_selection_migration_preflight_[0-9]*_[0-9]*) ;;
+    *) echo "unsafe migration preflight schema name" >&2; exit 64 ;;
+esac
+
+cleanup() {
+    cleanup_failed=0
+    if [ "$container_created" -eq 1 ] && docker inspect "$container_name" >/dev/null 2>&1; then
+        actual_label="$(docker inspect --format '{{ index .Config.Labels "devsam.core2026.task" }}' "$container_name")"
+        if [ "$actual_label" != "$task_label" ]; then
+            echo "refusing to remove container with unexpected ownership label: $container_name" >&2
+            cleanup_failed=1
+        elif ! docker rm -f "$container_name" >/dev/null; then
+            cleanup_failed=1
+        fi
+    fi
+    case "$work_dir" in
+        /tmp/sammo-npc-migration-preflight.*)
+            if [ -d "$work_dir" ] && ! rm -r -- "$work_dir"; then
+                cleanup_failed=1
+            fi
+            ;;
+        *)
+            echo "refusing to remove unsafe migration preflight work directory: $work_dir" >&2
+            cleanup_failed=1
+            ;;
+    esac
+    if docker inspect "$container_name" >/dev/null 2>&1; then
+        echo "migration preflight container remains after cleanup: $container_name" >&2
+        cleanup_failed=1
+    fi
+    return "$cleanup_failed"
+}
+
+handle_exit() {
+    exit_status=$?
+    trap - EXIT HUP INT TERM
+    if ! cleanup && [ "$exit_status" -eq 0 ]; then
+        exit_status=1
+    fi
+    exit "$exit_status"
+}
+
+trap handle_exit EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+command -v docker >/dev/null 2>&1 || {
+    echo "docker is required" >&2
+    exit 69
+}
+command -v pnpm >/dev/null 2>&1 || {
+    echo "pnpm is required" >&2
+    exit 69
+}
+[ -d "$prisma_dir/migrations/$target_migration" ] || {
+    echo "target migration is missing: $target_migration" >&2
+    exit 66
+}
+
+umask 077
+password="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')"
+password_file="$work_dir/postgres_password"
+printf '%s\n' "$password" >"$password_file"
+chmod 0600 "$password_file"
+
+docker run -d \
+    --name "$container_name" \
+    --label "devsam.core2026.task=$task_label" \
+    --tmpfs /var/lib/postgresql:rw,nodev,nosuid,size=1g \
+    --mount "type=bind,source=$password_file,target=/run/secrets/postgres_password,readonly" \
+    -e POSTGRES_DB=sammo \
+    -e POSTGRES_USER=sammo \
+    -e POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password \
+    -p 127.0.0.1::5432 \
+    postgres:18.4-bookworm >/dev/null
+container_created=1
+
+if [ -n "$(docker inspect --format '{{ range .Mounts }}{{ if eq .Type "volume" }}volume{{ end }}{{ end }}' "$container_name")" ]; then
+    echo "migration preflight container unexpectedly owns a Docker volume" >&2
+    exit 1
+fi
+
+attempt=0
+until docker exec "$container_name" pg_isready -U sammo -d sammo >/dev/null 2>&1; do
+    attempt=$((attempt + 1))
+    if [ "$attempt" -ge 60 ]; then
+        docker logs --tail 100 "$container_name" >&2
+        exit 1
+    fi
+    sleep 1
+done
+
+published_port="$(docker port "$container_name" 5432/tcp)"
+published_port="${published_port##*:}"
+case "$published_port" in
+    ''|*[!0-9]*) echo "could not resolve PostgreSQL host port" >&2; exit 1 ;;
+esac
+
+export POSTGRES_HOST=127.0.0.1
+export POSTGRES_PORT="$published_port"
+export POSTGRES_DB=sammo
+export POSTGRES_USER=sammo
+export POSTGRES_PASSWORD="$password"
+export POSTGRES_SCHEMA="$schema_name"
+unset DATABASE_URL DATABASE_SCHEMA
+
+stage_prisma="$work_dir/prisma"
+mkdir -p "$stage_prisma/migrations"
+cp "$prisma_dir/game.prisma" "$stage_prisma/game.prisma"
+found_target=0
+for migration_dir in "$prisma_dir"/migrations/[0-9]*; do
+    migration_name="$(basename "$migration_dir")"
+    if [ "$migration_name" = "$target_migration" ]; then
+        found_target=1
+        break
+    fi
+    cp -R "$migration_dir" "$stage_prisma/migrations/$migration_name"
+done
+[ "$found_target" -eq 1 ] || {
+    echo "target migration was not found in migration order" >&2
+    exit 1
+}
+
+cd "$package_dir"
+if ! PRISMA_SCHEMA="$stage_prisma/game.prisma" \
+    pnpm exec prisma migrate deploy --schema "$stage_prisma/game.prisma" \
+        >"$work_dir/predecessor-deploy.log" 2>&1; then
+    sed -n '1,160p' "$work_dir/predecessor-deploy.log" >&2
+    exit 1
+fi
+
+docker exec -i "$container_name" psql -v ON_ERROR_STOP=1 -U sammo -d sammo >/dev/null <<SQL
+SET search_path TO "$schema_name";
+INSERT INTO "general" ("id", "user_id", "name", "turn_time") VALUES
+    (900001, 'npc-migration-duplicate-owner', '중복장수1', CURRENT_TIMESTAMP),
+    (900002, 'npc-migration-duplicate-owner', '중복장수2', CURRENT_TIMESTAMP);
+SQL
+
+failure_log="$work_dir/duplicate-deploy.log"
+if PRISMA_SCHEMA="$prisma_dir/game.prisma" \
+    pnpm exec prisma migrate deploy --schema "$prisma_dir/game.prisma" >"$failure_log" 2>&1; then
+    echo "migration unexpectedly accepted duplicate general owners" >&2
+    exit 1
+fi
+grep -Fq 'current transaction is aborted' "$failure_log"
+
+objects_after_failure="$(
+    docker exec "$container_name" psql -v ON_ERROR_STOP=1 -U sammo -d sammo -tAc "
+        SELECT
+            to_regclass('\"$schema_name\".\"select_npc_token\"') IS NULL
+            AND to_regclass('\"$schema_name\".\"select_npc_token_valid_until_idx\"') IS NULL
+            AND to_regclass('\"$schema_name\".\"general_user_id_key\"') IS NULL;
+    "
+)"
+[ "$objects_after_failure" = "t" ] || {
+    echo "NPC selection DDL survived the failed transactional migration" >&2
+    exit 1
+}
+
+failed_metadata="$(
+    docker exec "$container_name" psql -v ON_ERROR_STOP=1 -U sammo -d sammo -tAc "
+        SELECT
+            count(*),
+            bool_and(finished_at IS NULL),
+            bool_and(rolled_back_at IS NULL),
+            bool_and(logs IS NULL),
+            sum(applied_steps_count)
+        FROM \"$schema_name\".\"_prisma_migrations\"
+        WHERE migration_name = '$target_migration';
+    "
+)"
+[ "$failed_metadata" = "1|t|t|t|0" ] || {
+    echo "unexpected target migration metadata after transactional failure: $failed_metadata" >&2
+    exit 1
+}
+
+retry_log="$work_dir/retry-without-resolve.log"
+if PRISMA_SCHEMA="$prisma_dir/game.prisma" \
+    pnpm exec prisma migrate deploy --schema "$prisma_dir/game.prisma" >"$retry_log" 2>&1; then
+    echo "Prisma unexpectedly retried a failed migration without resolve" >&2
+    exit 1
+fi
+if ! grep -Fq 'Error: P3009' "$retry_log"; then
+    echo "migration retry was not blocked by the unfinished Prisma history row" >&2
+    sed -n '1,200p' "$retry_log" >&2
+    exit 1
+fi
+
+docker exec -i "$container_name" psql -v ON_ERROR_STOP=1 -U sammo -d sammo >/dev/null <<SQL
+SET search_path TO "$schema_name";
+DELETE FROM "general" WHERE "id" = 900002;
+SQL
+
+if ! PRISMA_SCHEMA="$prisma_dir/game.prisma" \
+    pnpm exec prisma migrate resolve --rolled-back "$target_migration" \
+        --schema "$prisma_dir/game.prisma" >"$work_dir/resolve.log" 2>&1; then
+    sed -n '1,200p' "$work_dir/resolve.log" >&2
+    exit 1
+fi
+
+PRISMA_SCHEMA="$prisma_dir/game.prisma" \
+    pnpm exec prisma migrate deploy --schema "$prisma_dir/game.prisma" >"$work_dir/recovered-deploy.log"
+PRISMA_SCHEMA="$prisma_dir/game.prisma" \
+    pnpm exec prisma migrate deploy --schema "$prisma_dir/game.prisma" >"$work_dir/noop-deploy.log"
+grep -Fq 'No pending migrations to apply' "$work_dir/noop-deploy.log"
+PRISMA_SCHEMA="$prisma_dir/game.prisma" \
+    pnpm exec prisma migrate status --schema "$prisma_dir/game.prisma" >"$work_dir/status.log"
+grep -Fq 'Database schema is up to date' "$work_dir/status.log"
+
+objects_after_recovery="$(
+    docker exec "$container_name" psql -v ON_ERROR_STOP=1 -U sammo -d sammo -tAc "
+        SELECT
+            to_regclass('\"$schema_name\".\"select_npc_token\"') IS NOT NULL
+            AND to_regclass('\"$schema_name\".\"select_npc_token_valid_until_idx\"') IS NOT NULL
+            AND to_regclass('\"$schema_name\".\"general_user_id_key\"') IS NOT NULL;
+    "
+)"
+[ "$objects_after_recovery" = "t" ] || {
+    echo "NPC selection table or indexes are missing after recovery" >&2
+    exit 1
+}
+
+migration_rows_after_recovery="$(
+    docker exec "$container_name" psql -v ON_ERROR_STOP=1 -U sammo -d sammo -tAc "
+        SELECT
+            count(*) FILTER (WHERE rolled_back_at IS NOT NULL),
+            count(*) FILTER (WHERE finished_at IS NOT NULL)
+        FROM \"$schema_name\".\"_prisma_migrations\"
+        WHERE migration_name = '$target_migration';
+    "
+)"
+[ "$migration_rows_after_recovery" = "1|1" ] || {
+    echo "unexpected Prisma migration history after recovery: $migration_rows_after_recovery" >&2
+    exit 1
+}
+
+echo "NPC selection migration duplicate preflight and recovery passed"