Files
core2026/tools/run-conditional-integration.sh
T

604 lines
23 KiB
Bash
Executable File

#!/bin/sh
set -eu
usage() {
echo "usage: $0 [env-file]" >&2
exit 64
}
[ "$#" -le 1 ] || usage
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
workspace_root=$(CDPATH= cd -- "$script_dir/.." && pwd)
registry_file="$script_dir/conditional-integration-registry.tsv"
env_file=${1:-"$workspace_root/.env.ci"}
if [ ! -f "$env_file" ]; then
echo "missing integration environment file: $env_file" >&2
exit 66
fi
if [ ! -f "$registry_file" ]; then
echo "missing integration marker registry: $registry_file" >&2
exit 66
fi
env_file=$(CDPATH= cd -- "$(dirname -- "$env_file")" && pwd)/$(basename -- "$env_file")
# .env.ci is generated by the sam_rebuild development Compose helper and is a
# shell-compatible, credential-bearing local file. Never print its contents.
set -a
# shellcheck disable=SC1090
. "$env_file"
set +a
: "${POSTGRES_HOST:?POSTGRES_HOST is required in $env_file}"
: "${POSTGRES_PORT:?POSTGRES_PORT is required in $env_file}"
: "${POSTGRES_DB:?POSTGRES_DB is required in $env_file}"
: "${POSTGRES_USER:?POSTGRES_USER is required in $env_file}"
: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required in $env_file}"
: "${REDIS_URL:?REDIS_URL is required in $env_file}"
node_tag=$(printf '%s' "${CI_NODE_INDEX:-local}" | tr -cd 'a-zA-Z0-9_' | tr 'A-Z' 'a-z' | cut -c1-8)
run_id=$(date -u +%m%d%H%M%S)_$$_${node_tag}
export CONDITIONAL_INTEGRATION_RUN_ID=$run_id
schema_ownership_token="sammo-conditional-integration:$run_id"
supported_registry_modes="core create_general gateway_runtime immediate_action npc_possession reference_live_sortie reference_npc_possession select_pool"
term_grace_seconds=${CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS:-10}
case "$term_grace_seconds" in
''|*[!0-9]*)
echo "CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS must be an integer from 1 to 60" >&2
exit 64
;;
esac
if [ "$term_grace_seconds" -lt 1 ] || [ "$term_grace_seconds" -gt 60 ]; then
echo "CONDITIONAL_INTEGRATION_TERM_GRACE_SECONDS must be an integer from 1 to 60" >&2
exit 64
fi
integration_schema=${CONDITIONAL_INTEGRATION_SCHEMA:-ci_${run_id}_integration}
scenario_schema=${SCENARIO_SEED_INTEGRATION_SCHEMA:-ci_${run_id}_scenario_seed}
npc_possession_schema=${NPC_POSSESSION_INTEGRATION_SCHEMA:-ci_${run_id}_npc_possession_integration}
create_general_schema=${CREATE_GENERAL_INTEGRATION_SCHEMA:-ci_${run_id}_create_general_integration}
select_pool_schema=${SELECT_POOL_INTEGRATION_SCHEMA:-ci_${run_id}_select_pool_integration}
immediate_action_schema=${IMMEDIATE_ACTION_INTEGRATION_SCHEMA:-ci_${run_id}_immediate_action_integration}
gateway_runtime_schema=${GATEWAY_RUNTIME_INTEGRATION_SCHEMA:-ci_${run_id}_gateway_runtime_integration}
npc_possession_differential_schema=${NPC_POSSESSION_DIFFERENTIAL_SCHEMA:-ci_${run_id}_npc_possession_differential}
live_sortie_schema=${LIVE_SORTIE_PERSISTENCE_SCHEMA:-ci_${run_id}_live_sortie_persistence}
for schema in \
"$integration_schema" \
"$scenario_schema" \
"$npc_possession_schema" \
"$create_general_schema" \
"$select_pool_schema" \
"$immediate_action_schema" \
"$gateway_runtime_schema" \
"$npc_possession_differential_schema" \
"$live_sortie_schema"; do
case "$schema" in
''|[!a-z_]*|*[!a-z0-9_]*)
echo "integration schema must be a lowercase PostgreSQL identifier: $schema" >&2
exit 64
;;
esac
if [ "${#schema}" -gt 63 ]; then
echo "integration schema exceeds PostgreSQL's 63-byte identifier limit: $schema" >&2
exit 64
fi
done
report_dir=$(mktemp -d)
summary_file="$report_dir/summary.tsv"
validated_registry_file="$report_dir/validated-registry.tsv"
active_process_group=
cleanup_resources_started=0
build_database_url() {
SCHEMA_NAME=$1 node --input-type=module -e '
const required = [
"POSTGRES_HOST",
"POSTGRES_PORT",
"POSTGRES_DB",
"POSTGRES_USER",
"POSTGRES_PASSWORD",
"SCHEMA_NAME",
];
for (const name of required) {
if (!process.env[name]) {
throw new Error(`missing ${name}`);
}
}
const url = new URL("postgresql://localhost");
url.hostname = process.env.POSTGRES_HOST;
url.port = process.env.POSTGRES_PORT;
url.username = process.env.POSTGRES_USER;
url.password = process.env.POSTGRES_PASSWORD;
url.pathname = `/${process.env.POSTGRES_DB}`;
url.searchParams.set("schema", process.env.SCHEMA_NAME);
process.stdout.write(url.href);
'
}
base_database_url=$(build_database_url public)
create_owned_schema() {
schema=$1
SCHEMA_NAME=$schema SCHEMA_OWNERSHIP_TOKEN=$schema_ownership_token \
DATABASE_URL=$base_database_url \
pnpm --filter @sammo-ts/infra exec node --input-type=module -e '
import pg from "pg";
const client = new pg.Client({ connectionString: process.env.DATABASE_URL });
const quoteIdentifier = (value) => `"${value.replaceAll("\"", "\"\"")}"`;
const apostrophe = String.fromCharCode(39);
const quoteLiteral = (value) =>
`${apostrophe}${value.replaceAll(apostrophe, apostrophe.repeat(2))}${apostrophe}`;
await client.connect();
try {
await client.query("BEGIN");
await client.query(`CREATE SCHEMA ${quoteIdentifier(process.env.SCHEMA_NAME)}`);
await client.query(
`COMMENT ON SCHEMA ${quoteIdentifier(process.env.SCHEMA_NAME)} IS ${quoteLiteral(
process.env.SCHEMA_OWNERSHIP_TOKEN
)}`
);
await client.query("COMMIT");
} catch (error) {
await client.query("ROLLBACK");
throw error;
} finally {
await client.end();
}
'
}
drop_owned_schemas() {
SCHEMA_OWNERSHIP_TOKEN=$schema_ownership_token DATABASE_URL=$base_database_url \
pnpm --filter @sammo-ts/infra exec node --input-type=module -e '
import pg from "pg";
const client = new pg.Client({ connectionString: process.env.DATABASE_URL });
const quoteIdentifier = (value) => `"${value.replaceAll("\"", "\"\"")}"`;
await client.connect();
try {
const result = await client.query(
`SELECT nspname
FROM pg_namespace
WHERE obj_description(oid, $$pg_namespace$$) = $1
ORDER BY nspname`,
[process.env.SCHEMA_OWNERSHIP_TOKEN]
);
for (const { nspname } of result.rows) {
await client.query(`DROP SCHEMA ${quoteIdentifier(nspname)} CASCADE`);
}
} finally {
await client.end();
}
' >/dev/null
}
delete_owned_redis_keys() {
CONDITIONAL_INTEGRATION_RUN_ID=$run_id REDIS_URL=$REDIS_URL \
pnpm --filter @sammo-ts/infra exec node --input-type=module -e '
import { createClient } from "redis";
const client = createClient({ url: process.env.REDIS_URL });
const runId = process.env.CONDITIONAL_INTEGRATION_RUN_ID;
const patterns = [
`sammo:game:*:che:security-http-${runId}:*`,
`sammo:game:*:che:nation-html-${runId}:*`,
`sammo:che:battle-sim-e2e-${runId}-*:battle-sim:*`,
];
await client.connect();
try {
for (const pattern of patterns) {
for await (const keys of client.scanIterator({ MATCH: pattern, COUNT: 100 })) {
if (keys.length > 0) {
await client.del(keys);
}
}
}
} finally {
await client.quit();
}
' >/dev/null
}
terminate_active_process_group() {
[ -n "$active_process_group" ] || return 0
process_group=$active_process_group
kill -TERM "-$process_group" 2>/dev/null || true
waited_seconds=0
while kill -0 "-$process_group" 2>/dev/null &&
[ "$waited_seconds" -lt "$term_grace_seconds" ]; do
sleep 1
waited_seconds=$((waited_seconds + 1))
done
if kill -0 "-$process_group" 2>/dev/null; then
echo "conditional integration process group did not stop after ${term_grace_seconds}s; sending SIGKILL" >&2
kill -KILL "-$process_group" 2>/dev/null || true
fi
wait "$process_group" 2>/dev/null || true
active_process_group=
}
print_summary() {
if [ ! -s "$summary_file" ]; then
echo "conditional integration summary: 0 passed files, 0 skipped files, 0 failed files"
return
fi
awk -F '\t' '
{
passed += $2;
skipped += $3;
failed += $4;
}
END {
printf "conditional integration summary: %d passed files, %d skipped files, %d failed files\n",
passed, skipped, failed;
}
' "$summary_file"
}
handle_exit() {
exit_status=$?
trap - EXIT HUP INT TERM
terminate_active_process_group
if [ "$cleanup_resources_started" -eq 1 ]; then
if ! drop_owned_schemas && [ "$exit_status" -eq 0 ]; then
exit_status=1
fi
if ! delete_owned_redis_keys && [ "$exit_status" -eq 0 ]; then
exit_status=1
fi
fi
print_summary
rm -rf "$report_dir"
exit "$exit_status"
}
trap handle_exit EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
validate_marker_registry() {
discovered_file="$report_dir/discovered-markers.txt"
registered_file="$report_dir/registered-markers.txt"
registry_errors_file="$report_dir/registry-errors.txt"
missing_file="$report_dir/unregistered-markers.txt"
stale_file="$report_dir/stale-markers.txt"
(
cd "$workspace_root"
rg -o --no-filename \
'process\.env\.[A-Z0-9_]+_DATABASE_URL' \
app/game-api/test \
app/game-engine/test \
tools/integration-tests/test \
-g '*.integration.test.ts' |
sed 's/process\.env\.//' |
sort -u
) >"$discovered_file"
: >"$validated_registry_file"
awk -F '\t' -v supported_modes="$supported_registry_modes" \
-v validated_registry_file="$validated_registry_file" '
BEGIN {
split(supported_modes, mode_names, " ");
for (mode_index in mode_names) {
allowed_modes[mode_names[mode_index]] = 1;
}
}
/^#/ || /^[[:space:]]*$/ { next }
NF != 2 {
printf "line %d must contain exactly one tab-separated marker and mode\n", NR;
failed = 1;
next;
}
$1 !~ /^[A-Z][A-Z0-9_]*_DATABASE_URL$/ {
printf "line %d has invalid marker: %s\n", NR, $1;
failed = 1;
}
!($2 in allowed_modes) {
printf "line %d has unsupported execution mode: %s\n", NR, $2;
failed = 1;
}
seen[$1]++ {
printf "line %d duplicates marker: %s\n", NR, $1;
failed = 1;
}
{
print $1 "\t" $2 > validated_registry_file;
}
END { exit failed }
' "$registry_file" >"$registry_errors_file" || {
echo "invalid conditional integration marker registry:" >&2
sed 's/^/ /' "$registry_errors_file" >&2
exit 65
}
sort -u "$validated_registry_file" -o "$validated_registry_file"
cut -f1 "$validated_registry_file" >"$registered_file"
comm -23 "$discovered_file" "$registered_file" >"$missing_file"
if [ -s "$missing_file" ]; then
echo "unregistered conditional database marker(s):" >&2
sed 's/^/ /' "$missing_file" >&2
exit 65
fi
comm -13 "$discovered_file" "$registered_file" >"$stale_file"
if [ -s "$stale_file" ]; then
echo "stale conditional database marker(s):" >&2
sed 's/^/ /' "$stale_file" >&2
exit 65
fi
}
markers_for_mode() {
mode=$1
awk -F '\t' -v mode="$mode" '$2 == mode { print $1 }' "$validated_registry_file" |
paste -sd '|' -
}
record_vitest_result() {
label=$1
result_file=$2
counts=$(node --input-type=module -e '
import fs from "node:fs";
const result = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
const files = Array.isArray(result.testResults) ? result.testResults : [];
const isSkipped = ({ assertionResults = [], status }) =>
status === "pending" ||
(assertionResults.length > 0 &&
assertionResults.every(({ status: assertionStatus }) =>
assertionStatus === "pending" || assertionStatus === "todo"
));
const skipped = files.filter(isSkipped).length;
const failed = files.filter(({ status }) => status === "failed").length;
const passed = files.length - skipped - failed;
process.stdout.write(`${passed}\t${skipped}\t${failed}`);
' "$result_file")
printf '%s\t%s\n' "$label" "$counts" >>"$summary_file"
skipped=$(printf '%s' "$counts" | cut -f2)
if [ "$skipped" -ne 0 ]; then
echo "$label left $skipped integration test file(s) skipped" >&2
return 1
fi
}
run_vitest() {
package_dir=$1
label=$2
shift 2
result_file="$report_dir/$(printf '%s' "$label" | tr -cd 'a-zA-Z0-9_' | tr 'A-Z' 'a-z').json"
echo "running $label tests in $package_dir"
(
cd "$workspace_root/$package_dir"
exec setsid pnpm exec vitest run \
--no-file-parallelism \
--maxWorkers=1 \
--reporter=default \
--reporter=json \
--outputFile.json="$result_file" \
"$@"
) &
active_process_group=$!
test_status=0
wait "$active_process_group" || test_status=$?
active_process_group=
result_status=0
if [ -f "$result_file" ]; then
record_vitest_result "$label" "$result_file" || result_status=$?
else
printf '%s\t0\t0\t1\n' "$label" >>"$summary_file"
result_status=1
fi
if [ "$test_status" -eq 0 ] && [ "$result_status" -ne 0 ]; then
test_status=$result_status
fi
return "$test_status"
}
run_marked_tests() {
package_dir=$1
marker=$2
label=$3
test_files=$(cd "$workspace_root/$package_dir" && rg -l "$marker" test -g '*.integration.test.ts' | sort)
if [ -z "$test_files" ]; then
echo "no $label tests found under $package_dir" >&2
exit 65
fi
# Integration test paths cannot contain whitespace in this repository.
# shellcheck disable=SC2086
run_vitest "$package_dir" "$label" $test_files
}
run_redis_only_tests() {
package_dir=app/game-api
database_marker=$1
test_files=$(
cd "$workspace_root/$package_dir"
redis_files=$(rg -l 'process\.env\.REDIS_URL' test -g '*.integration.test.ts' | sort)
# Files already selected through a database marker run once in that
# database group, where Redis is also available.
# shellcheck disable=SC2086
rg --files-without-match "$database_marker" $redis_files
)
if [ -z "$test_files" ]; then
echo "no Redis-only integration tests found under $package_dir" >&2
exit 65
fi
# shellcheck disable=SC2086
run_vitest "$package_dir" "game_api_redis" $test_files
}
export PATH
cd "$workspace_root"
validate_marker_registry
pnpm install --frozen-lockfile
pnpm --filter @sammo-ts/infra prisma:generate
pnpm --filter @sammo-ts/common build
pnpm --filter @sammo-ts/infra build
cleanup_resources_started=1
create_owned_schema "$integration_schema"
create_owned_schema "$npc_possession_schema"
create_owned_schema "$scenario_schema"
database_url=$(build_database_url "$integration_schema")
export POSTGRES_SCHEMA=$integration_schema
export DATABASE_URL=$database_url
export INPUT_EVENT_DATABASE_URL=$database_url
export GENERAL_LIFECYCLE_DATABASE_URL=$database_url
export TURN_DAEMON_LEASE_DATABASE_URL=$database_url
export TURN_DIFFERENTIAL_DATABASE_URL=$database_url
export RESERVED_TURN_DATABASE_URL=$database_url
pnpm --filter @sammo-ts/infra prisma:db:push:game
core_database_markers=$(markers_for_mode core)
run_marked_tests app/game-api "$core_database_markers" "game_api_postgresql"
run_marked_tests app/game-engine "$core_database_markers" "game_engine_postgresql"
run_marked_tests tools/integration-tests "$core_database_markers" "snapshot_postgresql"
create_owned_schema "$create_general_schema"
create_general_database_url=$(build_database_url "$create_general_schema")
(
export POSTGRES_SCHEMA=$create_general_schema
export DATABASE_URL=$create_general_database_url
pnpm --filter @sammo-ts/infra prisma:db:push:game
)
export CREATE_GENERAL_DATABASE_URL=$create_general_database_url
run_marked_tests app/game-api \
"$(markers_for_mode create_general)" \
"create_general_postgresql"
create_owned_schema "$select_pool_schema"
select_pool_database_url=$(build_database_url "$select_pool_schema")
(
export POSTGRES_SCHEMA=$select_pool_schema
export DATABASE_URL=$select_pool_database_url
pnpm --filter @sammo-ts/infra prisma:db:push:game
)
export SELECT_POOL_DATABASE_URL=$select_pool_database_url
run_marked_tests app/game-api \
"$(markers_for_mode select_pool)" \
"select_pool_api_postgresql"
run_marked_tests app/game-engine \
"$(markers_for_mode select_pool)" \
"select_pool_engine_postgresql"
create_owned_schema "$immediate_action_schema"
immediate_action_database_url=$(build_database_url "$immediate_action_schema")
(
export POSTGRES_SCHEMA=$immediate_action_schema
export DATABASE_URL=$immediate_action_database_url
pnpm --filter @sammo-ts/infra prisma:db:push:game
)
export IMMEDIATE_ACTION_DATABASE_URL=$immediate_action_database_url
run_marked_tests app/game-engine \
"$(markers_for_mode immediate_action)" \
"immediate_action_postgresql"
create_owned_schema "$gateway_runtime_schema"
gateway_runtime_database_url=$(build_database_url "$gateway_runtime_schema")
(
export POSTGRES_SCHEMA=$gateway_runtime_schema
export DATABASE_URL=$gateway_runtime_database_url
export GATEWAY_DATABASE_URL=$gateway_runtime_database_url
pnpm --filter @sammo-ts/infra prisma:db:push:gateway
)
export GATEWAY_RUNTIME_ACTION_DATABASE_URL=$gateway_runtime_database_url
run_marked_tests app/game-engine \
"$(markers_for_mode gateway_runtime)" \
"gateway_runtime_postgresql"
npc_possession_database_url=$(build_database_url "$npc_possession_schema")
(
export POSTGRES_SCHEMA=$npc_possession_schema
export DATABASE_URL=$npc_possession_database_url
pnpm --filter @sammo-ts/infra prisma:migrate:deploy:game
pnpm --filter @sammo-ts/infra prisma:migrate:deploy:game
)
export NPC_POSSESSION_DATABASE_URL=$npc_possession_database_url
run_marked_tests app/game-api \
"$(markers_for_mode npc_possession)" \
"npc_possession_postgresql"
if [ "${TURN_DIFFERENTIAL_REFERENCE:-}" = "1" ]; then
reference_workspace_root=${TURN_DIFFERENTIAL_WORKSPACE_ROOT:-"$workspace_root/.."}
reference_stack=${TURN_DIFFERENTIAL_STACK_DIR:-"$reference_workspace_root/docker_compose_files/reference"}
if [ ! -f "$reference_stack/compose.yml" ]; then
echo "reference compose file is missing; set TURN_DIFFERENTIAL_WORKSPACE_ROOT or TURN_DIFFERENTIAL_STACK_DIR" >&2
exit 69
fi
if [ ! -f "$reference_workspace_root/ref/sam/d_setting/RootDB.php" ] ||
[ ! -f "$reference_workspace_root/ref/sam/hwe/d_setting/DB.php" ]; then
echo "reference runtime database configuration is missing under $reference_workspace_root/ref/sam" >&2
exit 69
fi
if ! docker compose -f "$reference_stack/compose.yml" ps --status running --services | grep -qx db ||
! docker compose -f "$reference_stack/compose.yml" exec -T db healthcheck.sh --connect --innodb_initialized \
>/dev/null 2>&1; then
echo "TURN_DIFFERENTIAL_REFERENCE=1 requires a healthy bootstrapped reference db stack" >&2
exit 69
fi
reference_tables=$(
docker compose -f "$reference_stack/compose.yml" exec -T db sh -c '
export MYSQL_PWD="$(cat /run/secrets/db_root_password)"
mariadb --batch --skip-column-names -u root -e "
SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA = '\''$MARIADB_DATABASE'\'' AND TABLE_NAME = '\''member'\'';
SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA = '\''$REF_HWE_DB_NAME'\'' AND TABLE_NAME IN ('\''general'\'', '\''select_npc_token'\'');
"
' 2>/dev/null
) || {
echo "failed to inspect reference database bootstrap state" >&2
exit 69
}
if [ "$reference_tables" != "$(printf '1\n2')" ]; then
echo "reference root/HWE databases are not bootstrapped" >&2
exit 69
fi
create_owned_schema "$npc_possession_differential_schema"
npc_possession_differential_database_url=$(build_database_url "$npc_possession_differential_schema")
(
export POSTGRES_SCHEMA=$npc_possession_differential_schema
export DATABASE_URL=$npc_possession_differential_database_url
export NPC_POSSESSION_DIFFERENTIAL_DATABASE_URL=$npc_possession_differential_database_url
pnpm --filter @sammo-ts/infra prisma:db:push:game
)
export NPC_POSSESSION_DIFFERENTIAL_DATABASE_URL=$npc_possession_differential_database_url
export TURN_DIFFERENTIAL_WORKSPACE_ROOT=$reference_workspace_root
export TURN_DIFFERENTIAL_STACK_DIR=$reference_stack
run_marked_tests tools/integration-tests \
"$(markers_for_mode reference_npc_possession)" \
"npc_possession_reference"
create_owned_schema "$live_sortie_schema"
live_sortie_database_url=$(build_database_url "$live_sortie_schema")
(
export POSTGRES_SCHEMA=$live_sortie_schema
export DATABASE_URL=$live_sortie_database_url
pnpm --filter @sammo-ts/infra prisma:db:push:game
)
export POSTGRES_SCHEMA=$live_sortie_schema
export DATABASE_URL=$live_sortie_database_url
export LIVE_SORTIE_PERSISTENCE_DATABASE_URL=$live_sortie_database_url
run_marked_tests tools/integration-tests \
"$(markers_for_mode reference_live_sortie)" \
"live_sortie_postgresql"
export POSTGRES_SCHEMA=$integration_schema
export DATABASE_URL=$database_url
fi
run_redis_only_tests "$core_database_markers"
scenario_database_url=$(build_database_url "$scenario_schema")
export POSTGRES_SCHEMA=$scenario_schema
export DATABASE_URL=$scenario_database_url
pnpm --filter @sammo-ts/infra prisma:db:push:game
run_vitest app/game-engine "scenario_seed_postgresql" test/scenarioSeeder.test.ts
echo "conditional PostgreSQL and Redis integration tests passed"