feat: manage uploaded image retention
This commit is contained in:
@@ -2,6 +2,10 @@
|
|||||||
IMAGE_BIND_ADDRESS=127.0.0.1
|
IMAGE_BIND_ADDRESS=127.0.0.1
|
||||||
IMAGE_PORT=8191
|
IMAGE_PORT=8191
|
||||||
IMAGE_REPOSITORY_PATH=/home/letrhee/sam_rebuild/image
|
IMAGE_REPOSITORY_PATH=/home/letrhee/sam_rebuild/image
|
||||||
|
# Persistent service state and SQLite metadata, normally on SSD storage.
|
||||||
|
IMAGE_RUNTIME_PATH=./runtime-data
|
||||||
|
# Optional separate bind for uploaded images, for example a ZFS dataset.
|
||||||
|
IMAGE_UPLOAD_PATH=./runtime-data/uploads
|
||||||
IMAGE_UID=1000
|
IMAGE_UID=1000
|
||||||
IMAGE_GID=1000
|
IMAGE_GID=1000
|
||||||
# Source CIDRs observed inside image-web. A host-native proxy normally appears
|
# Source CIDRs observed inside image-web. A host-native proxy normally appears
|
||||||
@@ -17,7 +21,16 @@ IMAGE_PUBLIC_BASES=https://sam.hided.net/image,https://sam-image.hided.net
|
|||||||
|
|
||||||
GITEA_WEBHOOK_SECRET_FILE=./secrets/gitea_webhook_secret
|
GITEA_WEBHOOK_SECRET_FILE=./secrets/gitea_webhook_secret
|
||||||
IMAGE_ADMIN_SECRET_FILE=./secrets/image_admin_secret
|
IMAGE_ADMIN_SECRET_FILE=./secrets/image_admin_secret
|
||||||
|
IMAGE_ADMIN_PANEL_PASSWORD_FILE=./secrets/image_admin_panel_password
|
||||||
|
IMAGE_ADMIN_PANEL_SESSION_SECRET_FILE=./secrets/image_admin_panel_session_secret
|
||||||
IMAGE_SYNC_CORE_SECRET_FILE=./secrets/image_sync_core_secret
|
IMAGE_SYNC_CORE_SECRET_FILE=./secrets/image_sync_core_secret
|
||||||
IMAGE_SYNC_CORE2026_SECRET_FILE=./secrets/image_sync_core2026_secret
|
IMAGE_SYNC_CORE2026_SECRET_FILE=./secrets/image_sync_core2026_secret
|
||||||
IMAGE_UPLOAD_CORE_SECRET_FILE=./secrets/image_upload_core_secret
|
IMAGE_UPLOAD_CORE_SECRET_FILE=./secrets/image_upload_core_secret
|
||||||
IMAGE_UPLOAD_CORE2026_SECRET_FILE=./secrets/image_upload_core2026_secret
|
IMAGE_UPLOAD_CORE2026_SECRET_FILE=./secrets/image_upload_core2026_secret
|
||||||
|
|
||||||
|
# Account icons are permanent. These values apply only to Tiptap content images.
|
||||||
|
IMAGE_CONTENT_RETENTION_DAYS=730
|
||||||
|
IMAGE_CONTENT_QUARANTINE_DAYS=30
|
||||||
|
IMAGE_ASSET_MAINTENANCE_SECONDS=21600
|
||||||
|
IMAGE_ASSET_TOUCH_FLUSH_SECONDS=60
|
||||||
|
IMAGE_ADMIN_PANEL_SESSION_HOURS=8
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ Never give either caller `image_admin_secret`, which also authorizes explicit
|
|||||||
branch changes. This fallback handles webhook delivery outages; if the image
|
branch changes. This fallback handles webhook delivery outages; if the image
|
||||||
service itself is stopped, restore it and run the caller command again.
|
service itself is stopped, restore it and run the caller command again.
|
||||||
|
|
||||||
### Short-lived user-icon uploads
|
### User uploads
|
||||||
|
|
||||||
Core and Core2026 can store validated account icons and editor attachments
|
Core and Core2026 can store validated account icons and editor attachments
|
||||||
through this service with
|
through this service with
|
||||||
@@ -158,10 +158,59 @@ the matching `image_upload_core_secret` or `image_upload_core2026_secret` on the
|
|||||||
game server. The shared secrets stay server-side in Docker secrets; they are
|
game server. The shared secrets stay server-side in Docker secrets; they are
|
||||||
not returned to browsers or forwarded to Cloudflare.
|
not returned to browsers or forwarded to Cloudflare.
|
||||||
|
|
||||||
Run `deploy/scripts/init-secrets.sh` before the first Compose start so
|
Run `deploy/scripts/init-secrets.sh` before the first Compose start so the upload
|
||||||
`runtime-data/uploads` exists with permissions that allow the hook container to
|
directory exists with permissions that allow the hook container to write and
|
||||||
write and the Nginx container to read. Back up this directory independently of
|
the Nginx container to read. `IMAGE_UPLOAD_PATH` can point only the uploaded
|
||||||
the Git repository when moving servers.
|
images at a separate ZFS dataset. Keep `IMAGE_RUNTIME_PATH` on SSD storage for
|
||||||
|
the small SQLite database and service state. Back up both paths independently
|
||||||
|
of the Git repository when moving servers.
|
||||||
|
|
||||||
|
### Retention and image administration
|
||||||
|
|
||||||
|
Account icons below `/icons/users/` are permanent. They are inventoried in the
|
||||||
|
metadata database but never become deletion candidates. Tiptap images below
|
||||||
|
`/uploads/` become candidates after 730 days without a request reaching the
|
||||||
|
origin Nginx. Nginx mirrors a small internal notification to Node for those
|
||||||
|
requests; Node deduplicates paths in memory and writes each path to SQLite at
|
||||||
|
most once per flush interval (60 seconds by default).
|
||||||
|
|
||||||
|
Cloudflare and browser cache hits do not necessarily reach the origin. The
|
||||||
|
two-year interval is therefore an operational expiry policy, not proof that an
|
||||||
|
old HTML document no longer contains the URL. Confirm that the Cloudflare edge
|
||||||
|
TTL does not exceed one year before enabling this policy.
|
||||||
|
|
||||||
|
Candidates remain publicly available until an administrator quarantines them.
|
||||||
|
Quarantine moves the file into a hidden directory on the same upload bind and
|
||||||
|
is reversible. Permanent deletion is disabled until the 30-day quarantine
|
||||||
|
period has elapsed. A request received while an image is only a candidate
|
||||||
|
returns it to active state; requests do not restore quarantined files.
|
||||||
|
|
||||||
|
Open the minimal administration panel at:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://sam-image.hided.net/admin/
|
||||||
|
```
|
||||||
|
|
||||||
|
Log in with the value in the ignored `secrets/image_admin_panel_password` file.
|
||||||
|
The panel can filter and preview assets, show storage and retention timestamps,
|
||||||
|
quarantine candidates, restore quarantined files, and permanently delete files
|
||||||
|
whose grace period has elapsed. It uses a separate short-lived, HttpOnly,
|
||||||
|
SameSite session; the stronger `image_admin_secret` used for branch deployment
|
||||||
|
is never sent to the browser. Rotate the panel password by replacing its secret
|
||||||
|
file and recreating `image-hook`.
|
||||||
|
|
||||||
|
Retention settings are available in `.env`:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
IMAGE_CONTENT_RETENTION_DAYS=730
|
||||||
|
IMAGE_CONTENT_QUARANTINE_DAYS=30
|
||||||
|
IMAGE_ASSET_TOUCH_FLUSH_SECONDS=60
|
||||||
|
IMAGE_ASSET_MAINTENANCE_SECONDS=21600
|
||||||
|
```
|
||||||
|
|
||||||
|
The SQLite database uses WAL and synchronous writes. Back it up with the service
|
||||||
|
stopped or with a SQLite-aware backup; do not copy only the main `.sqlite3` file
|
||||||
|
while the service is running because committed data can still be in `-wal`.
|
||||||
|
|
||||||
Legacy HTTP mutation is disabled by default. An emergency PHP rollback must
|
Legacy HTTP mutation is disabled by default. An emergency PHP rollback must
|
||||||
first stop `image-hook`, then create the ignored `hook/legacy-enabled` sentinel
|
first stop `image-hook`, then create the ignored `hook/legacy-enabled` sentinel
|
||||||
@@ -181,9 +230,13 @@ curl -fsS https://sam-image.hided.net/game/back.jpg -o /dev/null
|
|||||||
curl -fsS https://sam-image.hided.net/image/icons/default.jpg -o /dev/null
|
curl -fsS https://sam-image.hided.net/image/icons/default.jpg -o /dev/null
|
||||||
```
|
```
|
||||||
|
|
||||||
Nginx serves only `game/`, `icons/`, `hook/list.json`, and
|
Then log in to `/admin/` and verify that an existing account icon is shown as
|
||||||
`hook/inventory.v2.json`. The API inventory additionally reports the deployed
|
permanent (`user-icons`, `active`) and a Tiptap upload is shown as `content`.
|
||||||
branch, full commit, generation time, asset list, and both public base URLs.
|
|
||||||
|
Nginx serves the static `game/`, `icons/`, and `uploads/` paths, the two public
|
||||||
|
inventory files, and the authenticated `/admin/` panel. The API inventory
|
||||||
|
additionally reports the deployed branch, full commit, generation time, asset
|
||||||
|
list, and both public base URLs.
|
||||||
|
|
||||||
### Explicit branch deployment
|
### Explicit branch deployment
|
||||||
|
|
||||||
@@ -195,7 +248,9 @@ signed administration command:
|
|||||||
./deploy/scripts/admin-deploy.sh <branch> [expected-commit]
|
./deploy/scripts/admin-deploy.sh <branch> [expected-commit]
|
||||||
```
|
```
|
||||||
|
|
||||||
The administration route is not exposed through the public reverse proxy.
|
The signed branch-deployment route under `/v1/admin/` is not exposed through
|
||||||
|
the public reverse proxy. It is separate from the password-authenticated image
|
||||||
|
management panel at `/admin/`.
|
||||||
|
|
||||||
### Tests and rollback
|
### Tests and rollback
|
||||||
|
|
||||||
|
|||||||
+21
-4
@@ -4,7 +4,7 @@ services:
|
|||||||
image-hook:
|
image-hook:
|
||||||
build:
|
build:
|
||||||
context: ./node-hook
|
context: ./node-hook
|
||||||
image: sam-image-hook:1.2.0
|
image: sam-image-hook:1.3.0
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
user: "${IMAGE_UID:-1000}:${IMAGE_GID:-1000}"
|
user: "${IMAGE_UID:-1000}:${IMAGE_GID:-1000}"
|
||||||
read_only: true
|
read_only: true
|
||||||
@@ -20,8 +20,16 @@ services:
|
|||||||
IMAGE_STATE_PATH: /var/lib/image-hook/state.json
|
IMAGE_STATE_PATH: /var/lib/image-hook/state.json
|
||||||
IMAGE_UPLOAD_ROOT: /var/lib/image-hook/uploads
|
IMAGE_UPLOAD_ROOT: /var/lib/image-hook/uploads
|
||||||
IMAGE_UPLOAD_STATE_PATH: /var/lib/image-hook/upload-state.json
|
IMAGE_UPLOAD_STATE_PATH: /var/lib/image-hook/upload-state.json
|
||||||
|
IMAGE_ASSET_DB_PATH: /var/lib/image-hook/image-assets.sqlite3
|
||||||
|
IMAGE_CONTENT_RETENTION_DAYS: ${IMAGE_CONTENT_RETENTION_DAYS:-730}
|
||||||
|
IMAGE_CONTENT_QUARANTINE_DAYS: ${IMAGE_CONTENT_QUARANTINE_DAYS:-30}
|
||||||
|
IMAGE_ASSET_MAINTENANCE_SECONDS: ${IMAGE_ASSET_MAINTENANCE_SECONDS:-21600}
|
||||||
|
IMAGE_ASSET_TOUCH_FLUSH_SECONDS: ${IMAGE_ASSET_TOUCH_FLUSH_SECONDS:-60}
|
||||||
|
IMAGE_ADMIN_PANEL_SESSION_HOURS: ${IMAGE_ADMIN_PANEL_SESSION_HOURS:-8}
|
||||||
GITEA_WEBHOOK_SECRET_FILE: /run/secrets/gitea_webhook_secret
|
GITEA_WEBHOOK_SECRET_FILE: /run/secrets/gitea_webhook_secret
|
||||||
IMAGE_ADMIN_SECRET_FILE: /run/secrets/image_admin_secret
|
IMAGE_ADMIN_SECRET_FILE: /run/secrets/image_admin_secret
|
||||||
|
IMAGE_ADMIN_PANEL_PASSWORD_FILE: /run/secrets/image_admin_panel_password
|
||||||
|
IMAGE_ADMIN_PANEL_SESSION_SECRET_FILE: /run/secrets/image_admin_panel_session_secret
|
||||||
IMAGE_SYNC_CLIENT_SECRET_FILES: core=/run/secrets/image_sync_core_secret,core2026=/run/secrets/image_sync_core2026_secret
|
IMAGE_SYNC_CLIENT_SECRET_FILES: core=/run/secrets/image_sync_core_secret,core2026=/run/secrets/image_sync_core2026_secret
|
||||||
IMAGE_UPLOAD_CLIENT_SECRET_FILES: core=/run/secrets/image_upload_core_secret,core2026=/run/secrets/image_upload_core2026_secret
|
IMAGE_UPLOAD_CLIENT_SECRET_FILES: core=/run/secrets/image_upload_core_secret,core2026=/run/secrets/image_upload_core2026_secret
|
||||||
MAX_UPLOAD_BYTES: "51200"
|
MAX_UPLOAD_BYTES: "51200"
|
||||||
@@ -30,10 +38,15 @@ services:
|
|||||||
- type: bind
|
- type: bind
|
||||||
source: ${IMAGE_REPOSITORY_PATH:-.}
|
source: ${IMAGE_REPOSITORY_PATH:-.}
|
||||||
target: /data/image
|
target: /data/image
|
||||||
- ./runtime-data:/var/lib/image-hook
|
- ${IMAGE_RUNTIME_PATH:-./runtime-data}:/var/lib/image-hook
|
||||||
|
- type: bind
|
||||||
|
source: ${IMAGE_UPLOAD_PATH:-./runtime-data/uploads}
|
||||||
|
target: /var/lib/image-hook/uploads
|
||||||
secrets:
|
secrets:
|
||||||
- gitea_webhook_secret
|
- gitea_webhook_secret
|
||||||
- image_admin_secret
|
- image_admin_secret
|
||||||
|
- image_admin_panel_password
|
||||||
|
- image_admin_panel_session_secret
|
||||||
- image_sync_core_secret
|
- image_sync_core_secret
|
||||||
- image_sync_core2026_secret
|
- image_sync_core2026_secret
|
||||||
- image_upload_core_secret
|
- image_upload_core_secret
|
||||||
@@ -57,7 +70,7 @@ services:
|
|||||||
image-web:
|
image-web:
|
||||||
build:
|
build:
|
||||||
context: ./deploy/nginx
|
context: ./deploy/nginx
|
||||||
image: sam-image-web:1.2.0
|
image: sam-image-web:1.3.0
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
depends_on:
|
depends_on:
|
||||||
image-hook:
|
image-hook:
|
||||||
@@ -73,7 +86,7 @@ services:
|
|||||||
target: /srv/image
|
target: /srv/image
|
||||||
read_only: true
|
read_only: true
|
||||||
- type: bind
|
- type: bind
|
||||||
source: ./runtime-data/uploads
|
source: ${IMAGE_UPLOAD_PATH:-./runtime-data/uploads}
|
||||||
target: /srv/uploads
|
target: /srv/uploads
|
||||||
read_only: true
|
read_only: true
|
||||||
tmpfs:
|
tmpfs:
|
||||||
@@ -102,6 +115,10 @@ secrets:
|
|||||||
file: ${GITEA_WEBHOOK_SECRET_FILE:-./secrets/gitea_webhook_secret}
|
file: ${GITEA_WEBHOOK_SECRET_FILE:-./secrets/gitea_webhook_secret}
|
||||||
image_admin_secret:
|
image_admin_secret:
|
||||||
file: ${IMAGE_ADMIN_SECRET_FILE:-./secrets/image_admin_secret}
|
file: ${IMAGE_ADMIN_SECRET_FILE:-./secrets/image_admin_secret}
|
||||||
|
image_admin_panel_password:
|
||||||
|
file: ${IMAGE_ADMIN_PANEL_PASSWORD_FILE:-./secrets/image_admin_panel_password}
|
||||||
|
image_admin_panel_session_secret:
|
||||||
|
file: ${IMAGE_ADMIN_PANEL_SESSION_SECRET_FILE:-./secrets/image_admin_panel_session_secret}
|
||||||
image_sync_core_secret:
|
image_sync_core_secret:
|
||||||
file: ${IMAGE_SYNC_CORE_SECRET_FILE:-./secrets/image_sync_core_secret}
|
file: ${IMAGE_SYNC_CORE_SECRET_FILE:-./secrets/image_sync_core_secret}
|
||||||
image_sync_core2026_secret:
|
image_sync_core2026_secret:
|
||||||
|
|||||||
@@ -82,6 +82,32 @@ http {
|
|||||||
}
|
}
|
||||||
|
|
||||||
location ^~ /v1/admin/ { return 404; }
|
location ^~ /v1/admin/ { return 404; }
|
||||||
|
|
||||||
|
location = /admin {
|
||||||
|
proxy_pass http://image-hook:8081/admin;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
}
|
||||||
|
|
||||||
|
location ^~ /admin/ {
|
||||||
|
client_max_body_size 8k;
|
||||||
|
proxy_pass http://image-hook:8081;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_request_buffering on;
|
||||||
|
}
|
||||||
|
|
||||||
|
location = /v1/internal/content-access {
|
||||||
|
internal;
|
||||||
|
proxy_pass http://image-hook:8081/v1/internal/content-access;
|
||||||
|
proxy_pass_request_body off;
|
||||||
|
proxy_set_header Content-Length "";
|
||||||
|
proxy_set_header X-Image-Path $request_uri;
|
||||||
|
access_log off;
|
||||||
|
}
|
||||||
|
|
||||||
location = /image { return 404; }
|
location = /image { return 404; }
|
||||||
location = /image/ { return 404; }
|
location = /image/ { return 404; }
|
||||||
location ^~ /image/ { rewrite ^/image/(.*)$ /$1 last; }
|
location ^~ /image/ { rewrite ^/image/(.*)$ /$1 last; }
|
||||||
@@ -105,6 +131,8 @@ http {
|
|||||||
|
|
||||||
location ~ "^/uploads/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" {
|
location ~ "^/uploads/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" {
|
||||||
alias /srv/uploads/content/$1/$2;
|
alias /srv/uploads/content/$1/$2;
|
||||||
|
mirror /v1/internal/content-access;
|
||||||
|
mirror_request_body off;
|
||||||
etag on;
|
etag on;
|
||||||
expires 1y;
|
expires 1y;
|
||||||
add_header Cache-Control "public, immutable" always;
|
add_header Cache-Control "public, immutable" always;
|
||||||
|
|||||||
@@ -4,13 +4,14 @@ set -eu
|
|||||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||||
repository_dir=$(CDPATH= cd -- "$script_dir/../.." && pwd)
|
repository_dir=$(CDPATH= cd -- "$script_dir/../.." && pwd)
|
||||||
secret_dir="$repository_dir/secrets"
|
secret_dir="$repository_dir/secrets"
|
||||||
state_dir="$repository_dir/runtime-data"
|
state_dir=${IMAGE_RUNTIME_PATH:-$repository_dir/runtime-data}
|
||||||
|
upload_dir=${IMAGE_UPLOAD_PATH:-$state_dir/uploads}
|
||||||
|
|
||||||
umask 077
|
umask 077
|
||||||
mkdir -p "$secret_dir"
|
mkdir -p "$secret_dir"
|
||||||
mkdir -p "$state_dir"
|
mkdir -p "$state_dir"
|
||||||
mkdir -p "$state_dir/uploads"
|
mkdir -p "$upload_dir"
|
||||||
for name in gitea_webhook_secret image_admin_secret image_sync_core_secret image_sync_core2026_secret image_upload_core_secret image_upload_core2026_secret; do
|
for name in gitea_webhook_secret image_admin_secret image_admin_panel_password image_admin_panel_session_secret image_sync_core_secret image_sync_core2026_secret image_upload_core_secret image_upload_core2026_secret; do
|
||||||
path="$secret_dir/$name"
|
path="$secret_dir/$name"
|
||||||
if [ ! -e "$path" ]; then
|
if [ ! -e "$path" ]; then
|
||||||
openssl rand -hex 32 > "$path"
|
openssl rand -hex 32 > "$path"
|
||||||
@@ -18,6 +19,6 @@ for name in gitea_webhook_secret image_admin_secret image_sync_core_secret image
|
|||||||
chmod 600 "$path"
|
chmod 600 "$path"
|
||||||
done
|
done
|
||||||
chmod 700 "$state_dir"
|
chmod 700 "$state_dir"
|
||||||
chmod 755 "$state_dir/uploads"
|
chmod 755 "$upload_dir"
|
||||||
|
|
||||||
echo "Secret files are ready in $secret_dir (values not printed)."
|
echo "Secret files are ready in $secret_dir (values not printed)."
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "sam-image-hook",
|
"name": "sam-image-hook",
|
||||||
"version": "1.2.0",
|
"version": "1.3.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"engines": {
|
"engines": {
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,355 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { mkdir, readFile, readdir, rename, stat, unlink } from 'node:fs/promises';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
|
import { DeploymentError } from './git-service.mjs';
|
||||||
|
|
||||||
|
const CLIENT_PATTERN = /^[a-z0-9][a-z0-9_-]{1,31}$/;
|
||||||
|
const FILE_PATTERN = /^[a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif)$/;
|
||||||
|
const CONTENT_PATH_PATTERN = /^uploads\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$/;
|
||||||
|
|
||||||
|
function publicPath(category, client, filename) {
|
||||||
|
return category === 'user-icons'
|
||||||
|
? `icons/users/${client}/${filename}`
|
||||||
|
: `uploads/${client}/${filename}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function fileLocation(root, category, client, filename) {
|
||||||
|
return join(root, category, client, filename);
|
||||||
|
}
|
||||||
|
|
||||||
|
function rowToAsset(row, now, retentionMs, quarantineMs) {
|
||||||
|
if (!row) return null;
|
||||||
|
return {
|
||||||
|
path: row.path,
|
||||||
|
category: row.category,
|
||||||
|
client: row.client,
|
||||||
|
filename: row.filename,
|
||||||
|
sizeBytes: row.size_bytes,
|
||||||
|
digest: row.digest,
|
||||||
|
createdAt: new Date(row.created_at).toISOString(),
|
||||||
|
lastSeenAt: new Date(row.last_seen_at).toISOString(),
|
||||||
|
state: row.state,
|
||||||
|
candidateAt: row.candidate_at === null ? null : new Date(row.candidate_at).toISOString(),
|
||||||
|
quarantinedAt: row.quarantined_at === null ? null : new Date(row.quarantined_at).toISOString(),
|
||||||
|
deletedAt: row.deleted_at === null ? null : new Date(row.deleted_at).toISOString(),
|
||||||
|
eligibleAt: row.category === 'content'
|
||||||
|
? new Date(row.last_seen_at + retentionMs).toISOString()
|
||||||
|
: null,
|
||||||
|
deleteAvailableAt: row.quarantined_at === null
|
||||||
|
? null
|
||||||
|
: new Date(row.quarantined_at + quarantineMs).toISOString(),
|
||||||
|
deleteAvailable: row.state === 'quarantined' && row.quarantined_at + quarantineMs <= now,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AssetStore {
|
||||||
|
constructor(config, options = {}) {
|
||||||
|
this.root = config.uploadRoot;
|
||||||
|
this.dbPath = config.assetDbPath;
|
||||||
|
this.retentionMs = config.contentRetentionMs;
|
||||||
|
this.quarantineMs = config.contentQuarantineMs;
|
||||||
|
this.maintenanceIntervalMs = config.assetMaintenanceIntervalMs;
|
||||||
|
this.touchFlushIntervalMs = config.assetTouchFlushIntervalMs;
|
||||||
|
this.now = options.now ?? (() => Date.now());
|
||||||
|
this.pendingTouches = new Set();
|
||||||
|
this.touchTimer = null;
|
||||||
|
this.maintenanceTimer = null;
|
||||||
|
this.db = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async initialize() {
|
||||||
|
await mkdir(this.root, { recursive: true });
|
||||||
|
await mkdir(dirname(this.dbPath), { recursive: true });
|
||||||
|
this.db = new DatabaseSync(this.dbPath);
|
||||||
|
this.db.exec(`
|
||||||
|
PRAGMA journal_mode = WAL;
|
||||||
|
PRAGMA synchronous = FULL;
|
||||||
|
PRAGMA busy_timeout = 5000;
|
||||||
|
CREATE TABLE IF NOT EXISTS asset (
|
||||||
|
path TEXT PRIMARY KEY,
|
||||||
|
category TEXT NOT NULL CHECK (category IN ('user-icons', 'content')),
|
||||||
|
client TEXT NOT NULL,
|
||||||
|
filename TEXT NOT NULL,
|
||||||
|
size_bytes INTEGER NOT NULL,
|
||||||
|
digest TEXT,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
last_seen_at INTEGER NOT NULL,
|
||||||
|
state TEXT NOT NULL DEFAULT 'active'
|
||||||
|
CHECK (state IN ('active', 'candidate', 'quarantined', 'deleted')),
|
||||||
|
candidate_at INTEGER,
|
||||||
|
quarantined_at INTEGER,
|
||||||
|
deleted_at INTEGER
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS asset_state_seen_idx ON asset(category, state, last_seen_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS asset_client_created_idx ON asset(client, created_at DESC);
|
||||||
|
`);
|
||||||
|
await this.#inventoryExisting();
|
||||||
|
this.markCandidates();
|
||||||
|
this.maintenanceTimer = setInterval(() => {
|
||||||
|
try {
|
||||||
|
this.flushTouches();
|
||||||
|
this.markCandidates();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(JSON.stringify({ level: 'error', message: 'asset maintenance failed', reason: error.message }));
|
||||||
|
}
|
||||||
|
}, this.maintenanceIntervalMs);
|
||||||
|
this.maintenanceTimer.unref();
|
||||||
|
}
|
||||||
|
|
||||||
|
register({ category, client, filename, body, digest, createdAt = this.now() }) {
|
||||||
|
this.#assertOpen();
|
||||||
|
if (!['user-icons', 'content'].includes(category) || !CLIENT_PATTERN.test(client) || !FILE_PATTERN.test(filename)) {
|
||||||
|
throw new DeploymentError('Invalid asset path', 400);
|
||||||
|
}
|
||||||
|
const path = publicPath(category, client, filename);
|
||||||
|
const sizeBytes = body?.length ?? 0;
|
||||||
|
const sha256 = digest ?? (body ? createHash('sha256').update(body).digest('hex') : null);
|
||||||
|
this.db.prepare(`
|
||||||
|
INSERT INTO asset(path, category, client, filename, size_bytes, digest, created_at, last_seen_at, state)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active')
|
||||||
|
ON CONFLICT(path) DO UPDATE SET
|
||||||
|
size_bytes = excluded.size_bytes,
|
||||||
|
digest = COALESCE(asset.digest, excluded.digest)
|
||||||
|
WHERE asset.state != 'deleted'
|
||||||
|
`).run(path, category, client, filename, sizeBytes, sha256, createdAt, createdAt);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
touch(path) {
|
||||||
|
if (!CONTENT_PATH_PATTERN.test(path)) return false;
|
||||||
|
this.pendingTouches.add(path);
|
||||||
|
if (!this.touchTimer) {
|
||||||
|
this.touchTimer = setTimeout(() => {
|
||||||
|
this.touchTimer = null;
|
||||||
|
try {
|
||||||
|
this.flushTouches();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(JSON.stringify({ level: 'error', message: 'asset access flush failed', reason: error.message }));
|
||||||
|
}
|
||||||
|
}, this.touchFlushIntervalMs);
|
||||||
|
this.touchTimer.unref();
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
flushTouches() {
|
||||||
|
this.#assertOpen();
|
||||||
|
if (this.pendingTouches.size === 0) return 0;
|
||||||
|
const paths = [...this.pendingTouches];
|
||||||
|
this.pendingTouches.clear();
|
||||||
|
const touchedAt = this.now();
|
||||||
|
const update = this.db.prepare(`
|
||||||
|
UPDATE asset
|
||||||
|
SET last_seen_at = ?,
|
||||||
|
state = CASE WHEN state = 'candidate' THEN 'active' ELSE state END,
|
||||||
|
candidate_at = CASE WHEN state = 'candidate' THEN NULL ELSE candidate_at END
|
||||||
|
WHERE path = ? AND category = 'content' AND state IN ('active', 'candidate')
|
||||||
|
`);
|
||||||
|
this.db.exec('BEGIN IMMEDIATE');
|
||||||
|
try {
|
||||||
|
let changed = 0;
|
||||||
|
for (const path of paths) changed += Number(update.run(touchedAt, path).changes);
|
||||||
|
this.db.exec('COMMIT');
|
||||||
|
return changed;
|
||||||
|
} catch (error) {
|
||||||
|
this.db.exec('ROLLBACK');
|
||||||
|
for (const path of paths) this.pendingTouches.add(path);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
markCandidates() {
|
||||||
|
this.#assertOpen();
|
||||||
|
const now = this.now();
|
||||||
|
const cutoff = now - this.retentionMs;
|
||||||
|
return Number(this.db.prepare(`
|
||||||
|
UPDATE asset
|
||||||
|
SET state = 'candidate', candidate_at = ?
|
||||||
|
WHERE category = 'content' AND state = 'active' AND last_seen_at <= ?
|
||||||
|
`).run(now, cutoff).changes);
|
||||||
|
}
|
||||||
|
|
||||||
|
summary() {
|
||||||
|
this.#assertOpen();
|
||||||
|
const grouped = this.db.prepare(`
|
||||||
|
SELECT category, state, COUNT(*) AS count, COALESCE(SUM(size_bytes), 0) AS bytes
|
||||||
|
FROM asset
|
||||||
|
GROUP BY category, state
|
||||||
|
`).all();
|
||||||
|
const result = { totalCount: 0, totalBytes: 0, groups: {} };
|
||||||
|
for (const row of grouped) {
|
||||||
|
const key = `${row.category}:${row.state}`;
|
||||||
|
result.groups[key] = { count: Number(row.count), bytes: Number(row.bytes) };
|
||||||
|
if (row.state !== 'deleted') {
|
||||||
|
result.totalCount += Number(row.count);
|
||||||
|
result.totalBytes += Number(row.bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
list({ category, state, client, search = '', limit = 100, offset = 0 } = {}) {
|
||||||
|
this.#assertOpen();
|
||||||
|
const where = [];
|
||||||
|
const values = [];
|
||||||
|
if (category) { where.push('category = ?'); values.push(category); }
|
||||||
|
if (state) { where.push('state = ?'); values.push(state); }
|
||||||
|
if (client) { where.push('client = ?'); values.push(client); }
|
||||||
|
if (search) { where.push('(path LIKE ? OR digest LIKE ?)'); values.push(`%${search}%`, `%${search}%`); }
|
||||||
|
const clause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
|
||||||
|
const boundedLimit = Math.max(1, Math.min(200, Number(limit) || 100));
|
||||||
|
const boundedOffset = Math.max(0, Number(offset) || 0);
|
||||||
|
const total = Number(this.db.prepare(`SELECT COUNT(*) AS count FROM asset ${clause}`).get(...values).count);
|
||||||
|
const rows = this.db.prepare(`
|
||||||
|
SELECT * FROM asset ${clause}
|
||||||
|
ORDER BY CASE state WHEN 'candidate' THEN 0 WHEN 'quarantined' THEN 1 ELSE 2 END,
|
||||||
|
last_seen_at ASC, path ASC
|
||||||
|
LIMIT ? OFFSET ?
|
||||||
|
`).all(...values, boundedLimit, boundedOffset);
|
||||||
|
const now = this.now();
|
||||||
|
return {
|
||||||
|
total,
|
||||||
|
limit: boundedLimit,
|
||||||
|
offset: boundedOffset,
|
||||||
|
assets: rows.map((row) => rowToAsset(row, now, this.retentionMs, this.quarantineMs)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
get(path) {
|
||||||
|
this.#assertOpen();
|
||||||
|
return rowToAsset(
|
||||||
|
this.db.prepare('SELECT * FROM asset WHERE path = ?').get(path),
|
||||||
|
this.now(),
|
||||||
|
this.retentionMs,
|
||||||
|
this.quarantineMs,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async preview(path) {
|
||||||
|
const asset = this.get(path);
|
||||||
|
if (!asset || !['active', 'candidate'].includes(asset.state)) {
|
||||||
|
throw new DeploymentError('Asset preview is not available', 404);
|
||||||
|
}
|
||||||
|
const contentTypeByExtension = {
|
||||||
|
avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif',
|
||||||
|
};
|
||||||
|
const extension = asset.filename.slice(asset.filename.lastIndexOf('.') + 1);
|
||||||
|
return {
|
||||||
|
body: await readFile(fileLocation(this.root, asset.category, asset.client, asset.filename)),
|
||||||
|
contentType: contentTypeByExtension[extension],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async quarantine(path) {
|
||||||
|
this.flushTouches();
|
||||||
|
const asset = this.get(path);
|
||||||
|
if (!asset || asset.category !== 'content' || asset.state !== 'candidate') {
|
||||||
|
throw new DeploymentError('Only content deletion candidates can be quarantined', 409);
|
||||||
|
}
|
||||||
|
const source = fileLocation(this.root, 'content', asset.client, asset.filename);
|
||||||
|
const destination = fileLocation(this.root, '.trash/content', asset.client, asset.filename);
|
||||||
|
await mkdir(dirname(destination), { recursive: true });
|
||||||
|
await rename(source, destination);
|
||||||
|
const now = this.now();
|
||||||
|
try {
|
||||||
|
this.db.prepare(`
|
||||||
|
UPDATE asset SET state = 'quarantined', quarantined_at = ? WHERE path = ? AND state = 'candidate'
|
||||||
|
`).run(now, path);
|
||||||
|
} catch (error) {
|
||||||
|
await rename(destination, source).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return this.get(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
async restore(path) {
|
||||||
|
const asset = this.get(path);
|
||||||
|
if (!asset || asset.category !== 'content' || asset.state !== 'quarantined') {
|
||||||
|
throw new DeploymentError('Only quarantined content can be restored', 409);
|
||||||
|
}
|
||||||
|
const source = fileLocation(this.root, '.trash/content', asset.client, asset.filename);
|
||||||
|
const destination = fileLocation(this.root, 'content', asset.client, asset.filename);
|
||||||
|
await mkdir(dirname(destination), { recursive: true });
|
||||||
|
await rename(source, destination);
|
||||||
|
const now = this.now();
|
||||||
|
try {
|
||||||
|
this.db.prepare(`
|
||||||
|
UPDATE asset
|
||||||
|
SET state = 'active', last_seen_at = ?, candidate_at = NULL, quarantined_at = NULL
|
||||||
|
WHERE path = ? AND state = 'quarantined'
|
||||||
|
`).run(now, path);
|
||||||
|
} catch (error) {
|
||||||
|
await rename(destination, source).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return this.get(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(path) {
|
||||||
|
const asset = this.get(path);
|
||||||
|
if (!asset || asset.category !== 'content' || asset.state !== 'quarantined') {
|
||||||
|
throw new DeploymentError('Only quarantined content can be deleted', 409);
|
||||||
|
}
|
||||||
|
if (!asset.deleteAvailable) {
|
||||||
|
throw new DeploymentError('Quarantine grace period has not elapsed', 409);
|
||||||
|
}
|
||||||
|
await unlink(fileLocation(this.root, '.trash/content', asset.client, asset.filename));
|
||||||
|
const now = this.now();
|
||||||
|
this.db.prepare(`
|
||||||
|
UPDATE asset SET state = 'deleted', deleted_at = ? WHERE path = ? AND state = 'quarantined'
|
||||||
|
`).run(now, path);
|
||||||
|
return this.get(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
close() {
|
||||||
|
if (this.touchTimer) clearTimeout(this.touchTimer);
|
||||||
|
if (this.maintenanceTimer) clearInterval(this.maintenanceTimer);
|
||||||
|
this.touchTimer = null;
|
||||||
|
this.maintenanceTimer = null;
|
||||||
|
if (this.db) {
|
||||||
|
this.flushTouches();
|
||||||
|
this.db.close();
|
||||||
|
this.db = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async #inventoryExisting() {
|
||||||
|
for (const category of ['user-icons', 'content']) {
|
||||||
|
const categoryRoot = join(this.root, category);
|
||||||
|
let clients;
|
||||||
|
try {
|
||||||
|
clients = await readdir(categoryRoot, { withFileTypes: true });
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') continue;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
for (const clientEntry of clients) {
|
||||||
|
if (!clientEntry.isDirectory() || !CLIENT_PATTERN.test(clientEntry.name)) continue;
|
||||||
|
const files = await readdir(join(categoryRoot, clientEntry.name), { withFileTypes: true });
|
||||||
|
for (const fileEntry of files) {
|
||||||
|
if (!fileEntry.isFile() || !FILE_PATTERN.test(fileEntry.name)) continue;
|
||||||
|
const info = await stat(join(categoryRoot, clientEntry.name, fileEntry.name));
|
||||||
|
const createdAt = Math.trunc(info.birthtimeMs > 0 ? Math.min(info.birthtimeMs, info.mtimeMs) : info.mtimeMs);
|
||||||
|
this.db.prepare(`
|
||||||
|
INSERT INTO asset(path, category, client, filename, size_bytes, digest, created_at, last_seen_at, state)
|
||||||
|
VALUES (?, ?, ?, ?, ?, NULL, ?, ?, 'active')
|
||||||
|
ON CONFLICT(path) DO NOTHING
|
||||||
|
`).run(
|
||||||
|
publicPath(category, clientEntry.name, fileEntry.name),
|
||||||
|
category,
|
||||||
|
clientEntry.name,
|
||||||
|
fileEntry.name,
|
||||||
|
info.size,
|
||||||
|
createdAt,
|
||||||
|
createdAt,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#assertOpen() {
|
||||||
|
if (!this.db) throw new Error('AssetStore is not initialized');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -47,11 +47,22 @@ function clientSecrets(variableName) {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function positiveNumber(name, fallback) {
|
||||||
|
const value = Number(text(name, fallback));
|
||||||
|
if (!Number.isFinite(value) || value <= 0) {
|
||||||
|
throw new Error(`${name} must be a positive number`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
export function loadConfig() {
|
export function loadConfig() {
|
||||||
const webhookSecret = secret('GITEA_WEBHOOK_SECRET', 'GITEA_WEBHOOK_SECRET_FILE');
|
const webhookSecret = secret('GITEA_WEBHOOK_SECRET', 'GITEA_WEBHOOK_SECRET_FILE');
|
||||||
const adminSecret = secret('IMAGE_ADMIN_SECRET', 'IMAGE_ADMIN_SECRET_FILE');
|
const adminSecret = secret('IMAGE_ADMIN_SECRET', 'IMAGE_ADMIN_SECRET_FILE');
|
||||||
if (webhookSecret.length < 32 || adminSecret.length < 32) {
|
const adminPanelPassword = secret('IMAGE_ADMIN_PANEL_PASSWORD', 'IMAGE_ADMIN_PANEL_PASSWORD_FILE');
|
||||||
throw new Error('Webhook and admin secrets must be at least 32 characters');
|
const adminPanelSessionSecret = secret('IMAGE_ADMIN_PANEL_SESSION_SECRET', 'IMAGE_ADMIN_PANEL_SESSION_SECRET_FILE');
|
||||||
|
if (webhookSecret.length < 32 || adminSecret.length < 32
|
||||||
|
|| adminPanelPassword.length < 16 || adminPanelSessionSecret.length < 32) {
|
||||||
|
throw new Error('Webhook, deployment, and session secrets need 32 characters; panel password needs 16');
|
||||||
}
|
}
|
||||||
|
|
||||||
const allowedBranches = text('IMAGE_ALLOWED_BRANCHES', 'master')
|
const allowedBranches = text('IMAGE_ALLOWED_BRANCHES', 'master')
|
||||||
@@ -80,5 +91,13 @@ export function loadConfig() {
|
|||||||
maxContentUploadBytes: Number(text('MAX_CONTENT_UPLOAD_BYTES', '1048576')),
|
maxContentUploadBytes: Number(text('MAX_CONTENT_UPLOAD_BYTES', '1048576')),
|
||||||
uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'),
|
uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'),
|
||||||
uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'),
|
uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'),
|
||||||
|
assetDbPath: text('IMAGE_ASSET_DB_PATH', '/var/lib/image-hook/image-assets.sqlite3'),
|
||||||
|
contentRetentionMs: positiveNumber('IMAGE_CONTENT_RETENTION_DAYS', '730') * 86_400_000,
|
||||||
|
contentQuarantineMs: positiveNumber('IMAGE_CONTENT_QUARANTINE_DAYS', '30') * 86_400_000,
|
||||||
|
assetMaintenanceIntervalMs: positiveNumber('IMAGE_ASSET_MAINTENANCE_SECONDS', '21600') * 1000,
|
||||||
|
assetTouchFlushIntervalMs: positiveNumber('IMAGE_ASSET_TOUCH_FLUSH_SECONDS', '60') * 1000,
|
||||||
|
adminPanelPassword,
|
||||||
|
adminPanelSessionSecret,
|
||||||
|
adminPanelSessionTtlMs: positiveNumber('IMAGE_ADMIN_PANEL_SESSION_HOURS', '8') * 3_600_000,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import { createServer } from 'node:http';
|
import { createServer } from 'node:http';
|
||||||
import { readFile } from 'node:fs/promises';
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { handleAdminPanel } from './admin-panel.mjs';
|
||||||
|
import { AssetStore } from './asset-store.mjs';
|
||||||
import { loadConfig } from './config.mjs';
|
import { loadConfig } from './config.mjs';
|
||||||
import { verifyAdminSignature, verifyHexHmac, verifyUploadSignature } from './auth.mjs';
|
import { verifyAdminSignature, verifyHexHmac, verifyUploadSignature } from './auth.mjs';
|
||||||
import { DeploymentError, GitService } from './git-service.mjs';
|
import { DeploymentError, GitService } from './git-service.mjs';
|
||||||
@@ -49,13 +51,18 @@ function hasImageSignature(body, extension) {
|
|||||||
|
|
||||||
export async function createApp(config = loadConfig(), dependencies = {}) {
|
export async function createApp(config = loadConfig(), dependencies = {}) {
|
||||||
const service = dependencies.service ?? new GitService(config);
|
const service = dependencies.service ?? new GitService(config);
|
||||||
const uploadStore = dependencies.uploadStore ?? new UploadStore(config);
|
const assetStore = dependencies.assetStore ?? (dependencies.uploadStore ? {
|
||||||
|
async initialize() {}, touch() { return false; }, close() {},
|
||||||
|
} : new AssetStore(config));
|
||||||
|
const uploadStore = dependencies.uploadStore ?? new UploadStore(config, assetStore);
|
||||||
await service.initialize();
|
await service.initialize();
|
||||||
|
await assetStore.initialize();
|
||||||
await uploadStore.initialize();
|
await uploadStore.initialize();
|
||||||
|
|
||||||
const server = createServer(async (request, response) => {
|
const server = createServer(async (request, response) => {
|
||||||
const url = new URL(request.url, 'http://image-hook');
|
const url = new URL(request.url, 'http://image-hook');
|
||||||
try {
|
try {
|
||||||
|
if (await handleAdminPanel({ request, response, url, config, assetStore, readBody })) return;
|
||||||
if (request.method === 'GET' && url.pathname === '/healthz') {
|
if (request.method === 'GET' && url.pathname === '/healthz') {
|
||||||
return json(response, 200, { ok: true });
|
return json(response, 200, { ok: true });
|
||||||
}
|
}
|
||||||
@@ -66,6 +73,23 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
|
|||||||
const inventory = JSON.parse(await readFile(`${config.repositoryPath}/hook/inventory.v2.json`, 'utf8'));
|
const inventory = JSON.parse(await readFile(`${config.repositoryPath}/hook/inventory.v2.json`, 'utf8'));
|
||||||
return json(response, 200, inventory);
|
return json(response, 200, inventory);
|
||||||
}
|
}
|
||||||
|
if (['GET', 'HEAD'].includes(request.method) && url.pathname === '/v1/internal/content-access') {
|
||||||
|
const original = request.headers['x-image-path'];
|
||||||
|
let path;
|
||||||
|
try {
|
||||||
|
path = typeof original === 'string'
|
||||||
|
? new URL(original, 'http://image').pathname.replace(/^\/(?:image\/)?/, '')
|
||||||
|
: null;
|
||||||
|
} catch {
|
||||||
|
path = null;
|
||||||
|
}
|
||||||
|
if (!path || !assetStore.touch(path)) {
|
||||||
|
throw new DeploymentError('Invalid content image path', 400);
|
||||||
|
}
|
||||||
|
response.writeHead(204, { 'cache-control': 'no-store' });
|
||||||
|
response.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (request.method === 'POST' && url.pathname === '/v1/hooks/gitea') {
|
if (request.method === 'POST' && url.pathname === '/v1/hooks/gitea') {
|
||||||
if (!request.headers['content-type']?.toLowerCase().startsWith('application/json')) {
|
if (!request.headers['content-type']?.toLowerCase().startsWith('application/json')) {
|
||||||
throw new DeploymentError('Content-Type must be application/json', 415);
|
throw new DeploymentError('Content-Type must be application/json', 415);
|
||||||
@@ -205,7 +229,9 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
return { server, service, uploadStore };
|
server.once('close', () => assetStore.close?.());
|
||||||
|
|
||||||
|
return { server, service, uploadStore, assetStore };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (process.argv[1] === new URL(import.meta.url).pathname) {
|
if (process.argv[1] === new URL(import.meta.url).pathname) {
|
||||||
@@ -214,4 +240,14 @@ if (process.argv[1] === new URL(import.meta.url).pathname) {
|
|||||||
server.listen(config.port, '0.0.0.0', () => {
|
server.listen(config.port, '0.0.0.0', () => {
|
||||||
console.log(JSON.stringify({ level: 'info', message: 'image hook listening', port: config.port }));
|
console.log(JSON.stringify({ level: 'info', message: 'image hook listening', port: config.port }));
|
||||||
});
|
});
|
||||||
|
let stopping = false;
|
||||||
|
const shutdown = (signal) => {
|
||||||
|
if (stopping) return;
|
||||||
|
stopping = true;
|
||||||
|
console.log(JSON.stringify({ level: 'info', message: 'image hook stopping', signal }));
|
||||||
|
server.close(() => process.exit(0));
|
||||||
|
setTimeout(() => process.exit(1), 10_000).unref();
|
||||||
|
};
|
||||||
|
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||||
|
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,9 +4,10 @@ import { dirname, join } from 'node:path';
|
|||||||
import { DeploymentError } from './git-service.mjs';
|
import { DeploymentError } from './git-service.mjs';
|
||||||
|
|
||||||
export class UploadStore {
|
export class UploadStore {
|
||||||
constructor(config) {
|
constructor(config, assetStore = null) {
|
||||||
this.root = config.uploadRoot;
|
this.root = config.uploadRoot;
|
||||||
this.statePath = config.uploadStatePath;
|
this.statePath = config.uploadStatePath;
|
||||||
|
this.assetStore = assetStore;
|
||||||
this.queue = Promise.resolve();
|
this.queue = Promise.resolve();
|
||||||
this.uploads = [];
|
this.uploads = [];
|
||||||
}
|
}
|
||||||
@@ -39,6 +40,7 @@ export class UploadStore {
|
|||||||
if (previous.path !== path || previous.digest !== digest) {
|
if (previous.path !== path || previous.digest !== digest) {
|
||||||
throw new DeploymentError('Upload request ID was already used', 409);
|
throw new DeploymentError('Upload request ID was already used', 409);
|
||||||
}
|
}
|
||||||
|
this.assetStore?.register({ category, client, filename, body, digest });
|
||||||
return { duplicate: true, path: previous.path };
|
return { duplicate: true, path: previous.path };
|
||||||
}
|
}
|
||||||
const destination = join(this.root, relativePath);
|
const destination = join(this.root, relativePath);
|
||||||
@@ -50,6 +52,7 @@ export class UploadStore {
|
|||||||
throw new DeploymentError('Upload path already exists', 409);
|
throw new DeploymentError('Upload path already exists', 409);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
this.assetStore?.register({ category, client, filename, body, digest });
|
||||||
this.uploads = [...this.uploads.slice(-999), { key: requestKey, path, digest }];
|
this.uploads = [...this.uploads.slice(-999), { key: requestKey, path, digest }];
|
||||||
await this.#save();
|
await this.#save();
|
||||||
return { duplicate: false, path };
|
return { duplicate: false, path };
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { AssetStore } from '../src/asset-store.mjs';
|
||||||
|
|
||||||
|
const DAY = 86_400_000;
|
||||||
|
|
||||||
|
async function fixture(t) {
|
||||||
|
const root = await mkdtemp(join(tmpdir(), 'image-assets-'));
|
||||||
|
let now = Date.UTC(2028, 0, 1);
|
||||||
|
const config = {
|
||||||
|
uploadRoot: join(root, 'uploads'),
|
||||||
|
assetDbPath: join(root, 'metadata', 'assets.sqlite3'),
|
||||||
|
contentRetentionMs: 730 * DAY,
|
||||||
|
contentQuarantineMs: 30 * DAY,
|
||||||
|
assetMaintenanceIntervalMs: DAY,
|
||||||
|
assetTouchFlushIntervalMs: 60_000,
|
||||||
|
};
|
||||||
|
const store = new AssetStore(config, { now: () => now });
|
||||||
|
await store.initialize();
|
||||||
|
t.after(async () => {
|
||||||
|
store.close();
|
||||||
|
await rm(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
return { root, config, store, setNow(value) { now = value; }, getNow() { return now; } };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function addFile(f, category, client, filename, body, createdAt) {
|
||||||
|
const directory = join(f.config.uploadRoot, category, client);
|
||||||
|
await mkdir(directory, { recursive: true });
|
||||||
|
await writeFile(join(directory, filename), body);
|
||||||
|
f.store.register({ category, client, filename, body, createdAt });
|
||||||
|
}
|
||||||
|
|
||||||
|
test('user icons remain permanent while old content becomes a deletion candidate', async (t) => {
|
||||||
|
const f = await fixture(t);
|
||||||
|
const old = f.getNow() - 731 * DAY;
|
||||||
|
const icon = `${'a'.repeat(32)}.png`;
|
||||||
|
const content = `${'b'.repeat(32)}.webp`;
|
||||||
|
await addFile(f, 'user-icons', 'core2026', icon, Buffer.from('icon'), old);
|
||||||
|
await addFile(f, 'content', 'core2026', content, Buffer.from('content'), old);
|
||||||
|
|
||||||
|
assert.equal(f.store.markCandidates(), 1);
|
||||||
|
assert.equal(f.store.get(`icons/users/core2026/${icon}`).state, 'active');
|
||||||
|
assert.equal(f.store.get(`uploads/core2026/${content}`).state, 'candidate');
|
||||||
|
assert.equal((await f.store.preview(`uploads/core2026/${content}`)).body.toString(), 'content');
|
||||||
|
assert.equal(f.store.get(`uploads/core2026/${content}`).state, 'candidate');
|
||||||
|
assert.deepEqual(f.store.summary().groups['user-icons:active'], { count: 1, bytes: 4 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('content access is batched, renews retention, and cancels candidate state', async (t) => {
|
||||||
|
const f = await fixture(t);
|
||||||
|
const path = `uploads/core/${'c'.repeat(32)}.png`;
|
||||||
|
await addFile(f, 'content', 'core', `${'c'.repeat(32)}.png`, Buffer.from('content'), f.getNow() - 731 * DAY);
|
||||||
|
f.store.markCandidates();
|
||||||
|
assert.equal(f.store.get(path).state, 'candidate');
|
||||||
|
|
||||||
|
assert.equal(f.store.touch(path), true);
|
||||||
|
assert.equal(f.store.touch(path), true);
|
||||||
|
assert.equal(f.store.flushTouches(), 1);
|
||||||
|
assert.equal(f.store.get(path).state, 'active');
|
||||||
|
assert.equal(f.store.get(path).lastSeenAt, new Date(f.getNow()).toISOString());
|
||||||
|
assert.equal(f.store.touch('icons/users/core/not-content.png'), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('candidate quarantine is recoverable and permanent deletion requires the grace period', async (t) => {
|
||||||
|
const f = await fixture(t);
|
||||||
|
const filename = `${'d'.repeat(32)}.gif`;
|
||||||
|
const path = `uploads/core2026/${filename}`;
|
||||||
|
const body = Buffer.from('content');
|
||||||
|
await addFile(f, 'content', 'core2026', filename, body, f.getNow() - 731 * DAY);
|
||||||
|
f.store.markCandidates();
|
||||||
|
|
||||||
|
await f.store.quarantine(path);
|
||||||
|
await assert.rejects(f.store.delete(path), /grace period/);
|
||||||
|
await f.store.restore(path);
|
||||||
|
assert.equal(await readFile(join(f.config.uploadRoot, 'content', 'core2026', filename), 'utf8'), 'content');
|
||||||
|
assert.equal(f.store.get(path).state, 'active');
|
||||||
|
|
||||||
|
f.setNow(f.getNow() + 731 * DAY);
|
||||||
|
f.store.markCandidates();
|
||||||
|
await f.store.quarantine(path);
|
||||||
|
f.setNow(f.getNow() + 31 * DAY);
|
||||||
|
const deleted = await f.store.delete(path);
|
||||||
|
assert.equal(deleted.state, 'deleted');
|
||||||
|
assert.equal(f.store.summary().totalCount, 0);
|
||||||
|
});
|
||||||
@@ -176,3 +176,96 @@ test('sync endpoint rejects unknown callers and body fields outside the sync con
|
|||||||
});
|
});
|
||||||
assert.equal(extraField.status, 400);
|
assert.equal(extraField.status, 400);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('internal content access endpoint batches only valid uploaded content paths', async (t) => {
|
||||||
|
const service = { async initialize() {}, async recordError() {} };
|
||||||
|
const touched = [];
|
||||||
|
const assetStore = {
|
||||||
|
async initialize() {},
|
||||||
|
touch(path) { touched.push(path); return path.startsWith('uploads/'); },
|
||||||
|
close() {},
|
||||||
|
};
|
||||||
|
const { server } = await createApp({ maxBodyBytes: 4096 }, {
|
||||||
|
service, uploadStore: noUploadStore, assetStore,
|
||||||
|
});
|
||||||
|
server.listen(0, '127.0.0.1');
|
||||||
|
await once(server, 'listening');
|
||||||
|
t.after(() => server.close());
|
||||||
|
const address = server.address();
|
||||||
|
const validPath = `/uploads/core2026/${'e'.repeat(32)}.png?cache=1`;
|
||||||
|
const valid = await fetch(`http://127.0.0.1:${address.port}/v1/internal/content-access`, {
|
||||||
|
headers: { 'x-image-path': validPath },
|
||||||
|
});
|
||||||
|
assert.equal(valid.status, 204);
|
||||||
|
assert.deepEqual(touched, [`uploads/core2026/${'e'.repeat(32)}.png`]);
|
||||||
|
|
||||||
|
const legacyBase = await fetch(`http://127.0.0.1:${address.port}/v1/internal/content-access`, {
|
||||||
|
method: 'HEAD',
|
||||||
|
headers: { 'x-image-path': `/image/uploads/core/${'f'.repeat(32)}.webp` },
|
||||||
|
});
|
||||||
|
assert.equal(legacyBase.status, 204);
|
||||||
|
assert.deepEqual(touched, [
|
||||||
|
`uploads/core2026/${'e'.repeat(32)}.png`,
|
||||||
|
`uploads/core/${'f'.repeat(32)}.webp`,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('admin panel uses a separate login session and CSRF-protected asset actions', async (t) => {
|
||||||
|
const service = { async initialize() {}, async recordError() {} };
|
||||||
|
const actions = [];
|
||||||
|
const asset = {
|
||||||
|
path: `uploads/core/${'f'.repeat(32)}.webp`, category: 'content', client: 'core',
|
||||||
|
filename: `${'f'.repeat(32)}.webp`, sizeBytes: 10, digest: null,
|
||||||
|
createdAt: new Date(0).toISOString(), lastSeenAt: new Date(0).toISOString(),
|
||||||
|
state: 'candidate', candidateAt: new Date(0).toISOString(), quarantinedAt: null,
|
||||||
|
deletedAt: null, eligibleAt: new Date(0).toISOString(), deleteAvailableAt: null, deleteAvailable: false,
|
||||||
|
};
|
||||||
|
const assetStore = {
|
||||||
|
async initialize() {}, touch() { return false; }, close() {},
|
||||||
|
list() { return { total: 1, limit: 100, offset: 0, assets: [asset] }; },
|
||||||
|
summary() { return { totalCount: 1, totalBytes: 10, groups: { 'content:candidate': { count: 1, bytes: 10 } } }; },
|
||||||
|
async quarantine(path) { actions.push(['quarantine', path]); return { ...asset, state: 'quarantined' }; },
|
||||||
|
};
|
||||||
|
const config = {
|
||||||
|
maxBodyBytes: 4096,
|
||||||
|
adminPanelPassword: 'panel-password-value',
|
||||||
|
adminPanelSessionSecret: 'p'.repeat(32),
|
||||||
|
adminPanelSessionTtlMs: 8 * 3_600_000,
|
||||||
|
};
|
||||||
|
const { server } = await createApp(config, { service, uploadStore: noUploadStore, assetStore });
|
||||||
|
server.listen(0, '127.0.0.1');
|
||||||
|
await once(server, 'listening');
|
||||||
|
t.after(() => server.close());
|
||||||
|
const base = `http://127.0.0.1:${server.address().port}`;
|
||||||
|
|
||||||
|
const anonymousApi = await fetch(`${base}/admin/api/assets`);
|
||||||
|
assert.equal(anonymousApi.status, 401);
|
||||||
|
const loginBody = new URLSearchParams({ password: config.adminPanelPassword });
|
||||||
|
const login = await fetch(`${base}/admin/login`, {
|
||||||
|
method: 'POST', redirect: 'manual',
|
||||||
|
headers: { 'content-type': 'application/x-www-form-urlencoded' }, body: loginBody,
|
||||||
|
});
|
||||||
|
assert.equal(login.status, 303);
|
||||||
|
const cookie = login.headers.get('set-cookie').split(';', 1)[0];
|
||||||
|
assert.match(login.headers.get('set-cookie'), /HttpOnly; Secure; SameSite=Strict/);
|
||||||
|
const dashboard = await fetch(`${base}/admin/`, { headers: { cookie } });
|
||||||
|
assert.equal(dashboard.status, 200);
|
||||||
|
assert.match(dashboard.headers.get('content-security-policy'), /frame-ancestors 'none'/);
|
||||||
|
const html = await dashboard.text();
|
||||||
|
const csrf = html.match(/name="csrf-token" content="([^"]+)"/)[1];
|
||||||
|
|
||||||
|
const list = await fetch(`${base}/admin/api/assets`, { headers: { cookie } });
|
||||||
|
assert.equal(list.status, 200);
|
||||||
|
assert.equal((await list.json()).assets[0].state, 'candidate');
|
||||||
|
const rejected = await fetch(`${base}/admin/api/assets/action`, {
|
||||||
|
method: 'POST', headers: { cookie, 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify({ path: asset.path, action: 'quarantine' }),
|
||||||
|
});
|
||||||
|
assert.equal(rejected.status, 403);
|
||||||
|
const accepted = await fetch(`${base}/admin/api/assets/action`, {
|
||||||
|
method: 'POST', headers: { cookie, 'content-type': 'application/json', 'x-csrf-token': csrf },
|
||||||
|
body: JSON.stringify({ path: asset.path, action: 'quarantine' }),
|
||||||
|
});
|
||||||
|
assert.equal(accepted.status, 200);
|
||||||
|
assert.deepEqual(actions, [['quarantine', asset.path]]);
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user