From dc0e1a6da96d627a7a2445daa3e3f95c6edd1a02 Mon Sep 17 00:00:00 2001 From: hided62 Date: Thu, 6 Aug 2026 15:52:54 +0000 Subject: [PATCH] feat: store editor images in upload bind --- README.md | 9 ++++--- compose.yaml | 3 ++- deploy/nginx/templates/default.conf.template | 23 +++++++++++++---- node-hook/src/config.mjs | 1 + node-hook/src/server.mjs | 22 +++++++++------- node-hook/src/upload-store.mjs | 13 ++++++---- node-hook/test/server.test.mjs | 27 ++++++++++++++++++++ node-hook/test/upload-store.test.mjs | 20 ++++++++++++--- 8 files changed, 91 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index e8aca49..4a461e0 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,9 @@ service itself is stopped, restore it and run the caller command again. ### Short-lived user-icon uploads -Core and Core2026 can store validated account icons in this repository through -`PUT /v1/uploads/user-icons//.`. Each game +Core and Core2026 can store validated account icons and editor attachments +through this service with +`PUT /v1/uploads///.`. Each game server validates the authenticated user and image first, then sends the raw image body with `X-Image-Client`, `X-Image-Expires`, `X-Image-Request-Id`, and `X-Image-Signature` headers. @@ -148,7 +149,9 @@ five minutes in the future, so a grant cannot be reused for another path, content type, body, or later upload. The service also checks the image magic, caller scope, and request replay before it writes one immutable file below the host bind directory `runtime-data/uploads`. User uploads are deliberately not -added to Git; Nginx exposes that bind read-only at `/icons/users/`. +added to Git; Nginx exposes that bind read-only at `/icons/users/` for account +icons and `/uploads/` for editor content. User icons retain the 50KB limit; +editor content retains the existing 1MB limit. Create separate upload secrets with `deploy/scripts/init-secrets.sh`. Mount only the matching `image_upload_core_secret` or `image_upload_core2026_secret` on the diff --git a/compose.yaml b/compose.yaml index 4289b39..fbfa973 100644 --- a/compose.yaml +++ b/compose.yaml @@ -25,6 +25,7 @@ services: IMAGE_SYNC_CLIENT_SECRET_FILES: core=/run/secrets/image_sync_core_secret,core2026=/run/secrets/image_sync_core2026_secret IMAGE_UPLOAD_CLIENT_SECRET_FILES: core=/run/secrets/image_upload_core_secret,core2026=/run/secrets/image_upload_core2026_secret MAX_UPLOAD_BYTES: "51200" + MAX_CONTENT_UPLOAD_BYTES: "1048576" volumes: - type: bind source: ${IMAGE_REPOSITORY_PATH:-.} @@ -73,7 +74,7 @@ services: read_only: true - type: bind source: ./runtime-data/uploads - target: /srv/user-icons + target: /srv/uploads read_only: true tmpfs: - /tmp:size=8m,mode=1777 diff --git a/deploy/nginx/templates/default.conf.template b/deploy/nginx/templates/default.conf.template index c13e4ec..3d4a2ee 100644 --- a/deploy/nginx/templates/default.conf.template +++ b/deploy/nginx/templates/default.conf.template @@ -69,9 +69,9 @@ http { proxy_request_buffering on; } - location ^~ /v1/uploads/user-icons/ { + location ^~ /v1/uploads/ { limit_except PUT { deny all; } - client_max_body_size 50k; + client_max_body_size 1m; proxy_pass http://image-hook:8081; proxy_set_header Host $host; proxy_set_header X-Image-Client $http_x_image_client; @@ -92,8 +92,8 @@ http { add_header X-Content-Type-Options nosniff always; } - location ^~ /icons/users/ { - alias /srv/user-icons/; + location ~ "^/icons/users/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" { + alias /srv/uploads/user-icons/$1/$2; etag on; expires 1y; add_header Cache-Control "public, immutable" always; @@ -101,7 +101,20 @@ http { add_header X-Content-Type-Options nosniff always; } - location ^~ /icons/ { + location /icons/users/ { return 404; } + + location ~ "^/uploads/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" { + alias /srv/uploads/content/$1/$2; + etag on; + expires 1y; + add_header Cache-Control "public, immutable" always; + add_header Access-Control-Allow-Origin "*" always; + add_header X-Content-Type-Options nosniff always; + } + + location /uploads/ { return 404; } + + location /icons/ { try_files $uri =404; add_header Access-Control-Allow-Origin "*" always; add_header X-Content-Type-Options nosniff always; diff --git a/node-hook/src/config.mjs b/node-hook/src/config.mjs index 53e60fa..7497599 100644 --- a/node-hook/src/config.mjs +++ b/node-hook/src/config.mjs @@ -77,6 +77,7 @@ export function loadConfig() { uploadClientSecrets: clientSecrets('IMAGE_UPLOAD_CLIENT_SECRET_FILES'), maxBodyBytes: Number(text('MAX_BODY_BYTES', '1048576')), maxUploadBytes: Number(text('MAX_UPLOAD_BYTES', '51200')), + maxContentUploadBytes: Number(text('MAX_CONTENT_UPLOAD_BYTES', '1048576')), uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'), uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'), }; diff --git a/node-hook/src/server.mjs b/node-hook/src/server.mjs index 51e8552..6684e07 100644 --- a/node-hook/src/server.mjs +++ b/node-hook/src/server.mjs @@ -38,7 +38,7 @@ function parseJson(body) { function hasImageSignature(body, extension) { if (extension === 'png') return body.length >= 8 && body.subarray(0, 8).equals(Buffer.from('89504e470d0a1a0a', 'hex')); - if (extension === 'jpg') return body.length >= 3 && body[0] === 0xff && body[1] === 0xd8 && body[2] === 0xff; + if (extension === 'jpg' || extension === 'jpeg') return body.length >= 3 && body[0] === 0xff && body[1] === 0xd8 && body[2] === 0xff; if (extension === 'gif') return body.length >= 6 && ['GIF87a', 'GIF89a'].includes(body.subarray(0, 6).toString('ascii')); if (extension === 'webp') return body.length >= 12 && body.subarray(0, 4).toString('ascii') === 'RIFF' && body.subarray(8, 12).toString('ascii') === 'WEBP'; @@ -149,23 +149,26 @@ export async function createApp(config = loadConfig(), dependencies = {}) { }); return json(response, 200, { ok: true, ...result }); } - if (request.method === 'PUT' && url.pathname.startsWith('/v1/uploads/user-icons/')) { + if (request.method === 'PUT' && url.pathname.startsWith('/v1/uploads/')) { const client = request.headers['x-image-client']; const expires = request.headers['x-image-expires']; const requestId = request.headers['x-image-request-id']; const contentType = request.headers['content-type']?.toLowerCase() ?? ''; const knownClient = typeof client === 'string' && Object.hasOwn(config.uploadClientSecrets, client); - const match = url.pathname.match(/^\/v1\/uploads\/user-icons\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32})\.(avif|webp|jpg|png|gif)$/); - if (!match || match[1] !== client) { + const match = url.pathname.match(/^\/v1\/uploads\/(user-icons|content)\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32})\.(avif|webp|jpe?g|png|gif)$/); + if (!match || match[2] !== client) { throw new DeploymentError('Invalid upload path', 400); } const mimeByExtension = { - avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', png: 'image/png', gif: 'image/gif', + avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', }; - if (contentType !== mimeByExtension[match[3]]) { + if (contentType !== mimeByExtension[match[4]]) { throw new DeploymentError('Content-Type does not match upload path', 415); } - const body = await readBody(request, config.maxUploadBytes); + const body = await readBody( + request, + match[1] === 'user-icons' ? config.maxUploadBytes : config.maxContentUploadBytes, + ); const signatureValid = verifyUploadSignature({ secret: knownClient ? config.uploadClientSecrets[client] : 'invalid-client-secret'.padEnd(32, '!'), expires, @@ -178,13 +181,14 @@ export async function createApp(config = loadConfig(), dependencies = {}) { if (!knownClient || !signatureValid) { return json(response, 401, { ok: false, reason: 'invalid or expired upload grant' }); } - if (!hasImageSignature(body, match[3])) { + if (!hasImageSignature(body, match[4])) { throw new DeploymentError('Body is not the declared image format', 400); } const result = await uploadStore.store({ requestKey: `${client}:${requestId}`, + category: match[1], client, - filename: `${match[2]}.${match[3]}`, + filename: `${match[3]}.${match[4]}`, body, }); const urls = config.publicBases.map((base) => `${base}/${result.path}`); diff --git a/node-hook/src/upload-store.mjs b/node-hook/src/upload-store.mjs index 6bef08e..4313f3d 100644 --- a/node-hook/src/upload-store.mjs +++ b/node-hook/src/upload-store.mjs @@ -22,14 +22,17 @@ export class UploadStore { } } - store({ requestKey, client, filename, body }) { + store({ requestKey, category, client, filename, body }) { const operation = this.queue.then(async () => { - if (!/^[a-z0-9][a-z0-9_-]{1,31}$/.test(client) - || !/^[a-f0-9]{32}\.(?:avif|webp|jpg|png|gif)$/.test(filename)) { + if (!['user-icons', 'content'].includes(category) + || !/^[a-z0-9][a-z0-9_-]{1,31}$/.test(client) + || !/^[a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif)$/.test(filename)) { throw new DeploymentError('Invalid upload path', 400); } - const relativePath = `${client}/${filename}`; - const path = `icons/users/${relativePath}`; + const relativePath = `${category}/${client}/${filename}`; + const path = category === 'user-icons' + ? `icons/users/${client}/${filename}` + : `uploads/${client}/${filename}`; const digest = createHash('sha256').update(body).digest('hex'); const previous = this.uploads.find((upload) => upload.key === requestKey); if (previous) { diff --git a/node-hook/test/server.test.mjs b/node-hook/test/server.test.mjs index a0e2f1c..c3dcb2e 100644 --- a/node-hook/test/server.test.mjs +++ b/node-hook/test/server.test.mjs @@ -61,6 +61,7 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay const config = { maxBodyBytes: 4096, maxUploadBytes: 51200, + maxContentUploadBytes: 1048576, syncClientSecrets: { core: 's'.repeat(32) }, uploadClientSecrets: { core2026: secret }, publicBases: ['https://sam-image.hided.net', 'https://sam.hided.net/image'], @@ -86,6 +87,7 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay assert.equal(accepted.status, 201); assert.deepEqual(calls[0], { requestKey: `core2026:${requestId}`, + category: 'user-icons', client: 'core2026', filename: `${'a'.repeat(32)}.png`, body, @@ -95,6 +97,31 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay `https://sam.hided.net/image/icons/users/core2026/${'a'.repeat(32)}.png`, ]); + const contentPath = `/v1/uploads/content/core2026/${'b'.repeat(32)}.webp`; + const contentBody = Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(4), Buffer.from('WEBP'), Buffer.alloc(4)]); + const contentRequestId = 'content-request-1234'; + const contentSignature = uploadSignature(secret, { + expires, + requestId: contentRequestId, + pathname: contentPath, + contentType: 'image/webp', + body: contentBody, + }); + const contentResponse = await fetch(`http://127.0.0.1:${address.port}${contentPath}`, { + method: 'PUT', + headers: { + 'content-type': 'image/webp', + 'x-image-client': 'core2026', + 'x-image-expires': expires, + 'x-image-request-id': contentRequestId, + 'x-image-signature': contentSignature, + }, + body: contentBody, + }); + assert.equal(contentResponse.status, 201); + assert.equal(calls[1].category, 'content'); + assert.equal(calls[1].filename, `${'b'.repeat(32)}.webp`); + const tampered = await fetch(`http://127.0.0.1:${address.port}${pathname}`, { method: 'PUT', headers, body: Buffer.from('89504e470d0a1a0affffffff', 'hex'), }); diff --git a/node-hook/test/upload-store.test.mjs b/node-hook/test/upload-store.test.mjs index fd71b8b..bbe2476 100644 --- a/node-hook/test/upload-store.test.mjs +++ b/node-hook/test/upload-store.test.mjs @@ -13,19 +13,20 @@ test('stores uploads only in the bind directory and persists replay state', asyn const filename = `${'a'.repeat(32)}.png`; const store = new UploadStore(config); await store.initialize(); - const first = await store.store({ requestKey: 'core2026:request-1', client: 'core2026', filename, body }); + const first = await store.store({ requestKey: 'core2026:request-1', category: 'user-icons', client: 'core2026', filename, body }); assert.deepEqual(first, { duplicate: false, path: `icons/users/core2026/${filename}` }); - assert.equal(await readFile(join(root, 'uploads', 'core2026', filename), 'utf8'), 'immutable image bytes'); + assert.equal(await readFile(join(root, 'uploads', 'user-icons', 'core2026', filename), 'utf8'), 'immutable image bytes'); const restarted = new UploadStore(config); await restarted.initialize(); assert.deepEqual( - await restarted.store({ requestKey: 'core2026:request-1', client: 'core2026', filename, body }), + await restarted.store({ requestKey: 'core2026:request-1', category: 'user-icons', client: 'core2026', filename, body }), { duplicate: true, path: `icons/users/core2026/${filename}` }, ); await assert.rejects( restarted.store({ requestKey: 'core2026:request-1', + category: 'user-icons', client: 'core2026', filename: `${'b'.repeat(32)}.png`, body, @@ -33,7 +34,18 @@ test('stores uploads only in the bind directory and persists replay state', asyn /already used/, ); await assert.rejects( - restarted.store({ requestKey: 'core2026:request-2', client: '../escape', filename, body }), + restarted.store({ requestKey: 'core2026:request-2', category: 'user-icons', client: '../escape', filename, body }), /Invalid upload path/, ); + const contentFilename = `${'c'.repeat(32)}.webp`; + assert.deepEqual( + await restarted.store({ + requestKey: 'core2026:content-1', + category: 'content', + client: 'core2026', + filename: contentFilename, + body: Buffer.from('content image'), + }), + { duplicate: false, path: `uploads/core2026/${contentFilename}` }, + ); });