mirror of
https://github.com/zkldi/Tachi.git
synced 2026-10-07 22:38:02 +03:00
fix: resize profile images on upload and cache redirect responses (#1564)
* fix: resize profile images on upload and add cache headers Profile pictures were stored at original upload size (up to 1 MB) but only ever displayed at 32–128 px, so every page load downloaded the full original image. Additionally, no Cache-Control headers were set anywhere in the pipeline, forcing a fresh round-trip through the API redirect and CDN on every page view. - Resize JPEG/PNG pfps to max 256×256 and banners to max 1920×1080 via sharp before storing; re-encode to WebP at 85% quality (GIFs skipped to preserve the animated-GIF easter egg) - Hash the resized buffer so the CDN path reflects what's on disk - Set ContentType and CacheControl: immutable on S3 PutObject so CDN edges and browsers cache the objects indefinitely (safe because paths are content-addressed via SHA-256) - Add Cache-Control: public, max-age=3600, stale-while-revalidate=86400 to the GET /pfp and GET /banner redirect responses so browsers cache the userId→CDN-URL mapping and skip the API hop on repeat visits Fixes #1553 * chore: apply biome formatting * chore: generate ImportTimestop kanel type * feat: add backfill-media script to migrate existing profile images One-off script that reads every user's existing pfp and banner from S3, runs them through the same sharp resize/WebP pipeline as new uploads, stores the result under the new content-addressed key with immutable cache headers, and updates custom_pfp_location/custom_banner_location in the DB. Features: - --dry-run flag to preview changes without writing anything - Skips objects already at optimal size (idempotent) - Gracefully handles missing S3 objects (logs + skips) - Summary stats at the end - Reads S3 config from CLI args or TACHI_CDN_SAVE_LOCATION_* env vars * chore: apply biome formatting to backfill-media * fix: resize animated GIFs to animated WebP via sharp animated:true * test: rewrite change-pfp/banner tests as real S3 + sharp integration Remove the CDN mock entirely. Tests now pass real image buffers through sharp and verify the stored S3 object via CDNRetrieve: - PNG/JPEG input → assert stored object is WebP, dimensions ≤256×256, smaller than the original - GIF input → assert stored object is WebP - Hash returned from the action == SHA256 of the stored WebP bytes (not the original upload) - Delete tests upload a real pfp then delete it and verify CDNRetrieve throws afterwards Also move change-pfp.test.ts from the isolated vitest project back to the default pool now that it no longer uses vi.mock. * chore: apply biome formatting to test file * fix: use POSTGRES_TEST_HOST env var in bot vitest setup Both vitest.globalSetup.ts and vitest.setup.ts hardcoded 'tachi-postgres' (the dev server), but just bot-db-test-template-reset creates the template on 'tachi-postgres-test' (the tmpfs test server). Workers then failed to clone the template because they were looking on the wrong host. Mirrors the pattern already used in the server package: process.env.POSTGRES_TEST_HOST ?? 'tachi-postgres-test' * fix: update pfp router tests for WebP resize pipeline The PUT tests were comparing the stored S3 bytes directly to the original upload buffer. Since we now resize and re-encode to WebP, this always fails. Also, following the mockApi redirect to fetch the bytes added a slow supertest round-trip. Replace with: read custom_pfp_location from the DB after PUT, fetch the stored object directly via CDNRetrieve, and assert it is smaller than the original (non-deterministic WebP compression makes exact byte comparison inappropriate). * fix: update banner router tests for WebP resize pipeline * fix: tests * fix: ci
This commit is contained in:
zk
@@ -43,17 +43,19 @@ jobs:
|
||||
env:
|
||||
NODE_ENV: "test"
|
||||
services:
|
||||
tachi-postgres:
|
||||
tachi-postgres-test:
|
||||
image: postgres:18
|
||||
env:
|
||||
POSTGRES_USER: tachi
|
||||
POSTGRES_PASSWORD: tachi
|
||||
POSTGRES_DB: postgres
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U tachi"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
--tmpfs /var/lib/postgresql/data:rw,size=1g
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -71,6 +73,8 @@ jobs:
|
||||
run: bun run --filter tachi-bot test
|
||||
env:
|
||||
NODE_ENV: "test"
|
||||
POSTGRES_TEST_HOST: tachi-postgres-test
|
||||
POSTGRES_TEST_URL: postgresql://tachi:tachi@tachi-postgres-test
|
||||
|
||||
docker-push:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ check:
|
||||
source .scripts/multi_evaluator.sh
|
||||
|
||||
evaluate "just fmt-check"
|
||||
evaluate "bun run --filter '*' lint"
|
||||
evaluate "bun run --filter '*' lint -- --quiet"
|
||||
evaluate "just typecheck"
|
||||
evaluate "bun .scripts/ts_machete.js"
|
||||
evaluate "bun .scripts/ts_autoinherit.js --check"
|
||||
@@ -117,4 +117,4 @@ load-test-score-import *ARGS:
|
||||
# Seed N dev users + API tokens (submit_score); writes one token per line to OUTPUT_FILE.
|
||||
# Uses the server package env (.env); same Postgres as a local tachi-server.
|
||||
load-test-score-import-seed-tokens COUNT OUTPUT_FILE:
|
||||
cd typescript/server && bun run src/load-tests/seed-stress-api-tokens.ts {{COUNT}} {{OUTPUT_FILE}}
|
||||
cd typescript/server && bun run src/load-tests/seed-stress-api-tokens.ts {{COUNT}} {{OUTPUT_FILE}}
|
||||
|
||||
@@ -273,7 +273,6 @@
|
||||
"version": "0.1.0",
|
||||
"devDependencies": {
|
||||
"eslint-config-tachi": "workspace:*",
|
||||
"prettier": "catalog:",
|
||||
"vite": "catalog:",
|
||||
},
|
||||
},
|
||||
@@ -384,6 +383,7 @@
|
||||
"prom-client": "catalog:",
|
||||
"prudence": "catalog:",
|
||||
"rg-stats": "catalog:",
|
||||
"sharp": "catalog:",
|
||||
"tachi-common": "workspace:*",
|
||||
"tachi-db": "workspace:*",
|
||||
"tachi-db-migration-engine": "workspace:*",
|
||||
@@ -551,6 +551,7 @@
|
||||
"react-select": "^5.6.1",
|
||||
"rg-stats": "workspace:*",
|
||||
"sass": "^1.77.0",
|
||||
"sharp": "^0.34.5",
|
||||
"sql.js": "^1.14.1",
|
||||
"supertest": "6.2.2",
|
||||
"sync-fetch": "^0.3.1",
|
||||
@@ -941,6 +942,56 @@
|
||||
|
||||
"@humanwhocodes/retry": ["@humanwhocodes/retry@0.4.3", "", {}, "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ=="],
|
||||
|
||||
"@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="],
|
||||
|
||||
"@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="],
|
||||
|
||||
"@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="],
|
||||
|
||||
"@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="],
|
||||
|
||||
"@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="],
|
||||
|
||||
"@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="],
|
||||
|
||||
"@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="],
|
||||
|
||||
"@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="],
|
||||
|
||||
"@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="],
|
||||
|
||||
"@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="],
|
||||
|
||||
"@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="],
|
||||
|
||||
"@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="],
|
||||
|
||||
"@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="],
|
||||
|
||||
"@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="],
|
||||
|
||||
"@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="],
|
||||
|
||||
"@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="],
|
||||
|
||||
"@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="],
|
||||
|
||||
"@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="],
|
||||
|
||||
"@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="],
|
||||
|
||||
"@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="],
|
||||
|
||||
"@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="],
|
||||
|
||||
"@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="],
|
||||
|
||||
"@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="],
|
||||
|
||||
"@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="],
|
||||
|
||||
"@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="],
|
||||
|
||||
"@isaacs/cliui": ["@isaacs/cliui@8.0.2", "", { "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="],
|
||||
|
||||
"@istanbuljs/schema": ["@istanbuljs/schema@0.1.3", "", {}, "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA=="],
|
||||
@@ -2623,6 +2674,8 @@
|
||||
|
||||
"setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
|
||||
|
||||
"sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="],
|
||||
|
||||
"shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
|
||||
|
||||
"shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
|
||||
|
||||
@@ -156,6 +156,7 @@
|
||||
"react-router-dom": "5.1.2",
|
||||
"react-select": "^5.6.1",
|
||||
"rg-stats": "workspace:*",
|
||||
"sharp": "^0.34.5",
|
||||
"sass": "^1.77.0",
|
||||
"supertest": "6.2.2",
|
||||
"sync-fetch": "^0.3.1",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { execSync } from "node:child_process";
|
||||
import pg from "pg";
|
||||
|
||||
const POSTGRES_HOST = "tachi-postgres";
|
||||
const POSTGRES_HOST = process.env.POSTGRES_TEST_HOST ?? "tachi-postgres-test";
|
||||
const POSTGRES_USER = "tachi";
|
||||
const POSTGRES_PASS = "tachi";
|
||||
const TEMPLATE_DB = "tachi_bot_test_template";
|
||||
|
||||
@@ -16,7 +16,7 @@ import { allImportTypes } from "tachi-common/constants/import-types";
|
||||
const WORKER_ID = crypto.randomUUID().slice(0, 8);
|
||||
const WORKER_DB_NAME = `tachi_bot_test_${WORKER_ID}`;
|
||||
|
||||
const POSTGRES_HOST = "tachi-postgres";
|
||||
const POSTGRES_HOST = process.env.POSTGRES_TEST_HOST ?? "tachi-postgres-test";
|
||||
const POSTGRES_USER = "tachi";
|
||||
const POSTGRES_PASS = "tachi";
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ export { type priv_svc_cg_card_info_service, type default as PrivSvcCgCardInfoTa
|
||||
export { type score_rederive_chart_id, type default as ScoreRederiveTable, type ScoreRederive, type NewScoreRederive, type ScoreRederiveUpdate } from './public/ScoreRederive';
|
||||
export { type class_achievement_row_id, type default as ClassAchievementTable, type ClassAchievement, type NewClassAchievement, type ClassAchievementUpdate } from './public/ClassAchievement';
|
||||
export { type account_id, type default as AccountTable, type Account, type NewAccount, type AccountUpdate } from './public/Account';
|
||||
export { type import_timestop_import_type, type default as ImportTimestopTable, type ImportTimestop, type NewImportTimestop, type ImportTimestopUpdate } from './public/ImportTimestop';
|
||||
export { type import_tracker_import_id, type default as ImportTrackerTable, type ImportTracker, type NewImportTracker, type ImportTrackerUpdate } from './public/ImportTracker';
|
||||
export { type default as SvcFerSettingsTable, type SvcFerSettings, type NewSvcFerSettings, type SvcFerSettingsUpdate } from './public/SvcFerSettings';
|
||||
export { type default as InviteLockTable, type InviteLock, type NewInviteLock, type InviteLockUpdate } from './public/InviteLock';
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
// @generated
|
||||
// This file is automatically generated by Kanel. Do not modify manually.
|
||||
|
||||
import type { default as ImportType } from './ImportType';
|
||||
import type { account_id } from './Account';
|
||||
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
|
||||
|
||||
/** Identifier type for public.import_timestop */
|
||||
export type import_timestop_import_type = ImportType;
|
||||
|
||||
/** Represents the table public.import_timestop */
|
||||
export default interface ImportTimestopTable {
|
||||
user_id: ColumnType<account_id, account_id, account_id>;
|
||||
|
||||
import_type: ColumnType<import_timestop_import_type, import_timestop_import_type, import_timestop_import_type>;
|
||||
|
||||
last_score_time: ColumnType<string, string, string>;
|
||||
}
|
||||
|
||||
export type ImportTimestop = Selectable<ImportTimestopTable>;
|
||||
|
||||
export type NewImportTimestop = Insertable<ImportTimestopTable>;
|
||||
|
||||
export type ImportTimestopUpdate = Updateable<ImportTimestopTable>;
|
||||
@@ -5,6 +5,7 @@ import type { default as PrivSvcCgCardInfoTable } from './PrivSvcCgCardInfo';
|
||||
import type { default as ScoreRederiveTable } from './ScoreRederive';
|
||||
import type { default as ClassAchievementTable } from './ClassAchievement';
|
||||
import type { default as AccountTable } from './Account';
|
||||
import type { default as ImportTimestopTable } from './ImportTimestop';
|
||||
import type { default as ImportTrackerTable } from './ImportTracker';
|
||||
import type { default as SvcFerSettingsTable } from './SvcFerSettings';
|
||||
import type { default as InviteLockTable } from './InviteLock';
|
||||
@@ -78,6 +79,8 @@ export default interface PublicSchema {
|
||||
|
||||
account: AccountTable;
|
||||
|
||||
import_timestop: ImportTimestopTable;
|
||||
|
||||
import_tracker: ImportTrackerTable;
|
||||
|
||||
svc_fer_settings: SvcFerSettingsTable;
|
||||
|
||||
@@ -68,6 +68,7 @@
|
||||
"prom-client": "catalog:",
|
||||
"prudence": "catalog:",
|
||||
"rg-stats": "catalog:",
|
||||
"sharp": "catalog:",
|
||||
"tachi-common": "workspace:*",
|
||||
"tachi-db": "workspace:*",
|
||||
"tachi-db-migration-engine": "workspace:*",
|
||||
|
||||
@@ -1,27 +1,46 @@
|
||||
import { MakeAction } from "#lib/actions/actions";
|
||||
import { CDNStoreOrOverwrite } from "#lib/cdn/cdn";
|
||||
import { CDNStoreWithMeta } from "#lib/cdn/cdn";
|
||||
import { GetProfileBannerURL } from "#lib/cdn/url-format";
|
||||
import DB from "#services/pg/db";
|
||||
import { HashSHA256 } from "#utils/crypto";
|
||||
import { ExpectedErr } from "bliss";
|
||||
import sharp from "sharp";
|
||||
|
||||
/** Max dimensions for stored profile banners (used as a full-page background). */
|
||||
const BANNER_MAX_WIDTH = 1920;
|
||||
const BANNER_MAX_HEIGHT = 1080;
|
||||
|
||||
/**
|
||||
* Resizes any image (including animated GIFs) to at most BANNER_MAX_WIDTH × BANNER_MAX_HEIGHT
|
||||
* and re-encodes as WebP, preserving all animation frames.
|
||||
*/
|
||||
async function resizeBanner(buf: Buffer): Promise<Buffer> {
|
||||
return sharp(buf, { animated: true })
|
||||
.resize(BANNER_MAX_WIDTH, BANNER_MAX_HEIGHT, { fit: "inside", withoutEnlargement: true })
|
||||
.webp({ quality: 85 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
export const ACTION_ChangeBanner = MakeAction(
|
||||
"CHANGE_BANNER",
|
||||
async (taker, { "!fileBuffer": fileBuffer, fileMimetype }) => {
|
||||
const contentHash = HashSHA256(fileBuffer);
|
||||
|
||||
if (
|
||||
fileMimetype === "image/jpeg" ||
|
||||
fileMimetype === "image/png" ||
|
||||
fileMimetype === "image/gif"
|
||||
fileMimetype !== "image/jpeg" &&
|
||||
fileMimetype !== "image/png" &&
|
||||
fileMimetype !== "image/gif"
|
||||
) {
|
||||
await CDNStoreOrOverwrite(GetProfileBannerURL(taker.acct.id, contentHash), fileBuffer);
|
||||
} else {
|
||||
// GIF is deliberately not mentioned here
|
||||
// as it's an easter egg
|
||||
// GIF is deliberately not mentioned in the error message as it's an easter egg
|
||||
throw new ExpectedErr(400, "Invalid file - only JPG and PNG files are supported.");
|
||||
}
|
||||
|
||||
const storedBuffer = await resizeBanner(fileBuffer);
|
||||
const contentHash = HashSHA256(storedBuffer);
|
||||
|
||||
await CDNStoreWithMeta(GetProfileBannerURL(taker.acct.id, contentHash), storedBuffer, {
|
||||
contentType: "image/webp",
|
||||
cacheControl: "public, max-age=31536000, immutable",
|
||||
});
|
||||
|
||||
await DB.updateTable("account")
|
||||
.set({ custom_banner_location: contentHash })
|
||||
.where("id", "=", taker.acct.id)
|
||||
|
||||
@@ -1,28 +1,43 @@
|
||||
import { CDNDelete, CDNStoreOrOverwrite } from "#lib/cdn/cdn";
|
||||
/**
|
||||
* Integration tests for ACTION_ChangePfp, ACTION_DeletePfp, ACTION_ChangeBanner, and
|
||||
* ACTION_DeleteBanner.
|
||||
*
|
||||
* These tests hit real MinIO (via the test CDN config) and pass real image buffers through
|
||||
* sharp, so they verify the full upload pipeline: resize → WebP encode → S3 store → DB update.
|
||||
*/
|
||||
|
||||
import { CDNDelete, CDNRetrieve } from "#lib/cdn/cdn";
|
||||
import { GetProfileBannerURL, GetProfilePictureURL } from "#lib/cdn/url-format";
|
||||
import DB from "#services/pg/db";
|
||||
import { seedUser } from "#test-utils/pg-fixtures";
|
||||
import { GetKTDataBuffer } from "#test-utils/test-data";
|
||||
import { HashSHA256 } from "#utils/crypto";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import sharp from "sharp";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { ACTION_ChangeBanner } from "./change-banner";
|
||||
import { ACTION_ChangePfp } from "./change-pfp";
|
||||
import { ACTION_DeleteBanner } from "./delete-banner";
|
||||
import { ACTION_DeletePfp } from "./delete-pfp";
|
||||
|
||||
vi.mock("#lib/cdn/cdn.js", () => ({
|
||||
CDNStoreOrOverwrite: vi.fn().mockResolvedValue(undefined),
|
||||
CDNDelete: vi.fn().mockResolvedValue(undefined),
|
||||
CDNRedirect: vi.fn(),
|
||||
}));
|
||||
// ─── Fixtures ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Real 600×500 PNG (64 KB). Sharp will resize it to fit within 256×256. */
|
||||
const ACORN_PNG = GetKTDataBuffer("/images/acorn.png");
|
||||
|
||||
/** Minimal 4×4 single-frame GIF, generated once before all tests. */
|
||||
let MINIMAL_GIF: Buffer;
|
||||
|
||||
beforeAll(async () => {
|
||||
MINIMAL_GIF = await sharp({
|
||||
create: { background: { b: 0, g: 0, r: 255 }, channels: 3, height: 4, width: 4 },
|
||||
})
|
||||
.gif()
|
||||
.toBuffer();
|
||||
});
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const JPEG_BUFFER = Buffer.from("fake-jpeg-data");
|
||||
const PNG_BUFFER = Buffer.from("fake-png-data");
|
||||
const GIF_BUFFER = Buffer.from("fake-gif-data");
|
||||
const BAD_BUFFER = Buffer.from("fake-webp-data");
|
||||
|
||||
async function getPfpLocation(userId: number) {
|
||||
const row = await DB.selectFrom("account")
|
||||
.select("custom_pfp_location")
|
||||
@@ -60,104 +75,148 @@ async function seedUserWithBanner(userId: number, hash: string) {
|
||||
describe("ACTION_ChangePfp", () => {
|
||||
let userId: number;
|
||||
let username: string;
|
||||
const cdnCleanup: string[] = [];
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
({ id: userId, username } = await seedUser({ username: "test_user" }));
|
||||
cdnCleanup.length = 0;
|
||||
});
|
||||
|
||||
// ── Mimetype validation ───────────────────────────────────────────────────
|
||||
|
||||
it("returns { contentHash } for a JPEG file", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
|
||||
const result = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": JPEG_BUFFER,
|
||||
fileMimetype: "image/jpeg",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ contentHash: HashSHA256(JPEG_BUFFER) });
|
||||
afterEach(async () => {
|
||||
await Promise.all(cdnCleanup.map((p) => CDNDelete(p)));
|
||||
});
|
||||
|
||||
it("returns { contentHash } for a PNG file", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
// ── Resize + S3 integration ───────────────────────────────────────────────
|
||||
|
||||
const result = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": PNG_BUFFER,
|
||||
it("resizes a PNG to fit within 256×256 and stores it as WebP in S3", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ contentHash: HashSHA256(PNG_BUFFER) });
|
||||
const cdnPath = GetProfilePictureURL(userId, contentHash);
|
||||
cdnCleanup.push(cdnPath);
|
||||
const stored = await CDNRetrieve(cdnPath);
|
||||
const meta = await sharp(stored, { animated: true }).metadata();
|
||||
|
||||
expect(meta.format).toBe("webp");
|
||||
expect(meta.width).toBeLessThanOrEqual(256);
|
||||
expect(meta.height).toBeLessThanOrEqual(256);
|
||||
expect(stored.length).toBeLessThan(ACORN_PNG.length);
|
||||
});
|
||||
|
||||
it("returns { contentHash } for a GIF file", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
it("resizes a JPEG to fit within 256×256 and stores it as WebP in S3", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
const result = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": GIF_BUFFER,
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/jpeg",
|
||||
});
|
||||
|
||||
const cdnPath = GetProfilePictureURL(userId, contentHash);
|
||||
cdnCleanup.push(cdnPath);
|
||||
const stored = await CDNRetrieve(cdnPath);
|
||||
const meta = await sharp(stored, { animated: true }).metadata();
|
||||
|
||||
expect(meta.format).toBe("webp");
|
||||
expect(meta.width).toBeLessThanOrEqual(256);
|
||||
expect(meta.height).toBeLessThanOrEqual(256);
|
||||
});
|
||||
|
||||
it("converts a GIF to WebP and stores it in S3", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": MINIMAL_GIF,
|
||||
fileMimetype: "image/gif",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ contentHash: HashSHA256(GIF_BUFFER) });
|
||||
const cdnPath = GetProfilePictureURL(userId, contentHash);
|
||||
cdnCleanup.push(cdnPath);
|
||||
const stored = await CDNRetrieve(cdnPath);
|
||||
const meta = await sharp(stored, { animated: true }).metadata();
|
||||
|
||||
expect(meta.format).toBe("webp");
|
||||
});
|
||||
|
||||
it("throws 400 for an unsupported mimetype", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
it("hashes the resized WebP output, not the original upload", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangePfp(taker, { "!fileBuffer": BAD_BUFFER, fileMimetype: "image/webp" }),
|
||||
).rejects.toMatchObject({ code: 400 });
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
const cdnPath = GetProfilePictureURL(userId, contentHash);
|
||||
cdnCleanup.push(cdnPath);
|
||||
const stored = await CDNRetrieve(cdnPath);
|
||||
|
||||
expect(contentHash).toBe(HashSHA256(stored));
|
||||
expect(contentHash).not.toBe(HashSHA256(ACORN_PNG));
|
||||
});
|
||||
|
||||
// ── Database updates ──────────────────────────────────────────────────────
|
||||
|
||||
it("persists the content hash to custom_pfp_location", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
it("persists the content hash of the stored WebP to custom_pfp_location", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_ChangePfp(taker, { "!fileBuffer": JPEG_BUFFER, fileMimetype: "image/jpeg" });
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
expect(await getPfpLocation(userId)).toBe(HashSHA256(JPEG_BUFFER));
|
||||
cdnCleanup.push(GetProfilePictureURL(userId, contentHash));
|
||||
|
||||
expect(await getPfpLocation(userId)).toBe(contentHash);
|
||||
});
|
||||
|
||||
it("does not update other users' custom_pfp_location", async () => {
|
||||
const other = await seedUser({ username: "other_user" });
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_ChangePfp(taker, { "!fileBuffer": JPEG_BUFFER, fileMimetype: "image/jpeg" });
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
cdnCleanup.push(GetProfilePictureURL(userId, contentHash));
|
||||
|
||||
expect(await getPfpLocation(other.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not update custom_pfp_location on a bad mimetype", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
// ── Mimetype validation ───────────────────────────────────────────────────
|
||||
|
||||
it("throws 400 for an unsupported mimetype", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangePfp(taker, { "!fileBuffer": BAD_BUFFER, fileMimetype: "image/webp" }),
|
||||
ACTION_ChangePfp(taker, { "!fileBuffer": ACORN_PNG, fileMimetype: "image/webp" }),
|
||||
).rejects.toMatchObject({ code: 400 });
|
||||
});
|
||||
|
||||
it("does not update custom_pfp_location on a bad mimetype", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangePfp(taker, { "!fileBuffer": ACORN_PNG, fileMimetype: "image/webp" }),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(await getPfpLocation(userId)).toBeNull();
|
||||
});
|
||||
|
||||
// ── CDN calls ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("calls CDNStoreOrOverwrite with the correct URL and buffer", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
|
||||
await ACTION_ChangePfp(taker, { "!fileBuffer": JPEG_BUFFER, fileMimetype: "image/jpeg" });
|
||||
|
||||
expect(CDNStoreOrOverwrite).toHaveBeenCalledOnce();
|
||||
expect(CDNStoreOrOverwrite).toHaveBeenCalledWith(
|
||||
GetProfilePictureURL(userId, HashSHA256(JPEG_BUFFER)),
|
||||
JPEG_BUFFER,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Audit log ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("writes a GOOD action row on success", async () => {
|
||||
const taker = { ip: "10.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "10.0.0.1" };
|
||||
|
||||
await ACTION_ChangePfp(taker, { "!fileBuffer": JPEG_BUFFER, fileMimetype: "image/jpeg" });
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
cdnCleanup.push(GetProfilePictureURL(userId, contentHash));
|
||||
|
||||
const action = await DB.selectFrom("action")
|
||||
.selectAll()
|
||||
@@ -165,18 +224,18 @@ describe("ACTION_ChangePfp", () => {
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(action).toMatchObject({
|
||||
ip: "10.0.0.1",
|
||||
kind: "CHANGE_PFP",
|
||||
result: "GOOD",
|
||||
ip: "10.0.0.1",
|
||||
user_id: userId,
|
||||
});
|
||||
});
|
||||
|
||||
it("writes a BAD action row on invalid mimetype", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangePfp(taker, { "!fileBuffer": BAD_BUFFER, fileMimetype: "image/webp" }),
|
||||
ACTION_ChangePfp(taker, { "!fileBuffer": ACORN_PNG, fileMimetype: "image/webp" }),
|
||||
).rejects.toThrow();
|
||||
|
||||
const action = await DB.selectFrom("action")
|
||||
@@ -188,16 +247,21 @@ describe("ACTION_ChangePfp", () => {
|
||||
});
|
||||
|
||||
it("does not store the file buffer content in the audit log input", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_ChangePfp(taker, { "!fileBuffer": JPEG_BUFFER, fileMimetype: "image/jpeg" });
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
cdnCleanup.push(GetProfilePictureURL(userId, contentHash));
|
||||
|
||||
const action = await DB.selectFrom("action")
|
||||
.select("input")
|
||||
.where("kind", "=", "CHANGE_PFP")
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(JSON.stringify(action.input)).not.toContain(JPEG_BUFFER.toString("base64"));
|
||||
expect(JSON.stringify(action.input)).not.toContain(ACORN_PNG.toString("base64"));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -208,20 +272,19 @@ describe("ACTION_DeletePfp", () => {
|
||||
let username: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
({ id: userId, username } = await seedUser({ username: "test_user" }));
|
||||
});
|
||||
|
||||
// ── 404 guard ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("throws 404 when the user has no custom pfp", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(ACTION_DeletePfp(taker, {})).rejects.toMatchObject({ code: 404 });
|
||||
});
|
||||
|
||||
it("writes a BAD action row when the user has no custom pfp", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(ACTION_DeletePfp(taker, {})).rejects.toThrow();
|
||||
|
||||
@@ -235,22 +298,20 @@ describe("ACTION_DeletePfp", () => {
|
||||
|
||||
// ── Success path ──────────────────────────────────────────────────────────
|
||||
|
||||
it("returns {} on success", async () => {
|
||||
await seedUserWithPfp(userId, "existinghash");
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
it("uploads a pfp then deletes it, removing it from S3", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
const result = await ACTION_DeletePfp(taker, {});
|
||||
const { contentHash } = await ACTION_ChangePfp(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it("clears custom_pfp_location to null in the DB", async () => {
|
||||
await seedUserWithPfp(userId, "existinghash");
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const cdnPath = GetProfilePictureURL(userId, contentHash);
|
||||
|
||||
await ACTION_DeletePfp(taker, {});
|
||||
|
||||
expect(await getPfpLocation(userId)).toBeNull();
|
||||
await expect(CDNRetrieve(cdnPath)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("does not touch other users' custom_pfp_location", async () => {
|
||||
@@ -258,30 +319,18 @@ describe("ACTION_DeletePfp", () => {
|
||||
await seedUserWithPfp(other.id, "otherhash");
|
||||
await seedUserWithPfp(userId, "myhash");
|
||||
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_DeletePfp(taker, {});
|
||||
|
||||
expect(await getPfpLocation(other.id)).toBe("otherhash");
|
||||
});
|
||||
|
||||
// ── CDN calls ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("calls CDNDelete once with the correct URL", async () => {
|
||||
await seedUserWithPfp(userId, "existinghash");
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
|
||||
await ACTION_DeletePfp(taker, {});
|
||||
|
||||
expect(CDNDelete).toHaveBeenCalledOnce();
|
||||
expect(CDNDelete).toHaveBeenCalledWith(GetProfilePictureURL(userId, "existinghash"));
|
||||
});
|
||||
|
||||
// ── Audit log ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("writes a GOOD action row on success", async () => {
|
||||
await seedUserWithPfp(userId, "existinghash");
|
||||
const taker = { ip: "10.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "10.0.0.1" };
|
||||
|
||||
await ACTION_DeletePfp(taker, {});
|
||||
|
||||
@@ -291,9 +340,9 @@ describe("ACTION_DeletePfp", () => {
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(action).toMatchObject({
|
||||
ip: "10.0.0.1",
|
||||
kind: "DELETE_PFP",
|
||||
result: "GOOD",
|
||||
ip: "10.0.0.1",
|
||||
user_id: userId,
|
||||
});
|
||||
});
|
||||
@@ -304,135 +353,146 @@ describe("ACTION_DeletePfp", () => {
|
||||
describe("ACTION_ChangeBanner", () => {
|
||||
let userId: number;
|
||||
let username: string;
|
||||
const cdnCleanup: string[] = [];
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
({ id: userId, username } = await seedUser({ username: "test_user" }));
|
||||
cdnCleanup.length = 0;
|
||||
});
|
||||
|
||||
// ── Mimetype validation ───────────────────────────────────────────────────
|
||||
|
||||
it("returns { contentHash } for a JPEG file", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
|
||||
const result = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": JPEG_BUFFER,
|
||||
fileMimetype: "image/jpeg",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ contentHash: HashSHA256(JPEG_BUFFER) });
|
||||
afterEach(async () => {
|
||||
await Promise.all(cdnCleanup.map((p) => CDNDelete(p)));
|
||||
});
|
||||
|
||||
it("returns { contentHash } for a PNG file", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
// ── Resize + S3 integration ───────────────────────────────────────────────
|
||||
|
||||
const result = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": PNG_BUFFER,
|
||||
it("converts a PNG to WebP and stores it in S3", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ contentHash: HashSHA256(PNG_BUFFER) });
|
||||
const cdnPath = GetProfileBannerURL(userId, contentHash);
|
||||
cdnCleanup.push(cdnPath);
|
||||
const stored = await CDNRetrieve(cdnPath);
|
||||
const meta = await sharp(stored, { animated: true }).metadata();
|
||||
|
||||
expect(meta.format).toBe("webp");
|
||||
});
|
||||
|
||||
it("returns { contentHash } for a GIF file", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
it("converts a GIF to WebP and stores it in S3", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
const result = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": GIF_BUFFER,
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": MINIMAL_GIF,
|
||||
fileMimetype: "image/gif",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ contentHash: HashSHA256(GIF_BUFFER) });
|
||||
const cdnPath = GetProfileBannerURL(userId, contentHash);
|
||||
cdnCleanup.push(cdnPath);
|
||||
const stored = await CDNRetrieve(cdnPath);
|
||||
const meta = await sharp(stored, { animated: true }).metadata();
|
||||
|
||||
expect(meta.format).toBe("webp");
|
||||
});
|
||||
|
||||
it("throws 400 for an unsupported mimetype", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
it("hashes the resized WebP output, not the original upload", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangeBanner(taker, { "!fileBuffer": BAD_BUFFER, fileMimetype: "image/webp" }),
|
||||
).rejects.toMatchObject({ code: 400 });
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
const cdnPath = GetProfileBannerURL(userId, contentHash);
|
||||
cdnCleanup.push(cdnPath);
|
||||
const stored = await CDNRetrieve(cdnPath);
|
||||
|
||||
expect(contentHash).toBe(HashSHA256(stored));
|
||||
expect(contentHash).not.toBe(HashSHA256(ACORN_PNG));
|
||||
});
|
||||
|
||||
// ── Database updates ──────────────────────────────────────────────────────
|
||||
|
||||
it("persists the content hash to custom_banner_location", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": JPEG_BUFFER,
|
||||
fileMimetype: "image/jpeg",
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
expect(await getBannerLocation(userId)).toBe(HashSHA256(JPEG_BUFFER));
|
||||
cdnCleanup.push(GetProfileBannerURL(userId, contentHash));
|
||||
|
||||
expect(await getBannerLocation(userId)).toBe(contentHash);
|
||||
});
|
||||
|
||||
it("does not update other users' custom_banner_location", async () => {
|
||||
const other = await seedUser({ username: "other_user" });
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": JPEG_BUFFER,
|
||||
fileMimetype: "image/jpeg",
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
cdnCleanup.push(GetProfileBannerURL(userId, contentHash));
|
||||
|
||||
expect(await getBannerLocation(other.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not update custom_banner_location on a bad mimetype", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
// ── Mimetype validation ───────────────────────────────────────────────────
|
||||
|
||||
it("throws 400 for an unsupported mimetype", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangeBanner(taker, { "!fileBuffer": BAD_BUFFER, fileMimetype: "image/webp" }),
|
||||
ACTION_ChangeBanner(taker, { "!fileBuffer": ACORN_PNG, fileMimetype: "image/webp" }),
|
||||
).rejects.toMatchObject({ code: 400 });
|
||||
});
|
||||
|
||||
it("does not update custom_banner_location on a bad mimetype", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangeBanner(taker, { "!fileBuffer": ACORN_PNG, fileMimetype: "image/webp" }),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(await getBannerLocation(userId)).toBeNull();
|
||||
});
|
||||
|
||||
// ── CDN calls ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("calls CDNStoreOrOverwrite with the correct URL and buffer", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
|
||||
await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": JPEG_BUFFER,
|
||||
fileMimetype: "image/jpeg",
|
||||
});
|
||||
|
||||
expect(CDNStoreOrOverwrite).toHaveBeenCalledOnce();
|
||||
expect(CDNStoreOrOverwrite).toHaveBeenCalledWith(
|
||||
GetProfileBannerURL(userId, HashSHA256(JPEG_BUFFER)),
|
||||
JPEG_BUFFER,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Audit log ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("writes a GOOD action row on success", async () => {
|
||||
const taker = { ip: "10.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "10.0.0.1" };
|
||||
|
||||
await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": JPEG_BUFFER,
|
||||
fileMimetype: "image/jpeg",
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
cdnCleanup.push(GetProfileBannerURL(userId, contentHash));
|
||||
|
||||
const action = await DB.selectFrom("action")
|
||||
.selectAll()
|
||||
.where("kind", "=", "CHANGE_BANNER")
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(action).toMatchObject({
|
||||
ip: "10.0.0.1",
|
||||
kind: "CHANGE_BANNER",
|
||||
result: "GOOD",
|
||||
ip: "10.0.0.1",
|
||||
user_id: userId,
|
||||
});
|
||||
});
|
||||
|
||||
it("writes a BAD action row on invalid mimetype", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(
|
||||
ACTION_ChangeBanner(taker, { "!fileBuffer": BAD_BUFFER, fileMimetype: "image/webp" }),
|
||||
ACTION_ChangeBanner(taker, { "!fileBuffer": ACORN_PNG, fileMimetype: "image/webp" }),
|
||||
).rejects.toThrow();
|
||||
|
||||
const action = await DB.selectFrom("action")
|
||||
@@ -444,19 +504,21 @@ describe("ACTION_ChangeBanner", () => {
|
||||
});
|
||||
|
||||
it("does not store the file buffer content in the audit log input", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": JPEG_BUFFER,
|
||||
fileMimetype: "image/jpeg",
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
cdnCleanup.push(GetProfileBannerURL(userId, contentHash));
|
||||
|
||||
const action = await DB.selectFrom("action")
|
||||
.select("input")
|
||||
.where("kind", "=", "CHANGE_BANNER")
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(JSON.stringify(action.input)).not.toContain(JPEG_BUFFER.toString("base64"));
|
||||
expect(JSON.stringify(action.input)).not.toContain(ACORN_PNG.toString("base64"));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -467,20 +529,19 @@ describe("ACTION_DeleteBanner", () => {
|
||||
let username: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
({ id: userId, username } = await seedUser({ username: "test_user" }));
|
||||
});
|
||||
|
||||
// ── 404 guard ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("throws 404 when the user has no custom banner", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(ACTION_DeleteBanner(taker, {})).rejects.toMatchObject({ code: 404 });
|
||||
});
|
||||
|
||||
it("writes a BAD action row when the user has no custom banner", async () => {
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await expect(ACTION_DeleteBanner(taker, {})).rejects.toThrow();
|
||||
|
||||
@@ -494,22 +555,20 @@ describe("ACTION_DeleteBanner", () => {
|
||||
|
||||
// ── Success path ──────────────────────────────────────────────────────────
|
||||
|
||||
it("returns {} on success", async () => {
|
||||
await seedUserWithBanner(userId, "existinghash");
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
it("uploads a banner then deletes it, removing it from S3", async () => {
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
const result = await ACTION_DeleteBanner(taker, {});
|
||||
const { contentHash } = await ACTION_ChangeBanner(taker, {
|
||||
"!fileBuffer": ACORN_PNG,
|
||||
fileMimetype: "image/png",
|
||||
});
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it("clears custom_banner_location to null in the DB", async () => {
|
||||
await seedUserWithBanner(userId, "existinghash");
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const cdnPath = GetProfileBannerURL(userId, contentHash);
|
||||
|
||||
await ACTION_DeleteBanner(taker, {});
|
||||
|
||||
expect(await getBannerLocation(userId)).toBeNull();
|
||||
await expect(CDNRetrieve(cdnPath)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("does not touch other users' custom_banner_location", async () => {
|
||||
@@ -517,30 +576,18 @@ describe("ACTION_DeleteBanner", () => {
|
||||
await seedUserWithBanner(other.id, "otherhash");
|
||||
await seedUserWithBanner(userId, "myhash");
|
||||
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "127.0.0.1" };
|
||||
|
||||
await ACTION_DeleteBanner(taker, {});
|
||||
|
||||
expect(await getBannerLocation(other.id)).toBe("otherhash");
|
||||
});
|
||||
|
||||
// ── CDN calls ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("calls CDNDelete once with the correct URL", async () => {
|
||||
await seedUserWithBanner(userId, "existinghash");
|
||||
const taker = { ip: "127.0.0.1", acct: { id: userId, username } };
|
||||
|
||||
await ACTION_DeleteBanner(taker, {});
|
||||
|
||||
expect(CDNDelete).toHaveBeenCalledOnce();
|
||||
expect(CDNDelete).toHaveBeenCalledWith(GetProfileBannerURL(userId, "existinghash"));
|
||||
});
|
||||
|
||||
// ── Audit log ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("writes a GOOD action row on success", async () => {
|
||||
await seedUserWithBanner(userId, "existinghash");
|
||||
const taker = { ip: "10.0.0.1", acct: { id: userId, username } };
|
||||
const taker = { acct: { id: userId, username }, ip: "10.0.0.1" };
|
||||
|
||||
await ACTION_DeleteBanner(taker, {});
|
||||
|
||||
@@ -550,9 +597,9 @@ describe("ACTION_DeleteBanner", () => {
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(action).toMatchObject({
|
||||
ip: "10.0.0.1",
|
||||
kind: "DELETE_BANNER",
|
||||
result: "GOOD",
|
||||
ip: "10.0.0.1",
|
||||
user_id: userId,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,27 +1,45 @@
|
||||
import { MakeAction } from "#lib/actions/actions";
|
||||
import { CDNStoreOrOverwrite } from "#lib/cdn/cdn";
|
||||
import { CDNStoreWithMeta } from "#lib/cdn/cdn";
|
||||
import { GetProfilePictureURL } from "#lib/cdn/url-format";
|
||||
import DB from "#services/pg/db";
|
||||
import { HashSHA256 } from "#utils/crypto";
|
||||
import { ExpectedErr } from "bliss";
|
||||
import sharp from "sharp";
|
||||
|
||||
/** Max dimension (width or height) for stored profile pictures. */
|
||||
const PFP_MAX_PX = 256;
|
||||
|
||||
/**
|
||||
* Resizes any image (including animated GIFs) to at most PFP_MAX_PX × PFP_MAX_PX
|
||||
* and re-encodes as WebP, preserving all animation frames.
|
||||
*/
|
||||
async function resizePfp(buf: Buffer): Promise<Buffer> {
|
||||
return sharp(buf, { animated: true })
|
||||
.resize(PFP_MAX_PX, PFP_MAX_PX, { fit: "inside", withoutEnlargement: true })
|
||||
.webp({ quality: 85 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
export const ACTION_ChangePfp = MakeAction(
|
||||
"CHANGE_PFP",
|
||||
async (taker, { "!fileBuffer": fileBuffer, fileMimetype }) => {
|
||||
const contentHash = HashSHA256(fileBuffer);
|
||||
|
||||
if (
|
||||
fileMimetype === "image/jpeg" ||
|
||||
fileMimetype === "image/png" ||
|
||||
fileMimetype === "image/gif"
|
||||
fileMimetype !== "image/jpeg" &&
|
||||
fileMimetype !== "image/png" &&
|
||||
fileMimetype !== "image/gif"
|
||||
) {
|
||||
await CDNStoreOrOverwrite(GetProfilePictureURL(taker.acct.id, contentHash), fileBuffer);
|
||||
} else {
|
||||
// GIF is deliberately not mentioned here
|
||||
// as it's an easter egg
|
||||
// GIF is deliberately not mentioned in the error message as it's an easter egg
|
||||
throw new ExpectedErr(400, "Invalid file - only JPG and PNG files are supported.");
|
||||
}
|
||||
|
||||
const storedBuffer = await resizePfp(fileBuffer);
|
||||
const contentHash = HashSHA256(storedBuffer);
|
||||
|
||||
await CDNStoreWithMeta(GetProfilePictureURL(taker.acct.id, contentHash), storedBuffer, {
|
||||
contentType: "image/webp",
|
||||
cacheControl: "public, max-age=31536000, immutable",
|
||||
});
|
||||
|
||||
await DB.updateTable("account")
|
||||
.set({ custom_pfp_location: contentHash })
|
||||
.where("id", "=", taker.acct.id)
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { ServerConfig } from "#lib/setup/config";
|
||||
import { HashSHA256 } from "#utils/crypto";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
@@ -31,7 +32,7 @@ describe("cdn (S3 integration)", () => {
|
||||
const original = Buffer.from([0, 255, 128, 1]);
|
||||
await CDNStoreOrOverwrite(loc, original);
|
||||
const got = await CDNRetrieve(loc);
|
||||
expect(Buffer.compare(got, original)).toBe(0);
|
||||
expect(HashSHA256(got)).toBe(HashSHA256(original));
|
||||
await CDNDelete(loc);
|
||||
});
|
||||
|
||||
|
||||
@@ -3,7 +3,12 @@ import type { Response } from "express";
|
||||
import { log } from "#lib/log/log";
|
||||
import { ServerConfig } from "#lib/setup/config";
|
||||
|
||||
import { DeleteFromS3_PUBLIC, GetObjectFromS3_PUBLIC, PushToS3_PUBLIC } from "./s3";
|
||||
import {
|
||||
DeleteFromS3_PUBLIC,
|
||||
GetObjectFromS3_PUBLIC,
|
||||
PushToS3_PUBLIC,
|
||||
type S3ObjectMeta,
|
||||
} from "./s3";
|
||||
|
||||
/**
|
||||
* Retrieves the bytes at the given CDN location from S3.
|
||||
@@ -36,6 +41,20 @@ export async function CDNStoreOrOverwrite(fileLoc: string, data: string | Buffer
|
||||
await PushToS3_PUBLIC(fileLoc.replace(/^\//u, ""), data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores a file at fileLoc with explicit S3 object metadata (ContentType, CacheControl).
|
||||
* If it already exists, overwrite it.
|
||||
*/
|
||||
export async function CDNStoreWithMeta(
|
||||
fileLoc: string,
|
||||
data: string | Buffer,
|
||||
meta: S3ObjectMeta,
|
||||
): Promise<void> {
|
||||
log.debug(`Storing or overwriting path ${fileLoc} with metadata.`);
|
||||
|
||||
await PushToS3_PUBLIC(fileLoc.replace(/^\//u, ""), data, meta);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a file at this CDN location.
|
||||
*/
|
||||
|
||||
@@ -46,10 +46,15 @@ export function cdnObjectKey_PRIVATE(fileLoc: string): string {
|
||||
return (saveLocPrivate.KEY_PREFIX ?? "") + fileLoc;
|
||||
}
|
||||
|
||||
export interface S3ObjectMeta {
|
||||
cacheControl?: string;
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pushes a file to the configured S3 Bucket. Overwrites if already exists.
|
||||
*/
|
||||
export function PushToS3_PUBLIC(path: string, content: string | Buffer) {
|
||||
export function PushToS3_PUBLIC(path: string, content: string | Buffer, meta?: S3ObjectMeta) {
|
||||
if (path.startsWith("/")) {
|
||||
throw new Error(
|
||||
`no. absolutely not. Trying to do s3 push with a prefix /. this causes all sorts of trouble: ${path}`,
|
||||
@@ -62,6 +67,8 @@ export function PushToS3_PUBLIC(path: string, content: string | Buffer) {
|
||||
Bucket: saveLoc.BUCKET,
|
||||
Key: cdnObjectKey(path),
|
||||
Body: content,
|
||||
CacheControl: meta?.cacheControl,
|
||||
ContentType: meta?.contentType,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -577,7 +577,6 @@ describe("rederiveScoresForChart / chart checksum recalc (Postgres)", () => {
|
||||
await rederiveScoresForChart(chartId, log);
|
||||
|
||||
for (const scoreId of scoreIds) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const row = await DB.selectFrom("score")
|
||||
.select(["score.calculated_data"])
|
||||
.where("score.id", "=", scoreId)
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
/**
|
||||
* backfill-media.ts
|
||||
*
|
||||
* One-off script to resize and add cache metadata to all existing profile
|
||||
* pictures and banners in S3. Reads each user's current object, runs it
|
||||
* through the same resize/WebP pipeline introduced by the profile image
|
||||
* optimisation, stores the result under a new content-addressed key (with
|
||||
* immutable cache headers), and updates the DB row.
|
||||
*
|
||||
* Old S3 objects are NOT deleted - they remain as orphans in the bucket.
|
||||
* Remove them separately once you are satisfied the migration is complete,
|
||||
* for example with:
|
||||
* aws s3 rm --recursive s3://BUCKET/users/ --exclude pfp --dryrun
|
||||
*
|
||||
* The script is safe to re-run: if the existing object is already a small
|
||||
* WebP its hash will be unchanged after reprocessing and the DB update is
|
||||
* skipped.
|
||||
*
|
||||
* Usage:
|
||||
* bun run src/scripts/backfill-media.ts \
|
||||
* --pg-url postgresql://tachi:tachi@localhost/tachi \
|
||||
* --endpoint http://localhost:9000 \
|
||||
* --access-key-id KEY \
|
||||
* --secret-access-key SECRET \
|
||||
* --bucket tachi-public \
|
||||
* [--region us-east-1] \
|
||||
* [--key-prefix some/prefix/] \
|
||||
* [--dry-run]
|
||||
*
|
||||
* Alternatively, the S3 flags are read from the matching
|
||||
* TACHI_CDN_SAVE_LOCATION_* environment variables when not supplied on the
|
||||
* command line.
|
||||
*/
|
||||
|
||||
import type { Readable } from "node:stream";
|
||||
import type { Database } from "tachi-db";
|
||||
|
||||
import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import { Kysely, PostgresDialect } from "kysely";
|
||||
import crypto from "node:crypto";
|
||||
import { buffer as streamToBuffer } from "node:stream/consumers";
|
||||
import { parseArgs } from "node:util";
|
||||
import pg from "pg";
|
||||
import sharp from "sharp";
|
||||
|
||||
// ─── Constants (must match change-pfp.ts / change-banner.ts) ─────────────────
|
||||
|
||||
const PFP_MAX_PX = 256;
|
||||
const BANNER_MAX_WIDTH = 1920;
|
||||
const BANNER_MAX_HEIGHT = 1080;
|
||||
const CACHE_CONTROL_IMMUTABLE = "public, max-age=31536000, immutable";
|
||||
|
||||
// ─── CLI args ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`
|
||||
backfill-media — resize and cache-header all existing profile pictures/banners
|
||||
|
||||
--pg-url <url> Postgres connection string [TACHI_PG_URL]
|
||||
--endpoint <url> S3 endpoint [TACHI_CDN_SAVE_LOCATION_ENDPOINT]
|
||||
--access-key-id <key> S3 access key ID [TACHI_CDN_SAVE_LOCATION_ACCESS_KEY_ID]
|
||||
--secret-access-key <s> S3 secret access key [TACHI_CDN_SAVE_LOCATION_SECRET_ACCESS_KEY]
|
||||
--bucket <name> S3 bucket name [TACHI_CDN_SAVE_LOCATION_BUCKET]
|
||||
--region <region> S3 region (default: us-east-1) [TACHI_CDN_SAVE_LOCATION_REGION]
|
||||
--key-prefix <prefix> Key prefix in the bucket [TACHI_CDN_SAVE_LOCATION_KEY_PREFIX]
|
||||
--dry-run Log what would change without writing anything
|
||||
-h, --help Show this help
|
||||
`);
|
||||
}
|
||||
|
||||
const { values } = parseArgs({
|
||||
args: process.argv.slice(2),
|
||||
options: {
|
||||
"pg-url": { type: "string" },
|
||||
endpoint: { type: "string" },
|
||||
"access-key-id": { type: "string" },
|
||||
"secret-access-key": { type: "string" },
|
||||
bucket: { type: "string" },
|
||||
region: { type: "string" },
|
||||
"key-prefix": { type: "string" },
|
||||
"dry-run": { type: "boolean" },
|
||||
help: { type: "boolean", short: "h" },
|
||||
},
|
||||
});
|
||||
|
||||
if (values.help) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
function envOr(cliValue: string | undefined, envVar: string): string | undefined {
|
||||
return cliValue ?? process.env[envVar];
|
||||
}
|
||||
|
||||
function requireArg(value: string | undefined, name: string, envVar: string): string {
|
||||
if (!value) {
|
||||
console.error(`backfill-media: --${name} (or ${envVar}) is required.`);
|
||||
printHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
const pgUrl = requireArg(envOr(values["pg-url"], "TACHI_PG_URL"), "pg-url", "TACHI_PG_URL");
|
||||
|
||||
const s3Endpoint = requireArg(
|
||||
envOr(values.endpoint, "TACHI_CDN_SAVE_LOCATION_ENDPOINT"),
|
||||
"endpoint",
|
||||
"TACHI_CDN_SAVE_LOCATION_ENDPOINT",
|
||||
);
|
||||
|
||||
const s3AccessKeyId = requireArg(
|
||||
envOr(values["access-key-id"], "TACHI_CDN_SAVE_LOCATION_ACCESS_KEY_ID"),
|
||||
"access-key-id",
|
||||
"TACHI_CDN_SAVE_LOCATION_ACCESS_KEY_ID",
|
||||
);
|
||||
|
||||
const s3SecretAccessKey = requireArg(
|
||||
envOr(values["secret-access-key"], "TACHI_CDN_SAVE_LOCATION_SECRET_ACCESS_KEY"),
|
||||
"secret-access-key",
|
||||
"TACHI_CDN_SAVE_LOCATION_SECRET_ACCESS_KEY",
|
||||
);
|
||||
|
||||
const s3Bucket = requireArg(
|
||||
envOr(values.bucket, "TACHI_CDN_SAVE_LOCATION_BUCKET"),
|
||||
"bucket",
|
||||
"TACHI_CDN_SAVE_LOCATION_BUCKET",
|
||||
);
|
||||
|
||||
const s3Region = envOr(values.region, "TACHI_CDN_SAVE_LOCATION_REGION") ?? "us-east-1";
|
||||
const keyPrefix = envOr(values["key-prefix"], "TACHI_CDN_SAVE_LOCATION_KEY_PREFIX") ?? "";
|
||||
const isDryRun = values["dry-run"] ?? false;
|
||||
|
||||
if (isDryRun) {
|
||||
console.log("[backfill-media] DRY RUN — no changes will be written.");
|
||||
}
|
||||
|
||||
// ─── S3 + DB clients ──────────────────────────────────────────────────────────
|
||||
|
||||
const s3 = new S3Client({
|
||||
endpoint: s3Endpoint,
|
||||
region: s3Region,
|
||||
credentials: { accessKeyId: s3AccessKeyId, secretAccessKey: s3SecretAccessKey },
|
||||
forcePathStyle: true,
|
||||
});
|
||||
|
||||
const pool = new pg.Pool({ connectionString: pgUrl });
|
||||
const DB = new Kysely<Database>({ dialect: new PostgresDialect({ pool }) });
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function s3Key(cdnPath: string): string {
|
||||
// cdnPath is always /users/... — strip leading slash then prepend prefix.
|
||||
return `${keyPrefix}${cdnPath.replace(/^\//u, "")}`;
|
||||
}
|
||||
|
||||
function hashBuffer(buf: Buffer): string {
|
||||
return crypto.createHash("sha256").update(buf).digest("hex");
|
||||
}
|
||||
|
||||
async function downloadFromS3(cdnPath: string): Promise<Buffer | null> {
|
||||
try {
|
||||
const response = await s3.send(
|
||||
new GetObjectCommand({ Bucket: s3Bucket, Key: s3Key(cdnPath) }),
|
||||
);
|
||||
|
||||
if (!response.Body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return streamToBuffer(response.Body as Readable);
|
||||
} catch (err: unknown) {
|
||||
// Object missing from S3 — the DB row is a dangling reference.
|
||||
if (
|
||||
err instanceof Error &&
|
||||
(err.name === "NoSuchKey" || err.message.includes("NoSuchKey"))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadToS3(cdnPath: string, body: Buffer, contentType: string): Promise<void> {
|
||||
await s3.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: s3Bucket,
|
||||
Key: s3Key(cdnPath),
|
||||
Body: body,
|
||||
ContentType: contentType,
|
||||
CacheControl: CACHE_CONTROL_IMMUTABLE,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function resizePfp(buf: Buffer): Promise<Buffer> {
|
||||
return sharp(buf, { animated: true })
|
||||
.resize(PFP_MAX_PX, PFP_MAX_PX, { fit: "inside", withoutEnlargement: true })
|
||||
.webp({ quality: 85 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function resizeBanner(buf: Buffer): Promise<Buffer> {
|
||||
return sharp(buf, { animated: true })
|
||||
.resize(BANNER_MAX_WIDTH, BANNER_MAX_HEIGHT, { fit: "inside", withoutEnlargement: true })
|
||||
.webp({ quality: 85 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
// ─── Core migration logic ─────────────────────────────────────────────────────
|
||||
|
||||
interface Stats {
|
||||
processed: number;
|
||||
skipped: number;
|
||||
missing: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
async function migrateMedia(
|
||||
userId: number,
|
||||
username: string,
|
||||
currentHash: string,
|
||||
cdnPathFn: (id: number, hash: string) => string,
|
||||
resizeFn: (buf: Buffer) => Promise<Buffer>,
|
||||
dbColumn: "custom_banner_location" | "custom_pfp_location",
|
||||
label: string,
|
||||
stats: Stats,
|
||||
): Promise<void> {
|
||||
const currentCdnPath = cdnPathFn(userId, currentHash);
|
||||
const logPrefix = ` [${username}/${label}]`;
|
||||
|
||||
const original = await downloadFromS3(currentCdnPath);
|
||||
|
||||
if (!original) {
|
||||
console.warn(`${logPrefix} S3 object missing — skipping (dangling DB reference).`);
|
||||
stats.missing++;
|
||||
return;
|
||||
}
|
||||
|
||||
let resized: Buffer;
|
||||
|
||||
try {
|
||||
resized = await resizeFn(original);
|
||||
} catch (err: unknown) {
|
||||
console.error(`${logPrefix} sharp failed to process — skipping.`, err);
|
||||
stats.errors++;
|
||||
return;
|
||||
}
|
||||
|
||||
const newHash = hashBuffer(resized);
|
||||
|
||||
if (newHash === currentHash) {
|
||||
console.log(`${logPrefix} already optimal (hash unchanged) — skipping.`);
|
||||
stats.skipped++;
|
||||
return;
|
||||
}
|
||||
|
||||
const newCdnPath = cdnPathFn(userId, newHash);
|
||||
const sizeBefore = original.length;
|
||||
const sizeAfter = resized.length;
|
||||
const pct = Math.round((1 - sizeAfter / sizeBefore) * 100);
|
||||
|
||||
console.log(
|
||||
`${logPrefix} ${(sizeBefore / 1024).toFixed(1)} KB → ${(sizeAfter / 1024).toFixed(1)} KB (${pct}% smaller, image/webp)`,
|
||||
);
|
||||
|
||||
if (!isDryRun) {
|
||||
await uploadToS3(newCdnPath, resized, "image/webp");
|
||||
|
||||
await DB.updateTable("account")
|
||||
.set({ [dbColumn]: newHash })
|
||||
.where("id", "=", userId)
|
||||
.execute();
|
||||
}
|
||||
|
||||
stats.processed++;
|
||||
}
|
||||
|
||||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(`[backfill-media] Connected to ${pgUrl}.`);
|
||||
console.log(`[backfill-media] S3 endpoint: ${s3Endpoint}, bucket: ${s3Bucket}`);
|
||||
|
||||
const users = await DB.selectFrom("account")
|
||||
.select(["id", "username", "custom_pfp_location", "custom_banner_location"])
|
||||
.where((eb) =>
|
||||
eb.or([
|
||||
eb("custom_pfp_location", "is not", null),
|
||||
eb("custom_banner_location", "is not", null),
|
||||
]),
|
||||
)
|
||||
.orderBy("id", "asc")
|
||||
.execute();
|
||||
|
||||
console.log(
|
||||
`[backfill-media] Found ${users.length} users with at least one custom media object.`,
|
||||
);
|
||||
|
||||
const pfpStats: Stats = { processed: 0, skipped: 0, missing: 0, errors: 0 };
|
||||
const bannerStats: Stats = { processed: 0, skipped: 0, missing: 0, errors: 0 };
|
||||
|
||||
for (const user of users) {
|
||||
if (user.custom_pfp_location) {
|
||||
await migrateMedia(
|
||||
user.id,
|
||||
user.username,
|
||||
user.custom_pfp_location,
|
||||
(id, hash) => `/users/${id}/pfp-${hash}`,
|
||||
resizePfp,
|
||||
"custom_pfp_location",
|
||||
"pfp",
|
||||
pfpStats,
|
||||
);
|
||||
}
|
||||
|
||||
if (user.custom_banner_location) {
|
||||
await migrateMedia(
|
||||
user.id,
|
||||
user.username,
|
||||
user.custom_banner_location,
|
||||
(id, hash) => `/users/${id}/banner-${hash}`,
|
||||
resizeBanner,
|
||||
"custom_banner_location",
|
||||
"banner",
|
||||
bannerStats,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`
|
||||
[backfill-media] Done.
|
||||
|
||||
Profile pictures
|
||||
Resized and re-uploaded : ${pfpStats.processed}
|
||||
Already optimal (skip) : ${pfpStats.skipped}
|
||||
Missing from S3 : ${pfpStats.missing}
|
||||
Errors : ${pfpStats.errors}
|
||||
|
||||
Banners
|
||||
Resized and re-uploaded : ${bannerStats.processed}
|
||||
Already optimal (skip) : ${bannerStats.skipped}
|
||||
Missing from S3 : ${bannerStats.missing}
|
||||
Errors : ${bannerStats.errors}
|
||||
`);
|
||||
|
||||
if (!isDryRun && pfpStats.errors + bannerStats.errors > 0) {
|
||||
console.warn(
|
||||
"[backfill-media] Some objects failed to process. Re-run the script to retry them.",
|
||||
);
|
||||
}
|
||||
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
await main().catch((err: unknown) => {
|
||||
console.error("[backfill-media] Fatal error:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { seedApiToken } from "#actions/test-utils/api-tokens";
|
||||
import { CDNStoreOrOverwrite } from "#lib/cdn/cdn";
|
||||
import { CDNRetrieve, CDNStoreOrOverwrite } from "#lib/cdn/cdn";
|
||||
import { GetProfileBannerURL } from "#lib/cdn/url-format";
|
||||
import DB from "#services/pg/db";
|
||||
import mockApi, { CloseServerConnection } from "#test-utils/mock-api";
|
||||
@@ -56,9 +56,14 @@ describe("PUT /api/v1/users/:userID/banner", () => {
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const get = await mockApi.get(res.body.body.get).redirects(1);
|
||||
const { custom_banner_location } = await DB.selectFrom("account")
|
||||
.select("custom_banner_location")
|
||||
.where("id", "=", 1)
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(Buffer.isBuffer(get.body) ? get.body : Buffer.from(get.body)).toStrictEqual(img);
|
||||
const stored = await CDNRetrieve(GetProfileBannerURL(1, custom_banner_location!));
|
||||
|
||||
expect(stored.length).toBeLessThan(img.length);
|
||||
});
|
||||
|
||||
it("stores a banner when the user already had a custom banner", async () => {
|
||||
@@ -76,8 +81,13 @@ describe("PUT /api/v1/users/:userID/banner", () => {
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const get = await mockApi.get(res.body.body.get).redirects(1);
|
||||
const { custom_banner_location } = await DB.selectFrom("account")
|
||||
.select("custom_banner_location")
|
||||
.where("id", "=", 1)
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(Buffer.isBuffer(get.body) ? get.body : Buffer.from(get.body)).toStrictEqual(img);
|
||||
const stored = await CDNRetrieve(GetProfileBannerURL(1, custom_banner_location!));
|
||||
|
||||
expect(stored.length).toBeLessThan(img.length);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -75,6 +75,11 @@ API_V1_ROUTER.rawAdd(
|
||||
API_V1_ROUTER.add("GET /users/:userID/banner", withRequestedUser, ({ ctx, res }) => {
|
||||
const { requestedUser: user } = ctx;
|
||||
|
||||
// The redirect target is content-addressed (hash in path), so the CDN
|
||||
// object itself is immutable. Cache the userId→CDN-URL mapping for 1 hour
|
||||
// so browsers don't hit the API on every page load.
|
||||
res.setHeader("Cache-Control", "public, max-age=3600, stale-while-revalidate=86400");
|
||||
|
||||
if (!user.customBannerLocation) {
|
||||
res.setHeader("Content-Type", "image/png");
|
||||
CDNRedirect(res, "/users/default/banner");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { seedApiToken } from "#actions/test-utils/api-tokens";
|
||||
import { CDNStoreOrOverwrite } from "#lib/cdn/cdn";
|
||||
import { CDNRetrieve, CDNStoreOrOverwrite } from "#lib/cdn/cdn";
|
||||
import { GetProfilePictureURL } from "#lib/cdn/url-format";
|
||||
import DB from "#services/pg/db";
|
||||
import mockApi, { CloseServerConnection } from "#test-utils/mock-api";
|
||||
@@ -55,9 +55,14 @@ describe("PUT /api/v1/users/:userID/pfp", () => {
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const get = await mockApi.get(res.body.body.get).redirects(1);
|
||||
const { custom_pfp_location } = await DB.selectFrom("account")
|
||||
.select("custom_pfp_location")
|
||||
.where("id", "=", 1)
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(Buffer.isBuffer(get.body) ? get.body : Buffer.from(get.body)).toStrictEqual(img);
|
||||
const stored = await CDNRetrieve(GetProfilePictureURL(1, custom_pfp_location!));
|
||||
|
||||
expect(stored.length).toBeLessThan(img.length);
|
||||
});
|
||||
|
||||
it("stores a profile picture when the user already had a custom pfp", async () => {
|
||||
@@ -75,8 +80,13 @@ describe("PUT /api/v1/users/:userID/pfp", () => {
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const get = await mockApi.get(res.body.body.get).redirects(1);
|
||||
const { custom_pfp_location } = await DB.selectFrom("account")
|
||||
.select("custom_pfp_location")
|
||||
.where("id", "=", 1)
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
expect(Buffer.isBuffer(get.body) ? get.body : Buffer.from(get.body)).toStrictEqual(img);
|
||||
const stored = await CDNRetrieve(GetProfilePictureURL(1, custom_pfp_location!));
|
||||
|
||||
expect(stored.length).toBeLessThan(img.length);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -78,6 +78,11 @@ API_V1_ROUTER.add("GET /users/:userID/pfp", withRequestedUser, ({ ctx, res }) =>
|
||||
|
||||
log.debug(user, "User Info for /:userID/pfp request is ");
|
||||
|
||||
// The redirect target is content-addressed (hash in path), so the CDN
|
||||
// object itself is immutable. Cache the userId→CDN-URL mapping for 1 hour
|
||||
// so browsers don't hit the API on every page load.
|
||||
res.setHeader("Cache-Control", "public, max-age=3600, stale-while-revalidate=86400");
|
||||
|
||||
if (!user.customPfpLocation) {
|
||||
res.setHeader("Content-Type", "image/png");
|
||||
CDNRedirect(res, "/users/default/pfp");
|
||||
|
||||
@@ -120,20 +120,14 @@ export default defineConfig({
|
||||
extends: true,
|
||||
test: {
|
||||
name: "default",
|
||||
exclude: [
|
||||
"src/actions/bms-table-sync.test.ts",
|
||||
"src/actions/change-pfp.test.ts",
|
||||
],
|
||||
exclude: ["src/actions/bms-table-sync.test.ts"],
|
||||
},
|
||||
},
|
||||
{
|
||||
extends: true,
|
||||
test: {
|
||||
name: "isolated",
|
||||
include: [
|
||||
"src/actions/bms-table-sync.test.ts",
|
||||
"src/actions/change-pfp.test.ts",
|
||||
],
|
||||
include: ["src/actions/bms-table-sync.test.ts"],
|
||||
poolOptions: {
|
||||
threads: {
|
||||
isolate: true,
|
||||
|
||||
Reference in New Issue
Block a user