diff --git a/.cursor/skills/db-formats/SKILL.md b/.cursor/skills/db-formats/SKILL.md index 9c2173d85..d3cf35997 100644 --- a/.cursor/skills/db-formats/SKILL.md +++ b/.cursor/skills/db-formats/SKILL.md @@ -128,5 +128,5 @@ const { clientSecret: _secret, ...publicDoc } = ToAPIClientDocument(row); | `api-token.ts` | `priv_api_token` | `APITokenDocument` | | `user.ts` | `account` | `UserDocument` | | `user-settings.ts` | `account_settings` | `UserSettingsDocument` | -| `game-stats.ts` | `game_stats` | `UserGameStats` | +| `game-profiles.ts` | `game_profile` | `UserGameStats` (ratings + classes); UGPT preferences and showcase JSON also live on `game_profile` | | `kshook-sv6c-settings.ts` | `svc_kshook_sv6c_settings` | `KsHookSettingsDocument` | diff --git a/.scripts/ts_autoinherit.js b/.scripts/ts_autoinherit.js index ac2fca57a..af0e33a10 100755 --- a/.scripts/ts_autoinherit.js +++ b/.scripts/ts_autoinherit.js @@ -10,7 +10,9 @@ process.chdir(cwd); const rootPkg = JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8")); for (const workspacePattern of rootPkg.workspaces) { - let glob = new Glob(workspacePattern + "*/package.json"); + // "typescript/*" + "/package.json" => "typescript/*/package.json" (one workspace dir only). + // "typescript/*" + "*/package.json" would merge *+* into ** and match nested paths / node_modules. + let glob = new Glob(workspacePattern + "/package.json"); for (const pkgPath of glob.scanSync(".")) { console.log(pkgPath); diff --git a/Justfile-apps b/Justfile-apps index da7ac994d..366337979 100644 --- a/Justfile-apps +++ b/Justfile-apps @@ -1,9 +1,9 @@ start: #!/bin/bash + set -euo pipefail ARGS=(server client seeds-webui) echo "Starting: ${ARGS[@]}" - set -e pids=() for target in ${ARGS[@]}; do (just "$target") & @@ -14,10 +14,33 @@ start: server: - cd typescript/server && \ - VERSION=$(jq -r .version package.json) \ - COMMIT_HASH=$(git rev-parse --short HEAD) \ - bun --watch src/main.ts + #!/bin/bash + set -euo pipefail + cd typescript/server + VERSION="${VERSION:-"$(jq -r .version package.json)"}" + COMMIT_HASH="${COMMIT_HASH:-"$(git rev-parse --short HEAD)"}" + export VERSION COMMIT_HASH + sidecar_pids=() + # Crons: Postgres-backed scheduler. Jobs: `job_queue` (SKIP LOCKED; run multiple for load). + if [[ -z "${TACHI_SERVER_NO_SIDECARS:-}" ]]; then + bun run cron-worker & sidecar_pids+=($!) + n="${TACHI_SERVER_JOB_WORKER_COUNT:-1}" + [[ "$n" =~ ^[0-9]+$ ]] && [[ "$n" -ge 1 ]] || n=1 + for _ in $(seq 1 "$n"); do + bun run job-queue-worker & sidecar_pids+=($!) + done + fi + _cleanup_sidecars() { + trap - INT TERM EXIT + local s=$? + if ((${#sidecar_pids[@]})); then + kill -TERM "${sidecar_pids[@]}" 2>/dev/null || true + wait "${sidecar_pids[@]}" 2>/dev/null || true + fi + exit "$s" + } + trap _cleanup_sidecars INT TERM EXIT + bun --watch src/main.ts client: cd typescript/client && bun vite diff --git a/Justfile-db b/Justfile-db index 94d7e5483..09c30f812 100644 --- a/Justfile-db +++ b/Justfile-db @@ -7,6 +7,10 @@ db POSTGRES_DB=DEFAULT_DB: db-cheap-reset POSTGRES_DB=DEFAULT_DB: #!/bin/bash + set -euo pipefail + # Disconnect all sessions using this DB so DROP DATABASE can succeed. + psql "{{POSTGRES_URL}}/postgres" -v ON_ERROR_STOP=1 -c \ + "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '{{POSTGRES_DB}}'" export POSTGRES_URL="{{POSTGRES_URL}}/{{POSTGRES_DB}}" tachidb database drop tachidb database create @@ -15,6 +19,7 @@ db-cheap-reset POSTGRES_DB=DEFAULT_DB: # Drop and recreate the database so migrations can be re-applied from scratch. db-reset POSTGRES_DB=DEFAULT_DB: #!/bin/bash + set -euo pipefail just db-cheap-reset "{{POSTGRES_DB}}" just db-load-seeds "{{POSTGRES_DB}}" diff --git a/Justfile-test b/Justfile-test index 587b93b02..ce3276cc2 100644 --- a/Justfile-test +++ b/Justfile-test @@ -85,6 +85,7 @@ test-parity suite="": # Workers clone from this template rather than re-running migrations each time. bot-db-test-template-reset: #!/bin/bash + set -euo pipefail export POSTGRES_URL="{{POSTGRES_URL}}/tachi_bot_test_template" tachidb database drop || true tachidb database create @@ -94,9 +95,20 @@ bot-db-test-template-reset: # Workers clone from this template rather than re-running migrations each time. server-db-test-template-reset: #!/bin/bash + set -euo pipefail export POSTGRES_URL="{{POSTGRES_URL}}/tachi_server_test_template" tachidb database drop || true tachidb database create tachidb migrate run -# <== End of parallel db test justscripts \ No newline at end of file +# <== End of parallel db test justscripts + +# Score-import HTTP load tests against a live server (multipart /api/v1/import/file). +# See typescript/server/src/load-tests/README.md +load-test-score-import *ARGS: + cd typescript/server && bun run load-test:score-import -- {{ARGS}} + +# Seed N dev users + API tokens (submit_score); writes one token per line to OUTPUT_FILE. +# Uses the server package env (.env); same Postgres as a local tachi-server. +load-test-score-import-seed-tokens COUNT OUTPUT_FILE: + cd typescript/server && bun run src/load-tests/seed-stress-api-tokens.ts {{COUNT}} {{OUTPUT_FILE}} \ No newline at end of file diff --git a/bun.lock b/bun.lock index 79da9ed50..300e8d005 100644 --- a/bun.lock +++ b/bun.lock @@ -313,13 +313,13 @@ "name": "tachi-seeds-webui", "version": "0.1.0-see-project-root", "dependencies": { - "@codemirror/lang-sql": "^6.10.0", - "@codemirror/language": "^6.12.3", - "@lezer/highlight": "^1.2.3", - "@sqlite.org/sqlite-wasm": "^3.51.2-build9", - "codemirror": "^6.0.2", - "comlink": "^4.4.2", - "fast-json-patch": "^3.1.1", + "@codemirror/lang-sql": "catalog:", + "@codemirror/language": "catalog:", + "@lezer/highlight": "catalog:", + "@sqlite.org/sqlite-wasm": "catalog:", + "codemirror": "catalog:", + "comlink": "catalog:", + "fast-json-patch": "catalog:", "nanoid": "catalog:", "natural-compare": "catalog:", "react": "catalog:", @@ -371,6 +371,7 @@ "bunyan": "catalog:", "commander": "catalog:", "connect-redis": "catalog:", + "cron-parser": "catalog:", "csv-parse": "catalog:", "deepmerge": "catalog:", "dotenv": "catalog:", @@ -452,6 +453,8 @@ }, "catalog": { "@aws-sdk/client-s3": "3.49.0", + "@codemirror/lang-sql": "^6.10.0", + "@codemirror/language": "^6.12.3", "@discordjs/builders": "0.5.0", "@discordjs/rest": "0.1.0-canary.0", "@eslint/js": "^9.20.0", @@ -459,11 +462,13 @@ "@fullcalendar/core": "^6.0.0", "@fullcalendar/daygrid": "^6.0.0", "@fullcalendar/react": "^6.0.0", + "@lezer/highlight": "^1.2.3", "@nivo/bar": "0.79.1", "@nivo/core": "0.79.0", "@nivo/line": "0.79.1", "@octokit/app": "15.1.0", "@octokit/webhooks-types": "7.5.1", + "@sqlite.org/sqlite-wasm": "^3.51.2-build9", "@types/bcryptjs": "^2.4.2", "@types/bunyan": "^1.8.8", "@types/connect-redis": "0.0.18", @@ -509,8 +514,11 @@ "bunyan": "1.8.15", "chalk": "4", "cheerio": "^1.0.0-rc.12", + "codemirror": "^6.0.2", + "comlink": "^4.4.2", "commander": "^8.2.0", "connect-redis": "6.1.1", + "cron-parser": "5.0.2", "csv-parse": "^5.0.3", "deepmerge": "^4.2.2", "discord-api-types": "0.22.0", @@ -531,6 +539,7 @@ "express-prom-bundle": "^7.0.0", "express-rate-limit": "5.5.1", "express-session": "1.17.2", + "fast-json-patch": "^3.1.1", "fast-json-stable-hash": "^1.0.3", "fast-xml-parser": "^4.2.5", "formik": "2.2.9", @@ -1688,7 +1697,7 @@ "crelt": ["crelt@1.0.6", "", {}, "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g=="], - "cron-parser": ["cron-parser@2.18.0", "", { "dependencies": { "is-nan": "^1.3.0", "moment-timezone": "^0.5.31" } }, "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg=="], + "cron-parser": ["cron-parser@5.0.2", "", { "dependencies": { "luxon": "^3.5.0" } }, "sha512-RXXr5WuvLInay/DstwRD087/DpLOm24Y3mrkRmOdrMjbdGfQyPiMsx1Ad+SaJF2zN0tN78ckBjcEkIHVd+MX7Q=="], "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], @@ -3258,12 +3267,16 @@ "broadcast-channel/rimraf": ["rimraf@3.0.2", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "bin.js" } }, "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA=="], + "bullmq/cron-parser": ["cron-parser@2.18.0", "", { "dependencies": { "is-nan": "^1.3.0", "moment-timezone": "^0.5.31" } }, "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg=="], + "bullmq/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "bullmq/tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="], "cheerio-select/css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + "cron-parser/luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="], + "css-select/domhandler": ["domhandler@4.3.1", "", { "dependencies": { "domelementtype": "^2.2.0" } }, "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ=="], "css-select/domutils": ["domutils@2.8.0", "", { "dependencies": { "dom-serializer": "^1.0.1", "domelementtype": "^2.2.0", "domhandler": "^4.2.0" } }, "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A=="], diff --git a/db/migrations/20260301154256_genesis.sql b/db/migrations/20260301154256_genesis.sql index 21cc13ffb..86ebe2f8d 100644 --- a/db/migrations/20260301154256_genesis.sql +++ b/db/migrations/20260301154256_genesis.sql @@ -541,21 +541,6 @@ CREATE TABLE "folder_chart_lookup" ( CREATE INDEX ON "folder_chart_lookup" (folder_id); CREATE INDEX ON "folder_chart_lookup" (chart_id); -CREATE TABLE "game_settings" ( - user_id BIGINT REFERENCES account(id) NOT NULL, - game GAME NOT NULL, - PRIMARY KEY (user_id, game), - - pf_preferred_score_alg TEXT, - pf_preferred_session_alg TEXT, - pf_preferred_profile_alg TEXT, - pf_preferred_default_enum TEXT, - pf_default_table TEXT, - pf_preferred_ranking TEXT CHECK (pf_preferred_ranking IN ('global', 'rival')), - - data JSONB NOT NULL -- game specific payload -); - CREATE TABLE "game_rival" ( user_id BIGINT REFERENCES account(id) NOT NULL, game GAME NOT NULL, @@ -566,17 +551,7 @@ CREATE TABLE "game_rival" ( CHECK (user_id != rival) ); -CREATE TABLE "game_settings_showcase" ( - user_id BIGINT REFERENCES account(id) NOT NULL, - game GAME NOT NULL, - - PRIMARY KEY (user_id, game), - - -- kinda ridiculous but { mode: "chart", etc. } or - -- { mode: "folder" } - data JSONB NOT NULL -); - +-- Per-user per-game: ratings/classes (import-derived) + UGPT preferences + showcase JSON (Tachi3 #47). CREATE TABLE "game_profile" ( user_id BIGINT REFERENCES account(id) NOT NULL, game GAME NOT NULL, @@ -584,7 +559,20 @@ CREATE TABLE "game_profile" ( PRIMARY KEY (user_id, game), ratings JSONB NOT NULL, - classes JSONB NOT NULL + classes JSONB NOT NULL, + + pf_preferred_score_alg TEXT, + pf_preferred_session_alg TEXT, + pf_preferred_profile_alg TEXT, + pf_preferred_default_enum TEXT, + pf_default_table TEXT, + pf_preferred_ranking TEXT, + + data JSONB NOT NULL DEFAULT '{}'::jsonb, + showcase JSONB NOT NULL DEFAULT '[]'::jsonb, + + CONSTRAINT game_profile_pf_preferred_ranking_check + CHECK (pf_preferred_ranking IS NULL OR pf_preferred_ranking IN ('global', 'rival')) ); CREATE TABLE "game_stats_snapshot" ( @@ -770,6 +758,20 @@ CREATE TABLE "score_rederive" ( enqueued_at TIMESTAMPTZ NOT NULL DEFAULT now() ); +-- Session aggregate `calculated_data` must be refreshed when committed scores in that session change. +CREATE TABLE session_dirty ( + session_id TEXT NOT NULL PRIMARY KEY, + enqueued_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Per-playtype game stats (`game_profile`) must be refreshed when committed scores change (ratings use PBs). +CREATE TABLE game_profile_dirty ( + user_id BIGINT NOT NULL, + game GAME NOT NULL, + enqueued_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, game) +); + -- Trigger function: on any score INSERT/UPDATE/DELETE, mark the (user_id, chart_id) -- pair as needing PB recalculation. CREATE FUNCTION enqueue_pb_dirty() RETURNS trigger AS $$ @@ -791,6 +793,72 @@ CREATE TRIGGER "score_pb_dirty" AFTER INSERT OR UPDATE OR DELETE ON score FOR EACH ROW EXECUTE FUNCTION enqueue_pb_dirty(); +-- Staging scores (committed = false) skip these queues until commit. +CREATE FUNCTION enqueue_session_dirty() RETURNS trigger AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.committed AND OLD.session_id IS NOT NULL THEN + INSERT INTO session_dirty (session_id) + VALUES (OLD.session_id) + ON CONFLICT DO NOTHING; + END IF; + ELSIF TG_OP = 'UPDATE' THEN + IF NEW.committed THEN + IF OLD.session_id IS NOT NULL AND OLD.session_id IS DISTINCT FROM NEW.session_id THEN + INSERT INTO session_dirty (session_id) + VALUES (OLD.session_id) + ON CONFLICT DO NOTHING; + END IF; + IF NEW.session_id IS NOT NULL THEN + INSERT INTO session_dirty (session_id) + VALUES (NEW.session_id) + ON CONFLICT DO NOTHING; + END IF; + END IF; + ELSE + IF NEW.committed AND NEW.session_id IS NOT NULL THEN + INSERT INTO session_dirty (session_id) + VALUES (NEW.session_id) + ON CONFLICT DO NOTHING; + END IF; + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER "score_session_dirty" + AFTER INSERT OR UPDATE OR DELETE ON score + FOR EACH ROW EXECUTE FUNCTION enqueue_session_dirty(); + +CREATE FUNCTION enqueue_game_profile_dirty() RETURNS trigger AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.committed THEN + INSERT INTO game_profile_dirty (user_id, game) + VALUES (OLD.user_id, OLD.game) + ON CONFLICT DO NOTHING; + END IF; + ELSIF TG_OP = 'UPDATE' THEN + IF NEW.committed THEN + INSERT INTO game_profile_dirty (user_id, game) + VALUES (NEW.user_id, NEW.game) + ON CONFLICT DO NOTHING; + END IF; + ELSE + IF NEW.committed THEN + INSERT INTO game_profile_dirty (user_id, game) + VALUES (NEW.user_id, NEW.game) + ON CONFLICT DO NOTHING; + END IF; + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER "score_game_profile_dirty" + AFTER INSERT OR UPDATE OR DELETE ON score + FOR EACH ROW EXECUTE FUNCTION enqueue_game_profile_dirty(); + -- Trigger function: on chart UPDATE, if derivation_checksum changed, enqueue the chart -- for score re-derivation. CREATE FUNCTION enqueue_score_rederive() RETURNS trigger AS $$ diff --git a/db/migrations/20260422130000_grant_grafana_ro_public_read.sql b/db/migrations/20260422130000_grant_grafana_ro_public_read.sql new file mode 100644 index 000000000..5bbe5d80b --- /dev/null +++ b/db/migrations/20260422130000_grant_grafana_ro_public_read.sql @@ -0,0 +1,17 @@ +-- Grafana local-dev role (see dev/postgres-init.sql): init runs before migrations, so +-- GRANT SELECT ON ALL TABLES saw no tables; default privileges can also miss edge cases. +-- Re-apply read access to all existing objects; no-op when grafana_ro is absent. +DO $grant$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'grafana_ro') THEN + EXECUTE 'GRANT USAGE ON SCHEMA public TO grafana_ro'; + EXECUTE 'GRANT SELECT ON ALL TABLES IN SCHEMA public TO grafana_ro'; + EXECUTE 'GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO grafana_ro'; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'grafana_ro') + AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tachi') THEN + EXECUTE 'ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON TABLES TO grafana_ro'; + EXECUTE 'ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON SEQUENCES TO grafana_ro'; + END IF; +END +$grant$; diff --git a/db/migrations/20260423120000_job_queue_dequeue_index.sql b/db/migrations/20260423120000_job_queue_dequeue_index.sql new file mode 100644 index 000000000..948087d70 --- /dev/null +++ b/db/migrations/20260423120000_job_queue_dequeue_index.sql @@ -0,0 +1,4 @@ +-- Fast dequeue of queued (status = 0) jobs ordered by when they are due. +CREATE INDEX job_queue_dequeue_idx +ON job_queue (scheduled_for ASC, created_at ASC) +WHERE status = 0; diff --git a/dev/functions.fish b/dev/functions.fish index b5f7682e2..50d632b84 100644 --- a/dev/functions.fish +++ b/dev/functions.fish @@ -54,9 +54,15 @@ function fish_greeting echo $(rgb "This terminal is set up inside a Linux Machine!" ffffff 000000) echo $(rgb "This machine comes pre-installed with Tachi and helpful tools." ffffff 000000) echo "" - echo "Type $(cmd "just start") to start up a frontend and backend." - echo " $(rgb "The server will start on http://localhost:3000." ffff00 000000)" - echo " $(rgb "Use Ctrl+C to stop the server." ffff00 000000)" + echo "Type $(cmd "just start") to start up tachi." + echo " $(rgb "The site will start on http://localhost:3000." ffff00 000000)" + echo " $(rgb "The seeds web UI will start on http://localhost:3100." ffff00 000000)" + echo " $(rgb "Use Ctrl+C to stop Tachi." ffff00 000000)" + echo "" + echo "You can also run:" + echo " $(cmd "just grafana") to view the Grafana dashboard." + echo " $(cmd "just mailpit") to view emails that have been sent by Tachi." + echo " $(cmd "just seeds-webui") to view the seeds web UI." echo "" echo "Type $(cmd "seeds") to run seeds scripts." echo " $(rgb "Create new script files in typescript/seeds-scripts." ffff00 000000)" diff --git a/dev/observability/grafana/provisioning/datasources/datasources.yml b/dev/observability/grafana/provisioning/datasources/datasources.yml index 874e10b36..9e91d6a97 100644 --- a/dev/observability/grafana/provisioning/datasources/datasources.yml +++ b/dev/observability/grafana/provisioning/datasources/datasources.yml @@ -12,9 +12,9 @@ datasources: type: postgres access: proxy url: tachi-postgres:5432 - user: tachi + user: grafana_ro secureJsonData: - password: tachi + password: grafana_ro jsonData: database: tachi_dev sslmode: disable diff --git a/dev/observability/setup-datasources.sh b/dev/observability/setup-datasources.sh index e42295def..95e34c6b4 100755 --- a/dev/observability/setup-datasources.sh +++ b/dev/observability/setup-datasources.sh @@ -2,8 +2,8 @@ set -euo pipefail GRAFANA_URL="${GRAFANA_URL:-http://tachi-grafana:3000}" -GRAFANA_USER="${GRAFANA_USER:-admin}" -GRAFANA_PASS="${GRAFANA_PASS:-admin}" +GRAFANA_USER="${GRAFANA_USER:-tachi}" +GRAFANA_PASS="${GRAFANA_PASS:-tachi}" PROM_URL="${PROM_URL:-http://tachi-prometheus:9090}" ALLOY_URL="${ALLOY_URL:-http://tachi-alloy:12345}" diff --git a/dev/postgres-init.sql b/dev/postgres-init.sql index 2d84ca6ef..669dc5d94 100644 --- a/dev/postgres-init.sql +++ b/dev/postgres-init.sql @@ -15,4 +15,6 @@ END$$; GRANT CONNECT ON DATABASE tachi_dev TO grafana_ro; GRANT USAGE ON SCHEMA public TO grafana_ro; GRANT SELECT ON ALL TABLES IN SCHEMA public TO grafana_ro; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO grafana_ro; ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON TABLES TO grafana_ro; +ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON SEQUENCES TO grafana_ro; diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 124cb3996..4e0439979 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -106,8 +106,8 @@ services: ports: - "3005:3000" environment: - GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: admin + GF_SECURITY_ADMIN_USER: tachi + GF_SECURITY_ADMIN_PASSWORD: tachi GF_USERS_ALLOW_SIGN_UP: "false" GF_SERVER_HTTP_ADDR: 0.0.0.0 GF_SERVER_HTTP_PORT: "3000" diff --git a/docs/docs/api/routes/admin.md b/docs/docs/api/routes/admin.md index 023184349..6e763994c 100644 --- a/docs/docs/api/routes/admin.md +++ b/docs/docs/api/routes/admin.md @@ -65,14 +65,11 @@ Empty Object. ***** -## Resynchronise all PBs that match the given query or users. +## Re-run PB processing for every scored user+chart (synchronous). -`POST /api/v1/admin/resync-pbs` +`POST /api/v1/admin/recalc-pbs` -!!! info - This is intended to be used in the case that PBs fall out of - sync with what they should be. This could be due to a - database migration going awry, or anything else. +Inserts every distinct `(user_id, chart_id)` from the **`score`** table into **`pb_dirty`**, then **drains** `pb_dirty` and downstream **`session_dirty`** / **`game_profile_dirty`** queues until nothing remains (same batching as the background worker, but the HTTP request waits until idle). Intended when PBs may be out of sync (e.g. after a bad migration). There is **no request body** and no filter—always all distinct pairs that appear on scores. ### Permissions @@ -80,34 +77,29 @@ Empty Object. ### Parameters -| Property | Type | Description | -| :: | :: | :: | -| `userIDs` | Array<Integer> (Optional) | The list of userIDs to resynchronise PBs for. | -| `filters` | Mongo Query for the PBs Collection. (Optional) | A query to reduce the amount of PBs that get reprocessed. | +None (send `{}` if your client requires a body). ### Response -Empty Object. +Empty object (standard success wrapper with `body`). ### Example #### Request + ``` -POST /api/v1/admin/resync-pbs +POST /api/v1/admin/recalc-pbs ``` ```js -{ - "filter": { - "game": {$in: ["iidx","sdvx"]} - }, - "userIDs": [1,2,3,4] -} +{} ``` #### Response -Nothing. +```js +{} +``` ***** @@ -195,43 +187,38 @@ Empty Object. ***** -## Perform a site recalc on this set of scores. +## Re-derive all scores site-wide (synchronous). `POST /api/v1/admin/recalc` +Enqueues **every chart** into **`score_rederive`**, then **drains** `score_rederive` and downstream **`pb_dirty`**, **`session_dirty`**, and **`game_profile_dirty`** queues until nothing remains (the request waits until idle). Re-runs `scoreDeriver` and `scoreCalcs` for every score. There is **no request body** and no filter—always all charts. + ### Permissions - Admin ### Parameters -| Property | Type | Description | -| :: | :: | :: | -| `` | Mongo Query for scores | Filters the amount of scores recalced. If not provided, defaults to every score on the site. | +None (send an empty JSON object `{}` if your client requires a body). ### Response -| Property | Type | Description | -| :: | :: | :: | -| `scoresRecalced` | Integer | The amount of scores recalced. | +Empty object (standard success wrapper with `body`). ### Example #### Request + ``` POST /api/v1/admin/recalc ``` + ```js -{ - "game": "iidx", - "scoreData.percent": {$gt: 90}, -} +{} ``` #### Response ```js -{ - "scoresRecalced": 174 -} +{} ``` diff --git a/docs/docs/api/routes/import.md b/docs/docs/api/routes/import.md index ef41b54e6..8ea1a048c 100644 --- a/docs/docs/api/routes/import.md +++ b/docs/docs/api/routes/import.md @@ -242,6 +242,36 @@ Returns rows from `orphan_score` for the authenticated user (scores that failed ***** +## Get one orphaned score (full payload) + +`GET /api/v1/import/orphans/:orphanID` + +Returns a single `orphan_score` row for the authenticated user, including raw `data` and `context` JSON (for debugging unmatched imports). `404` if the row does not exist or belongs to another user. + +### Permissions + +- submit_score + +### Parameters + +| Parameter | Type | Description | +| :: | :: | :: | +| `orphanID` | Path | The orphan’s `orphanID` (same as in `GET /import/orphans` or import errors). | + +### Response + +| Property | Type | Description | +| :: | :: | :: | +| `orphanID` | String | Stable orphan identifier. | +| `importType` | String | Import type that produced the orphan. | +| `gameGroup` | String | Game group. | +| `timeInserted` | Number | Unix time in ms when the row was stored. | +| `message` | String or null | Stored error / context message. | +| `data` | Object | Raw import datapoint (shape depends on `importType`). | +| `context` | Object | Raw import context. | + +***** + ## Delete one orphaned score `DELETE /api/v1/import/orphans/:orphanID` diff --git a/docs/docs/api/routes/users.md b/docs/docs/api/routes/users.md index 78ee4d7f0..3c370a68c 100644 --- a/docs/docs/api/routes/users.md +++ b/docs/docs/api/routes/users.md @@ -158,9 +158,9 @@ GET /api/v1/users/me IF authenticated as userID 1. --- -## Retrieve user's statistics on all games. +## Retrieve per-game profiles for a user. -`GET /api/v1/users/:userID/game-stats` +`GET /api/v1/users/:userID/game-profiles` ### Parameters @@ -170,7 +170,7 @@ None. | Property | Type | Description | | :------: | :--------------------------------------------------: | :-----------------------------------------: | -| `` | Array<UserGameStatsDocument & \_\_rankingData> | The array of User Game Stats this user has. | +| `` | Array<UserGameStatsDocument & \_\_rankingData> | The array of per-game profile documents (ratings and classes) this user has. | !!! info For UI reasons, the UserGameStatsDocuments here have an additional `__rankingData` property, which contains leaderboard ranking information for this user. @@ -182,7 +182,7 @@ For UI reasons, the UserGameStatsDocuments here have an additional `__rankingDat ``` GET /api/v1/users/zkldime-stats OR -GET /api/v1/users/1/game-stats +GET /api/v1/users/1/game-profiles ``` #### Response diff --git a/package.json b/package.json index ca31a6cff..6832acb78 100644 --- a/package.json +++ b/package.json @@ -198,7 +198,15 @@ "@types/tap": "^15.0.5", "tap": "^15.1.6", "@types/istanbul-lib-coverage": "^2.0.6", - "istanbul-lib-coverage": "^3.2.2" + "istanbul-lib-coverage": "^3.2.2", + "@codemirror/lang-sql": "^6.10.0", + "@codemirror/language": "^6.12.3", + "@lezer/highlight": "^1.2.3", + "@sqlite.org/sqlite-wasm": "^3.51.2-build9", + "codemirror": "^6.0.2", + "comlink": "^4.4.2", + "fast-json-patch": "^3.1.1", + "cron-parser": "5.0.2" }, "overrides": { "@types/mongodb": "3.6.20", diff --git a/typescript/client/src/app/pages/admin/AdminOperationsPage.tsx b/typescript/client/src/app/pages/admin/AdminOperationsPage.tsx index 42f705d00..f38f99db9 100644 --- a/typescript/client/src/app/pages/admin/AdminOperationsPage.tsx +++ b/typescript/client/src/app/pages/admin/AdminOperationsPage.tsx @@ -19,9 +19,6 @@ export default function AdminOperationsPage() { const [folderId, setFolderId] = useState(""); - const [resyncBody, setResyncBody] = useState("{}"); - const [recalcBody, setRecalcBody] = useState("{}"); - const [supporterUser, setSupporterUser] = useState(""); const announcementGameConfig = announcementGame ? GetGameGroupConfig(announcementGame) : null; @@ -226,34 +223,22 @@ export default function AdminOperationsPage() { - Resync PBs + Recalc PBs - - JSON body - setResyncBody(e.target.value)} - placeholder='{} or { "userIDs": [1, 2], "filter": { ... } }' - rows={5} - style={{ fontFamily: "monospace", fontSize: "0.85rem" }} - value={resyncBody} - /> - +

+ Enqueues every distinct user+chart that has at least one score into + pb_dirty, then drains that queue and + downstream session/profile queues until idle (all games). This request + waits until processing finishes. +

@@ -271,32 +256,19 @@ export default function AdminOperationsPage() { Recalc scores - - Mongo filter (JSON object) - setRecalcBody(e.target.value)} - placeholder="{}" - rows={5} - style={{ fontFamily: "monospace", fontSize: "0.85rem" }} - value={recalcBody} - /> - +

+ Enqueues every chart for full score re-derivation (all games), then + drains score and downstream queues until idle. This request waits until + processing finishes; can take a long time on large databases. +

diff --git a/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTLeaderboardsPage.tsx b/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTLeaderboardsPage.tsx index 5409fe888..90b5ff8f1 100644 --- a/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTLeaderboardsPage.tsx +++ b/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTLeaderboardsPage.tsx @@ -16,7 +16,11 @@ import { type UserLeaderboardReturns } from "#types/api-returns"; import { type GamePT } from "#types/react"; import { type UGSDataset } from "#types/tables"; import { CreateUserMap } from "#util/data"; -import { FormatGPTProfileRating, FormatGPTProfileRatingName } from "#util/misc"; +import { + FormatGPTProfileRating, + FormatGPTProfileRatingName, + getProfileRatingAlgKeysInDisplayOrder, +} from "#util/misc"; import { NumericSOV, StrSOV } from "#util/sorts"; import React, { useState } from "react"; import { Col, Form, Row } from "react-bootstrap"; @@ -25,7 +29,6 @@ import { type Classes, FormatGame, GameToGameGroup, - GetGameConfig, GetGameGroupConfig, type V3Game, } from "tachi-common"; @@ -64,16 +67,18 @@ export default function GPTLeaderboardsPage({ game }: GamePT) { } function ProfileLeaderboard({ game }: GamePT) { - const gameConfig = GetGameConfig(game); - const defaultAlg = useProfileRatingAlg(game); const [alg, setAlg] = useState(defaultAlg); + const profileAlgKeys = getProfileRatingAlgKeysInDisplayOrder( + game, + ) as Array; + const SelectComponent = - Object.keys(gameConfig.profileRatingAlgs).length > 1 ? ( + profileAlgKeys.length > 1 ? ( setAlg(e.target.value as any)} value={alg}> - {Object.keys(gameConfig.profileRatingAlgs).map((e) => ( + {profileAlgKeys.map((e) => ( @@ -107,12 +112,12 @@ function ProfileLeaderboard({ game }: GamePT) { const userDataset: UGSDataset = []; - for (const [index, gs] of data.gameStats.entries()) { + for (const gs of data.gameStats) { userDataset.push({ ...gs, __related: { user: userMap.get(gs.userID)!, - index, + index: gs.rank - 1, }, }); } @@ -125,11 +130,9 @@ function ProfileLeaderboard({ game }: GamePT) { dataset={userDataset} entryName="Rankers" headers={[ - ["Ranking", "Rank", NumericSOV((x) => x.__related.index)], + ["Ranking", "Rank", NumericSOV((x) => x.rank)], ["User", "User", StrSOV((x) => x.__related.user.username)], - ...( - Object.keys(gameConfig.profileRatingAlgs) as Array - ).map( + ...profileAlgKeys.map( (e) => [ FormatGPTProfileRatingName(game, e), @@ -143,9 +146,7 @@ function ProfileLeaderboard({ game }: GamePT) { - {( - Object.keys(gameConfig.profileRatingAlgs) as Array - ).map((e) => ( + {profileAlgKeys.map((e) => ( {r.ratings[e] ? FormatGPTProfileRating(game, e, r.ratings[e]!) diff --git a/typescript/client/src/app/pages/dashboard/users/UserOrphansPage.tsx b/typescript/client/src/app/pages/dashboard/users/UserOrphansPage.tsx index 46635e960..c6e169ce1 100644 --- a/typescript/client/src/app/pages/dashboard/users/UserOrphansPage.tsx +++ b/typescript/client/src/app/pages/dashboard/users/UserOrphansPage.tsx @@ -3,28 +3,38 @@ import TachiTable from "#components/tables/components/TachiTable"; import Loading from "#components/util/Loading"; import { APIFetchV1 } from "#util/api"; import React, { useCallback, useEffect, useState } from "react"; -import { Alert, Button } from "react-bootstrap"; -import { GetGameGroupConfig, type GameGroup, type UserDocument } from "tachi-common"; +import { Alert, Button, Modal } from "react-bootstrap"; +import { type GameGroup, GetGameGroupConfig, type UserDocument } from "tachi-common"; type OrphanListItem = { + gameGroup: string; + importType: string; + message: string | null; orphanID: string; rowID: string; - importType: string; - gameGroup: string; - timeInserted: number; - message: string | null; summary: string | null; + timeInserted: number; }; -type ListBody = { orphans: OrphanListItem[]; hasMore: boolean }; +type ListBody = { hasMore: boolean; orphans: OrphanListItem[] }; type ReprocessBody = { + failed: number; processed: number; removed: number; - failed: number; success: number; }; +type OrphanDetailBody = { + context: unknown; + data: unknown; + gameGroup: string; + importType: string; + message: string | null; + orphanID: string; + timeInserted: number; +}; + export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument }) { useSetSubheader( ["Users", reqUser.username, "Orphan scores"], @@ -38,6 +48,10 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument }) const [loadingMore, setLoadingMore] = useState(false); const [reprocessBusy, setReprocessBusy] = useState(false); const [lastMessage, setLastMessage] = useState(null); + const [detailOpen, setDetailOpen] = useState(false); + const [detailLoading, setDetailLoading] = useState(false); + const [detailJson, setDetailJson] = useState(null); + const [detailTitle, setDetailTitle] = useState(""); const fetchPage = useCallback(async (afterRowID: string | undefined, append: boolean) => { const params = new URLSearchParams({ limit: "50" }); @@ -107,6 +121,26 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument }) } }; + const onOpenDetail = async (orphanID: string) => { + setDetailTitle(orphanID); + setDetailJson(null); + setDetailOpen(true); + setDetailLoading(true); + const res = await APIFetchV1( + `/import/orphans/${encodeURIComponent(orphanID)}`, + undefined, + false, + true, + ); + setDetailLoading(false); + if (!res.success) { + setDetailJson(null); + return; + } + const { data, context, ...meta } = res.body; + setDetailJson(JSON.stringify({ ...meta, data, context }, null, 2)); + }; + const onDelete = async (orphanID: string) => { if (!window.confirm(`Delete orphan ${orphanID}? This cannot be undone.`)) { return; @@ -128,24 +162,46 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument })

Orphan scores

- When an import cannot match a song or chart (SongOrChartNotFound), Tachi still stores - that datapoint as an orphan. Orphans are retried automatically around{" "} - 00:01 UTC each day, or you can run a full reprocess below. + When an import cannot match a song or chart (SongOrChartNotFound), Tachi still + stores that datapoint as an orphan. Orphans are retried + automatically around 00:01 UTC each day, or you can run a full + reprocess below.

- Deleting an orphan only removes that queued datapoint; it does not revert an entire - import. + Deleting an orphan only removes that queued datapoint; it does not revert an + entire import.

-
{lastMessage && {lastMessage}} + setDetailOpen(false)} show={detailOpen} size="lg"> + + Orphan details — {detailTitle} + + + {detailLoading ? ( + + ) : detailJson ? ( + 
+							{detailJson}
+
+ ) : ( +

Could not load details.

+ )} + + + {loading ? ( ) : orphans.length === 0 ? ( @@ -170,18 +226,28 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument }) {o.importType} - {GetGameGroupConfig(o.gameGroup as GameGroup)?.name ?? o.gameGroup} + {GetGameGroupConfig(o.gameGroup as GameGroup)?.name ?? + o.gameGroup} {new Date(o.timeInserted).toLocaleString()} {o.message ?? "—"} - +
+ + +
)} diff --git a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/LeaderboardsPage.tsx b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/LeaderboardsPage.tsx index c12e76478..497522e1d 100644 --- a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/LeaderboardsPage.tsx +++ b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/LeaderboardsPage.tsx @@ -24,7 +24,7 @@ import { type integer, type ProfileRatingAlgorithms, type UserDocument, - type UserGameStats, + type UserGameStatsWithProfileLeaderboardRank, type V3Game, } from "tachi-common"; @@ -111,7 +111,7 @@ function LeaderboardsPageContent({ const bestNearbyUser = stats.thisUsersRanking.ranking - stats.above.length - 1; - function LeaderboardRow({ s, i }: { i: integer; s: UserGameStats }) { + function LeaderboardRow({ s }: { s: UserGameStatsWithProfileLeaderboardRank }) { return ( - #{i} + #{s.rank} {reqUser.id === s.userID && ( /{stats.thisUsersRanking.outOf} @@ -177,18 +177,14 @@ function LeaderboardsPageContent({ {bestNearbyUser >= 1 && leaderboard.gameStats .slice(0, bestNearbyUser) - .map((s, i) => )} + .map((s) => )} {bestNearbyUser > 4 && ( ... )} - {[...stats.above, stats.thisUsersStats, ...stats.below].map((s, i) => ( - + {[...stats.above, stats.thisUsersStats, ...stats.below].map((s) => ( + ))} diff --git a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/OverviewPage.tsx b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/OverviewPage.tsx index 2c570d300..e57e20e79 100644 --- a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/OverviewPage.tsx +++ b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/OverviewPage.tsx @@ -15,7 +15,12 @@ import SelectButton from "#components/util/SelectButton"; import { useProfileRatingAlg } from "#components/util/useScoreRatingAlg"; import { type UGPTHistory } from "#types/api-returns"; import { type GamePT, type SetState, type UGPT } from "#types/react"; -import { FormatGPTProfileRating, FormatGPTProfileRatingName, UppercaseFirst } from "#util/misc"; +import { + FormatGPTProfileRating, + FormatGPTProfileRatingName, + getProfileRatingAlgKeysInDisplayOrder, + UppercaseFirst, +} from "#util/misc"; import { FormatDate, MillisToSince } from "#util/time"; import { DateTime } from "luxon"; import React, { useMemo, useState } from "react"; @@ -158,7 +163,7 @@ function UserHistory({ {mode === "ranking" ? ( <> - {Object.keys(gameConfig.profileRatingAlgs).length > 1 && ( + {getProfileRatingAlgKeysInDisplayOrder(game).length > 1 && (
@@ -166,7 +171,7 @@ function UserHistory({ } value={rating} > - {Object.keys(gameConfig.profileRatingAlgs).map((e) => ( + {getProfileRatingAlgKeysInDisplayOrder(game).map((e) => ( @@ -211,7 +216,7 @@ function UserHistory({ /> ) : ( <> - {Object.keys(gameConfig.profileRatingAlgs).length > 1 && ( + {getProfileRatingAlgKeysInDisplayOrder(game).length > 1 && (
@@ -219,7 +224,7 @@ function UserHistory({ } value={rating} > - {Object.keys(gameConfig.profileRatingAlgs).map((e) => ( + {getProfileRatingAlgKeysInDisplayOrder(game).map((e) => ( diff --git a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/SpecificSessionPage.tsx b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/SpecificSessionPage.tsx index db767b51c..8ca719b8f 100644 --- a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/SpecificSessionPage.tsx +++ b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/SpecificSessionPage.tsx @@ -73,7 +73,7 @@ function SessionPage({ data, game }: { data: SessionReturns } & UGPT) { const scoreDataset = useMemo(() => { const d = []; - for (const sci of data.scoreInfo) { + for (const sci of sessionData.scoreInfo) { const score = scoreMap.get(sci.scoreID); if (!score) { diff --git a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/UGPTSettingsPage.tsx b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/UGPTSettingsPage.tsx index 2c20af9cc..439d69bb9 100644 --- a/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/UGPTSettingsPage.tsx +++ b/typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/UGPTSettingsPage.tsx @@ -20,6 +20,7 @@ import { FormatGPTProfileRatingName, FormatGPTScoreRatingName, FormatGPTSessionRatingName, + getProfileRatingAlgKeysInDisplayOrder, ToFixedFloor, UppercaseFirst, } from "#util/misc"; @@ -254,7 +255,7 @@ function PreferencesForm({ )} - {Object.keys(gameConfig.profileRatingAlgs).length > 1 && ( + {getProfileRatingAlgKeysInDisplayOrder(game).length > 1 && ( Preferred Profile Algorithm - {Object.keys(gameConfig.profileRatingAlgs).map((e) => ( + {getProfileRatingAlgKeysInDisplayOrder(game).map((e) => ( diff --git a/typescript/client/src/app/routes/UserRoutes.tsx b/typescript/client/src/app/routes/UserRoutes.tsx index bcf4ac1ba..52c4d6606 100644 --- a/typescript/client/src/app/routes/UserRoutes.tsx +++ b/typescript/client/src/app/routes/UserRoutes.tsx @@ -9,9 +9,9 @@ import UGPTSettingsPage from "#app/pages/dashboard/users/games/_game/_playtype/U import UGPTUtilsPage from "#app/pages/dashboard/users/games/_game/_playtype/utils/UGPTUtilsPage"; import UserGamesPage from "#app/pages/dashboard/users/UserGamesPage"; import UserImportsPage from "#app/pages/dashboard/users/UserImportsPage"; -import UserOrphansPage from "#app/pages/dashboard/users/UserOrphansPage"; import UserIntegrationsPage from "#app/pages/dashboard/users/UserIntegrationsPage"; import UserInvitesPage from "#app/pages/dashboard/users/UserInvitesPage"; +import UserOrphansPage from "#app/pages/dashboard/users/UserOrphansPage"; import UserSettingsPage from "#app/pages/dashboard/users/UserSettingsPage"; import { ErrorPage } from "#app/pages/ErrorPage"; import RequireAuthAsUserParam from "#components/auth/RequireAuthAsUserParam"; diff --git a/typescript/client/src/components/imports/ImportInfo.tsx b/typescript/client/src/components/imports/ImportInfo.tsx index 472c90bbd..77b23ffcc 100644 --- a/typescript/client/src/components/imports/ImportInfo.tsx +++ b/typescript/client/src/components/imports/ImportInfo.tsx @@ -53,7 +53,7 @@ export default function ImportInfo({ return; } - APIFetchV1(`/users/${user!.id}/game-stats`).then((r) => { + APIFetchV1(`/users/${user!.id}/game-profiles`).then((r) => { if (!r.success) { console.warn(`Can't update user stats post-import. ${r.description}`); return; @@ -130,8 +130,8 @@ export default function ImportInfo({ are matched with data, all we have to display might be a hash.
SongOrChartNotFound means the score was still{" "} - saved as an orphan for nightly matching (around 00:01 UTC) or - manual reprocess — see{" "} + saved as an orphan for nightly matching (around 00:01 + UTC) or manual reprocess — see{" "} {user ? ( Orphan scores ) : ( @@ -157,7 +157,9 @@ export default function ImportInfo({
This may be stored as an orphan.{" "} {user ? ( - Open orphan queue + + Open orphan queue + ) : ( "Open orphan queue" )} diff --git a/typescript/client/src/components/layout/header/HeaderMenu.tsx b/typescript/client/src/components/layout/header/HeaderMenu.tsx index 085cbcd73..095b9a4b2 100644 --- a/typescript/client/src/components/layout/header/HeaderMenu.tsx +++ b/typescript/client/src/components/layout/header/HeaderMenu.tsx @@ -28,7 +28,7 @@ export function HeaderMenu({ const { data, error } = useApiQuery( // We should generate a valid url just in case the skip somehow fails - `/users/${user?.id ?? "me"}/game-stats`, + `/users/${user?.id ?? "me"}/game-profiles`, undefined, undefined, // We should skip if a user isn't logged in. diff --git a/typescript/client/src/components/tables/cells/IndexCell.tsx b/typescript/client/src/components/tables/cells/IndexCell.tsx index 1c1a5553b..f41e4682d 100644 --- a/typescript/client/src/components/tables/cells/IndexCell.tsx +++ b/typescript/client/src/components/tables/cells/IndexCell.tsx @@ -16,7 +16,7 @@ export default function IndexCell({ index }: { index: integer }) { {index + 1} diff --git a/typescript/client/src/components/tables/components/SelectableRating.tsx b/typescript/client/src/components/tables/components/SelectableRating.tsx index 11af61e9b..820f9056d 100644 --- a/typescript/client/src/components/tables/components/SelectableRating.tsx +++ b/typescript/client/src/components/tables/components/SelectableRating.tsx @@ -1,6 +1,11 @@ import Icon from "#components/util/Icon"; import { type SetState } from "#types/react"; -import { FormatGPTScoreRatingName, FormatGPTSessionRatingName } from "#util/misc"; +import { + FormatGPTProfileRatingName, + FormatGPTScoreRatingName, + FormatGPTSessionRatingName, + getProfileRatingAlgKeysInDisplayOrder, +} from "#util/misc"; import React from "react"; import { GetGameConfig, type V3Game } from "tachi-common"; @@ -33,6 +38,11 @@ export default function SelectableRating({ key = "sessionRatingAlgs"; } + const ratingKeys = + mode === "profile" + ? getProfileRatingAlgKeysInDisplayOrder(game) + : (Object.keys(gameConfig[key]) as string[]); + return (
@@ -41,11 +51,13 @@ export default function SelectableRating({ onChange={(v) => setRating(v.target.value as AllRatings)} value={rating} > - {Object.keys(gameConfig[key]).map((s) => ( + {ratingKeys.map((s) => ( ))} diff --git a/typescript/client/src/components/user/UGPTProfiles.tsx b/typescript/client/src/components/user/UGPTProfiles.tsx index ca89bc10b..349ec384d 100644 --- a/typescript/client/src/components/user/UGPTProfiles.tsx +++ b/typescript/client/src/components/user/UGPTProfiles.tsx @@ -48,7 +48,7 @@ const ContextualGamesInfo = memo(({ user }: { user: UserDocument }) => { function QueryGamesInfo({ reqUser }: { reqUser: UserDocument }) { const { data, error } = useApiQuery( - `/users/${reqUser.id}/game-stats`, + `/users/${reqUser.id}/game-profiles`, undefined, undefined, !reqUser, diff --git a/typescript/client/src/components/user/UGPTRankingData.tsx b/typescript/client/src/components/user/UGPTRankingData.tsx index b572c347b..92f09fdd9 100644 --- a/typescript/client/src/components/user/UGPTRankingData.tsx +++ b/typescript/client/src/components/user/UGPTRankingData.tsx @@ -1,6 +1,6 @@ import { useProfileRatingAlg } from "#components/util/useScoreRatingAlg"; import { type GamePT } from "#types/react"; -import { FormatGPTProfileRatingName } from "#util/misc"; +import { FormatGPTProfileRatingName, getProfileRatingAlgKeysInDisplayOrder } from "#util/misc"; import React from "react"; import { Link } from "react-router-dom"; import { type integer, type ProfileRatingAlgorithms, type V3Game } from "tachi-common"; @@ -22,19 +22,21 @@ export default function RankingData({ const extendData = []; - for (const k in rankingData) { + for (const k of getProfileRatingAlgKeysInDisplayOrder(game)) { const key = k as ProfileRatingAlgorithms[V3Game]; - if (key !== alg) { - extendData.push( -
- - {FormatGPTProfileRatingName(game, key)}: #{rankingData[key].ranking}/ - {rankingData[key].outOf} - -
, - ); + if (!(key in rankingData) || key === alg) { + continue; } + + extendData.push( +
+ + {FormatGPTProfileRatingName(game, key)}: #{rankingData[key].ranking}/ + {rankingData[key].outOf} + +
, + ); } return ( diff --git a/typescript/client/src/components/user/UGPTStatsOverview.tsx b/typescript/client/src/components/user/UGPTStatsOverview.tsx index 4cc9e6dd1..2bce34e61 100644 --- a/typescript/client/src/components/user/UGPTStatsOverview.tsx +++ b/typescript/client/src/components/user/UGPTStatsOverview.tsx @@ -6,23 +6,22 @@ import { FormatGPTProfileRating, FormatGPTProfileRatingName, FormatGPTScoreRatingName, + getProfileRatingAlgRowStyle, + sortProfileRatingEntries, UppercaseFirst, } from "#util/misc"; import { StrSOV } from "#util/sorts"; import React from "react"; -import { - type Classes, - GetGameConfig, - type ProfileRatingAlgorithms, - type UserGameStats, - type V3Game, -} from "tachi-common"; +import { type Classes, GetGameConfig, type UserGameStats, type V3Game } from "tachi-common"; export default function UGPTRatingsTable({ ugs }: { ugs: UserGameStats }) { const game = ugs.game; const gameConfig = GetGameConfig(game); - const ratings = Object.entries(ugs.ratings) as [ProfileRatingAlgorithms[V3Game], number][]; + const ratings = sortProfileRatingEntries( + game, + Object.entries(ugs.ratings) as [string, number][], + ); return ( @@ -79,7 +78,9 @@ export default function UGPTRatingsTable({ ugs }: { ugs: UserGameStats }) {
- {FormatGPTProfileRating(game, k as any, v)} + + {FormatGPTProfileRating(game, k as any, v)} + ))} diff --git a/typescript/client/src/components/util/UpdateUserGameStats.tsx b/typescript/client/src/components/util/UpdateUserGameStats.tsx index 5485e4399..4a8fa9689 100644 --- a/typescript/client/src/components/util/UpdateUserGameStats.tsx +++ b/typescript/client/src/components/util/UpdateUserGameStats.tsx @@ -3,7 +3,7 @@ import { APIFetchV1 } from "#util/api"; import { type UserGameStats } from "tachi-common"; export default async function UpdateUserGameStats(setUGS: SetState) { - const res = await APIFetchV1("/users/me/game-stats"); + const res = await APIFetchV1("/users/me/game-profiles"); if (!res.success) { setUGS(null); diff --git a/typescript/client/src/lib/games/iidx.tsx b/typescript/client/src/lib/games/iidx.tsx index 032cab206..a18d3f657 100644 --- a/typescript/client/src/lib/games/iidx.tsx +++ b/typescript/client/src/lib/games/iidx.tsx @@ -5,6 +5,7 @@ import RatingCell from "#components/tables/cells/RatingCell"; import ScoreCell from "#components/tables/cells/ScoreCell"; import { GetEnumColour } from "#lib/game-implementations"; import { type GPTClientImplementation } from "#lib/types"; +import { ChangeOpacity } from "#util/color-opacity"; import { NumericSOV } from "#util/sorts"; import React from "react"; import { COLOUR_SET, type GamesForGroup, IIDX_LAMPS, IIDXLIKE_GBOUNDARIES } from "tachi-common"; @@ -114,6 +115,17 @@ const IIDXCoreCells: GPTClientImplementation["scoreCoreCe ); +/** Applied to the profile stats table value cell only (see UGPTRatingsTable). */ +const IIDX_PROFILE_RATING_VALUE_CELL_STYLE: NonNullable< + GPTClientImplementation["profileRatingAlgRowStyle"] +> = { + ktLampRating: { backgroundColor: ChangeOpacity(COLOUR_SET.purple, 0.12) }, + ktLampRatingNC: { backgroundColor: ChangeOpacity(COLOUR_SET.blue, 0.14) }, + ktLampRatingHC: { backgroundColor: ChangeOpacity(COLOUR_SET.orange, 0.14) }, + ktLampRatingEXHC: { backgroundColor: ChangeOpacity(COLOUR_SET.gold, 0.12) }, + BPI: { backgroundColor: ChangeOpacity(COLOUR_SET.paleBlue, 0.12) }, +}; + const IIDXRatingCell: GPTClientImplementation["ratingCell"] = ({ sc, chart, @@ -169,6 +181,7 @@ export const IIDX_SP_IMPL: GPTClientImplementation<"iidx-sp"> = { classColours: IIDX_COLOURS, scoreCoreCells: IIDXCoreCells, ratingCell: IIDXRatingCell, + profileRatingAlgRowStyle: IIDX_PROFILE_RATING_VALUE_CELL_STYLE, }; export const IIDX_DP_IMPL: GPTClientImplementation<"iidx-dp"> = { @@ -194,4 +207,5 @@ export const IIDX_DP_IMPL: GPTClientImplementation<"iidx-dp"> = { classColours: IIDX_COLOURS, scoreCoreCells: IIDXCoreCells, ratingCell: IIDXRatingCell, + profileRatingAlgRowStyle: IIDX_PROFILE_RATING_VALUE_CELL_STYLE, }; diff --git a/typescript/client/src/lib/types.ts b/typescript/client/src/lib/types.ts index 4c6b523ad..7c1e5e8d2 100644 --- a/typescript/client/src/lib/types.ts +++ b/typescript/client/src/lib/types.ts @@ -106,6 +106,12 @@ export interface GPTClientImplementation { session?: Record; }; + /** + * Optional styles for the **value** cell (not the label) in the profile stats mini-table, + * keyed by profile rating algorithm id. Typically a tinted background. + */ + profileRatingAlgRowStyle?: Partial>; + /** * What headers should be used when rendering scores in a table for this game? */ diff --git a/typescript/client/src/types/api-returns.ts b/typescript/client/src/types/api-returns.ts index 468403e03..43f99d817 100644 --- a/typescript/client/src/types/api-returns.ts +++ b/typescript/client/src/types/api-returns.ts @@ -26,6 +26,7 @@ import { type UserDocument, type UserGameStats, type UserGameStatsSnapshotDocument, + type UserGameStatsWithProfileLeaderboardRank, type V3Game, } from "tachi-common"; @@ -45,10 +46,10 @@ export interface UGPTStatsReturn { } export interface UGPTLeaderboardAdjacent { - above: UserGameStats[]; - below: UserGameStats[]; + above: UserGameStatsWithProfileLeaderboardRank[]; + below: UserGameStatsWithProfileLeaderboardRank[]; users: UserDocument[]; - thisUsersStats: UserGameStats; + thisUsersStats: UserGameStatsWithProfileLeaderboardRank; thisUsersRanking: { outOf: integer; ranking: integer; @@ -56,7 +57,7 @@ export interface UGPTLeaderboardAdjacent { } export interface GPTLeaderboard { - gameStats: UserGameStats[]; + gameStats: UserGameStatsWithProfileLeaderboardRank[]; users: UserDocument[]; } @@ -179,7 +180,7 @@ export interface ScoreLeaderboardReturns { export interface UserLeaderboardReturns { users: UserDocument[]; - gameStats: UserGameStats[]; + gameStats: UserGameStatsWithProfileLeaderboardRank[]; } export interface UserRecentSummary { diff --git a/typescript/client/src/types/tables.ts b/typescript/client/src/types/tables.ts index 90ea9f107..e209db457 100644 --- a/typescript/client/src/types/tables.ts +++ b/typescript/client/src/types/tables.ts @@ -11,7 +11,7 @@ import { type ScoreDocument, type SongDocument, type UserDocument, - type UserGameStats, + type UserGameStatsWithProfileLeaderboardRank, type V3Game, } from "tachi-common"; @@ -53,7 +53,7 @@ export type UGSDataset = ({ index: integer; user: UserDocument; }; -} & UserGameStats)[]; +} & UserGameStatsWithProfileLeaderboardRank)[]; export type RivalChartDataset = ({ __related: { diff --git a/typescript/client/src/util/data.ts b/typescript/client/src/util/data.ts index b0a0fa37b..ea97bc656 100644 --- a/typescript/client/src/util/data.ts +++ b/typescript/client/src/util/data.ts @@ -2,11 +2,10 @@ import { type GoalsOnChartReturn, type GoalsOnFolderReturn } from "#types/api-re import { type ChartDocument, type GameGroup, - type GoalDocument, type integer, type ScoreDocument, - type SessionScoreInfo, type SEEDS_SongDocument, + type SessionScoreInfo, type SongDocument, type UserDocument, type V3Game, @@ -29,7 +28,7 @@ export function GetPBs(scoreInfo: SessionScoreInfo[]) { } export function CreateSongMap( - songs: Array | SEEDS_SongDocument>, + songs: Array | SongDocument>, ) { const songMap = new Map>(); diff --git a/typescript/client/src/util/misc.ts b/typescript/client/src/util/misc.ts index 83f629145..23b37d52f 100644 --- a/typescript/client/src/util/misc.ts +++ b/typescript/client/src/util/misc.ts @@ -1,5 +1,6 @@ import { GPT_CLIENT_IMPLEMENTATIONS } from "#lib/game-implementations"; import fjsh from "fast-json-stable-hash"; +import { type CSSProperties } from "react"; import toast from "react-hot-toast"; import { type useHistory } from "react-router-dom"; import { @@ -10,7 +11,6 @@ import { type GameConfig, GetGameConfig, type integer, - type QuestDocument, type QuestSubscriptionDocument, type ScoreDocument, type V3Game, @@ -105,6 +105,39 @@ export function FormatGPTScoreRatingName(game: V3Game, key: string) { return gameConfig.ratingAlgNameOverrides?.score?.[key] ?? UppercaseFirst(key); } +/** Lower `displayOrder` first; missing `displayOrder` sorts after set values, then by key. */ +export function compareProfileRatingAlgKeys(game: V3Game, a: string, b: string): number { + const cfg = GetGameConfig(game); + const oa = cfg.profileRatingAlgs[a as keyof typeof cfg.profileRatingAlgs]?.displayOrder; + const ob = cfg.profileRatingAlgs[b as keyof typeof cfg.profileRatingAlgs]?.displayOrder; + const va = typeof oa === "number" ? oa : 10_000; + const vb = typeof ob === "number" ? ob : 10_000; + if (va !== vb) { + return va - vb; + } + + return a.localeCompare(b); +} + +export function getProfileRatingAlgKeysInDisplayOrder(game: V3Game): string[] { + const cfg = GetGameConfig(game); + return (Object.keys(cfg.profileRatingAlgs) as string[]).sort((x, y) => + compareProfileRatingAlgKeys(game, x, y), + ); +} + +export function sortProfileRatingEntries( + game: V3Game, + entries: Array<[K, V]>, +): Array<[K, V]> { + return [...entries].sort(([a], [b]) => compareProfileRatingAlgKeys(game, a, b)); +} + +/** Styles for the profile rating value cell (right column), not the label. */ +export function getProfileRatingAlgRowStyle(game: V3Game, key: string): CSSProperties | undefined { + return GPT_CLIENT_IMPLEMENTATIONS[game]?.profileRatingAlgRowStyle?.[key]; +} + export function ReverseStr(str: string) { return str.split("").reverse().join(""); } diff --git a/typescript/client/src/util/seeds.ts b/typescript/client/src/util/seeds.ts index c5ae6a4ff..71200c9e7 100644 --- a/typescript/client/src/util/seeds.ts +++ b/typescript/client/src/util/seeds.ts @@ -17,6 +17,7 @@ import { CreateSongMap, DatabaseSeedNames, type GameGroup, + type SeedFolderRow, type SEEDS_BMSCourseDocument, type SEEDS_ChartDocument, type SEEDS_FolderDocument, @@ -24,7 +25,6 @@ import { type SEEDS_QuestDocument, type SEEDS_QuestlineDocument, type SEEDS_TableDocument, - type SeedFolderRow, type SongDocument, } from "tachi-common"; diff --git a/typescript/common/src/config/game-support/iidx.ts b/typescript/common/src/config/game-support/iidx.ts index 46febbe87..6d3dd91a7 100644 --- a/typescript/common/src/config/game-support/iidx.ts +++ b/typescript/common/src/config/game-support/iidx.ts @@ -172,6 +172,17 @@ export const GAME_IIDX_SP_CONF = { description: "A rating system that values your clear lamps on charts. Tierlist information is taken into account.", }, + ktLampRatingNC: { + description: + "Your normal clear performance. Tierlist information is taken into account.", + }, + ktLampRatingHC: { + description: "Your hard clear performance. Tierlist information is taken into account.", + }, + ktLampRatingEXHC: { + description: + "Your EX-hard clear performance. Tierlist information is taken into account.", + }, BPI: { description: "A rating system for Kaiden level play. Only applies to 11s and 12s. A BPI of 0 states the score is equal to the Kaiden Average for that chart. A BPI of 100 is equal to the world record.", @@ -182,11 +193,36 @@ export const GAME_IIDX_SP_CONF = { ktLampRating: { description: `An average of your best 20 ktLampRatings.`, associatedScoreAlgs: ["ktLampRating"], + displayOrder: 0, + }, + ktLampRatingNC: { + description: `An average of your best 20 ktLampRatingNCs.`, + associatedScoreAlgs: ["ktLampRatingNC"], + displayOrder: 1, + }, + ktLampRatingHC: { + description: `An average of your best 20 ktLampRatingHCs.`, + associatedScoreAlgs: ["ktLampRatingHC"], + displayOrder: 2, + }, + ktLampRatingEXHC: { + description: `An average of your best 20 ktLampRatingEXHCs.`, + associatedScoreAlgs: ["ktLampRatingEXHC"], + displayOrder: 3, + }, + BPI: { + description: `An average of your best 20 BPIs.`, + associatedScoreAlgs: ["BPI"], + displayOrder: 4, }, - BPI: { description: `An average of your best 20 BPIs.`, associatedScoreAlgs: ["BPI"] }, }, sessionRatingAlgs: { ktLampRating: { description: `An average of the best 10 ktLampRatings this session.` }, + ktLampRatingNC: { description: `An average of the best 10 ktLampRatingNCs this session.` }, + ktLampRatingHC: { description: `An average of the best 10 ktLampRatingHCs this session.` }, + ktLampRatingEXHC: { + description: `An average of the best 10 ktLampRatingEXHCs this session.`, + }, BPI: { description: `An average of the best 10 BPIs this session.` }, }, diff --git a/typescript/common/src/types/documents.ts b/typescript/common/src/types/documents.ts index cac0479d8..c8528b92f 100644 --- a/typescript/common/src/types/documents.ts +++ b/typescript/common/src/types/documents.ts @@ -338,6 +338,11 @@ export interface UserGameStats { classes: AnyClasses; } +/** `GET /games/:game/leaderboard` and `.../leaderboard-adjacent` (tie-aware profile rating rank). */ +export type UserGameStatsWithProfileLeaderboardRank = { + rank: integer; +} & UserGameStats; + export interface ChartTierlistInfo { text: string; value: number; diff --git a/typescript/common/src/types/game-config-utils.ts b/typescript/common/src/types/game-config-utils.ts index 0790615f4..6bc348fd6 100644 --- a/typescript/common/src/types/game-config-utils.ts +++ b/typescript/common/src/types/game-config-utils.ts @@ -21,6 +21,13 @@ export interface ProfileRatingAlgorithmConfig extends RatingAlgorithmConfig { * of this algorithm's description? */ associatedScoreAlgs: ReadonlyArray; + + /** + * Sort order in the UI when multiple profile ratings are listed (e.g. profile header + * table, settings). Lower numbers appear first; algorithms without this field sort + * after those that have it, then by key name. + */ + displayOrder?: number; } export interface ClassInfo { @@ -143,10 +150,10 @@ export interface FixedDifficulties { * Originally, I called this "SEMI_FIXED", which is the sort of ZK quirkiness you've came to know at * this point, but now there's no point, this is for chugekimai, and this makes it clear. */ -export interface ChuGekiMaiDifficulties { +export interface ChuGekiMaiDifficulties { type: "CHUGEKIMAI_STYLE"; - order: ReadonlyArray; + order: ReadonlyArray; /** * How should we format these difficulty names? @@ -154,9 +161,9 @@ export interface ChuGekiMaiDifficulties { * Dynamic ones (i.e. ones not in the Order set) are not formatted - they are printed * exactly as-is. */ - format: Partial>; + format: Partial>; - default: Difficulty; + default: TDifficulty; } export type DifficultyConfig = diff --git a/typescript/db/src/generated/index.ts b/typescript/db/src/generated/index.ts index 23bd3d2d8..28e28cefc 100644 --- a/typescript/db/src/generated/index.ts +++ b/typescript/db/src/generated/index.ts @@ -11,7 +11,6 @@ export { type default as InviteLockTable, type InviteLock, type NewInviteLock, t export { type orphan_chart_id, type default as OrphanChartTable, type OrphanChart, type NewOrphanChart, type OrphanChartUpdate } from './public/OrphanChart'; export { type game_rival_game, type default as GameRivalTable, type GameRival, type NewGameRival, type GameRivalUpdate } from './public/GameRival'; export { type goal_id, type default as GoalTable, type Goal, type NewGoal, type GoalUpdate } from './public/Goal'; -export { type game_settings_game, type default as GameSettingsTable, type GameSettings, type NewGameSettings, type GameSettingsUpdate } from './public/GameSettings'; export { type import_quest_row_id, type default as ImportQuestTable, type ImportQuest, type NewImportQuest, type ImportQuestUpdate } from './public/ImportQuest'; export { type cron_task_execution_id, type default as CronTaskExecutionTable, type CronTaskExecution, type NewCronTaskExecution, type CronTaskExecutionUpdate } from './public/CronTaskExecution'; export { type notification_row_id, type default as NotificationTable, type Notification, type NewNotification, type NotificationUpdate } from './public/Notification'; @@ -39,7 +38,7 @@ export { type default as MigrationTable, type Migration, type NewMigration, type export { type action_row_id, type default as ActionTable, type Action, type NewAction, type ActionUpdate } from './public/Action'; export { type priv_svc_kai_auth_token_service, type default as PrivSvcKaiAuthTokenTable, type PrivSvcKaiAuthToken, type NewPrivSvcKaiAuthToken, type PrivSvcKaiAuthTokenUpdate } from './public/PrivSvcKaiAuthToken'; export { type chart_id, type default as ChartTable, type Chart, type NewChart, type ChartUpdate } from './public/Chart'; -export { type game_settings_showcase_game, type default as GameSettingsShowcaseTable, type GameSettingsShowcase, type NewGameSettingsShowcase, type GameSettingsShowcaseUpdate } from './public/GameSettingsShowcase'; +export { type game_profile_dirty_user_id, type game_profile_dirty_game, type default as GameProfileDirtyTable, type GameProfileDirty, type NewGameProfileDirty, type GameProfileDirtyUpdate } from './public/GameProfileDirty'; export { type default as AccountFollowingTable, type AccountFollowing, type NewAccountFollowing, type AccountFollowingUpdate } from './public/AccountFollowing'; export { type import_id, type default as ImportTable, type Import, type NewImport, type ImportUpdate } from './public/Import'; export { type folder_id, type default as FolderTable, type Folder, type NewFolder, type FolderUpdate } from './public/Folder'; @@ -56,6 +55,7 @@ export { type priv_api_client_client_id, type default as PrivApiClientTable, typ export { type priv_api_token_token, type default as PrivApiTokenTable, type PrivApiToken, type NewPrivApiToken, type PrivApiTokenUpdate } from './public/PrivApiToken'; export { type default as SvcKshookSv6cSettingsTable, type SvcKshookSv6cSettings, type NewSvcKshookSv6cSettings, type SvcKshookSv6cSettingsUpdate } from './public/SvcKshookSv6cSettings'; export { type import_error_row_id, type default as ImportErrorTable, type ImportError, type NewImportError, type ImportErrorUpdate } from './public/ImportError'; +export { type session_dirty_session_id, type default as SessionDirtyTable, type SessionDirty, type NewSessionDirty, type SessionDirtyUpdate } from './public/SessionDirty'; export { type default as FolderChartLookupTable, type FolderChartLookup, type NewFolderChartLookup, type FolderChartLookupUpdate } from './public/FolderChartLookup'; export { type quest_id, type default as QuestTable, type Quest, type NewQuest, type QuestUpdate } from './public/Quest'; export { type default as AccountSettingsTable, type AccountSettings, type NewAccountSettings, type AccountSettingsUpdate } from './public/AccountSettings'; @@ -74,6 +74,8 @@ export { type default as ActionResult } from './public/ActionResult'; export { type default as GameGroup } from './public/GameGroup'; export { type default as ImportType } from './public/ImportType'; export { type enqueue_pb_dirty_params } from './public/enqueue_pb_dirty'; +export { type enqueue_game_profile_dirty_params } from './public/enqueue_game_profile_dirty'; export { type enqueue_score_rederive_params } from './public/enqueue_score_rederive'; +export { type enqueue_session_dirty_params } from './public/enqueue_session_dirty'; export { type default as PublicSchema } from './public/PublicSchema'; export { type default as Database } from './Database'; diff --git a/typescript/db/src/generated/public/GameProfile.ts b/typescript/db/src/generated/public/GameProfile.ts index 123987486..7409b12c8 100644 --- a/typescript/db/src/generated/public/GameProfile.ts +++ b/typescript/db/src/generated/public/GameProfile.ts @@ -17,6 +17,22 @@ export default interface GameProfileTable { ratings: ColumnType; classes: ColumnType; + + pf_preferred_score_alg: ColumnType; + + pf_preferred_session_alg: ColumnType; + + pf_preferred_profile_alg: ColumnType; + + pf_preferred_default_enum: ColumnType; + + pf_default_table: ColumnType; + + pf_preferred_ranking: ColumnType; + + data: ColumnType; + + showcase: ColumnType; } export type GameProfile = Selectable; diff --git a/typescript/db/src/generated/public/GameProfileDirty.ts b/typescript/db/src/generated/public/GameProfileDirty.ts new file mode 100644 index 000000000..72625fdc8 --- /dev/null +++ b/typescript/db/src/generated/public/GameProfileDirty.ts @@ -0,0 +1,26 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +import type { default as Game } from './Game'; +import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely'; + +/** Identifier type for public.game_profile_dirty */ +export type game_profile_dirty_user_id = number; + +/** Identifier type for public.game_profile_dirty */ +export type game_profile_dirty_game = Game; + +/** Represents the table public.game_profile_dirty */ +export default interface GameProfileDirtyTable { + user_id: ColumnType; + + game: ColumnType; + + enqueued_at: ColumnType; +} + +export type GameProfileDirty = Selectable; + +export type NewGameProfileDirty = Insertable; + +export type GameProfileDirtyUpdate = Updateable; diff --git a/typescript/db/src/generated/public/GameSettings.ts b/typescript/db/src/generated/public/GameSettings.ts deleted file mode 100644 index 19c4a229a..000000000 --- a/typescript/db/src/generated/public/GameSettings.ts +++ /dev/null @@ -1,36 +0,0 @@ -// @generated -// This file is automatically generated by Kanel. Do not modify manually. - -import type { default as Game } from './Game'; -import type { account_id } from './Account'; -import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely'; - -/** Identifier type for public.game_settings */ -export type game_settings_game = Game; - -/** Represents the table public.game_settings */ -export default interface GameSettingsTable { - user_id: ColumnType; - - game: ColumnType; - - pf_preferred_score_alg: ColumnType; - - pf_preferred_session_alg: ColumnType; - - pf_preferred_profile_alg: ColumnType; - - pf_preferred_default_enum: ColumnType; - - pf_default_table: ColumnType; - - pf_preferred_ranking: ColumnType; - - data: ColumnType; -} - -export type GameSettings = Selectable; - -export type NewGameSettings = Insertable; - -export type GameSettingsUpdate = Updateable; diff --git a/typescript/db/src/generated/public/GameSettingsShowcase.ts b/typescript/db/src/generated/public/GameSettingsShowcase.ts deleted file mode 100644 index 33e9f482f..000000000 --- a/typescript/db/src/generated/public/GameSettingsShowcase.ts +++ /dev/null @@ -1,24 +0,0 @@ -// @generated -// This file is automatically generated by Kanel. Do not modify manually. - -import type { default as Game } from './Game'; -import type { account_id } from './Account'; -import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely'; - -/** Identifier type for public.game_settings_showcase */ -export type game_settings_showcase_game = Game; - -/** Represents the table public.game_settings_showcase */ -export default interface GameSettingsShowcaseTable { - user_id: ColumnType; - - game: ColumnType; - - data: ColumnType; -} - -export type GameSettingsShowcase = Selectable; - -export type NewGameSettingsShowcase = Insertable; - -export type GameSettingsShowcaseUpdate = Updateable; diff --git a/typescript/db/src/generated/public/PublicSchema.ts b/typescript/db/src/generated/public/PublicSchema.ts index 7f1a622fc..773d43ac9 100644 --- a/typescript/db/src/generated/public/PublicSchema.ts +++ b/typescript/db/src/generated/public/PublicSchema.ts @@ -11,7 +11,6 @@ import type { default as InviteLockTable } from './InviteLock'; import type { default as OrphanChartTable } from './OrphanChart'; import type { default as GameRivalTable } from './GameRival'; import type { default as GoalTable } from './Goal'; -import type { default as GameSettingsTable } from './GameSettings'; import type { default as ImportQuestTable } from './ImportQuest'; import type { default as CronTaskExecutionTable } from './CronTaskExecution'; import type { default as NotificationTable } from './Notification'; @@ -39,7 +38,7 @@ import type { default as MigrationTable } from './Migration'; import type { default as ActionTable } from './Action'; import type { default as PrivSvcKaiAuthTokenTable } from './PrivSvcKaiAuthToken'; import type { default as ChartTable } from './Chart'; -import type { default as GameSettingsShowcaseTable } from './GameSettingsShowcase'; +import type { default as GameProfileDirtyTable } from './GameProfileDirty'; import type { default as AccountFollowingTable } from './AccountFollowing'; import type { default as ImportTable } from './Import'; import type { default as FolderTable } from './Folder'; @@ -56,6 +55,7 @@ import type { default as PrivApiClientTable } from './PrivApiClient'; import type { default as PrivApiTokenTable } from './PrivApiToken'; import type { default as SvcKshookSv6cSettingsTable } from './SvcKshookSv6cSettings'; import type { default as ImportErrorTable } from './ImportError'; +import type { default as SessionDirtyTable } from './SessionDirty'; import type { default as FolderChartLookupTable } from './FolderChartLookup'; import type { default as QuestTable } from './Quest'; import type { default as AccountSettingsTable } from './AccountSettings'; @@ -89,8 +89,6 @@ export default interface PublicSchema { goal: GoalTable; - game_settings: GameSettingsTable; - import_quest: ImportQuestTable; cron_task_execution: CronTaskExecutionTable; @@ -145,7 +143,7 @@ export default interface PublicSchema { chart: ChartTable; - game_settings_showcase: GameSettingsShowcaseTable; + game_profile_dirty: GameProfileDirtyTable; account_following: AccountFollowingTable; @@ -179,6 +177,8 @@ export default interface PublicSchema { import_error: ImportErrorTable; + session_dirty: SessionDirtyTable; + folder_chart_lookup: FolderChartLookupTable; quest: QuestTable; diff --git a/typescript/db/src/generated/public/SessionDirty.ts b/typescript/db/src/generated/public/SessionDirty.ts new file mode 100644 index 000000000..6136f8f56 --- /dev/null +++ b/typescript/db/src/generated/public/SessionDirty.ts @@ -0,0 +1,20 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely'; + +/** Identifier type for public.session_dirty */ +export type session_dirty_session_id = string; + +/** Represents the table public.session_dirty */ +export default interface SessionDirtyTable { + session_id: ColumnType; + + enqueued_at: ColumnType; +} + +export type SessionDirty = Selectable; + +export type NewSessionDirty = Insertable; + +export type SessionDirtyUpdate = Updateable; diff --git a/typescript/db/src/generated/public/enqueue_game_profile_dirty.ts b/typescript/db/src/generated/public/enqueue_game_profile_dirty.ts new file mode 100644 index 000000000..d137bba4a --- /dev/null +++ b/typescript/db/src/generated/public/enqueue_game_profile_dirty.ts @@ -0,0 +1,5 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +export interface enqueue_game_profile_dirty_params { +} diff --git a/typescript/db/src/generated/public/enqueue_session_dirty.ts b/typescript/db/src/generated/public/enqueue_session_dirty.ts new file mode 100644 index 000000000..0abdb6fdc --- /dev/null +++ b/typescript/db/src/generated/public/enqueue_session_dirty.ts @@ -0,0 +1,5 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +export interface enqueue_session_dirty_params { +} diff --git a/typescript/seeds-scripts/eslint.config.mjs b/typescript/seeds-scripts/eslint.config.mjs index 83b6907a9..478baf709 100644 --- a/typescript/seeds-scripts/eslint.config.mjs +++ b/typescript/seeds-scripts/eslint.config.mjs @@ -4,6 +4,13 @@ export default [ ...configTachi.base, configTachi.node, { - ignores: ["../../db/seeds/**", "node_modules/**", "js/**", "**/*.js"], + ignores: [ + "../../db/seeds/**", + "node_modules/**", + "js/**", + "**/*.js", + // Excluded from tsconfig.json; type-aware eslint would error on every file here. + "rerunners/**", + ], }, ]; diff --git a/typescript/seeds-scripts/test/schemas.ts b/typescript/seeds-scripts/test/schemas.ts index 93aec594a..b8fa30d37 100644 --- a/typescript/seeds-scripts/test/schemas.ts +++ b/typescript/seeds-scripts/test/schemas.ts @@ -1,6 +1,7 @@ import { ALL_GAMES, allSupportedGameGroups, + type GameGroup, SEEDS_BMS_COURSE_DOCUMENT_SCHEMA, SEEDS_CHART_DOCUMENT_SCHEMAS, SEEDS_FOLDER_DOCUMENT_SCHEMA, @@ -9,7 +10,6 @@ import { SEEDS_QUESTLINE_DOCUMENT_SCHEMA, SEEDS_SONG_DOCUMENT_SCHEMAS, SEEDS_TABLE_DOCUMENT_SCHEMA, - type GameGroup, type V3Game, } from "tachi-common"; import { type ZodType } from "zod"; diff --git a/typescript/seeds-webui/package.json b/typescript/seeds-webui/package.json index 68daa9a41..bcf79b1ba 100644 --- a/typescript/seeds-webui/package.json +++ b/typescript/seeds-webui/package.json @@ -15,13 +15,13 @@ "bench": "node -e \"process.exit(0)\"" }, "dependencies": { - "@codemirror/lang-sql": "^6.10.0", - "@codemirror/language": "^6.12.3", - "@lezer/highlight": "^1.2.3", - "@sqlite.org/sqlite-wasm": "^3.51.2-build9", - "codemirror": "^6.0.2", - "comlink": "^4.4.2", - "fast-json-patch": "^3.1.1", + "@codemirror/lang-sql": "catalog:", + "@codemirror/language": "catalog:", + "@lezer/highlight": "catalog:", + "@sqlite.org/sqlite-wasm": "catalog:", + "codemirror": "catalog:", + "comlink": "catalog:", + "fast-json-patch": "catalog:", "nanoid": "catalog:", "natural-compare": "catalog:", "react": "catalog:", diff --git a/typescript/seeds-webui/src/components/SqlEditor.tsx b/typescript/seeds-webui/src/components/SqlEditor.tsx index a916fef39..d9ced914d 100644 --- a/typescript/seeds-webui/src/components/SqlEditor.tsx +++ b/typescript/seeds-webui/src/components/SqlEditor.tsx @@ -2,8 +2,8 @@ import { sql, SQLite } from "@codemirror/lang-sql"; import { HighlightStyle, syntaxHighlighting } from "@codemirror/language"; import { Compartment, EditorState } from "@codemirror/state"; import { keymap } from "@codemirror/view"; -import { basicSetup, EditorView } from "codemirror"; import { tags } from "@lezer/highlight"; +import { basicSetup, EditorView } from "codemirror"; import { useCallback, useEffect, useRef, useState } from "react"; /** diff --git a/typescript/seeds-webui/src/components/SqliteWorkspaceGate.tsx b/typescript/seeds-webui/src/components/SqliteWorkspaceGate.tsx index b3bedbde7..8e99b6096 100644 --- a/typescript/seeds-webui/src/components/SqliteWorkspaceGate.tsx +++ b/typescript/seeds-webui/src/components/SqliteWorkspaceGate.tsx @@ -1,6 +1,7 @@ -import { useIngest } from "#lib/ingest/IngestProvider"; import type { ReactNode } from "react"; +import { useIngest } from "#lib/ingest/IngestProvider"; + type SqliteWorkspaceGateProps = { children: ReactNode }; /** @@ -55,12 +56,12 @@ export function SqliteWorkspaceGate({ children }: SqliteWorkspaceGateProps) {
{progress.total > 0 ? (
diff --git a/typescript/server/.env b/typescript/server/.env index 354d991dc..51b3c88ed 100644 --- a/typescript/server/.env +++ b/typescript/server/.env @@ -39,7 +39,7 @@ TACHI_OUR_URL=http://localhost:3000 ; TACHI_INVITE_CODE_BATCH_SIZE=2 ; TACHI_INVITE_CODE_INVITE_CAP=100 ; TACHI_INVITE_CODE_BETA_USER_BONUS=5 -TACHI_CDN_WEB_LOCATION=http://tachi-s3:9000/tachi-public +TACHI_CDN_WEB_LOCATION=http://127.0.0.1:9000/tachi-public TACHI_CDN_SAVE_LOCATION_ENDPOINT=http://tachi-s3:9000 TACHI_CDN_SAVE_LOCATION_ACCESS_KEY_ID=minio TACHI_CDN_SAVE_LOCATION_SECRET_ACCESS_KEY=password diff --git a/typescript/server/package.json b/typescript/server/package.json index e99947f77..bc2d7e895 100644 --- a/typescript/server/package.json +++ b/typescript/server/package.json @@ -11,7 +11,10 @@ "build": "tsgo -b tsconfig.build.json -v", "typecheck": "tsgo --noEmit", "lint": "eslint ./src", - "lint-fix": "eslint ./src --fix" + "lint-fix": "eslint ./src --fix", + "cron-worker": "bun run src/cron-worker.ts", + "job-queue-worker": "bun run src/job-queue-worker.ts", + "load-test:score-import": "bun run src/load-tests/score-import-load-cli.ts" }, "author": "zk", "license": "AGPL3", @@ -81,7 +84,8 @@ "tachi-db-migration-engine": "workspace:*", "ts-node": "catalog:", "typescript": "catalog:", - "zod": "catalog:" + "zod": "catalog:", + "cron-parser": "catalog:" }, "nyc": { "reporter": [ diff --git a/typescript/server/src/actions/customise-score.test.ts b/typescript/server/src/actions/customise-score.test.ts index e37295ed4..ccf619eac 100644 --- a/typescript/server/src/actions/customise-score.test.ts +++ b/typescript/server/src/actions/customise-score.test.ts @@ -1,5 +1,5 @@ import { LoadScoreDocumentById } from "#lib/db-formats/score"; -import { mongoScoreDataToPg, pgScoreDataToMongo } from "#lib/v3/migration-tools"; +import { mongoScoreDataToPg, pgScoreDataToAPI } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; import { seedUser } from "#test-utils/pg-fixtures"; import { type ScoreData } from "tachi-common"; @@ -20,7 +20,7 @@ describe("mergeScoreDataFromPg", () => { } as ScoreData<"iidx-sp">; const pg = mongoScoreDataToPg("iidx-sp", { ...original, judgements: {} }); - const back = pgScoreDataToMongo("iidx-sp", pg); + const back = pgScoreDataToAPI("iidx-sp", pg); expect(back).toMatchObject({ grade: "F", diff --git a/typescript/server/src/actions/patch-ugpt-settings.test.ts b/typescript/server/src/actions/patch-ugpt-settings.test.ts index 111454453..af794956f 100644 --- a/typescript/server/src/actions/patch-ugpt-settings.test.ts +++ b/typescript/server/src/actions/patch-ugpt-settings.test.ts @@ -1,5 +1,9 @@ import { SELECT_ACTION } from "#lib/db-formats/action"; -import { GetUGPTSettingsDocument, SELECT_GAME_SETTINGS } from "#lib/db-formats/ugpt-settings"; +import { + GetUGPTSettingsDocument, + SELECT_GAME_PROFILE_SETTINGS, +} from "#lib/db-formats/ugpt-settings"; +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import DB from "#services/pg/db"; import { seedUser } from "#test-utils/pg-fixtures"; import { beforeEach, describe, expect, it } from "vitest"; @@ -13,17 +17,13 @@ describe("ACTION_PatchUGPTSettings", () => { beforeEach(async () => { ({ id: userId, username } = await seedUser({ username: `ugpt_set_${Date.now()}` })); - await DB.insertInto("game_settings") + await DB.insertInto("game_profile") .values({ user_id: userId, game: "iidx-sp", - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...newGameProfilePreferenceColumns("iidx-sp"), }) .execute(); }); @@ -41,10 +41,10 @@ describe("ACTION_PatchUGPTSettings", () => { expect(settings?.preferences.preferredScoreAlg).toBe("ktLampRating"); - const row = await DB.selectFrom("game_settings") - .select(SELECT_GAME_SETTINGS) - .where("game_settings.user_id", "=", userId) - .where("game_settings.game", "=", "iidx-sp") + const row = await DB.selectFrom("game_profile") + .select(SELECT_GAME_PROFILE_SETTINGS) + .where("game_profile.user_id", "=", userId) + .where("game_profile.game", "=", "iidx-sp") .executeTakeFirstOrThrow(); expect(row.pf_preferred_score_alg).toBe("ktLampRating"); diff --git a/typescript/server/src/actions/patch-ugpt-settings.ts b/typescript/server/src/actions/patch-ugpt-settings.ts index f17068c9a..cbc69c2d0 100644 --- a/typescript/server/src/actions/patch-ugpt-settings.ts +++ b/typescript/server/src/actions/patch-ugpt-settings.ts @@ -1,10 +1,13 @@ import { MakeAction } from "#lib/actions/actions"; -import { GetUGPTSettingsDocument, SELECT_GAME_SETTINGS } from "#lib/db-formats/ugpt-settings"; +import { + GetUGPTSettingsDocument, + SELECT_GAME_PROFILE_SETTINGS, +} from "#lib/db-formats/ugpt-settings"; import DB from "#services/pg/db"; import { IsUserAdmin } from "#utils/user"; import { ExpectedErr } from "bliss"; import { type UGPTSettingsDocument } from "tachi-common"; -import { type GameSettingsUpdate } from "tachi-db"; +import { type GameProfileUpdate } from "tachi-db"; export const ACTION_PatchUGPTSettings = MakeAction("PATCH_UGPT_SETTINGS", async (taker, input) => { const { userID, game, preferences } = input; @@ -52,17 +55,17 @@ export const ACTION_PatchUGPTSettings = MakeAction("PATCH_UGPT_SETTINGS", async return { settings }; } - const row = await DB.selectFrom("game_settings") - .select(SELECT_GAME_SETTINGS) - .where("game_settings.user_id", "=", userID) - .where("game_settings.game", "=", game) + const row = await DB.selectFrom("game_profile") + .select(SELECT_GAME_PROFILE_SETTINGS) + .where("game_profile.user_id", "=", userID) + .where("game_profile.game", "=", game) .executeTakeFirst(); if (!row) { throw new ExpectedErr(404, "You do not have an account for this game."); } - const set: GameSettingsUpdate = {}; + const set: GameProfileUpdate = {}; if (body.preferredScoreAlg !== undefined) { set.pf_preferred_score_alg = body.preferredScoreAlg; @@ -99,10 +102,10 @@ export const ACTION_PatchUGPTSettings = MakeAction("PATCH_UGPT_SETTINGS", async return { settings }; } - await DB.updateTable("game_settings") + await DB.updateTable("game_profile") .set(set) - .where("user_id", "=", userID) - .where("game", "=", game) + .where("game_profile.user_id", "=", userID) + .where("game_profile.game", "=", game) .execute(); const settings = await GetUGPTSettingsDocument(userID, game); diff --git a/typescript/server/src/actions/score-import.ts b/typescript/server/src/actions/score-import.ts new file mode 100644 index 000000000..a3902553d --- /dev/null +++ b/typescript/server/src/actions/score-import.ts @@ -0,0 +1,57 @@ +import type { ScoreImportJobData } from "#lib/score-import/worker/types"; + +import { MakeAction } from "#lib/actions/actions"; +import { GetInputParser } from "#lib/score-import/framework/common/get-input-parser"; +import ScoreImportFatalError from "#lib/score-import/framework/score-importing/score-import-error"; +import ScoreImportMain from "#lib/score-import/framework/score-importing/score-import-main"; +import { + EndTrackingImport, + MarkImportAsFailed, + StartTrackingImport, +} from "#lib/score-import/framework/status-tracking/import-status-tracking"; +import { ExpectedErr } from "bliss"; +import { type ImportTypes } from "tachi-common"; + +/** + * Authoritative score-import mutation: tracking (unless worker already did), parse + convert, + * `EndTrackingImport` or `MarkImportAsFailed`, and `action` table audit. User-facing + * `ScoreImportFatalError` is turned into `ExpectedErr` for correct audit (`BAD` / not `THROW`). + */ +export const ACTION_ScoreImport = MakeAction("SCORE_IMPORT", async (taker, input) => { + const { importID, importType, userIntent, skipStartTracking } = input; + const parserArguments = input[ + "!parserArguments" + ] as ScoreImportJobData["parserArguments"]; + + const jobData: ScoreImportJobData = { + importID, + importType: importType as ImportTypes, + userID: taker.acct.id, + userIntent, + parserArguments, + }; + + if (!skipStartTracking) { + await StartTrackingImport(jobData); + } + + try { + const InputParser = GetInputParser(jobData); + await ScoreImportMain( + taker.acct.id, + userIntent, + importType as ImportTypes, + InputParser, + importID, + ); + await EndTrackingImport(importID); + return { importID }; + } catch (e) { + const err = e as Error | ScoreImportFatalError; + await MarkImportAsFailed(importID, err); + if (err instanceof ScoreImportFatalError) { + throw new ExpectedErr(err.statusCode, err.message); + } + throw e; + } +}); diff --git a/typescript/server/src/actions/set-rivals.test.ts b/typescript/server/src/actions/set-rivals.test.ts index 43593050a..0e9bfaaf3 100644 --- a/typescript/server/src/actions/set-rivals.test.ts +++ b/typescript/server/src/actions/set-rivals.test.ts @@ -1,3 +1,4 @@ +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import { ServerConfig } from "#lib/setup/config"; import DB from "#services/pg/db"; import { seedUser } from "#test-utils/pg-fixtures"; @@ -14,21 +15,23 @@ describe("ACTION_SetRivals", () => { ({ id: userId, username } = await seedUser({ username: `rival_set_${Date.now()}` })); rivalId = (await seedUser({ username: `rival_target_${Date.now()}` })).id; - const ugptRow = { - game: "iidx-sp" as const, - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), - }; - - await DB.insertInto("game_settings") + const prefs = newGameProfilePreferenceColumns("iidx-sp"); + await DB.insertInto("game_profile") .values([ - { user_id: userId, ...ugptRow }, - { user_id: rivalId, ...ugptRow }, + { + user_id: userId, + game: "iidx-sp", + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...prefs, + }, + { + user_id: rivalId, + game: "iidx-sp", + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...prefs, + }, ]) .execute(); }); @@ -102,21 +105,23 @@ describe("ACTION_SetRivals", () => { ), ); - const ugptRow = { - game: "iidx-sp" as const, - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), - }; - - await DB.insertInto("game_settings") + const prefs = newGameProfilePreferenceColumns("iidx-sp"); + await DB.insertInto("game_profile") .values([ - { user_id: main.id, ...ugptRow }, - ...rivalUsers.map((u) => ({ user_id: u.id, ...ugptRow })), + { + user_id: main.id, + game: "iidx-sp", + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...prefs, + }, + ...rivalUsers.map((u) => ({ + user_id: u.id, + game: "iidx-sp" as const, + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...prefs, + })), ]) .execute(); diff --git a/typescript/server/src/actions/update-ugpt-showcase.ts b/typescript/server/src/actions/update-ugpt-showcase.ts index 08b837f50..b0ef38545 100644 --- a/typescript/server/src/actions/update-ugpt-showcase.ts +++ b/typescript/server/src/actions/update-ugpt-showcase.ts @@ -14,27 +14,22 @@ export const ACTION_UpdateUgptShowcase = MakeAction( throw new ExpectedErr(403, "You are not authorised to modify this user's showcase."); } - const settingsRow = await DB.selectFrom("game_settings") - .select("user_id") - .where("user_id", "=", userID) - .where("game", "=", game) + const profileRow = await DB.selectFrom("game_profile") + .select("game_profile.user_id") + .where("game_profile.user_id", "=", userID) + .where("game_profile.game", "=", game) .executeTakeFirst(); - if (!settingsRow) { + if (!profileRow) { throw new ExpectedErr(404, "You do not have a profile for this game."); } const payload = stats as Array; - await DB.insertInto("game_settings_showcase") - .values({ - user_id: userID, - game, - data: JSON.stringify(payload), - }) - .onConflict((oc) => - oc.columns(["user_id", "game"]).doUpdateSet({ data: JSON.stringify(payload) }), - ) + await DB.updateTable("game_profile") + .set({ showcase: JSON.stringify(payload) }) + .where("game_profile.user_id", "=", userID) + .where("game_profile.game", "=", game) .execute(); const newSettings = await GetUGPTSettingsDocument(userID, game); diff --git a/typescript/server/src/cron-worker.ts b/typescript/server/src/cron-worker.ts new file mode 100644 index 000000000..81728bcd1 --- /dev/null +++ b/typescript/server/src/cron-worker.ts @@ -0,0 +1,47 @@ +import { loadServerEnvFile } from "#lib/setup/load-server-env"; +loadServerEnvFile(process.env.NODE_ENV === "test" ? ".env.test" : ".env"); + +import { runCronTickOnce } from "#lib/jobs/cron/cron-service"; +import { log } from "#lib/log/log"; +import { Env } from "#lib/setup/config"; +import { ClosePgConnection } from "#services/pg/db"; +import { applyMigrations } from "tachi-db-migration-engine"; + +const TICK_MS = 5_000; + +void bootstrap(); + +/** + * Often started alongside the API by `just server` (or `bun run cron-worker` alone). + * Single active scheduler (Postgres `cron_task`); extra processes no-op when the advisory lock is held. + */ +async function bootstrap() { + await applyMigrations(Env.POSTGRES_URL, Env.MIGRATIONS_DIR); + log.info({ bootInfo: true }, "tachi cron worker starting."); + let stopping = false; + const shutdown = () => { + stopping = true; + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); + + // eslint-disable-next-line no-unmodified-loop-condition + while (!stopping) { + try { + // eslint-disable-next-line no-await-in-loop + await runCronTickOnce(); + } catch (e) { + log.error(e, "Cron tick error."); + } + if (stopping) { + break; + } + // eslint-disable-next-line no-await-in-loop + await new Promise((r) => { + setTimeout(r, TICK_MS); + }); + } + log.info("Cron worker stopped."); + await ClosePgConnection(); + process.exit(0); +} diff --git a/typescript/server/src/game-implementations/games/iidx.impl.test.ts b/typescript/server/src/game-implementations/games/iidx.impl.test.ts index c0f3b5c3c..1fc537286 100644 --- a/typescript/server/src/game-implementations/games/iidx.impl.test.ts +++ b/typescript/server/src/game-implementations/games/iidx.impl.test.ts @@ -236,6 +236,110 @@ describe("IIDX_IMPL (unit)", () => { }); }); + describe("scoreCalcs ktLampRatingNC / ktLampRatingHC / ktLampRatingEXHC", () => { + it("IIDX SP", () => { + const run = ( + scoreData: Partial>, + chartData: Partial, + ) => + IIDX_SP_IMPL.scoreCalcs( + dmf(TestingIIDXSPScore.scoreData, scoreData), + IIDX_SP_IMPL.scoreDeriver( + dmf(TestingIIDXSPScore.scoreData, scoreData), + dmf(Testing511SPA, { data: chartData as never }), + ), + dmf(Testing511SPA, { data: chartData as never }), + ); + + function mkTier(v: number) { + return { value: v, text: "whatever", individualDifference: false }; + } + + const tiered = { ncTier: mkTier(15), hcTier: mkTier(16), exhcTier: mkTier(17) }; + + expect(run({ lamp: "FAILED" }, {})).toMatchObject({ + ktLampRatingNC: 0, + ktLampRatingHC: 0, + ktLampRatingEXHC: 0, + }); + expect(run({ lamp: "EASY CLEAR" }, {})).toMatchObject({ + ktLampRatingNC: 0, + ktLampRatingHC: 0, + ktLampRatingEXHC: 0, + }); + expect(run({ lamp: "CLEAR" }, tiered)).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 0, + ktLampRatingEXHC: 0, + }); + expect(run({ lamp: "HARD CLEAR" }, tiered)).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 16, + ktLampRatingEXHC: 0, + }); + expect(run({ lamp: "EX HARD CLEAR" }, tiered)).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 16, + ktLampRatingEXHC: 17, + }); + expect(run({ lamp: "FULL COMBO" }, tiered)).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 16, + ktLampRatingEXHC: 17, + }); + expect(run({ lamp: "HARD CLEAR" }, { ncTier: mkTier(15) })).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 15, + ktLampRatingEXHC: 0, + }); + }); + + it("IIDX DP", () => { + const run = ( + scoreData: Partial>, + chartData: Partial, + ) => + IIDX_DP_IMPL.scoreCalcs( + dmf(TestingIIDXSPScore.scoreData, scoreData), + IIDX_DP_IMPL.scoreDeriver( + dmf(TestingIIDXSPScore.scoreData, scoreData), + dmf(Testing511SPA, { data: chartData as never }) as never, + ), + dmf(Testing511SPA, { data: chartData as never }) as never, + ); + + function mkTier(v: number) { + return { value: v, text: "whatever", individualDifference: false }; + } + + expect(run({ lamp: "EASY CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({ + ktLampRatingNC: 0, + ktLampRatingHC: 0, + ktLampRatingEXHC: 0, + }); + expect(run({ lamp: "CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 0, + ktLampRatingEXHC: 0, + }); + expect(run({ lamp: "HARD CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 15, + ktLampRatingEXHC: 0, + }); + expect(run({ lamp: "EX HARD CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 15, + ktLampRatingEXHC: 15, + }); + expect(run({ lamp: "FULL COMBO" }, { dpTier: mkTier(15) })).toMatchObject({ + ktLampRatingNC: 15, + ktLampRatingHC: 15, + ktLampRatingEXHC: 15, + }); + }); + }); + describe("goal formatters", () => { describe.each([IIDX_SP_IMPL, IIDX_DP_IMPL] as const)("impl", (impl) => { it("criteria", () => { diff --git a/typescript/server/src/game-implementations/games/iidx.ts b/typescript/server/src/game-implementations/games/iidx.ts index 9c9d872c6..f49382cb3 100644 --- a/typescript/server/src/game-implementations/games/iidx.ts +++ b/typescript/server/src/game-implementations/games/iidx.ts @@ -29,15 +29,23 @@ type IIDXGames = "iidx-dp" | "iidx-sp"; const IIDX_SESSION_CALCS: GPTSessionCalcs = (arr) => ({ BPI: SessionAvgBest10For("BPI")(arr), ktLampRating: SessionAvgBest10For("ktLampRating")(arr), + ktLampRatingNC: SessionAvgBest10For("ktLampRatingNC")(arr), + ktLampRatingHC: SessionAvgBest10For("ktLampRatingHC")(arr), + ktLampRatingEXHC: SessionAvgBest10For("ktLampRatingEXHC")(arr), }); const IIDX_PROFILE_CALCS: GPTProfileCalcs = async (game, userID) => { - const [BPI, ktLampRating] = await Promise.all([ - ProfileAvgBestN("BPI", 20, true)(game, userID), - ProfileAvgBestN("ktLampRating", 20)(game, userID), - ]); + const [BPI, ktLampRating, ktLampRatingNC, ktLampRatingHC, ktLampRatingEXHC] = await Promise.all( + [ + ProfileAvgBestN("BPI", 20, true)(game, userID), + ProfileAvgBestN("ktLampRating", 20)(game, userID), + ProfileAvgBestN("ktLampRatingNC", 20)(game, userID), + ProfileAvgBestN("ktLampRatingHC", 20)(game, userID), + ProfileAvgBestN("ktLampRatingEXHC", 20)(game, userID), + ], + ); - return { BPI, ktLampRating }; + return { BPI, ktLampRating, ktLampRatingNC, ktLampRatingHC, ktLampRatingEXHC }; }; const IIDX_MERGERS: Array> = [ @@ -148,7 +156,29 @@ export const IIDX_SP_IMPL: GameImplementation<"iidx-sp"> = { ktLampRating = 0; } - return { BPI: bpi, ktLampRating }; + const atLeastNcClear = + scoreData.lamp === "CLEAR" || + scoreData.lamp === "HARD CLEAR" || + scoreData.lamp === "EX HARD CLEAR" || + scoreData.lamp === "FULL COMBO"; + const atLeastHcClear = + scoreData.lamp === "HARD CLEAR" || + scoreData.lamp === "EX HARD CLEAR" || + scoreData.lamp === "FULL COMBO"; + const atLeastExhcClear = + scoreData.lamp === "EX HARD CLEAR" || scoreData.lamp === "FULL COMBO"; + + const ktLampRatingNC = atLeastNcClear ? ncValue : 0; + const ktLampRatingHC = atLeastHcClear ? hcValue : 0; + const ktLampRatingEXHC = atLeastExhcClear ? exhcValue : 0; + + return { + BPI: bpi, + ktLampRating, + ktLampRatingNC, + ktLampRatingHC, + ktLampRatingEXHC, + }; }, sessionCalcs: IIDX_SESSION_CALCS, profileCalcs: IIDX_PROFILE_CALCS, @@ -205,7 +235,29 @@ export const IIDX_DP_IMPL: GameImplementation<"iidx-dp"> = { ktLampRating = 0; } - return { BPI: bpi, ktLampRating }; + const atLeastNcClear = + scoreData.lamp === "CLEAR" || + scoreData.lamp === "HARD CLEAR" || + scoreData.lamp === "EX HARD CLEAR" || + scoreData.lamp === "FULL COMBO"; + const atLeastHcClear = + scoreData.lamp === "HARD CLEAR" || + scoreData.lamp === "EX HARD CLEAR" || + scoreData.lamp === "FULL COMBO"; + const atLeastExhcClear = + scoreData.lamp === "EX HARD CLEAR" || scoreData.lamp === "FULL COMBO"; + + const ktLampRatingNC = atLeastNcClear ? ecValue : 0; + const ktLampRatingHC = atLeastHcClear ? ecValue : 0; + const ktLampRatingEXHC = atLeastExhcClear ? ecValue : 0; + + return { + BPI: bpi, + ktLampRating, + ktLampRatingNC, + ktLampRatingHC, + ktLampRatingEXHC, + }; }, sessionCalcs: IIDX_SESSION_CALCS, profileCalcs: IIDX_PROFILE_CALCS, diff --git a/typescript/server/src/job-queue-worker.ts b/typescript/server/src/job-queue-worker.ts new file mode 100644 index 000000000..a5115348c --- /dev/null +++ b/typescript/server/src/job-queue-worker.ts @@ -0,0 +1,64 @@ +/* eslint-disable no-await-in-loop */ +import { loadServerEnvFile } from "#lib/setup/load-server-env"; +loadServerEnvFile(process.env.NODE_ENV === "test" ? ".env.test" : ".env"); + +import { JOB_KIND_SCORE_IMPORT } from "#lib/jobs/job-queue/constants"; +import { ClaimNextJob, MarkJobDone, MarkJobFailed } from "#lib/jobs/job-queue/queue-ops"; +import { log } from "#lib/log/log"; +import { CloseScoreImportQueue } from "#lib/score-import/worker/queue"; +import { processScoreImportJobFromPayload } from "#lib/score-import/worker/score-import-job-processor"; +import { Env } from "#lib/setup/config"; +import { ClosePgConnection } from "#services/pg/db"; +import { CloseRedisConnection } from "#services/redis/redis"; +import { applyMigrations } from "tachi-db-migration-engine"; + +const POLL_MS = 250; + +void bootstrap(); + +/** + * Often run by `just server` (one or more via `TACHI_SERVER_JOB_WORKER_COUNT`); each process claims + * with `FOR UPDATE SKIP LOCKED`. + */ +async function bootstrap() { + await applyMigrations(Env.POSTGRES_URL, Env.MIGRATIONS_DIR); + log.info({ bootInfo: true }, "tachi job-queue worker starting (Postgres job_queue)."); + let stopping = false; + const shutdown = () => { + stopping = true; + }; + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); + // eslint-disable-next-line no-unmodified-loop-condition + while (!stopping) { + const job = await ClaimNextJob(); + if (!job) { + if (stopping) { + break; + } + await new Promise((r) => { + setTimeout(r, POLL_MS); + }); + continue; + } + try { + switch (job.job_kind) { + case JOB_KIND_SCORE_IMPORT: + await processScoreImportJobFromPayload(job.payload); + break; + default: + log.error({ job_kind: job.job_kind, row_id: job.row_id }, "Unknown job_kind."); + throw new Error(`Unknown job_kind ${String(job.job_kind)}`); + } + await MarkJobDone(job.row_id); + } catch (e) { + log.error(e, `Job ${job.row_id} failed.`); + await MarkJobFailed(job.row_id); + } + } + log.info("Job worker loop stopped, closing resources."); + await CloseScoreImportQueue(); + await CloseRedisConnection(); + await ClosePgConnection(); + process.exit(0); +} diff --git a/typescript/server/src/lib/actions/actions.ts b/typescript/server/src/lib/actions/actions.ts index 8c1668896..9fa4defd5 100644 --- a/typescript/server/src/lib/actions/actions.ts +++ b/typescript/server/src/lib/actions/actions.ts @@ -378,6 +378,18 @@ export const ActionSignatures = { }), output: z.object({}), }, + SCORE_IMPORT: { + input: z.object({ + importID: z.string(), + importType: z.string(), + userIntent: z.boolean(), + "!parserArguments": z.array(z.unknown()), + skipStartTracking: z.boolean().optional(), + }), + output: z.object({ + importID: z.string(), + }), + }, BMS_TABLE_SYNC: { input: z.object({}), output: z.object({}), diff --git a/typescript/server/src/lib/db-formats/pb.ts b/typescript/server/src/lib/db-formats/pb.ts index 5d4f7e979..70c0630cf 100644 --- a/typescript/server/src/lib/db-formats/pb.ts +++ b/typescript/server/src/lib/db-formats/pb.ts @@ -1,6 +1,6 @@ import type { Game } from "tachi-db"; -import { pgScoreDataToMongo } from "#lib/v3/migration-tools"; +import { pgScoreDataToAPI } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; import { EscapeForILIKE } from "#utils/misc"; import { ISO8601ToUnixMilliseconds } from "#utils/time"; @@ -103,13 +103,13 @@ export async function ToPbScoreDocument(row: PbDocumentJoinRow): Promise[1], + } as Parameters[1], ); const rawCd = row.calculated_data; diff --git a/typescript/server/src/lib/db-formats/score.ts b/typescript/server/src/lib/db-formats/score.ts index 7897dec0a..11f1705d7 100644 --- a/typescript/server/src/lib/db-formats/score.ts +++ b/typescript/server/src/lib/db-formats/score.ts @@ -1,4 +1,4 @@ -import { pgScoreDataToMongo } from "#lib/v3/migration-tools"; +import { pgScoreDataToAPI } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; import { ISO8601ToUnixMilliseconds } from "#utils/time"; import { type GameGroup, type ImportTypes, type ScoreDocument } from "tachi-common"; @@ -47,7 +47,7 @@ export interface ScoreDocumentJoinRow { } export function ToScoreDocument(row: ScoreDocumentJoinRow): ScoreDocument { - const scoreData = pgScoreDataToMongo(row.score_game, { + const scoreData = pgScoreDataToAPI(row.score_game, { data: row.score_data as any, derived: row.score_derived_data as any, judgements: row.score_judgements as any, diff --git a/typescript/server/src/lib/db-formats/ugpt-settings.ts b/typescript/server/src/lib/db-formats/ugpt-settings.ts index 1a064ec54..60790fed0 100644 --- a/typescript/server/src/lib/db-formats/ugpt-settings.ts +++ b/typescript/server/src/lib/db-formats/ugpt-settings.ts @@ -9,30 +9,32 @@ import { } from "tachi-common"; import { type Database } from "tachi-db"; -export const SELECT_GAME_SETTINGS = [ - "game_settings.user_id", - "game_settings.game", - "game_settings.pf_preferred_score_alg", - "game_settings.pf_preferred_session_alg", - "game_settings.pf_preferred_profile_alg", - "game_settings.pf_preferred_default_enum", - "game_settings.pf_default_table", - "game_settings.pf_preferred_ranking", - "game_settings.data", +export const SELECT_GAME_PROFILE_SETTINGS = [ + "game_profile.user_id", + "game_profile.game", + "game_profile.pf_preferred_score_alg", + "game_profile.pf_preferred_session_alg", + "game_profile.pf_preferred_profile_alg", + "game_profile.pf_preferred_default_enum", + "game_profile.pf_default_table", + "game_profile.pf_preferred_ranking", + "game_profile.data", + "game_profile.showcase", ] as const; -export type GameSettingsRow = Selection< +export type GameProfilePreferenceRow = Selection< Database, - "game_settings", - (typeof SELECT_GAME_SETTINGS)[number] + "game_profile", + (typeof SELECT_GAME_PROFILE_SETTINGS)[number] >; export function ToUGPTSettingsDocument( - row: GameSettingsRow, + row: GameProfilePreferenceRow, rivals: Array, - stats: Array, ): UGPTSettingsDocument { const gameSpecific = row.data as UGPTSettingsDocument["preferences"]["gameSpecific"]; + const rawShowcase = row.showcase as Array; + const stats = normalizeShowcaseStats(rawShowcase); return { userID: row.user_id, @@ -62,10 +64,10 @@ export async function GetUGPTSettingsDocument( userID: integer, game: V3Game, ): Promise { - const row = await DB.selectFrom("game_settings") - .select(SELECT_GAME_SETTINGS) - .where("game_settings.user_id", "=", userID) - .where("game_settings.game", "=", game) + const row = await DB.selectFrom("game_profile") + .select(SELECT_GAME_PROFILE_SETTINGS) + .where("game_profile.user_id", "=", userID) + .where("game_profile.game", "=", game) .executeTakeFirst(); if (!row) { @@ -80,18 +82,12 @@ export async function GetUGPTSettingsDocument( const rivals = rivalRows.map((r) => r.rival); - const showcaseRow = await DB.selectFrom("game_settings_showcase") - .select("data") - .where("user_id", "=", userID) - .where("game", "=", game) - .executeTakeFirst(); - - const stats = showcaseRow ? (showcaseRow.data as Array) : []; - - return ToUGPTSettingsDocument(row, rivals, normalizeShowcaseStats(stats)); + return ToUGPTSettingsDocument(row, rivals); } /** Strips legacy `metric` from chart entries stored before chart showcase was PB+playcount-only. */ +// TODO(zk): nonsense, lets just remove this +// with a migration? function normalizeShowcaseStats(raw: Array): Array { return raw.map((stat) => { if (stat.mode === "chart") { diff --git a/typescript/server/src/lib/folders/folders.ts b/typescript/server/src/lib/folders/folders.ts index d4208ac16..3986a23e4 100644 --- a/typescript/server/src/lib/folders/folders.ts +++ b/typescript/server/src/lib/folders/folders.ts @@ -10,7 +10,7 @@ import { LoadPbsForUserOnChartsByPgIds } from "#lib/db-formats/pb"; import { GetSongsByIDs } from "#lib/db-formats/song"; import { LoadTableDocumentByLegacyId } from "#lib/db-formats/table"; import { log } from "#lib/log/log"; -import { pgScoreDataToMongo } from "#lib/v3/migration-tools"; +import { pgScoreDataToAPI } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; import { GetFolderForIDGuaranteed } from "#utils/db"; import { ISO8601ToUnixMilliseconds, UnixMillisecondsToISO8601 } from "#utils/time"; @@ -293,7 +293,7 @@ export async function GetEnumDistForFolderAsOf( const maxByChart = new Map>(); for (const row of rows) { - const mongoData = pgScoreDataToMongo(v3Game, { + const mongoData = pgScoreDataToAPI(v3Game, { data: row.data as any, derived: row.derived_data as any, judgements: row.judgements as any, diff --git a/typescript/server/src/lib/folders/get-session-folder-raises.ts b/typescript/server/src/lib/folders/get-session-folder-raises.ts new file mode 100644 index 000000000..b4d288d62 --- /dev/null +++ b/typescript/server/src/lib/folders/get-session-folder-raises.ts @@ -0,0 +1,159 @@ +import { LoadFolderDocumentsByIds } from "#lib/db-formats/folders"; +import { GetEnumDistForFolderAsOf, GetFolderIDsForChartId } from "#lib/folders/folders"; +import { GetSessionData } from "#utils/queries/sessions"; +import { + type FolderDocument, + GetGameConfig, + GetScoreEnumConfs, + type integer, + type SessionDocument, + type V3Game, +} from "tachi-common"; + +export type SessionFolderRaisesPayload = { + folder: FolderDocument; + previousCount: integer; + raisedCharts: Array; + totalCharts: integer; + type: string; + value: string; +}; + +function bucketKey(folderId: string, metric: string, value: string): string { + return JSON.stringify([folderId, metric, value]); +} + +function parseBucketKey(key: string): [string, string, string] { + const parsed = JSON.parse(key) as [string, string, string]; + return parsed; +} + +/** + * Per-chart folder ids (lookup is identical for all scores on the same chart). + */ +async function folderIdsForChartCached( + chartId: string, + cache: Map>, +): Promise> { + let ids = cache.get(chartId); + + if (!ids) { + ids = await GetFolderIDsForChartId(chartId); + cache.set(chartId, ids); + } + + return ids; +} + +/** + * Folder raise rows for the session view: which folders gained which enum + * values on which charts, and the user's exact folder distribution before + * {@link SessionDocument.timeStarted}. + */ +export async function GetSessionFolderRaises( + session: SessionDocument, +): Promise> { + const { scores, scoreInfo } = await GetSessionData(session); + const scoreMap = new Map(scores.map((s) => [s.scoreID, s])); + const gameConfig = GetGameConfig(session.game as V3Game); + const enumMetrics = GetScoreEnumConfs(gameConfig); + + const chartFolderCache = new Map>(); + const bucket = new Map>(); + + for (const info of scoreInfo) { + const score = scoreMap.get(info.scoreID); + + if (!score) { + continue; + } + + for (const [metric, conf] of Object.entries(enumMetrics)) { + if (!info.isNewScore) { + const delta = info.deltas[metric]; + + if (delta === undefined || delta <= 0) { + continue; + } + } + + const enumIndexes = score.scoreData.enumIndexes; + const idx = enumIndexes?.[metric as keyof typeof enumIndexes]; + + if (idx === undefined) { + continue; + } + + if (idx <= conf.values.indexOf(conf.minimumRelevantValue)) { + continue; + } + + const valueRaw = (score.scoreData as Record)[metric]; + + if (typeof valueRaw !== "string") { + continue; + } + + const folderIds = await folderIdsForChartCached(score.chartID, chartFolderCache); + + for (const folderId of folderIds) { + const key = bucketKey(folderId, metric, valueRaw); + let set = bucket.get(key); + + if (!set) { + set = new Set(); + bucket.set(key, set); + } + + set.add(score.chartID); + } + } + } + + if (bucket.size === 0) { + return []; + } + + const folderIds = [...new Set([...bucket.keys()].map((k) => parseBucketKey(k)[0]))]; + const folderDocs = await LoadFolderDocumentsByIds(folderIds); + + const distCache = new Map>>(); + + await Promise.all( + folderIds.map(async (fid) => { + const dist = await GetEnumDistForFolderAsOf(session.userID, fid, session.timeStarted); + + distCache.set(fid, dist); + }), + ); + + const out: Array = []; + + for (const [key, chartSet] of bucket) { + const [folderId, metric, value] = parseBucketKey(key); + const folder = folderDocs.get(folderId); + + if (!folder || folder.game !== session.game) { + continue; + } + + const dist = distCache.get(folderId); + + if (!dist) { + continue; + } + + const previousCount = dist.enumDist[metric]?.[value] ?? 0; + + out.push({ + folder, + previousCount, + raisedCharts: [...chartSet], + totalCharts: dist.chartIDs.length, + type: metric, + value, + }); + } + + return out; +} diff --git a/typescript/server/src/lib/game-settings/create-game-settings.test.ts b/typescript/server/src/lib/game-settings/create-game-settings.test.ts index c7f0e0d62..2cc226d83 100644 --- a/typescript/server/src/lib/game-settings/create-game-settings.test.ts +++ b/typescript/server/src/lib/game-settings/create-game-settings.test.ts @@ -1,37 +1,18 @@ -import DB from "#services/pg/db"; -import { seedUser } from "#test-utils/pg-fixtures"; import { describe, expect, it } from "vitest"; -import { CreateGameSettings } from "./create-game-settings"; +import { newGameProfilePreferenceColumns } from "./create-game-settings"; -describe("CreateGameSettings", () => { - it("creates settings for a new user and game", async () => { - const { id: userId } = await seedUser(); - - await CreateGameSettings(userId, "bms-7k"); - - const row = await DB.selectFrom("game_settings") - .selectAll() - .where("user_id", "=", userId) - .where("game", "=", "bms-7k") - .executeTakeFirst(); - - expect(row).not.toBeUndefined(); - expect(row?.game).toBe("bms-7k"); +describe("newGameProfilePreferenceColumns", () => { + it("returns IIDX-specific defaults for iidx-sp", () => { + const cols = newGameProfilePreferenceColumns("iidx-sp"); + expect(JSON.parse(cols.data)).toEqual({ display2DXTra: false, bpiTarget: 0 }); + expect(JSON.parse(cols.showcase)).toEqual([]); + expect(cols.pf_preferred_score_alg).toBeNull(); }); - it("throws if settings already exist", async () => { - const { id } = await seedUser(); - const localUserId = id; - - try { - await CreateGameSettings(localUserId, "popn"); - await expect(CreateGameSettings(localUserId, "popn")).rejects.toThrow( - /Cannot create .* game-settings as one already exists/u, - ); - } finally { - await DB.deleteFrom("game_settings").where("user_id", "=", localUserId).execute(); - await DB.deleteFrom("account").where("id", "=", localUserId).execute(); - } + it("returns empty gameSpecific JSON for non-IIDX games", () => { + const cols = newGameProfilePreferenceColumns("bms-7k"); + expect(JSON.parse(cols.data)).toEqual({}); + expect(JSON.parse(cols.showcase)).toEqual([]); }); }); diff --git a/typescript/server/src/lib/game-settings/create-game-settings.ts b/typescript/server/src/lib/game-settings/create-game-settings.ts index 17f8e5428..94b2cd7fc 100644 --- a/typescript/server/src/lib/game-settings/create-game-settings.ts +++ b/typescript/server/src/lib/game-settings/create-game-settings.ts @@ -1,23 +1,9 @@ -import { log } from "#lib/log/log"; -import DB from "#services/pg/db"; -import { type integer, type V3Game } from "tachi-common"; +import { type V3Game } from "tachi-common"; /** - * Create GameSettings for a UGPT (which contains their preferences). + * Default preference columns for a new `game_profile` row (ratings/classes live alongside these). */ -export async function CreateGameSettings(userID: integer, game: V3Game) { - const exists = await DB.selectFrom("game_settings") - .select("user_id") - .where("user_id", "=", userID) - .where("game", "=", game) - .executeTakeFirst(); - - if (exists) { - log.error(`Cannot create ${userID} ${game} game-settings as one already exists?`); - - throw new Error(`Cannot create ${userID} ${game} game-settings as one already exists?`); - } - +export function newGameProfilePreferenceColumns(game: V3Game) { const gameSpecific = game === "iidx-sp" || game === "iidx-dp" ? { @@ -26,19 +12,14 @@ export async function CreateGameSettings(userID: integer, game: V3Game) { } : {}; - await DB.insertInto("game_settings") - .values({ - data: JSON.stringify(gameSpecific), - game, - pf_default_table: null, - pf_preferred_default_enum: null, - pf_preferred_profile_alg: null, - pf_preferred_ranking: null, - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - user_id: userID, - }) - .execute(); - - log.info(`Created game settings for ${userID} (${game}).`); + return { + pf_preferred_score_alg: null as string | null, + pf_preferred_session_alg: null as string | null, + pf_preferred_profile_alg: null as string | null, + pf_preferred_default_enum: null as string | null, + pf_default_table: null as string | null, + pf_preferred_ranking: null as string | null, + data: JSON.stringify(gameSpecific), + showcase: JSON.stringify([]), + }; } diff --git a/typescript/server/src/lib/jobs/cron/cron-registry.ts b/typescript/server/src/lib/jobs/cron/cron-registry.ts new file mode 100644 index 000000000..1d2926889 --- /dev/null +++ b/typescript/server/src/lib/jobs/cron/cron-registry.ts @@ -0,0 +1,125 @@ +import { ACTION_BacksyncBmsPmsSeeds } from "#actions/backsync-bms-pms-seeds"; +import { ACTION_BMSTableSync } from "#actions/bms-table-sync"; +import { ACTION_UGSSnapshot } from "#actions/ugs-snapshot"; +import { ACTION_UpdateBpiData } from "#actions/update-bpi-data"; +import { ACTION_UpdateDpTiers } from "#actions/update-dp-tiers"; +import { UpdateAILevels } from "#lib/jobs/bms-ai-table-sync"; +import { DefaultAdminUser } from "#lib/jobs/default-admin-user"; +import { DeorphanScoresMain } from "#lib/jobs/deorphan-scores"; +import { drainStatsQueuesInOrder } from "#lib/jobs/drain-dirty-queues"; +import { RebuildFolderChartLookupJob } from "#lib/jobs/rebuild-folder-chart-lookup"; +import { TachiConfig } from "#lib/setup/config"; +import { DedupeArr } from "#utils/misc"; + +export interface CronTaskDef { + /** `cron_task.id` primary key. */ + id: string; + /** 5-field cron in UTC. */ + schedule: string; + /** Shown in admin and synced to `cron_task.description`. */ + description: string; + run: () => Promise; +} + +function buildList(): Array { + const out: Array = [ + { + id: "rebuild_folder_chart_lookup", + schedule: "5 0 * * *", + description: "Rebuild folder chart lookup", + run: RebuildFolderChartLookupJob, + }, + { + id: "ugs_snapshot", + schedule: "0 0 * * *", + description: "Snapshot User Game Stats", + run: async () => { + const taker = await DefaultAdminUser.actionTaker(); + await ACTION_UGSSnapshot(taker, {}); + }, + }, + { + id: "deorphan_scores", + schedule: "1 0 * * *", + description: "De-Orphan Scores", + run: DeorphanScoresMain, + }, + { + id: "drain_stats_queues", + schedule: "* * * * *", + description: + "Drain score_rederive, pb_dirty, session_dirty, game_profile_dirty (ordered)", + run: drainStatsQueuesInOrder, + }, + ]; + + if (TachiConfig.TYPE !== "boku") { + out.push( + { + id: "update_bpi", + schedule: "2 0 * * *", + description: "Update BPI", + run: async () => { + const taker = await DefaultAdminUser.actionTaker(); + await ACTION_UpdateBpiData(taker, {}); + }, + }, + { + id: "update_dp_tiers", + schedule: "3 0 * * *", + description: "Update DP Tiers", + run: async () => { + const taker = await DefaultAdminUser.actionTaker(); + await ACTION_UpdateDpTiers(taker, {}); + }, + }, + ); + } + + if (TachiConfig.TYPE !== "kamai") { + out.push( + { + id: "update_ai_table", + schedule: "2 0 * * *", + description: "Update AI Table", + run: UpdateAILevels, + }, + { + id: "update_bms_tables", + schedule: "3 0 * * *", + description: "Update Tables (BMS)", + run: async () => { + const taker = await DefaultAdminUser.actionTaker(); + await ACTION_BMSTableSync(taker, {}); + }, + }, + { + id: "backsync_bms_pms", + schedule: "4 0 * * *", + description: "Backsync BMS + PMS", + run: async () => { + const taker = await DefaultAdminUser.actionTaker(); + await ACTION_BacksyncBmsPmsSeeds(taker, {}); + }, + }, + ); + } + + const names = out.map((e) => e.id); + if (DedupeArr(names).length !== names.length) { + throw new Error("cron task registry has duplicate id fields"); + } + return out; +} + +let cached: Array | undefined; + +/** + * In-memory cron definitions (code is authoritative for schedules; rows are upserted to Postgres). + */ +export function getCronTaskDefinitions(): Array { + if (!cached) { + cached = buildList(); + } + return cached; +} diff --git a/typescript/server/src/lib/jobs/cron/cron-service.ts b/typescript/server/src/lib/jobs/cron/cron-service.ts new file mode 100644 index 000000000..82c475d9c --- /dev/null +++ b/typescript/server/src/lib/jobs/cron/cron-service.ts @@ -0,0 +1,146 @@ +import { getCronTaskDefinitions } from "#lib/jobs/cron/cron-registry"; +import { log } from "#lib/log/log"; +import DB from "#services/pg/db"; +import CronExpressionParser from "cron-parser"; +import { sql } from "kysely"; + +/** Namespaces `pg_try_advisory_lock` for the cron scheduler. */ +const CRON_ADVISORY_KEY1 = 0x54_61_63_68; // "Tach" +const CRON_ADVISORY_KEY2 = 0x63_72_6f_6e; // "cron" + +/** + * Next cron fire strictly after `last`, that is still <= `now`. + * If none, `null` (not due). "Skip missed" to the latest such fire time. + */ +export function getDueFireTime(schedule: string, last: Date | null, now: Date): Date | null { + const it = CronExpressionParser.parse(schedule, { + currentDate: last ? new Date(last.getTime() + 1) : new Date(0), + }); + const first = it.next().toDate(); + if (first.getTime() > now.getTime()) { + return null; + } + let lastDue = first; + for (;;) { + const n = it.next().toDate(); + if (n.getTime() > now.getTime()) { + return lastDue; + } + lastDue = n; + } +} + +export async function syncCronTasksFromRegistry(): Promise { + const defs = getCronTaskDefinitions(); + const now = new Date().toISOString(); + for (const d of defs) { + await DB.insertInto("cron_task") + .values({ + id: d.id, + schedule: d.schedule, + description: d.description, + created_at: now, + updated_at: now, + last_scheduled_at: null, + }) + .onConflict((oc) => + oc.column("id").doUpdateSet({ + schedule: d.schedule, + description: d.description, + updated_at: now, + }), + ) + .execute(); + } +} + +async function tryAcquireCronTickLock(): Promise { + const r = await sql<{ acquired: boolean }>` + SELECT pg_try_advisory_lock(${CRON_ADVISORY_KEY1}, ${CRON_ADVISORY_KEY2}) AS acquired + `.execute(DB); + const row = r.rows[0] as { acquired: boolean } | undefined; + return row?.acquired === true; +} + +async function releaseCronTickLock(): Promise { + await sql`SELECT pg_advisory_unlock(${CRON_ADVISORY_KEY1}, ${CRON_ADVISORY_KEY2})`.execute(DB); +} + +export async function runCronTickOnce(): Promise { + const got = await tryAcquireCronTickLock(); + if (!got) { + return; + } + try { + await syncCronTasksFromRegistry(); + const now = new Date(); + const rows = await DB.selectFrom("cron_task") + .select(["cron_task.id", "cron_task.schedule", "cron_task.last_scheduled_at"]) + .execute(); + const byId = new Map(rows.map((r) => [r.id, r])); + for (const def of getCronTaskDefinitions()) { + const row = byId.get(def.id); + if (!row) { + continue; + } + const last = row.last_scheduled_at ? new Date(row.last_scheduled_at) : null; + const due = getDueFireTime(def.schedule, last, now); + if (!due) { + continue; + } + const scheduledAtIso = due.toISOString(); + const execRow = await DB.insertInto("cron_task_execution") + .values({ + task_id: def.id, + scheduled_at: scheduledAtIso, + status: "running", + completed_at: null, + output: null, + error: null, + }) + .returning("cron_task_execution.id") + .executeTakeFirstOrThrow(); + + try { + await def.run(); + await DB.updateTable("cron_task") + .set({ + last_scheduled_at: scheduledAtIso, + updated_at: new Date().toISOString(), + }) + .where("cron_task.id", "=", def.id) + .execute(); + await DB.updateTable("cron_task_execution") + .set({ + status: "success", + completed_at: new Date().toISOString(), + output: null, + error: null, + }) + .where("cron_task_execution.id", "=", execRow.id) + .execute(); + log.info(`Cron task ${def.id} completed for fire ${scheduledAtIso}.`); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.error({ err }, `Cron task ${def.id} failed.`); + await DB.updateTable("cron_task") + .set({ + last_scheduled_at: scheduledAtIso, + updated_at: new Date().toISOString(), + }) + .where("cron_task.id", "=", def.id) + .execute(); + await DB.updateTable("cron_task_execution") + .set({ + status: "failure", + completed_at: new Date().toISOString(), + error: message, + }) + .where("cron_task_execution.id", "=", execRow.id) + .execute(); + } + } + } finally { + await releaseCronTickLock(); + } +} diff --git a/typescript/server/src/lib/jobs/drain-dirty-queues.ts b/typescript/server/src/lib/jobs/drain-dirty-queues.ts index 08295418c..46d445a54 100644 --- a/typescript/server/src/lib/jobs/drain-dirty-queues.ts +++ b/typescript/server/src/lib/jobs/drain-dirty-queues.ts @@ -1,6 +1,15 @@ +import { + type ScoreDocumentJoinRow, + SELECT_SCORE_DOCUMENT, + ToScoreDocument, +} from "#lib/db-formats/score"; +import { SELECT_SESSION_DOCUMENT } from "#lib/db-formats/session"; import { log } from "#lib/log/log"; +import { CreateSessionCalcData } from "#lib/score-import/framework/calculated-data/session"; import { ProcessPBs } from "#lib/score-import/framework/pb/process-pbs"; import { rederiveScoresForChart } from "#lib/score-import/framework/pb/rederive-scores"; +import { scoreVisibleSql } from "#lib/score-import/framework/pg/score-visibility"; +import { UpdateUsersGamePlaytypeStats } from "#lib/score-import/framework/ugpt-stats/update-ugpt-stats"; import DB from "#services/pg/db"; import { type GameGroup, @@ -11,8 +20,12 @@ import { type V3Game, } from "tachi-common"; -const PB_DIRTY_BATCH = 1000; -const SCORE_REDERIVE_BATCH = 50; +const PB_DIRTY_BATCH = 5000; +const SCORE_REDERIVE_BATCH = 5000; +const SESSION_DIRTY_BATCH = 5000; +const GAME_PROFILE_DIRTY_BATCH = 500; +/** Safety cap for one cron tick across score_rederive + pb_dirty + session + game_profile drains. */ +const STATS_QUEUE_DRAIN_CAP = 100_000; /** * Drain the `pb_dirty` queue: group entries by (game, playtype, user_id), @@ -100,13 +113,280 @@ export async function drainScoreRederive(): Promise { .execute(); } - log.info( + log.debug( `Drained ${rows.length} score_rederive entries, re-derived ${totalScores} total scores.`, ); return rows.length; } +/** + * Drain `session_dirty`: recompute `session.calculated_data` from visible scores in that session. + */ +export async function drainSessionDirty(): Promise { + const rows = await DB.selectFrom("session_dirty") + .select(["session_dirty.session_id"]) + .orderBy("session_dirty.enqueued_at", "asc") + .limit(SESSION_DIRTY_BATCH) + .execute(); + + if (rows.length === 0) { + return 0; + } + + for (const row of rows) { + const sessionId = row.session_id; + + // eslint-disable-next-line no-await-in-loop + const scoreRows = await DB.selectFrom("score") + .innerJoin("chart", "chart.id", "score.chart_id") + .innerJoin("song", "song.id", "chart.song_id") + .leftJoin("import", "import.id", "score.import_id") + .select(SELECT_SCORE_DOCUMENT) + .where("score.session_id", "=", sessionId) + .where(scoreVisibleSql()) + .execute(); + + if (scoreRows.length === 0) { + // eslint-disable-next-line no-await-in-loop + await DB.deleteFrom("session_dirty") + .where("session_dirty.session_id", "=", sessionId) + .execute(); + continue; + } + + // eslint-disable-next-line no-await-in-loop + const sessionRow = await DB.selectFrom("session") + .select(SELECT_SESSION_DOCUMENT) + .where("session.id", "=", sessionId) + .executeTakeFirst(); + + if (!sessionRow) { + // eslint-disable-next-line no-await-in-loop + await DB.deleteFrom("session_dirty") + .where("session_dirty.session_id", "=", sessionId) + .execute(); + continue; + } + + const scoreDocs = scoreRows.map((r) => ToScoreDocument(r as ScoreDocumentJoinRow)); + const calculatedData = CreateSessionCalcData(sessionRow.game as V3Game, scoreDocs); + + // eslint-disable-next-line no-await-in-loop + await DB.updateTable("session") + .set({ + calculated_data: JSON.stringify(calculatedData), + }) + .where("session.id", "=", sessionId) + .execute(); + + // eslint-disable-next-line no-await-in-loop + await DB.deleteFrom("session_dirty") + .where("session_dirty.session_id", "=", sessionId) + .execute(); + } + + log.info(`Drained ${rows.length} session_dirty entries.`); + + return rows.length; +} + +/** + * Drain `game_profile_dirty`: recompute `game_profile` ratings/classes for each (user, playtype). + */ +export async function drainGameProfileDirty(): Promise { + const rows = await DB.selectFrom("game_profile_dirty") + .select(["game_profile_dirty.user_id", "game_profile_dirty.game"]) + .orderBy("game_profile_dirty.enqueued_at", "asc") + .limit(GAME_PROFILE_DIRTY_BATCH) + .execute(); + + if (rows.length === 0) { + return 0; + } + + for (const row of rows) { + const userId = row.user_id; + + // eslint-disable-next-line no-await-in-loop + await UpdateUsersGamePlaytypeStats(row.game as V3Game, userId, null, log); + + // eslint-disable-next-line no-await-in-loop + await DB.deleteFrom("game_profile_dirty") + .where("game_profile_dirty.user_id", "=", userId) + .where("game_profile_dirty.game", "=", row.game) + .execute(); + } + + log.info(`Drained ${rows.length} game_profile_dirty entries.`); + + return rows.length; +} + +/** + * Drain `score_rederive`, then `pb_dirty`, then `session_dirty`, then `game_profile_dirty`, + * repeating until a full pass does nothing or `STATS_QUEUE_DRAIN_CAP` row-processings is reached. + * PBs must run before game profiles (ratings read from `pb`). + */ +export async function drainStatsQueuesInOrder(): Promise { + let totalProcessed = 0; + + while (totalProcessed < STATS_QUEUE_DRAIN_CAP) { + let cycleMoved = 0; + + while (totalProcessed < STATS_QUEUE_DRAIN_CAP) { + // eslint-disable-next-line no-await-in-loop + const n = await drainScoreRederive(); + + if (n === 0) { + break; + } + + cycleMoved += n; + totalProcessed += n; + } + + while (totalProcessed < STATS_QUEUE_DRAIN_CAP) { + // eslint-disable-next-line no-await-in-loop + const n = await drainPbDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + totalProcessed += n; + } + + while (totalProcessed < STATS_QUEUE_DRAIN_CAP) { + // eslint-disable-next-line no-await-in-loop + const n = await drainSessionDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + totalProcessed += n; + } + + while (totalProcessed < STATS_QUEUE_DRAIN_CAP) { + // eslint-disable-next-line no-await-in-loop + const n = await drainGameProfileDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + totalProcessed += n; + } + + if (cycleMoved === 0) { + break; + } + } +} + +/** + * Drain `score_rederive`, then `pb_dirty`, `session_dirty`, and `game_profile_dirty`, + * repeating until a full pass moves nothing. No per-tick row cap (unlike the cron + * drain) — intended for admin synchronous recalc. + */ +export async function drainStatsQueuesFully(): Promise { + for (;;) { + let cycleMoved = 0; + + for (;;) { + const n = await drainScoreRederive(); + + if (n === 0) { + break; + } + + cycleMoved += n; + } + + for (;;) { + const n = await drainPbDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + } + + for (;;) { + const n = await drainSessionDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + } + + for (;;) { + const n = await drainGameProfileDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + } + + if (cycleMoved === 0) { + break; + } + } +} + +/** + * Drain `pb_dirty` then `session_dirty` and `game_profile_dirty`, repeating until + * idle. No per-tick row cap — intended for admin synchronous PB recalc. + */ +export async function drainPbDirtyAndDownstream(): Promise { + for (;;) { + let cycleMoved = 0; + + for (;;) { + const n = await drainPbDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + } + + for (;;) { + const n = await drainSessionDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + } + + for (;;) { + const n = await drainGameProfileDirty(); + + if (n === 0) { + break; + } + + cycleMoved += n; + } + + if (cycleMoved === 0) { + break; + } + } +} + /** * Delete `pb_dirty` rows for the given user + chart IDs. Call this after * a synchronous `ProcessPBs` to prevent the async worker from redundantly diff --git a/typescript/server/src/lib/jobs/inline-job-runner/job-runner.ts b/typescript/server/src/lib/jobs/inline-job-runner/job-runner.ts deleted file mode 100644 index 11f2d0a07..000000000 --- a/typescript/server/src/lib/jobs/inline-job-runner/job-runner.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { ACTION_BacksyncBmsPmsSeeds } from "#actions/backsync-bms-pms-seeds"; -import { ACTION_BMSTableSync } from "#actions/bms-table-sync"; -import { ACTION_UGSSnapshot } from "#actions/ugs-snapshot"; -import { ACTION_UpdateBpiData } from "#actions/update-bpi-data"; -import { ACTION_UpdateDpTiers } from "#actions/update-dp-tiers"; -import { DefaultAdminUser } from "#lib/jobs/default-admin-user"; -import { log } from "#lib/log/log"; -import { TachiConfig } from "#lib/setup/config"; -import { DedupeArr } from "#utils/misc"; -import { Queue, Worker } from "bullmq"; - -import { UpdateAILevels } from "../bms-ai-table-sync"; -import { DeorphanScoresMain } from "../deorphan-scores"; -import { drainPbDirty, drainScoreRederive } from "../drain-dirty-queues"; -import { RebuildFolderChartLookupJob } from "../rebuild-folder-chart-lookup"; - -interface Job { - name: string; - cronFormat: string; - run: () => Promise; -} - -const jobs: Array = [ - { - name: "Rebuild folder chart lookup", - cronFormat: "5 0 * * *", - run: RebuildFolderChartLookupJob, - }, - { - name: "Snapshot User Game Stats", - cronFormat: "0 0 * * *", - run: async () => { - const taker = await DefaultAdminUser.actionTaker(); - await ACTION_UGSSnapshot(taker, {}); - }, - }, - { - name: "De-Orphan Scores", - - // We run an hour after snapshotting UGS - // just to spread load out a bit. - cronFormat: "1 0 * * *", - run: DeorphanScoresMain, - }, - { - name: "Drain pb_dirty", - cronFormat: "* * * * *", - run: async () => { - let drained = 0; - - // Keep draining until the queue is empty or we hit a safety cap. - - while (true) { - // eslint-disable-next-line no-await-in-loop - const n = await drainPbDirty(); - - if (n === 0) { - break; - } - - drained += n; - - if (drained >= 10_000) { - break; - } - } - }, - }, - { - name: "Drain score_rederive", - cronFormat: "*/5 * * * *", - run: async () => { - while (true) { - // eslint-disable-next-line no-await-in-loop - const n = await drainScoreRederive(); - - if (n === 0) { - break; - } - } - }, - }, -]; - -// if kamaitachi or omnitachi -if (TachiConfig.TYPE !== "boku") { - jobs.push({ - name: "Update BPI", - cronFormat: "2 0 * * *", - run: async () => { - const taker = await DefaultAdminUser.actionTaker(); - await ACTION_UpdateBpiData(taker, {}); - }, - }); - - jobs.push({ - name: "Update DP Tiers", - cronFormat: "3 0 * * *", - run: async () => { - const taker = await DefaultAdminUser.actionTaker(); - await ACTION_UpdateDpTiers(taker, {}); - }, - }); -} - -// if bokutachi or omnitachi -if (TachiConfig.TYPE !== "kamai") { - jobs.push({ - name: "Update AI Table", - cronFormat: "2 0 * * *", - run: UpdateAILevels, - }); - - jobs.push({ - name: "Update Tables", - cronFormat: "3 0 * * *", - run: async () => { - const taker = await DefaultAdminUser.actionTaker(); - await ACTION_BMSTableSync(taker, {}); - }, - }); - - jobs.push({ - name: "Backsync BMS + PMS", - cronFormat: "4 0 * * *", - run: async () => { - const taker = await DefaultAdminUser.actionTaker(); - await ACTION_BacksyncBmsPmsSeeds(taker, {}); - }, - }); -} - -/** - * Initalises a tachi-server job runner. - * This runs the list of jobs defined in jobConfig.jobs. - */ -export function InitialiseJobRunner() { - log.info(`Booting up Job Runner.`); - - const names = jobs.map((e) => e.name); - - if (DedupeArr(names).length !== names.length) { - log.fatal(() => { - process.exit(1); - }, `Jobs has duplicate name fields, refusing to run.`); - } - - const JobQueue = new Queue("Job Runner"); - - const jobNameMap = new Map(); - - for (const job of jobs) { - void JobQueue.add(job.name, { jobName: job.name }, { repeat: { cron: job.cronFormat } }); - jobNameMap.set(job.name, job); - } - - const worker = new Worker("Job Runner", async (j) => { - const { jobName } = j.data as { jobName: string }; - - log.info(`Running job ${jobName}.`); - - const jobInfo = jobNameMap.get(jobName); - - if (!jobInfo) { - log.error(`Unknown job name ${jobName}, couldn't find a run function?`); - return false; - } - - await jobInfo.run(); - - return true; - }); - - log.info(`Initialised ${jobs.length} jobs (${jobs.map((e) => e.name).join(", ")}).`); - - return worker; -} - -if (require.main === module) { - InitialiseJobRunner(); -} diff --git a/typescript/server/src/lib/jobs/job-queue/constants.ts b/typescript/server/src/lib/jobs/job-queue/constants.ts new file mode 100644 index 000000000..500209aef --- /dev/null +++ b/typescript/server/src/lib/jobs/job-queue/constants.ts @@ -0,0 +1,8 @@ +/** Canonical `job_queue.job_kind` for score import jobs. */ +export const JOB_KIND_SCORE_IMPORT = "score_import" as const; + +/** Matches client adminConstants and Zenith `job_queue.status` values. */ +export const JOB_STATUS_QUEUED = 0; +export const JOB_STATUS_RUNNING = 1; +export const JOB_STATUS_DONE = 2; +export const JOB_STATUS_FAILED = 3; diff --git a/typescript/server/src/lib/jobs/job-queue/queue-ops.ts b/typescript/server/src/lib/jobs/job-queue/queue-ops.ts new file mode 100644 index 000000000..203f7a2fd --- /dev/null +++ b/typescript/server/src/lib/jobs/job-queue/queue-ops.ts @@ -0,0 +1,78 @@ +import type { JobQueue, NewJobQueue } from "tachi-db"; + +type EnqueueInput = Omit; +import { + JOB_STATUS_DONE, + JOB_STATUS_FAILED, + JOB_STATUS_QUEUED, + JOB_STATUS_RUNNING, +} from "#lib/jobs/job-queue/constants"; +import DB from "#services/pg/db"; +import { sql } from "kysely"; + +/** + * Enqueue a job (queued, due on or before `scheduled_for`). + */ +export async function EnqueueJob(row: EnqueueInput): Promise { + const r = await DB.insertInto("job_queue") + .values({ + ...row, + status: JOB_STATUS_QUEUED, + failed_attempts: 0, + }) + .returning("job_queue.row_id") + .executeTakeFirstOrThrow(); + return r.row_id; +} + +/** + * Claim the next job using `FOR UPDATE SKIP LOCKED` (fair multi-worker). + */ +export async function ClaimNextJob(): Promise { + const r = await sql` + WITH picked AS ( + SELECT "job_queue"."row_id" + FROM "job_queue" + WHERE "job_queue"."status" = ${JOB_STATUS_QUEUED} + AND "job_queue"."scheduled_for" <= NOW() + ORDER BY "job_queue"."scheduled_for" ASC, "job_queue"."created_at" ASC + FOR UPDATE OF "job_queue" SKIP LOCKED + LIMIT 1 + ) + UPDATE "job_queue" + SET + "status" = ${JOB_STATUS_RUNNING}, + "updated_at" = NOW() + FROM picked + WHERE "job_queue"."row_id" = "picked"."row_id" + RETURNING + "job_queue"."row_id", + "job_queue"."created_at", + "job_queue"."updated_at", + "job_queue"."scheduled_for", + "job_queue"."failed_attempts", + "job_queue"."status", + "job_queue"."scope", + "job_queue"."job_kind", + "job_queue"."payload" + `.execute(DB); + + if (r.rows.length === 0) { + return undefined; + } + return r.rows[0] as unknown as JobQueue; +} + +export async function MarkJobDone(rowId: string): Promise { + await DB.updateTable("job_queue") + .set({ status: JOB_STATUS_DONE, updated_at: new Date().toISOString() }) + .where("job_queue.row_id", "=", rowId) + .execute(); +} + +export async function MarkJobFailed(rowId: string): Promise { + await DB.updateTable("job_queue") + .set({ status: JOB_STATUS_FAILED, updated_at: new Date().toISOString() }) + .where("job_queue.row_id", "=", rowId) + .execute(); +} diff --git a/typescript/server/src/lib/rivals/rivals.ts b/typescript/server/src/lib/rivals/rivals.ts index 829c0aefc..54edcb0af 100644 --- a/typescript/server/src/lib/rivals/rivals.ts +++ b/typescript/server/src/lib/rivals/rivals.ts @@ -4,7 +4,7 @@ import { SetRivalsFailReasons } from "#lib/constants/err-codes"; import { log } from "#lib/log/log"; import { SendSetRivalNotification } from "#lib/notifications/notification-wrappers"; import { ServerConfig } from "#lib/setup/config"; -import { pgScoreDataToMongo } from "#lib/v3/migration-tools"; +import { pgScoreDataToAPI } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; import { ArrayDiff } from "#utils/misc"; import { GetUsersWithIDs, GetUserWithIDGuaranteed } from "#utils/user"; @@ -15,13 +15,13 @@ import { GetGameConfig, type integer, type PgScoreData, type V3Game } from "tach * Throws if the user hasn't played the GPT in question. */ export async function GetRivalIDs(userID: integer, game: V3Game) { - const settings = await DB.selectFrom("game_settings") - .select("user_id") - .where("user_id", "=", userID) - .where("game", "=", game) + const profile = await DB.selectFrom("game_profile") + .select("game_profile.user_id") + .where("game_profile.user_id", "=", userID) + .where("game_profile.game", "=", game) .executeTakeFirst(); - if (!settings) { + if (!profile) { throw new Error(`User ${userID} has not played ${game}. Cannot retrieve rivals.`); } @@ -73,7 +73,7 @@ function metricValueFromPbRow( derivedData: unknown, metricKey: string, ): number | null { - const scoreData = pgScoreDataToMongo(v3Game, { + const scoreData = pgScoreDataToAPI(v3Game, { data, derived: derivedData, judgements: {}, @@ -102,10 +102,10 @@ export async function setRivalsWithResult( return SetRivalsFailReasons.RIVALED_SELF; } - const { count } = await DB.selectFrom("game_settings") + const { count } = await DB.selectFrom("game_profile") .select(DB.fn.countAll().as("count")) - .where("game", "=", game) - .where("user_id", "in", newRivals) + .where("game_profile.game", "=", game) + .where("game_profile.user_id", "in", newRivals) .executeTakeFirstOrThrow(); const playedGPTCount = Number(count); @@ -114,19 +114,19 @@ export async function setRivalsWithResult( return SetRivalsFailReasons.RIVALS_HAVENT_PLAYED_GPT; } - const currentGameSettings = await DB.selectFrom("game_settings") - .select("user_id") - .where("user_id", "=", userID) - .where("game", "=", game) + const currentGameProfile = await DB.selectFrom("game_profile") + .select("game_profile.user_id") + .where("game_profile.user_id", "=", userID) + .where("game_profile.game", "=", game) .executeTakeFirst(); - if (!currentGameSettings) { + if (!currentGameProfile) { log.error( - `User ${userID} attempted to set rivals for ${game}, but doesn't have game settings. Was their account deleted in midair?`, + `User ${userID} attempted to set rivals for ${game}, but doesn't have a game profile. Was their account deleted in midair?`, ); throw new Error( - `User ${userID} attempted to set rivals for ${game}, but doesn't have game settings. Was their account deleted in midair?`, + `User ${userID} attempted to set rivals for ${game}, but doesn't have a game profile. Was their account deleted in midair?`, ); } diff --git a/typescript/server/src/lib/score-import/framework/express-wrapper.ts b/typescript/server/src/lib/score-import/framework/express-wrapper.ts index fc72b3685..efa197814 100644 --- a/typescript/server/src/lib/score-import/framework/express-wrapper.ts +++ b/typescript/server/src/lib/score-import/framework/express-wrapper.ts @@ -8,6 +8,7 @@ import type { import { log } from "#lib/log/log"; import { Random20Hex } from "#utils/misc"; +import { ExpectedErr } from "bliss"; import type { ParserArguments } from "../worker/types"; @@ -52,14 +53,17 @@ export async function ExpressWrappedScoreImportMain( }, }; } catch (err) { - // this is definitely fine, as the errors are emitted from the same place. - if (err instanceof ScoreImportFatalError) { - log.info(err.message); + // `ACTION_ScoreImport` throws `ExpectedErr` (mapped from `ScoreImportFatalError`); the + // external-worker guard in `MakeScoreImport` still throws `ScoreImportFatalError`. + if (ExpectedErr.is(err) || err instanceof ScoreImportFatalError) { + const description = ExpectedErr.is(err) ? err.reason : err.message; + const statusCode = ExpectedErr.is(err) ? err.code : err.statusCode; + log.info(description); return { - statusCode: err.statusCode, + statusCode, body: { success: false, - description: err.message, + description, }, }; } diff --git a/typescript/server/src/lib/score-import/framework/orphans/orphans.ts b/typescript/server/src/lib/score-import/framework/orphans/orphans.ts index 366332c3b..3b74f6a4e 100644 --- a/typescript/server/src/lib/score-import/framework/orphans/orphans.ts +++ b/typescript/server/src/lib/score-import/framework/orphans/orphans.ts @@ -48,13 +48,24 @@ async function deleteOrphanByOrphanId(orphanID: string): Promise { /** API-facing row for listing a user’s orphan_score entries. */ export type OrphanScoreListItem = { + gameGroup: string; + importType: string; + message: string | null; orphanID: string; rowID: string; - importType: string; - gameGroup: string; - timeInserted: number; - message: string | null; summary: string | null; + timeInserted: number; +}; + +/** API-facing detail for one orphan_score row (includes raw import payload). */ +export type OrphanScoreDetail = { + context: unknown; + data: unknown; + gameGroup: string; + importType: string; + message: string | null; + orphanID: string; + timeInserted: number; }; function summarizeOrphanRow(row: PgOrphanScoreRow): string | null { @@ -102,8 +113,38 @@ function orphanRowToListItem(row: PgOrphanScoreRow): OrphanScoreListItem { }; } +/** Loads one orphan_score row for the user, or null if none. */ +export async function getOrphanScoreDetailForUser( + orphanID: string, + userID: integer, +): Promise { + const row = await DB.selectFrom("orphan_score") + .select(SELECT_ORPHAN_SCORE) + .where("orphan_score.orphan_id", "=", orphanID) + .where("orphan_score.user_id", "=", userID) + .executeTakeFirst(); + + if (!row) { + return null; + } + + const msg = row.error_message.trim(); + return { + orphanID: row.orphan_id, + importType: row.import_type, + gameGroup: row.game_group, + timeInserted: new Date(row.time_inserted).getTime(), + message: msg.length > 0 ? msg : null, + data: row.data, + context: row.context, + }; +} + /** Deletes one orphan_score row if it belongs to the given user. Returns whether a row was removed. */ -export async function deleteOrphanScoreForUser(orphanID: string, userID: integer): Promise { +export async function deleteOrphanScoreForUser( + orphanID: string, + userID: integer, +): Promise { const result = await DB.deleteFrom("orphan_score") .where("orphan_score.orphan_id", "=", orphanID) .where("orphan_score.user_id", "=", userID) @@ -117,12 +158,12 @@ export async function deleteOrphanScoreForUser(orphanID: string, userID: integer * @param afterRowID — `row_id` of the last item from the previous page (omit on first page). */ export async function listOrphanScoresForUser(opts: { - userID: integer; - limit: number; afterRowID?: string; -}): Promise<{ orphans: OrphanScoreListItem[]; hasMore: boolean }> { + limit: number; + userID: integer; +}): Promise<{ hasMore: boolean; orphans: OrphanScoreListItem[] }> { const cap = Math.min(Math.max(opts.limit, 1), 100); - let anchor: { time_inserted: string; row_id: string } | undefined; + let anchor: { row_id: string; time_inserted: string } | undefined; if (opts.afterRowID !== undefined && opts.afterRowID.length > 0) { anchor = await DB.selectFrom("orphan_score") diff --git a/typescript/server/src/lib/score-import/framework/pb/rederive-scores.recalc.test.ts b/typescript/server/src/lib/score-import/framework/pb/rederive-scores.recalc.test.ts new file mode 100644 index 000000000..3e75aacf0 --- /dev/null +++ b/typescript/server/src/lib/score-import/framework/pb/rederive-scores.recalc.test.ts @@ -0,0 +1,501 @@ +import { ComputeChartStabilityChecksum } from "#game-implementations/utils/derivation-checksum"; +import { + type ScoreDocumentJoinRow, + SELECT_SCORE_DOCUMENT, + ToScoreDocument, +} from "#lib/db-formats/score"; +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; +import { + drainGameProfileDirty, + drainPbDirty, + drainScoreRederive, + drainSessionDirty, + drainStatsQueuesInOrder, +} from "#lib/jobs/drain-dirty-queues"; +import { log } from "#lib/log/log"; +import { CreateSessionCalcData } from "#lib/score-import/framework/calculated-data/session"; +import { rederiveScoresForChart } from "#lib/score-import/framework/pb/rederive-scores"; +import { scoreVisibleSql } from "#lib/score-import/framework/pg/score-visibility"; +import { mongoScoreDataToPg, pgScoreDataToAPI } from "#lib/v3/migration-tools"; +import DB from "#services/pg/db"; +import { seedUser } from "#test-utils/pg-fixtures"; +import { Testing511Song, Testing511SPA } from "#test-utils/test-data"; +import { type ChartDocument, type PgScoreData, type ScoreData } from "tachi-common"; +import { describe, expect, it } from "vitest"; + +let recalcSeedCounter = 0; + +function buildIidxSpChartDoc( + chartId: string, + songId: string, + levelNum: number, + notecount: number, +): ChartDocument<"iidx-sp"> { + return { + ...Testing511SPA, + chartID: chartId, + song: { ...Testing511Song, id: songId }, + levelNum, + level: String(levelNum), + data: { + ...Testing511SPA.data, + notecount, + }, + }; +} + +async function seedIidxSpGameProfile(userId: number) { + await DB.insertInto("game_profile") + .values({ + user_id: userId, + game: "iidx-sp", + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...newGameProfilePreferenceColumns("iidx-sp"), + }) + .execute(); +} + +async function insertSongAndChart(chartDoc: ChartDocument<"iidx-sp">) { + const songId = chartDoc.song.id; + const chartId = chartDoc.chartID; + const n = ++recalcSeedCounter; + + await DB.insertInto("song") + .values({ + id: songId, + legacy_id: 91_000 + n, + game_group: "iidx", + title: chartDoc.song.title, + artist: chartDoc.song.artist, + search_terms: chartDoc.song.searchTerms, + alt_titles: chartDoc.song.altTitles, + data: chartDoc.song.data as object, + fts_document: "", + }) + .execute(); + + await DB.insertInto("chart") + .values({ + id: chartId, + legacy_id: `legacy_recalc_${chartId}`, + game: "iidx-sp", + song_id: songId, + difficulty: chartDoc.difficulty, + level: chartDoc.level, + level_num: chartDoc.levelNum, + is_primary: chartDoc.isPrimary, + versions: chartDoc.versions, + data: chartDoc.data as object, + derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", chartDoc), + }) + .execute(); + + return { chartId, songId }; +} + +async function insertIidxScore(opts: { + chartId: string; + scoreData: ScoreData<"iidx-sp">; + sessionId?: string | null; + userId: number; +}) { + const now = new Date().toISOString(); + const { data, derived, judgements } = mongoScoreDataToPg("iidx-sp", opts.scoreData); + const scoreId = `sc-recalc-${opts.chartId}`; + + await DB.insertInto("score") + .values({ + id: scoreId, + user_id: opts.userId, + chart_id: opts.chartId, + game: "iidx-sp", + session_id: opts.sessionId ?? null, + import_id: null, + data: JSON.stringify(data), + derived_data: JSON.stringify(derived), + judgements: JSON.stringify(judgements), + calculated_data: JSON.stringify({}), + meta: JSON.stringify({}), + time_achieved: now, + time_added: now, + highlight: false, + comment: null, + }) + .execute(); + + return scoreId; +} + +function parseJsonb(v: unknown): T { + if (typeof v === "string") { + return JSON.parse(v) as T; + } + + return v as T; +} + +async function loadScoresForSession(sessionId: string) { + const scoreRows = await DB.selectFrom("score") + .innerJoin("chart", "chart.id", "score.chart_id") + .innerJoin("song", "song.id", "chart.song_id") + .leftJoin("import", "import.id", "score.import_id") + .select(SELECT_SCORE_DOCUMENT) + .where("score.session_id", "=", sessionId) + .where(scoreVisibleSql()) + .execute(); + + return scoreRows.map((r) => ToScoreDocument(r as ScoreDocumentJoinRow)); +} + +async function loadScorePayload(chartId: string) { + const row = await DB.selectFrom("score") + .select(["score.data", "score.derived_data", "score.judgements", "score.calculated_data"]) + .where("score.chart_id", "=", chartId) + .executeTakeFirstOrThrow(); + + return { + calculatedData: parseJsonb<{ BPI: number | null; ktLampRating: number }>( + row.calculated_data, + ), + scoreData: pgScoreDataToAPI("iidx-sp", { + data: parseJsonb(row.data), + derived: parseJsonb(row.derived_data), + judgements: parseJsonb(row.judgements), + } as PgScoreData<"iidx-sp">), + }; +} + +describe("rederiveScoresForChart / chart checksum recalc (Postgres)", () => { + it("enqueues score_rederive when chart derivation_checksum changes", async () => { + const { id: userId } = await seedUser(); + await seedIidxSpGameProfile(userId); + + const chartId = `C_RECALC_TRIG_${++recalcSeedCounter}`; + const songId = `S_RECALC_TRIG_${recalcSeedCounter}`; + const docV1 = buildIidxSpChartDoc(chartId, songId, 10, 786); + await insertSongAndChart(docV1); + + const docV2 = buildIidxSpChartDoc(chartId, songId, 11, 786); + const checksum2 = ComputeChartStabilityChecksum("iidx-sp", docV2); + + await DB.updateTable("chart") + .set({ + level: docV2.level, + level_num: docV2.levelNum, + derivation_checksum: checksum2, + }) + .where("chart.id", "=", chartId) + .execute(); + + const queued = await DB.selectFrom("score_rederive") + .select(["score_rederive.chart_id"]) + .where("score_rederive.chart_id", "=", chartId) + .executeTakeFirst(); + + expect(queued?.chart_id).toBe(chartId); + }); + + it("does not enqueue score_rederive when derivation_checksum is unchanged", async () => { + const chartId = `C_RECALC_NOTRIG_${++recalcSeedCounter}`; + const songId = `S_RECALC_NOTRIG_${recalcSeedCounter}`; + const doc = buildIidxSpChartDoc(chartId, songId, 10, 786); + await insertSongAndChart(doc); + + await DB.updateTable("chart") + .set({ legacy_id: `legacy_recalc_updated_${chartId}` }) + .where("chart.id", "=", chartId) + .execute(); + + const queued = await DB.selectFrom("score_rederive") + .select(["score_rederive.chart_id"]) + .where("score_rederive.chart_id", "=", chartId) + .executeTakeFirst(); + + expect(queued).toBeUndefined(); + }); + + it("updates calculated_data.ktLampRating when chart level_num changes (CLEAR lamp)", async () => { + const { id: userId } = await seedUser(); + await seedIidxSpGameProfile(userId); + + const chartId = `C_RECALC_LVL_${++recalcSeedCounter}`; + const songId = `S_RECALC_LVL_${recalcSeedCounter}`; + const doc10 = buildIidxSpChartDoc(chartId, songId, 10, 786); + await insertSongAndChart(doc10); + + await insertIidxScore({ + chartId, + userId, + scoreData: { + lamp: "CLEAR", + score: 1000, + grade: "AAA", + percent: 90, + optional: {}, + judgements: { pgreat: 500, great: 0 }, + } as ScoreData<"iidx-sp">, + }); + + await rederiveScoresForChart(chartId, log); + let { calculatedData } = await loadScorePayload(chartId); + expect(calculatedData.ktLampRating).toBe(10); + + const doc12 = buildIidxSpChartDoc(chartId, songId, 12, 786); + await DB.updateTable("chart") + .set({ + level: doc12.level, + level_num: doc12.levelNum, + derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc12), + }) + .where("chart.id", "=", chartId) + .execute(); + + await rederiveScoresForChart(chartId, log); + ({ calculatedData } = await loadScorePayload(chartId)); + expect(calculatedData.ktLampRating).toBe(12); + }); + + it("updates derived percent and grade when chart notecount changes (fixed EX score)", async () => { + const { id: userId } = await seedUser(); + await seedIidxSpGameProfile(userId); + + const chartId = `C_RECALC_NC_${++recalcSeedCounter}`; + const songId = `S_RECALC_NC_${recalcSeedCounter}`; + const doc1k = buildIidxSpChartDoc(chartId, songId, 10, 1000); + await insertSongAndChart(doc1k); + + await insertIidxScore({ + chartId, + userId, + scoreData: { + lamp: "CLEAR", + score: 1000, + grade: "AAA", + percent: 50, + optional: {}, + judgements: { pgreat: 500, great: 0 }, + } as ScoreData<"iidx-sp">, + }); + + await rederiveScoresForChart(chartId, log); + let { scoreData } = await loadScorePayload(chartId); + expect(scoreData.percent).toBeCloseTo(50, 5); + expect(scoreData.grade).toBe("C"); + + const doc500 = buildIidxSpChartDoc(chartId, songId, 10, 500); + await DB.updateTable("chart") + .set({ + data: doc500.data as object, + derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc500), + }) + .where("chart.id", "=", chartId) + .execute(); + + await rederiveScoresForChart(chartId, log); + ({ scoreData } = await loadScorePayload(chartId)); + expect(scoreData.percent).toBeCloseTo(100, 5); + expect(scoreData.grade).toBe("MAX"); + }); + + it("refreshes pb.calculated_data after rederive and drainPbDirty", async () => { + const { id: userId } = await seedUser(); + await seedIidxSpGameProfile(userId); + + const chartId = `C_RECALC_PB_${++recalcSeedCounter}`; + const songId = `S_RECALC_PB_${recalcSeedCounter}`; + const doc10 = buildIidxSpChartDoc(chartId, songId, 10, 786); + await insertSongAndChart(doc10); + + await insertIidxScore({ + chartId, + userId, + scoreData: { + lamp: "CLEAR", + score: 1000, + grade: "AAA", + percent: 50, + optional: {}, + judgements: { pgreat: 500, great: 0 }, + } as ScoreData<"iidx-sp">, + }); + + await rederiveScoresForChart(chartId, log); + await drainPbDirty(); + + const doc14 = buildIidxSpChartDoc(chartId, songId, 14, 786); + await DB.updateTable("chart") + .set({ + level: doc14.level, + level_num: doc14.levelNum, + derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc14), + }) + .where("chart.id", "=", chartId) + .execute(); + + await rederiveScoresForChart(chartId, log); + await drainPbDirty(); + + const scoreRow = await loadScorePayload(chartId); + const pbRow = await DB.selectFrom("pb") + .select(["pb.calculated_data"]) + .where("pb.user_id", "=", userId) + .where("pb.chart_id", "=", chartId) + .where("pb.lens", "is", null) + .executeTakeFirstOrThrow(); + + const pbCalc = parseJsonb<{ ktLampRating: number }>(pbRow.calculated_data); + expect(scoreRow.calculatedData.ktLampRating).toBe(14); + expect(pbCalc.ktLampRating).toBe(14); + }); + + it("drainScoreRederive runs rederive and clears score_rederive for the chart", async () => { + const { id: userId } = await seedUser(); + await seedIidxSpGameProfile(userId); + + const chartId = `C_RECALC_DRAIN_${++recalcSeedCounter}`; + const songId = `S_RECALC_DRAIN_${recalcSeedCounter}`; + const docA = buildIidxSpChartDoc(chartId, songId, 9, 786); + await insertSongAndChart(docA); + + await insertIidxScore({ + chartId, + userId, + scoreData: { + lamp: "CLEAR", + score: 1000, + grade: "AAA", + percent: 50, + optional: {}, + judgements: { pgreat: 500, great: 0 }, + } as ScoreData<"iidx-sp">, + }); + + const docB = buildIidxSpChartDoc(chartId, songId, 10, 786); + await DB.updateTable("chart") + .set({ + level: docB.level, + level_num: docB.levelNum, + derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", docB), + }) + .where("chart.id", "=", chartId) + .execute(); + + const n = await drainScoreRederive(); + expect(n).toBeGreaterThanOrEqual(1); + + const stillQueued = await DB.selectFrom("score_rederive") + .select(["score_rederive.chart_id"]) + .where("score_rederive.chart_id", "=", chartId) + .executeTakeFirst(); + + expect(stillQueued).toBeUndefined(); + + const { calculatedData } = await loadScorePayload(chartId); + expect(calculatedData.ktLampRating).toBe(10); + }); + + it("updates session.calculated_data and game_profile.ratings after rederive + pb + session + profile drains", async () => { + const { id: userId } = await seedUser(); + await seedIidxSpGameProfile(userId); + + const chartId = `C_RECALC_STATS_${++recalcSeedCounter}`; + const songId = `S_RECALC_STATS_${recalcSeedCounter}`; + const sessionId = `sess-recalc-stats-${chartId}`; + const doc10 = buildIidxSpChartDoc(chartId, songId, 10, 786); + await insertSongAndChart(doc10); + + const now = new Date().toISOString(); + + await DB.insertInto("session") + .values({ + id: sessionId, + user_id: userId, + game: "iidx-sp", + name: "Recalc stats test", + description: null, + time_inserted: now, + time_started: now, + time_ended: now, + calculated_data: JSON.stringify({}), + highlight: false, + }) + .execute(); + + await insertIidxScore({ + chartId, + userId, + sessionId, + scoreData: { + lamp: "CLEAR", + score: 1000, + grade: "AAA", + percent: 50, + optional: {}, + judgements: { pgreat: 500, great: 0 }, + } as ScoreData<"iidx-sp">, + }); + + await rederiveScoresForChart(chartId, log); + await drainPbDirty(); + await drainSessionDirty(); + await drainGameProfileDirty(); + + let sessRow = await DB.selectFrom("session") + .select(["session.calculated_data"]) + .where("session.id", "=", sessionId) + .executeTakeFirstOrThrow(); + + let scoreDocs = await loadScoresForSession(sessionId); + let expectedSession = CreateSessionCalcData("iidx-sp", scoreDocs); + expect(parseJsonb(sessRow.calculated_data)).toEqual(expectedSession); + + let gpRow = await DB.selectFrom("game_profile") + .select(["game_profile.ratings"]) + .where("game_profile.user_id", "=", userId) + .where("game_profile.game", "=", "iidx-sp") + .executeTakeFirstOrThrow(); + + let ratings = parseJsonb<{ ktLampRating: number | null }>(gpRow.ratings); + // ProfileAvgBestN(..., 20, returnMean): one PB => ktLampRating / 20. + expect(ratings.ktLampRating).toBeCloseTo(0.5, 5); + + const doc14 = buildIidxSpChartDoc(chartId, songId, 14, 786); + + await DB.updateTable("chart") + .set({ + level: doc14.level, + level_num: doc14.levelNum, + derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc14), + }) + .where("chart.id", "=", chartId) + .execute(); + + await rederiveScoresForChart(chartId, log); + await drainPbDirty(); + await drainSessionDirty(); + await drainGameProfileDirty(); + + sessRow = await DB.selectFrom("session") + .select(["session.calculated_data"]) + .where("session.id", "=", sessionId) + .executeTakeFirstOrThrow(); + + scoreDocs = await loadScoresForSession(sessionId); + expectedSession = CreateSessionCalcData("iidx-sp", scoreDocs); + expect(parseJsonb(sessRow.calculated_data)).toEqual(expectedSession); + + gpRow = await DB.selectFrom("game_profile") + .select(["game_profile.ratings"]) + .where("game_profile.user_id", "=", userId) + .where("game_profile.game", "=", "iidx-sp") + .executeTakeFirstOrThrow(); + + ratings = parseJsonb<{ ktLampRating: number | null }>(gpRow.ratings); + expect(ratings.ktLampRating).toBeCloseTo(0.7, 5); + }); + + it("drainStatsQueuesInOrder completes with empty queues", async () => { + await drainStatsQueuesInOrder(); + }); +}); diff --git a/typescript/server/src/lib/score-import/framework/pb/rederive-scores.ts b/typescript/server/src/lib/score-import/framework/pb/rederive-scores.ts index 6c801f859..f49852590 100644 --- a/typescript/server/src/lib/score-import/framework/pb/rederive-scores.ts +++ b/typescript/server/src/lib/score-import/framework/pb/rederive-scores.ts @@ -10,7 +10,7 @@ import { import { mongoScoreDataToPg } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; -const BATCH_SIZE = 500; +const BATCH_SIZE = 5000; /** * Re-derive `derived_data` and `calculated_data` for every score on the @@ -92,7 +92,9 @@ export async function rederiveScoresForChart(chartId: string, log: KtLogger): Pr } } - log.info({ chartId, totalUpdated }, `Re-derived ${totalUpdated} score(s) for chart.`); + if (totalUpdated > 0) { + log.debug({ chartId, totalUpdated }, `Re-derived ${totalUpdated} score(s) for chart.`); + } return totalUpdated; } diff --git a/typescript/server/src/lib/score-import/framework/score-import.ts b/typescript/server/src/lib/score-import/framework/score-import.ts index 6162dd20c..f4fc90a16 100644 --- a/typescript/server/src/lib/score-import/framework/score-import.ts +++ b/typescript/server/src/lib/score-import/framework/score-import.ts @@ -1,139 +1,47 @@ -import type { ImportDocument, ImportTypes, integer } from "tachi-common"; +import type { ImportDocument, ImportTypes } from "tachi-common"; -import { JOB_RETRY_COUNT } from "#lib/constants/tachi"; -import { log } from "#lib/log/log"; +import { ACTION_ScoreImport } from "#actions/score-import"; +import { LoadImportDocumentById } from "#lib/db-formats/import-document"; import { ServerConfig } from "#lib/setup/config"; -import { Sleep } from "#utils/misc"; +import { GetUserWithIDGuaranteed } from "#utils/user"; -import type { ScoreImportJobData, ScoreImportWorkerReturns } from "../worker/types"; +import type { ScoreImportJobData } from "../worker/types"; -import ScoreImportQueue, { ScoreImportQueueEvents } from "../worker/queue"; -/* eslint-disable no-await-in-loop */ -import { GetInputParser } from "./common/get-input-parser"; -import { UnsetOngoingImportLock } from "./import-locks/lock"; import ScoreImportFatalError from "./score-importing/score-import-error"; -import ScoreImportMain from "./score-importing/score-import-main"; -import { - EndTrackingImport, - MarkImportAsFailed, - StartTrackingImport, -} from "./status-tracking/import-status-tracking"; /** - * Makes a score import given ScoreImportJobData. - * If USE_EXTERNAL_SCORE_IMPORT_WORKER is set, then this will instead - * place this on the score import queue, and the worker will process it. + * Makes a score import given ScoreImportJobData (same process as the API). * - * Otherwise, it will just perform score importing on the same process. - * @returns An import document if awaited, however, you should not - * await this if you don't need the import document! Import Documents - * may take multiple minutes to generate for large imports. If you control - * the client, make it poll /api/v1/ongoing-imports/:importID. + * When `USE_EXTERNAL_SCORE_IMPORT_WORKER` is true, HTTP routes call + * {@link EnqueueScoreImportJob} instead — this function is only used for the + * inline (non-queued) path. */ export async function MakeScoreImport( jobData: ScoreImportJobData, ): Promise { - await StartTrackingImport(jobData); - - try { - const ImportDocument = await MakeScoreImportInner(jobData); - - await EndTrackingImport(jobData.importID); - - return ImportDocument; - } catch (e) { - const err = e as Error | ScoreImportFatalError; - - await MarkImportAsFailed(jobData.importID, err); - - throw err; - } -} - -/** - * Inner function that actually makes the score import. This is intended to be wrapped - * by the import-tracking code, as it's useful for errors. - */ -async function MakeScoreImportInner( - jobData: ScoreImportJobData, -): Promise { - if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER && process.env.IS_JOB === undefined) { - let timesAttempted = 1; - - // There's no chance this thing goes on 7 times. - // if it does, this import has been trying for the past 6 hours or so. - while (timesAttempted <= JOB_RETRY_COUNT) { - const job = await ScoreImportQueue.add( - `Import ${jobData.importID}${timesAttempted > 0 ? ` (TRY${timesAttempted})` : ""}`, - jobData, - { - jobId: `${jobData.importID}:TRY${timesAttempted}`, - }, - ); - - const data = (await job.waitUntilFinished( - ScoreImportQueueEvents, - )) as ScoreImportWorkerReturns; - - if (data.success) { - await EndTrackingImport(jobData.importID); - return data.ImportDocument; - } else if (data.statusCode !== 409) { - throw new ScoreImportFatalError(data.statusCode, data.description); - } - - const backoff = ExponentialBackoff(timesAttempted - 1); - - log.info( - `User ${jobData.userID} already had an import ongoing. (${ - jobData.importID - }) Backing off for ${(backoff / 1_000).toFixed(2)} seconds.`, - ); - - // If we get here, we were 409'd and the user already has an ongoing - // import. - // In the interest of not just throwing scores away, we'll back off a bit - // and then restart the job. - await Sleep(backoff); - - timesAttempted++; - } - - log.error( - jobData, - `User ${jobData.userID} didn't get an import through in around 6 hours. Has their lock gotten stuck?`, - ); - - await UnsetOngoingImportLock(jobData.userID); - - log.error(`Forcing off ${jobData.userID}'s import lock. Sketchy.`); - + if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { throw new ScoreImportFatalError( - 409, - "Couldn't get an import through in the past 6 hours, at all.", - ); - } else { - const InputParser = GetInputParser(jobData); - - return ScoreImportMain( - jobData.userID, - jobData.userIntent, - jobData.importType, - InputParser, - jobData.importID, + 500, + "MakeScoreImport may not be used when an external score-import worker is enabled; use EnqueueScoreImportJob instead.", ); } -} -function ExponentialBackoff(exponent: integer) { - // n | backoff - // 0 | 4 Seconds - // 1 | 16 Seconds - // 2 | 64 Seconds - // 3 | 256 Seconds - // 4 | 1024 Seconds - // ... - // ends at 7, which is around 4 hours. - - return Math.random() * 1000 * 4 ** (exponent + 1); + const user = await GetUserWithIDGuaranteed(jobData.userID); + await ACTION_ScoreImport( + { ip: null, acct: { id: user.id, username: user.username } }, + { + importID: jobData.importID, + importType: jobData.importType, + userIntent: jobData.userIntent, + "!parserArguments": jobData.parserArguments as Array, + }, + ); + const importDocument = await LoadImportDocumentById(jobData.importID); + if (!importDocument) { + throw new ScoreImportFatalError( + 500, + "Import completed but the import document could not be loaded.", + ); + } + return importDocument; } diff --git a/typescript/server/src/lib/score-import/framework/sessions/sessions.ts b/typescript/server/src/lib/score-import/framework/sessions/sessions.ts index 1ac26a25c..edbdba4b7 100644 --- a/typescript/server/src/lib/score-import/framework/sessions/sessions.ts +++ b/typescript/server/src/lib/score-import/framework/sessions/sessions.ts @@ -87,6 +87,9 @@ function ScoreToSessionScoreInfo( export async function GetSessionScoreInfo( session: SessionDocument, ): Promise> { + // TODO: Hard to implement efficiently + // need to get the PB for this chart BEFORE the + // given time T const scores = await DB.selectFrom("score") .innerJoin("chart", "chart.id", "score.chart_id") .innerJoin("song", "song.id", "chart.song_id") diff --git a/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.test.ts b/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.test.ts index d50b69776..5ac12e86f 100644 --- a/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.test.ts +++ b/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.test.ts @@ -112,7 +112,7 @@ describe("UpdateUsersGamePlaytypeStats (ported from update-ugpt-stats.oldtest.ts await insertPbFromTesting({ userId: 1, pb: TestingIIDXSPScorePB }); }); - it("creates game_profile and game_settings when the user has none", async () => { + it("creates game_profile with preference defaults when the user has none", async () => { const res = await UpdateUsersGamePlaytypeStats("iidx-sp", 1, null, log); expect(res).toEqual([]); @@ -124,14 +124,15 @@ describe("UpdateUsersGamePlaytypeStats (ported from update-ugpt-stats.oldtest.ts .executeTakeFirstOrThrow(); const ratings = typeof gp.ratings === "string" ? JSON.parse(gp.ratings) : gp.ratings; - expect(ratings).toMatchObject({ ktLampRating: expect.any(Number) }); + expect(ratings).toMatchObject({ + ktLampRating: expect.any(Number), + ktLampRatingNC: expect.any(Number), + ktLampRatingHC: expect.any(Number), + ktLampRatingEXHC: expect.any(Number), + }); - const settings = await DB.selectFrom("game_settings") - .selectAll() - .where("user_id", "=", 1) - .where("game", "=", "iidx-sp") - .executeTakeFirst(); - expect(settings).toBeDefined(); + const dataRaw = gp.data; + expect(dataRaw).toBeDefined(); }); it("updates ratings when game_profile already exists", async () => { @@ -172,8 +173,11 @@ describe("UpdateUsersGamePlaytypeStats (ported from update-ugpt-stats.oldtest.ts pb: deepmerge(TestingIIDXSPScorePB, { chartID: fakeChart, calculatedData: { - ktLampRating: e, BPI: 10.1, + ktLampRating: e, + ktLampRatingNC: e, + ktLampRatingHC: e, + ktLampRatingEXHC: e, }, }), }); diff --git a/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.ts b/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.ts index 585425ac3..967e0f35d 100644 --- a/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.ts +++ b/typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.ts @@ -1,6 +1,6 @@ import type { KtLogger } from "#lib/log/log"; -import { CreateGameSettings } from "#lib/game-settings/create-game-settings"; +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import DB from "#services/pg/db"; import { loadUserGameStats } from "#utils/class"; import { type ClassDelta, type integer, type V3Game } from "tachi-common"; @@ -80,9 +80,9 @@ export async function UpdateUsersGamePlaytypeStats( game, ratings: JSON.stringify(ratings), classes: JSON.stringify(classes), + ...newGameProfilePreferenceColumns(game), }) .execute(); - await CreateGameSettings(userID, game); } return deltas; diff --git a/typescript/server/src/lib/score-import/import-types/api/myt-wacca/class-handler.ts b/typescript/server/src/lib/score-import/import-types/api/myt-wacca/class-handler.ts index f97912c2e..5fd119bdb 100644 --- a/typescript/server/src/lib/score-import/import-types/api/myt-wacca/class-handler.ts +++ b/typescript/server/src/lib/score-import/import-types/api/myt-wacca/class-handler.ts @@ -15,7 +15,7 @@ export default async function CreateMytWACCAClassHandler( const req = create(DataRequestSchema, { apiId: titleApiId }); const data = await client.getData(req); - return (_gptString, _userID, _ratings, _logger) => { + return (_game, _userID, _ratings, _logger) => { // Currently (May 2025) Reverse and Plus are supported on Myt. // We look for both Reverse and PLUS version data, PLUS being prioritized if exists. // If / when custom dans are added, this will need to change. diff --git a/typescript/server/src/lib/score-import/worker/enqueue-pg.ts b/typescript/server/src/lib/score-import/worker/enqueue-pg.ts new file mode 100644 index 000000000..fc14dd2a1 --- /dev/null +++ b/typescript/server/src/lib/score-import/worker/enqueue-pg.ts @@ -0,0 +1,23 @@ +import type { ScoreImportJobData } from "#lib/score-import/worker/types"; +import type { ImportTypes } from "tachi-common"; + +import { JOB_KIND_SCORE_IMPORT } from "#lib/jobs/job-queue/constants"; +import { EnqueueJob } from "#lib/jobs/job-queue/queue-ops"; +import { StartTrackingImport } from "#lib/score-import/framework/status-tracking/import-status-tracking"; +import { jsonSerializeWithBuffers } from "#lib/score-import/worker/score-import-job-processor"; + +/** + * Enqueue a score import on the Postgres `job_queue` and begin tracking. Returns `job_queue.row_id`. + */ +export async function EnqueueScoreImportJob( + jobData: ScoreImportJobData, +): Promise { + await StartTrackingImport(jobData); + const payload = jsonSerializeWithBuffers(jobData); + return EnqueueJob({ + scheduled_for: new Date().toISOString(), + scope: `import:${jobData.importID}`, + job_kind: JOB_KIND_SCORE_IMPORT, + payload: JSON.parse(payload) as unknown, + }); +} diff --git a/typescript/server/src/lib/score-import/worker/queue.ts b/typescript/server/src/lib/score-import/worker/queue.ts index 2a91a0a5a..093c39e36 100644 --- a/typescript/server/src/lib/score-import/worker/queue.ts +++ b/typescript/server/src/lib/score-import/worker/queue.ts @@ -1,21 +1,11 @@ -import { Env, TachiConfig } from "#lib/setup/config"; -import { Queue, QueueEvents } from "bullmq"; +/** + * BullMQ score import queue (legacy). Replaced by Postgres `job_queue`. + * The API process no longer instantiates a Redis queue; keep a no-op close for shutdown paths. + */ +export default null; -const ScoreImportQueue = new Queue(`${TachiConfig.NAME} Score Import Queue`, { - connection: { host: Env.REDIS_URL, port: 6379 }, - defaultJobOptions: { - removeOnComplete: true, - removeOnFail: 10, // keep the last 10 failed jobs, but start pruning beyond that. - }, -}); +export const ScoreImportQueueEvents = null; -export default ScoreImportQueue; - -export const ScoreImportQueueEvents = new QueueEvents(ScoreImportQueue.name, { - connection: { host: Env.REDIS_URL, port: 6379 }, -}); - -export async function CloseScoreImportQueue() { - await ScoreImportQueueEvents.close(); - return ScoreImportQueue.close(); +export async function CloseScoreImportQueue(): Promise { + // no-op (Postgres `job_queue` is used for score imports) } diff --git a/typescript/server/src/lib/score-import/worker/score-import-job-processor.ts b/typescript/server/src/lib/score-import/worker/score-import-job-processor.ts new file mode 100644 index 000000000..fb04de9dc --- /dev/null +++ b/typescript/server/src/lib/score-import/worker/score-import-job-processor.ts @@ -0,0 +1,86 @@ +import type { ScoreImportJobData, ScoreImportWorkerReturns } from "#lib/score-import/worker/types"; +import type { ImportTypes } from "tachi-common"; + +import { ACTION_ScoreImport } from "#actions/score-import"; +import { LoadImportDocumentById } from "#lib/db-formats/import-document"; +import { log } from "#lib/log/log"; +import { GetUserWithID } from "#utils/user"; +import { ExpectedErr } from "bliss"; + +export function jsonSerializeWithBuffers(data: T): string { + return JSON.stringify(data, (_k, v) => { + if (Buffer.isBuffer(v)) { + return { type: "Buffer" as const, data: Array.from(v as ArrayLike & Buffer) }; + } + return v; + }); +} + +function decodeParserArguments(raw: Array): Array { + const out: Array = []; + + for (const arg of raw) { + if ( + arg && + typeof arg === "object" && + (arg as { buffer?: { data?: number[]; type?: string } }).buffer?.type === "Buffer" + ) { + const a = arg as { buffer: { data: number[] } }; + out.push({ ...a, buffer: Buffer.from(a.buffer.data) }); + } else { + out.push(arg); + } + } + return out; +} + +/** + * `payload` is JSONB from `job_queue` (object) or a JSON string. + */ +export async function processScoreImportJobFromPayload( + payload: unknown, +): Promise { + const parsed: unknown = + typeof payload === "string" ? (JSON.parse(payload) as unknown) : payload; + const data = parsed as ScoreImportJobData; + data.parserArguments = decodeParserArguments( + data.parserArguments as Array, + ) as ScoreImportJobData["parserArguments"]; + + const user = await GetUserWithID(data.userID); + if (!user) { + log.error(`Couldn't find user with ID ${data.userID} for import ${data.importID}.`); + throw new Error(`Couldn't find user with ID ${data.userID} for import ${data.importID}.`); + } + + log.debug(`Starting import ${data.importID}.`); + try { + await ACTION_ScoreImport( + { ip: null, acct: { id: user.id, username: user.username } }, + { + importID: data.importID, + importType: data.importType, + userIntent: data.userIntent, + "!parserArguments": data.parserArguments as Array, + skipStartTracking: true, + }, + ); + const importDocument = await LoadImportDocumentById(data.importID); + if (!importDocument) { + throw new Error( + `Import ${data.importID} completed but the import document could not be loaded.`, + ); + } + return { success: true, ImportDocument: importDocument }; + } catch (e) { + if (ExpectedErr.is(e)) { + log.info( + { err: e, importID: data.importID }, + `Import ${data.importID} hit ExpectedErr (user fault): ${e.reason}`, + ); + return { success: false, statusCode: e.code, description: e.reason }; + } + log.error(e, `Import ${data.importID} failed unexpectedly.`); + throw e; + } +} diff --git a/typescript/server/src/lib/score-import/worker/worker.ts b/typescript/server/src/lib/score-import/worker/worker.ts deleted file mode 100644 index 9d833b6ce..000000000 --- a/typescript/server/src/lib/score-import/worker/worker.ts +++ /dev/null @@ -1,139 +0,0 @@ -import type { ImportTypes } from "tachi-common"; - -import { HandleSIGTERMGracefully } from "#lib/handlers/sigterm"; -import { log } from "#lib/log/log"; -import { Env, ServerConfig } from "#lib/setup/config"; -import { GetUserWithID } from "#utils/user"; -import { Worker } from "bullmq"; -import { EventEmitter } from "events"; - -import type ScoreImportFatalError from "../framework/score-importing/score-import-error"; -import type { ScoreImportJob, ScoreImportJobData } from "./types"; - -import { GetInputParser } from "../framework/common/get-input-parser"; -import ScoreImportMain from "../framework/score-importing/score-import-main"; -import ScoreImportQueue from "./queue"; - -EventEmitter.defaultMaxListeners = 20; - -// For scaling performance, running score-importing in a separate worker is preferable -// as that way, other API calls don't get halted by particularly expensive imports on -// all cores. Parallelism can only get us so far in the same process. - -// You don't have to run this. If it is being ran, you need to set USE_EXTERNAL_SCORE_IMPORT_WORKER -// in env (TACHI_USE_EXTERNAL_SCORE_IMPORT_WORKER). That will ensure all score import jobs are thrown at redis and eventually -// end up here. - -// If you don't, score importing will happen on the same thread as your router. That's probably -// fine for lower throughputs, but hey. We're aiming a bit higher. - -// Explicitly set this before importing anything! -process.env.IS_SCORE_WORKER_SERVER = "true"; - -// Exit if we're not called with node. Think of this like if __name__ != "__main__" in python. -if (require.main !== module) { - log.fatal( - "The Score Import Worker was imported, instead of ran directly with node. This is a fatal error. Exiting.", - ); - process.exit(1); -} - -/** - * When a job is fired, this code will actually process the given data - * and import it into the codebase. - */ -export const worker = new Worker( - ScoreImportQueue.name, - async (job: ScoreImportJob) => { - const user = await GetUserWithID(job.data.userID); - - if (!user) { - log.error( - `Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).`, - ); - throw new Error( - `Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).`, - ); - } - - // Here's a hack. You cant pass buffers to bull workers, as content - // **must** be JSON serialisable. As such, all of our buffers get - // turned into nonsense objects. We need to "deJSONify" these buffers - // so lets do that now. - - const processedArgs: Array = []; - - // eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call - - for (const arg of job.data.parserArguments as Array) { - if (arg?.buffer?.type === "Buffer") { - processedArgs.push({ ...arg, buffer: Buffer.from(arg.buffer.data) }); - } else { - processedArgs.push(arg); - } - } - - job.data.parserArguments = processedArgs as ScoreImportJobData["parserArguments"]; - - log.debug({ job }, `Received score import job ${job.id}`); - - const InputParser = GetInputParser(job.data); - - log.debug(`Starting import.`); - - void job.updateProgress({ - description: "Importing Scores.", - }); - - try { - const ImportDocument = await ScoreImportMain( - user.id, - job.data.userIntent, - job.data.importType, - InputParser, - job.data.importID, - undefined, - job, - ); - - log.debug(`Finished import.`); - - return { success: true, ImportDocument }; - } catch (e) { - const err = e as Error | ScoreImportFatalError; - - // originally, we did `err instanceof ScoreImportFatalError`, but something in our toolchain - // has suddenly made all `instanceof` calls faulty. I still - to this day - have absolutely - // no idea why or how this broke, but now `instanceof` is considered a footgun so, great. - if ("statusCode" in err) { - log.info( - { id: job.id, err, jobData: job.data }, - `Job hit ScoreImportFatalError (User Fault) with message: ${err.message}`, - ); - return { success: false, statusCode: err.statusCode, description: err.message }; - } - - throw err; - } - }, - { - concurrency: ServerConfig.EXTERNAL_SCORE_IMPORT_WORKER_CONCURRENCY ?? 10, - connection: { - port: 6379, - host: Env.REDIS_URL, - }, - }, -); - -worker.on("failed", (job, err) => { - // any errors that escalate this far are unexpected, as they haven't been caught by previous calls. - log.error(err, `Job ${job.id} failed unexpectedly with message: ${err.message}`); -}); - -worker.on("completed", (job, result) => { - log.debug(result, `Job ${job.id} finished successfully.`); -}); - -process.on("SIGTERM", () => { - void HandleSIGTERMGracefully(); -}); diff --git a/typescript/server/src/lib/setup/config.ts b/typescript/server/src/lib/setup/config.ts index 3c3e5ec57..50cc41f30 100644 --- a/typescript/server/src/lib/setup/config.ts +++ b/typescript/server/src/lib/setup/config.ts @@ -60,6 +60,8 @@ const configSchema = z.object({ USE_EXTERNAL_SCORE_IMPORT_WORKER: z.boolean().default(false), EXTERNAL_SCORE_IMPORT_WORKER_CONCURRENCY: z.number().int().positive().optional(), ALLOW_RUNNING_OFFLINE: z.boolean().optional(), + /** Dev/stress: when true, score import HTTP routes use an unlimited score-import rate limiter. */ + DISABLE_SCORE_IMPORT_RATE_LIMIT: z.boolean().default(false), ENABLE_METRICS: z.boolean().default(true), EMAIL_CONFIG: z.object({ FROM: z.string(), @@ -425,6 +427,8 @@ const configFromEnv: unknown = { 10, ), ALLOW_RUNNING_OFFLINE: parseBool("TACHI_ALLOW_RUNNING_OFFLINE"), + DISABLE_SCORE_IMPORT_RATE_LIMIT: + parseBool("TACHI_DISABLE_SCORE_IMPORT_RATE_LIMIT", false) ?? false, ENABLE_METRICS: parseBool("TACHI_ENABLE_METRICS", true) ?? true, EMAIL_CONFIG: emailCfg, USC_QUEUE_SIZE: parseIntEnv("TACHI_USC_QUEUE_SIZE", 3), diff --git a/typescript/server/src/lib/showcase/get-stats.test.ts b/typescript/server/src/lib/showcase/get-stats.test.ts index de5901e49..f157b4f05 100644 --- a/typescript/server/src/lib/showcase/get-stats.test.ts +++ b/typescript/server/src/lib/showcase/get-stats.test.ts @@ -1,4 +1,5 @@ import { seedUser } from "#actions/test-utils/api-tokens"; +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import { mongoScoreDataToPg } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; import { @@ -63,20 +64,6 @@ async function seedFixtures() { .values({ folder_id: TestingIIDXFolderSP10.folderID, chart_id: Testing511SPA.chartID }) .execute(); - await DB.insertInto("game_settings") - .values({ - user_id: 1, - game: "iidx-sp", - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), - }) - .execute(); - const stats = [ { mode: "folder" as const, @@ -90,11 +77,14 @@ async function seedFixtures() { }, ]; - await DB.insertInto("game_settings_showcase") + await DB.insertInto("game_profile") .values({ user_id: 1, game: "iidx-sp", - data: JSON.stringify(stats), + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...newGameProfilePreferenceColumns("iidx-sp"), + showcase: JSON.stringify(stats), }) .execute(); @@ -142,9 +132,8 @@ describe("EvaluateUsersStatsShowcase (ported from get-stats.oldtest.ts)", () => expect(res[1]?.stat).toMatchObject({ mode: "chart", chartID: Testing511SPA.chartID }); }); - it("throws when the user has no game_settings row", async () => { - await DB.deleteFrom("game_settings").where("user_id", "=", 1).execute(); - await DB.deleteFrom("game_settings_showcase").where("user_id", "=", 1).execute(); + it("throws when the user has no game_profile row", async () => { + await DB.deleteFrom("game_profile").where("game_profile.user_id", "=", 1).execute(); await expect(EvaluateUsersStatsShowcase(1, "iidx-sp")).rejects.toThrow(); }); diff --git a/typescript/server/src/lib/showcase/get-stats.ts b/typescript/server/src/lib/showcase/get-stats.ts index 886f537d2..defbf988d 100644 --- a/typescript/server/src/lib/showcase/get-stats.ts +++ b/typescript/server/src/lib/showcase/get-stats.ts @@ -20,11 +20,11 @@ export async function EvaluateUsersStatsShowcase( if (!settings) { log.error( - `User ${getSettingsID} has no game-settings, yet a call to EvaluateUsersStatsShowcase was made.`, + `User ${getSettingsID} has no game profile row, yet a call to EvaluateUsersStatsShowcase was made.`, ); throw new Error( - `User ${getSettingsID} has no game-settings, yet a call to EvaluateUsersStatsShowcase was made.`, + `User ${getSettingsID} has no game profile row, yet a call to EvaluateUsersStatsShowcase was made.`, ); } diff --git a/typescript/server/src/lib/v3/migration-tools.test.ts b/typescript/server/src/lib/v3/migration-tools.test.ts index d356481e3..8cd2828c2 100644 --- a/typescript/server/src/lib/v3/migration-tools.test.ts +++ b/typescript/server/src/lib/v3/migration-tools.test.ts @@ -1,7 +1,7 @@ import { type PgScoreData, type ScoreData, SDVX_GRADES, SDVX_LAMPS } from "tachi-common"; import { describe, expect, it } from "vitest"; -import { mongoScoreDataToPg, pgScoreDataToMongo } from "./migration-tools"; +import { mongoScoreDataToPg, pgScoreDataToAPI } from "./migration-tools"; const sdvxScoreData: ScoreData<"sdvx"> = { score: 9_876_543, @@ -48,7 +48,7 @@ const pgSdvxScoreData: PgScoreData<"sdvx"> = { describe("pgScoreDataToMongo", () => { it("converts nicely", () => { - const merged = pgScoreDataToMongo("sdvx", pgSdvxScoreData); + const merged = pgScoreDataToAPI("sdvx", pgSdvxScoreData); expect(merged).toStrictEqual({ enumIndexes: { @@ -76,7 +76,7 @@ describe("pgScoreDataToMongo", () => { it("roundtrips as expected (sdvx)", () => { const pgScoreData = mongoScoreDataToPg("sdvx", sdvxScoreData); - const merged = pgScoreDataToMongo("sdvx", pgScoreData); + const merged = pgScoreDataToAPI("sdvx", pgScoreData); expect(merged).toStrictEqual(sdvxScoreData); }); diff --git a/typescript/server/src/lib/v3/migration-tools.ts b/typescript/server/src/lib/v3/migration-tools.ts index f261ab210..ad0151951 100644 --- a/typescript/server/src/lib/v3/migration-tools.ts +++ b/typescript/server/src/lib/v3/migration-tools.ts @@ -99,7 +99,7 @@ export function mongoScoreDataToPg( * Reconstruct API {@link ScoreData} from Postgres `data` / `derived_data` JSON blobs * and the `judgements` column (the inverse of {@link mongoScoreDataToPg}). */ -export function pgScoreDataToMongo( +export function pgScoreDataToAPI( game: TGame, scoreData: PgScoreData, ): ScoreData { diff --git a/typescript/server/src/load-tests/README.md b/typescript/server/src/load-tests/README.md new file mode 100644 index 000000000..012f7cc20 --- /dev/null +++ b/typescript/server/src/load-tests/README.md @@ -0,0 +1,100 @@ +# Score import load tests (HTTP) + +Stress **`POST /api/v1/import/file`** against a **running** Tachi instance using real fixtures from `src/test-utils/test-data/` (CSV, XML, JSON). + +## Prerequisites + +- Server reachable (e.g. `http://127.0.0.1:8080` when using local `PORT`). +- Accounts with **`submit_score`**: + - **Session:** login cookie (see below) — **one in-flight import per user**, so keep **`--concurrency 1`** unless you only care about rate limits. + - **API tokens:** one token per parallel slot. Generate many tokens with the seeder (below). + +**Rate limiting:** In dev, score imports are limited to **5 per minute per IP** unless you set: + +```bash +export TACHI_DISABLE_SCORE_IMPORT_RATE_LIMIT=true +``` + +Restart the server after changing this. Turn it off when you are done load testing. + +## Commands + +From repo root, prefer **`just`** (see `Justfile-test`). From **`typescript/server`**, you can use **`bun`** directly. + +### Run the load CLI + +```bash +# From repo root +just load-test-score-import -- \ + --url http://127.0.0.1:8080 \ + --token-file ./tokens.txt \ + --requests 40 \ + --concurrency 8 \ + --import-type file/eamusement-iidx-csv \ + --mutate-body +``` + +Equivalent from `typescript/server`: + +```bash +bun run load-test:score-import -- --url http://127.0.0.1:8080 --token-file ./tokens.txt ... +``` + +### Seed many API tokens (parallel imports) + +Creates N users in **your configured Postgres** (same DB as the server) and writes one bearer token per line: + +```bash +just load-test-score-import-seed-tokens 64 /tmp/tachi-load-tokens.txt +``` + +Then: + +```bash +just load-test-score-import -- \ + --url http://127.0.0.1:8080 \ + --token-file /tmp/tachi-load-tokens.txt \ + --requests 128 \ + --concurrency 64 \ + --mutate-body +``` + +### Session cookie (single-user, sequential) + +1. Log in (example: admin / dev password, captcha `test`): + + ```bash + curl -sS -c cookies.txt -X POST http://127.0.0.1:8080/api/v1/auth/login \ + -H "Content-Type: application/json" \ + -d '{"username":"admin","!password":"password","captcha":"test"}' + ``` + +2. Build the `Cookie` header from `cookies.txt` (Netscape format: the `Tachi_*_SESSION` line is tab-separated; do not use `grep -v '^#'` or you will drop `#HttpOnly_…` lines). + +3. Run with **`--cookie 'Tachi_…_SESSION=…'`** and **`--concurrency 1`**. + +## Useful flags + +| Flag | Purpose | +|------|--------| +| `--url` | Origin only, no trailing slash (required). | +| `--token-file` | One `Bearer` token per line (`#` comments ok). | +| `--requests` / `--concurrency` | Total uploads and parallel batch size (≤ token count for multi-user). | +| `--import-type` | `file/eamusement-iidx-csv` (default), `file/solid-state-squad`, `file/batch-manual`, etc. | +| `--file` | Override fixture path (defaults pick a file under `test-utils/test-data` per import type). | +| `--playtype` | `SP` / `DP` for IIDX CSV. | +| `--set key=value` | Extra multipart fields (repeatable). | +| `--mutate-body` | Slightly vary each upload (timestamps / scores) so payloads differ. | +| `--timeout-ms` | Per-request fetch timeout (`0` = none). | + +## Behaviour notes + +- **409** if the same user starts a second import before the first finishes. +- **429** from the score-import rate limiter unless disabled via env (above). +- **200** synchronous success; **202** if the server uses an external score-import worker (queued). +- Heavy runs (large CSV + high concurrency) will stress CPU, Postgres, and Redis; watch Grafana / local metrics. + +## Files + +- `score-import-load-cli.ts` — multipart client. +- `seed-stress-api-tokens.ts` — bulk token seeder for dev DBs. diff --git a/typescript/server/src/load-tests/score-import-load-cli.ts b/typescript/server/src/load-tests/score-import-load-cli.ts new file mode 100644 index 000000000..a3c555a5e --- /dev/null +++ b/typescript/server/src/load-tests/score-import-load-cli.ts @@ -0,0 +1,420 @@ +/** + * HTTP stress harness: multipart POST /api/v1/import/file against a live Tachi instance. + * + * Uses real fixtures under src/test-utils/test-data/ (CSV, XML, JSON) — not MER. + * + * One in-flight import per user (409 if you exceed). Use a token pool sized ≥ concurrency. + * + * @example + * bun run load-test:score-import -- \\ + * --url https://your.tachi.instance \\ + * --token-file ./tokens.txt \\ + * --requests 30 --concurrency 3 + * + * @example + * bun run load-test:score-import -- --url https://tachi.example \\ + * --token "$TACHI_TOKEN" --import-type file/solid-state-squad \\ + * --file src/test-utils/test-data/s3/large-example.xml --requests 10 + */ +import type { FileUploadImportTypes } from "tachi-common"; + +import { Command } from "commander"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { fileImportTypes } from "tachi-common/constants/import-types"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +/** Repo fixtures: typescript/server/src/test-utils/test-data */ +const TEST_DATA = path.join(__dirname, "../test-utils/test-data"); + +function defaultFixtureForType(t: FileUploadImportTypes): string { + switch (t) { + case "file/eamusement-iidx-csv": + return path.join(TEST_DATA, "eamusement-iidx-csv/post-leggendaria.csv"); + case "file/pli-iidx-csv": + return path.join(TEST_DATA, "eamusement-iidx-csv/small-hv-file.csv"); + case "file/eamusement-sdvx-csv": + return path.join(TEST_DATA, "eamusement-sdvx-csv/exceed-gear-score.csv"); + case "file/solid-state-squad": + return path.join(TEST_DATA, "s3/large-example.xml"); + case "file/batch-manual": + return path.join(TEST_DATA, "batch-manual/chunitachi.json"); + case "file/mypagescraper-records-csv": + return path.join(TEST_DATA, "wacca-mypage-scraper/records.csv"); + default: + throw new Error( + `No bundled default file for import type "${t}". Pass --file explicitly.`, + ); + } +} + +function assertFileImportType(s: string): FileUploadImportTypes { + if (!fileImportTypes.includes(s as FileUploadImportTypes)) { + console.error( + `Invalid --import-type "${s}". Expected one of:\n ${fileImportTypes.join("\n ")}`, + ); + process.exit(1); + } + return s as FileUploadImportTypes; +} + +interface RequestResult { + ok: boolean; + status: number; + durationMs: number; + note?: string; +} + +function percentile(sorted: number[], p: number): number { + if (sorted.length === 0) { + return 0; + } + const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); + return sorted[idx]!; +} + +function normalizeBaseUrl(url: string): string { + return url.replace(/\/+$/u, ""); +} + +function parseSetPairs(pairs: string[] | undefined): Record { + const r: Record = {}; + if (!pairs?.length) { + return r; + } + for (const pair of pairs) { + const i = pair.indexOf("="); + if (i <= 0) { + console.error(`--set expects key=value, got: ${pair}`); + process.exit(1); + } + r[pair.slice(0, i)] = pair.slice(i + 1); + } + return r; +} + +function buildFormFields( + importType: FileUploadImportTypes, + playtype: string, + extras: Record, +): Record { + const fields = { ...extras }; + if ( + (importType === "file/eamusement-iidx-csv" || importType === "file/pli-iidx-csv") && + fields.playtype === undefined + ) { + fields.playtype = playtype; + } + return fields; +} + +/** Optional: tweak file bytes so repeated uploads are less identical (CSV last column timestamps). */ +function maybeMutateFileBody( + filePath: string, + buf: Buffer, + mutate: boolean, + requestIndex: number, +): Buffer { + if (!mutate) { + return buf; + } + const base = path.basename(filePath).toLowerCase(); + if (base.endsWith(".csv")) { + let s = buf.toString("utf-8"); + const lines = s.split(/\r?\n/u); + for (let i = 1; i < lines.length; i++) { + const line = lines[i]; + if (!line?.trim()) { + continue; + } + lines[i] = line.replace( + /(\d{4}-\d{2}-\d{2} \d{2}:\d{2})$/u, + (_, ts: string) => `${ts}:${String((requestIndex + i * 7) % 60).padStart(2, "0")}`, + ); + if (lines[i] !== line) { + break; + } + } + s = lines.join("\n"); + return Buffer.from(s, "utf-8"); + } + if (base.endsWith(".json")) { + try { + const o = JSON.parse(buf.toString("utf-8")) as { scores?: Array<{ score?: number }> }; + if (Array.isArray(o.scores)) { + for (const row of o.scores) { + if (typeof row.score === "number") { + row.score = Math.max(0, row.score + (requestIndex % 97)); + } + } + } + return Buffer.from(`${JSON.stringify(o)}\n`, "utf-8"); + } catch { + return buf; + } + } + // XML / others: suffix a harmless byte that most XML parsers ignore after root — skip + return buf; +} + +async function postFileImport( + baseUrl: string, + authHeaders: Record, + filePath: string, + importType: FileUploadImportTypes, + formFields: Record, + userIntent: boolean, + requestIndex: number, + mutate: boolean, + timeoutMs: number | undefined, +): Promise { + const raw = readFileSync(filePath); + const bytes = maybeMutateFileBody(filePath, raw, mutate, requestIndex); + const fileName = path.basename(filePath); + const upload = new Blob([bytes], { type: "application/octet-stream" }); + + const form = new FormData(); + form.set("importType", importType); + form.set("scoreData", upload, fileName); + for (const [k, v] of Object.entries(formFields)) { + form.set(k, v); + } + + const headers: Record = { ...authHeaders }; + if (userIntent) { + headers["X-User-Intent"] = "true"; + } + + const t0 = performance.now(); + let res: Response; + try { + res = await fetch(`${baseUrl}/api/v1/import/file`, { + method: "POST", + headers, + body: form, + ...(timeoutMs !== undefined ? { signal: AbortSignal.timeout(timeoutMs) } : {}), + }); + } catch (e) { + const durationMs = performance.now() - t0; + const name = e instanceof Error ? e.name : ""; + const isAbort = name === "AbortError" || name === "TimeoutError"; + return { + ok: false, + status: 0, + durationMs, + note: isAbort + ? `timeout (>${timeoutMs}ms)` + : e instanceof Error + ? e.message + : String(e), + }; + } + const durationMs = performance.now() - t0; + const ok = res.status === 200 || res.status === 202; + + let note: string | undefined; + if (res.status === 409) { + note = "ongoing import (same user?)"; + } else if (res.status === 429) { + note = "rate limited"; + } else if (res.status === 401 || res.status === 403) { + note = "auth"; + } + + return { ok, status: res.status, durationMs, note }; +} + +function readTokenFile(p: string): string[] { + const raw = readFileSync(p, "utf-8"); + return raw + .split(/\r?\n/u) + .map((l) => l.trim()) + .filter((l) => l.length > 0 && !l.startsWith("#")); +} + +async function main() { + const program = new Command(); + program + .name("score-import-file-stress") + .description("POST /api/v1/import/file (multipart) for load testing.") + .requiredOption("--url ", "Tachi origin, e.g. https://example.com") + .option("-n, --requests ", "total uploads", "20") + .option("-c, --concurrency ", "parallel in-flight requests per batch", "1") + .option( + "--token ", + "Bearer API token (repeat for multiple users)", + (v, prev: string[] | undefined) => [...(prev ?? []), v], + ) + .option("--token-file ", "One bearer token per line") + .option("--cookie ", "Raw Cookie header; forces concurrency 1") + .option( + "-t, --import-type ", + `file/* import type (see tachi-common)`, + "file/eamusement-iidx-csv", + ) + .option("-f, --file ", "fixture path (default: test-utils file for --import-type)") + .option("--playtype ", "for IIDX CSV imports", "SP") + .option( + "--set ", + "extra multipart field (repeatable)", + (v, prev: string[] | undefined) => [...(prev ?? []), v], + ) + .option( + "--mutate-body", + "slightly change each upload (CSV time / JSON scores) so payloads differ", + ) + .option("--timeout-ms ", "per-request fetch timeout (0 = no limit)", "0") + .option("--no-user-intent", "omit X-User-Intent: true") + .parse(); + + const opts = program.opts() as { + concurrency: string; + cookie?: string; + file?: string; + importType: string; + mutateBody: boolean; + playtype: string; + requests: string; + set?: string[]; + timeoutMs: string; + token?: string[]; + tokenFile?: string; + url: string; + userIntent: boolean; + }; + + const timeoutParsed = Number.parseInt(opts.timeoutMs, 10); + const timeoutMs = + Number.isFinite(timeoutParsed) && timeoutParsed > 0 ? timeoutParsed : undefined; + + const importType = assertFileImportType(opts.importType); + const filePath = path.resolve(opts.file ?? defaultFixtureForType(importType)); + const formFields = buildFormFields(importType, opts.playtype, parseSetPairs(opts.set)); + + const baseUrl = normalizeBaseUrl(opts.url); + const total = Number.parseInt(opts.requests, 10); + let concurrency = Number.parseInt(opts.concurrency, 10); + + if (!Number.isFinite(total) || total < 1) { + console.error("--requests must be a positive integer"); + process.exit(1); + } + if (!Number.isFinite(concurrency) || concurrency < 1) { + console.error("--concurrency must be a positive integer"); + process.exit(1); + } + + const tokenPool: string[] = []; + if (opts.tokenFile) { + tokenPool.push(...readTokenFile(opts.tokenFile)); + } + if (opts.token?.length) { + tokenPool.push(...opts.token); + } + + let authHeaders: Record; + if (opts.cookie) { + if (tokenPool.length > 0) { + console.error("Use either --cookie or token(s), not both."); + process.exit(1); + } + concurrency = 1; + if (Number.parseInt(opts.concurrency, 10) > 1) { + console.warn("Cookie auth: concurrency forced to 1."); + } + authHeaders = { Cookie: opts.cookie }; + } else if (tokenPool.length > 0) { + if (concurrency > tokenPool.length) { + console.warn( + `Concurrency ${concurrency} > ${tokenPool.length} token(s); capping to ${tokenPool.length}.`, + ); + concurrency = tokenPool.length; + } + authHeaders = {}; + } else { + console.error("Provide --token, --token-file, or --cookie."); + process.exit(1); + } + + console.error(`Using fixture: ${filePath}`); + console.error(`importType=${importType} formFields=${JSON.stringify(formFields)}`); + + const results: RequestResult[] = []; + const tWall = performance.now(); + + /* eslint-disable no-await-in-loop -- batched parallel uploads */ + for (let start = 0; start < total; start += concurrency) { + const batch = Math.min(concurrency, total - start); + const batchOut = await Promise.all( + Array.from({ length: batch }, (_, slot) => { + const reqIndex = start + slot; + const headers = + opts.cookie !== undefined + ? authHeaders + : { Authorization: `Bearer ${tokenPool[slot]!}` }; + return postFileImport( + baseUrl, + headers, + filePath, + importType, + formFields, + opts.userIntent, + reqIndex, + opts.mutateBody, + timeoutMs, + ); + }), + ); + results.push(...batchOut); + } + /* eslint-enable no-await-in-loop */ + + const wallMs = performance.now() - tWall; + const okn = results.filter((r) => r.ok).length; + const lat = results.map((r) => r.durationMs).sort((a, b) => a - b); + const statusHistogram: Record = {}; + for (const r of results) { + const k = String(r.status); + statusHistogram[k] = (statusHistogram[k] ?? 0) + 1; + } + + console.log( + JSON.stringify( + { + endpoint: `${baseUrl}/api/v1/import/file`, + importType, + fixture: filePath, + requests: total, + concurrency: Math.min(concurrency, total), + accepted: okn, + failed: total - okn, + wallMs, + requestsPerSec: wallMs > 0 ? (total / wallMs) * 1000 : 0, + latencyMs: { + p50: percentile(lat, 50), + p95: percentile(lat, 95), + max: lat.length ? lat[lat.length - 1]! : 0, + }, + statusHistogram, + }, + null, + 2, + ), + ); + + const failedSamples = results.filter((r) => !r.ok).slice(0, 8); + if (failedSamples.length > 0) { + console.error("Sample failures:", failedSamples); + } + + if (okn < total) { + process.exit(1); + } +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/typescript/server/src/load-tests/seed-stress-api-tokens.ts b/typescript/server/src/load-tests/seed-stress-api-tokens.ts new file mode 100644 index 000000000..1a1303c4e --- /dev/null +++ b/typescript/server/src/load-tests/seed-stress-api-tokens.ts @@ -0,0 +1,53 @@ +/** + * Create N local-dev users + API tokens (submit_score) for parallel score-import load tests. + * + * @example + * bun run src/load-tests/seed-stress-api-tokens.ts 32 /tmp/stress-tokens.txt + */ +import { loadServerEnvFile } from "#lib/setup/load-server-env"; + +loadServerEnvFile(process.env.NODE_ENV === "test" ? ".env.test" : ".env"); + +import { seedApiToken } from "#actions/test-utils/api-tokens"; +import { seedUser } from "#test-utils/pg-fixtures"; +import { randomBytes } from "node:crypto"; +import { writeFileSync } from "node:fs"; + +async function main() { + const n = Number.parseInt(process.argv[2] ?? "", 10); + const outPath = process.argv[3]; + + if (!Number.isFinite(n) || n < 1) { + console.error( + "Usage: bun run src/load-tests/seed-stress-api-tokens.ts ", + ); + process.exit(1); + } + if (!outPath) { + console.error("Second argument must be output path for token lines."); + process.exit(1); + } + + const tokens: string[] = []; + const ts = Date.now(); + + for (let i = 0; i < n; i++) { + const u = await seedUser({ + username: `stress_${ts}_${i}`, + email: `stress_${ts}_${i}@stress.local`, + withCredential: true, + withSettings: true, + }); + const tok = `st_${ts}_${i}_${randomBytes(12).toString("hex")}`; + await seedApiToken({ token: tok, userId: u.id, submitScore: true }); + tokens.push(tok); + } + + writeFileSync(outPath, `${tokens.join("\n")}\n`, "utf-8"); + console.error(`Wrote ${n} tokens to ${outPath}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/typescript/server/src/main.ts b/typescript/server/src/main.ts index a47d33e63..453d3889a 100644 --- a/typescript/server/src/main.ts +++ b/typescript/server/src/main.ts @@ -11,8 +11,6 @@ import server, { metricsApp } from "#server/server"; import DB from "#services/pg/db"; import fetch from "#utils/fetch"; import { GetUserWithID } from "#utils/user"; -import { spawn } from "child_process"; -import path from "path"; import { applyMigrations } from "tachi-db-migration-engine"; log.info( @@ -83,39 +81,3 @@ if (metricsApp) { process.on("SIGTERM", () => { void HandleSIGTERMGracefully(instance, metricsInstance); }); - -if (process.env.INVOKE_JOB_RUNNER) { - log.info({ bootInfo: true }, `Spawning a tachi-server job runner inline.`); - - if (Env.NODE_ENV === "production") { - log.warn( - { bootInfo: true }, - `Spawning inline tachi-server job runner in production. This is bad for performance.`, - ); - } - - // Spawn as a separate process to avoid hogging the main thread. - const jobProcess = spawn( - "ts-node", - [ - // Note: Can't use -r tsconfig-paths/register here - // because that is rejected by some library called - // arg. - // I'm not sure why. - "--require=tsconfig-paths/register", - path.join(__dirname, "../src/lib/jobs/job-runner.ts"), - ], - { - stdio: "inherit", - }, - ); - - jobProcess.on("error", (err) => { - log.fatal({ err }, `Failed to spawn job runner. Terminating process.`); - }); - - process.on("beforeExit", () => { - log.info(`Killing Job Runner.`); - jobProcess.kill(); - }); -} diff --git a/typescript/server/src/scripts/migrate-to-postgres.ts b/typescript/server/src/scripts/migrate-to-postgres.ts index 1e2b72aa9..654bc63fb 100644 --- a/typescript/server/src/scripts/migrate-to-postgres.ts +++ b/typescript/server/src/scripts/migrate-to-postgres.ts @@ -26,8 +26,6 @@ import type { NewFolderView, NewGameProfile, NewGameRival, - NewGameSettings, - NewGameSettingsShowcase, NewGameStatsSnapshot, NewGoalSub, NewImport, @@ -771,22 +769,63 @@ async function main(): Promise { console.log(` ${notifications.length} notifications.`); } - // ── game_settings + game_settings_showcase + game_rival ────────────────── + // ── game_profile + game_rival ─────────────────────────────────────────────── + // Greenfield schema: one `game_profile` row per UGPT (stats + preferences + showcase JSON); + // see `db/migrations/20260301154256_genesis.sql`. Here we merge legacy Mongo `game-stats` + + // `game-settings` (+ rivals) into those rows for mongo-to-pg. { - console.log("\n[game_settings / game_settings_showcase / game_rival]"); + console.log("\n[game_profile / game_rival]"); const ugptSettings = await mongoDB.get("game-settings").find({}); + const gameStats = await mongoDB.get("game-stats").find({}); - const settingsRows: Array = []; - const showcaseRows: Array = []; - const rivalRows: Array = []; + type ProfileKey = `${number}:${PgGame}`; + const profileByKey = new Map(); + + const emptyPrefs = ( + game: PgGame, + ): Pick< + NewGameProfile, + | "data" + | "pf_default_table" + | "pf_preferred_default_enum" + | "pf_preferred_profile_alg" + | "pf_preferred_ranking" + | "pf_preferred_score_alg" + | "pf_preferred_session_alg" + | "showcase" + > => ({ + pf_preferred_score_alg: null, + pf_preferred_session_alg: null, + pf_preferred_profile_alg: null, + pf_preferred_default_enum: null, + pf_default_table: null, + pf_preferred_ranking: null, + data: JSON.stringify( + game === "iidx-sp" || game === "iidx-dp" + ? { display2DXTra: false, bpiTarget: 0 } + : {}, + ), + showcase: JSON.stringify([]), + }); + + for (const gs of gameStats) { + const game = mongoGameToPg(gs.game, (gs as { playtype?: string }).playtype); + const key = `${gs.userID}:${game}` as ProfileKey; + profileByKey.set(key, { + user_id: gs.userID, + game, + ratings: JSON.stringify(gs.ratings), + classes: JSON.stringify(gs.classes), + ...emptyPrefs(game), + }); + } for (const s of ugptSettings) { const game = mongoGameToPg(s.game, (s as { playtype?: string }).playtype); + const key = `${s.userID}:${game}` as ProfileKey; const prefs = s.preferences; - - settingsRows.push({ - user_id: s.userID, - game, + const existing = profileByKey.get(key); + const prefSlice = { pf_preferred_score_alg: (prefs.preferredScoreAlg as string | null) ?? null, pf_preferred_session_alg: (prefs.preferredSessionAlg as string | null) ?? null, pf_preferred_profile_alg: (prefs.preferredProfileAlg as string | null) ?? null, @@ -794,16 +833,26 @@ async function main(): Promise { pf_default_table: prefs.defaultTable, pf_preferred_ranking: prefs.preferredRanking, data: JSON.stringify(prefs.gameSpecific), - }); - - if (prefs.stats.length > 0) { - showcaseRows.push({ + showcase: JSON.stringify(prefs.stats), + }; + if (existing) { + profileByKey.set(key, { ...existing, ...prefSlice }); + } else { + profileByKey.set(key, { user_id: s.userID, game, - data: JSON.stringify(prefs.stats), + ratings: JSON.stringify({}), + classes: JSON.stringify({}), + ...prefSlice, }); } + } + const profileRows = [...profileByKey.values()]; + const rivalRows: Array = []; + + for (const s of ugptSettings) { + const game = mongoGameToPg(s.game, (s as { playtype?: string }).playtype); for (const rivalId of s.rivals) { if (rivalId !== s.userID) { rivalRows.push({ user_id: s.userID, game, rival: rivalId }); @@ -811,30 +860,13 @@ async function main(): Promise { } } - await batchInsert("game_settings", settingsRows); - await batchInsert("game_settings_showcase", showcaseRows); + await batchInsert("game_profile", profileRows); await batchInsert("game_rival", rivalRows); console.log( - ` ${settingsRows.length} game settings, ${showcaseRows.length} showcases, ${rivalRows.length} rivals.`, + ` ${profileRows.length} game profiles (${gameStats.length} stats docs merged with ${ugptSettings.length} settings docs), ${rivalRows.length} rivals.`, ); } - // ── game_profile ─────────────────────────────────────────────────────────── - { - console.log("\n[game_profile]"); - const gameStats = await mongoDB.get("game-stats").find({}); - - const statsRows: Array = gameStats.map((gs) => ({ - user_id: gs.userID, - game: mongoGameToPg(gs.game, (gs as { playtype?: string }).playtype), - ratings: JSON.stringify(gs.ratings), - classes: JSON.stringify(gs.classes), - })); - - await batchInsert("game_profile", statsRows); - console.log(` ${gameStats.length} game profiles.`); - } - // ── game_stats_snapshot ─────────────────────────────────────────────────── console.log("\n[game_stats_snapshot]"); await streamMigrate( diff --git a/typescript/server/src/security-audit/poc-folder-where-sql-raw.test.ts b/typescript/server/src/security-audit/poc-folder-where-sql-raw.test.ts deleted file mode 100644 index 26b0c202a..000000000 --- a/typescript/server/src/security-audit/poc-folder-where-sql-raw.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { BuildFolderQuery } from "#lib/folders/folders"; -import DB from "#services/pg/db"; -import { randomUUID } from "node:crypto"; -import { describe, expect, it } from "vitest"; - -/** - * POC for docs/security-audit-2026-04-05.md §2: `folder.where` is embedded with `sql.raw`. - * A writer who can set `where` to a tautology affects which charts belong to the folder. - * - * This is not an unauthenticated network exploit — it requires DB write access to `folder`. - */ -describe("POC: tautological folder.where via sql.raw", () => { - function ids(prefix: string) { - const u = randomUUID().replace(/-/gu, "").slice(0, 12); - - return { - folderId: `${prefix}-f-${u}`, - folderLegacy: `${prefix}-fl-${u}`, - songId: `${prefix}-s-${u}`, - chartA: `${prefix}-ca-${u}`, - chartB: `${prefix}-cb-${u}`, - }; - } - - it("matches every chart in the game when where is always true", async () => { - const { folderId, folderLegacy, songId, chartA, chartB } = ids("sqlraw"); - - const songLegacy = 9_800_000 + Math.floor(Math.random() * 99_000); - - await DB.insertInto("song") - .values({ - id: songId, - legacy_id: songLegacy, - game_group: "iidx", - title: "POC Song", - artist: "P", - search_terms: [], - alt_titles: [], - fts_document: "", - data: JSON.stringify({}), - }) - .execute(); - - await DB.insertInto("chart") - .values([ - { - id: chartA, - legacy_id: chartA, - game: "iidx-sp", - song_id: songId, - level: "9", - level_num: 9, - is_primary: true, - difficulty: "ANOTHER", - versions: [], - data: JSON.stringify({}), - }, - { - id: chartB, - legacy_id: chartB, - game: "iidx-sp", - song_id: songId, - level: "10", - level_num: 10, - is_primary: true, - difficulty: "HYPER", - versions: [], - data: JSON.stringify({}), - }, - ]) - .execute(); - - await DB.insertInto("folder") - .values({ - id: folderId, - legacy_id: folderLegacy, - game: "iidx-sp", - inactive: false, - title: "POC tautology", - slug: folderId, - where: "true", - version_filter: null, - search_terms: [], - }) - .execute(); - - const { folderQuery } = await BuildFolderQuery(folderId); - const { rows } = await folderQuery.execute(DB); - - const got = new Set(rows.map((r) => (r as { id: string }).id)); - - // Both charts match — `where: "true"` is an unrestricted tautology. - // A legitimate folder (e.g. `where: "chart.level_num = 10"`) would - // return only chartB. The tautology returns both. - expect(got.has(chartA)).toBe(true); - expect(got.has(chartB)).toBe(true); - expect(rows.length).toBeGreaterThanOrEqual(2); - }); -}); diff --git a/typescript/server/src/security-audit/poc-seeds-shell-injection.test.ts b/typescript/server/src/security-audit/poc-seeds-shell-injection.test.ts deleted file mode 100644 index 0ee49650e..000000000 --- a/typescript/server/src/security-audit/poc-seeds-shell-injection.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from "vitest"; - -/** - * POC for docs/security-audit-2026-04-05.md §1. - * - * The real handler is GET /api/v1/seeds/collections with ?revision=… - * (typescript/server/src/server/router/api/v1/seeds/router.ts). - * It only runs under RequireLocalDevelopment. - * - * The implementation builds a shell line like: - * PAGER=cat git show '${rev}:seeds/collections' | tail -n +3 - * and passes it to asyncExec (child_process.exec). - * - * Only ':' is rejected in revision — not ', ;, $(), etc. - */ -function buildGitShowTreeCommand(rev: string): string { - return `PAGER=cat git show '${rev}:seeds/collections' | tail -n +3`; -} - -describe("POC: seeds revision breaks out of single-quoted git argument (local dev)", () => { - it("injects shell separators when revision contains a single quote", () => { - const rev = "x'; id; echo 'y"; - const cmd = buildGitShowTreeCommand(rev); - // After the first ', the shell runs `; id; echo` before the trailing quote. - expect(cmd).toContain("'; id;"); - }); - - it("shows ':' alone is insufficient to prevent injection", () => { - const blocked = "abc:def"; - expect(blocked.includes(":")).toBe(true); - - const maliciousNoColon = "x'; whoami; echo 'z"; - expect(maliciousNoColon.includes(":")).toBe(false); - expect(buildGitShowTreeCommand(maliciousNoColon)).toMatch(/'; whoami;/u); - }); -}); diff --git a/typescript/server/src/server/middleware/rate-limiter.ts b/typescript/server/src/server/middleware/rate-limiter.ts index 1a11f1ce5..3c70c17ba 100644 --- a/typescript/server/src/server/middleware/rate-limiter.ts +++ b/typescript/server/src/server/middleware/rate-limiter.ts @@ -70,8 +70,8 @@ export const HyperAggressiveRateLimitMiddleware = rateLimit( // 5 requests every minute. This one has a tighter window, so it is less // vulnerable to bursting down the server. -// if we're in testing, disable this rate limit! +// Vitest: unlimited. Optional: TACHI_DISABLE_SCORE_IMPORT_RATE_LIMIT for local stress runs. export const ScoreImportRateLimiter = - Env.NODE_ENV === "test" + Env.NODE_ENV === "test" || ServerConfig.DISABLE_SCORE_IMPORT_RATE_LIMIT ? rateLimit(CreateRateLimitOptions(Infinity, "ScImport", ONE_MINUTE)) : rateLimit(CreateRateLimitOptions(5, "ScImport", ONE_MINUTE)); diff --git a/typescript/server/src/server/router/api/v1/admin/router.test.ts b/typescript/server/src/server/router/api/v1/admin/router.test.ts index 69541ef80..88acbed30 100644 --- a/typescript/server/src/server/router/api/v1/admin/router.test.ts +++ b/typescript/server/src/server/router/api/v1/admin/router.test.ts @@ -124,6 +124,182 @@ describe("POST /api/v1/admin/delete-score", () => { }); }); +describe("POST /api/v1/admin/recalc", () => { + it("returns 403 when the caller is not an admin", async () => { + await seedUser({ + username: "recalc_pleb", + email: "recalc_pleb@test.com", + withCredential: true, + withSettings: true, + }); + + const plebCookie = await loginAs("recalc_pleb"); + + const res = await mockApi.post("/api/v1/admin/recalc").set("Cookie", plebCookie).send({}); + + expect(res.status).toBe(403); + }); + + it("enqueues every chart for score re-derivation when the caller is an admin", async () => { + await seedUser({ + username: "recalc_admin", + email: "recalc_admin@test.com", + authLevel: "admin", + withCredential: true, + withSettings: true, + }); + + const adminCookie = await loginAs("recalc_admin"); + const chartId = Testing511SPA.chartID; + + await DB.insertInto("song") + .values({ + id: Testing511Song.id, + legacy_id: 1, + game_group: "iidx", + title: Testing511Song.title, + artist: Testing511Song.artist, + search_terms: [], + alt_titles: [], + data: Testing511Song.data, + fts_document: "", + }) + .execute(); + + await DB.insertInto("chart") + .values({ + id: chartId, + legacy_id: chartId, + game: "iidx-sp", + song_id: Testing511Song.id, + difficulty: Testing511SPA.difficulty, + level: Testing511SPA.level, + level_num: Testing511SPA.levelNum, + is_primary: true, + versions: Testing511SPA.versions, + data: Testing511SPA.data, + }) + .execute(); + + const res = await mockApi.post("/api/v1/admin/recalc").set("Cookie", adminCookie).send({}); + + expect(res.status).toBe(200); + + const stillQueued = await DB.selectFrom("score_rederive") + .select("chart_id") + .where("chart_id", "=", chartId) + .executeTakeFirst(); + + expect(stillQueued).toBeUndefined(); + }); +}); + +describe("POST /api/v1/admin/recalc-pbs", () => { + it("returns 403 when the caller is not an admin", async () => { + await seedUser({ + username: "recalc_pb_pleb", + email: "recalc_pb_pleb@test.com", + withCredential: true, + withSettings: true, + }); + + const plebCookie = await loginAs("recalc_pb_pleb"); + + const res = await mockApi + .post("/api/v1/admin/recalc-pbs") + .set("Cookie", plebCookie) + .send({}); + + expect(res.status).toBe(403); + }); + + it("enqueues pb_dirty for every distinct user+chart from scores when the caller is an admin", async () => { + await seedUser({ + username: "recalc_pb_admin", + email: "recalc_pb_admin@test.com", + authLevel: "admin", + withCredential: true, + withSettings: true, + }); + await seedUser({ + username: "recalc_pb_player", + email: "recalc_pb_player@test.com", + withCredential: true, + withSettings: true, + }); + + const adminCookie = await loginAs("recalc_pb_admin"); + const chartId = Testing511SPA.chartID; + const sd = TestingIIDXSPScore.scoreData as ScoreData<"iidx-sp">; + const { data, derived, judgements } = mongoScoreDataToPg("iidx-sp", sd); + const now = new Date().toISOString(); + + await DB.insertInto("song") + .values({ + id: Testing511Song.id, + legacy_id: 1, + game_group: "iidx", + title: Testing511Song.title, + artist: Testing511Song.artist, + search_terms: [], + alt_titles: [], + data: Testing511Song.data, + fts_document: "", + }) + .execute(); + + await DB.insertInto("chart") + .values({ + id: chartId, + legacy_id: chartId, + game: "iidx-sp", + song_id: Testing511Song.id, + difficulty: Testing511SPA.difficulty, + level: Testing511SPA.level, + level_num: Testing511SPA.levelNum, + is_primary: true, + versions: Testing511SPA.versions, + data: Testing511SPA.data, + }) + .execute(); + + await DB.insertInto("score") + .values({ + id: "recalc_pb_score", + user_id: 2, + chart_id: chartId, + game: "iidx-sp", + session_id: null, + import_id: null, + data: JSON.stringify(data), + derived_data: JSON.stringify(derived), + judgements: JSON.stringify(judgements), + calculated_data: JSON.stringify({}), + meta: JSON.stringify({}), + time_achieved: now, + time_added: now, + highlight: false, + comment: null, + }) + .execute(); + + const res = await mockApi + .post("/api/v1/admin/recalc-pbs") + .set("Cookie", adminCookie) + .send({}); + + expect(res.status).toBe(200); + + const stillDirty = await DB.selectFrom("pb_dirty") + .select(["pb_dirty.user_id", "pb_dirty.chart_id"]) + .where("pb_dirty.user_id", "=", 2) + .where("pb_dirty.chart_id", "=", chartId) + .executeTakeFirst(); + + expect(stillDirty).toBeUndefined(); + }); +}); + describe("POST /api/v1/admin/change-log-level", () => { it.todo("no route in router.ts (see router.oldtest.ts)"); }); diff --git a/typescript/server/src/server/router/api/v1/admin/router.ts b/typescript/server/src/server/router/api/v1/admin/router.ts index b12ad01f7..126dd0d26 100644 --- a/typescript/server/src/server/router/api/v1/admin/router.ts +++ b/typescript/server/src/server/router/api/v1/admin/router.ts @@ -10,22 +10,17 @@ import { GetJobQueue, } from "#lib/admin/admin-queries"; import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; +import { drainPbDirtyAndDownstream, drainStatsQueuesFully } from "#lib/jobs/drain-dirty-queues"; import { SendSiteAnnouncementNotification } from "#lib/notifications/notification-wrappers"; import { withAdmin } from "#lib/router/middleware"; import { success } from "#lib/router/typed-router"; import { TachiConfig } from "#lib/setup/config"; import DB from "#services/pg/db"; -import { IsValidPlaytype } from "#utils/misc"; +import { RecalcAllScores, UpdateAllPBs } from "#utils/calculations/recalc-scores"; import DestroyUserGameProfile from "#utils/reset-state/destroy-user-game-profile"; import { GetUserWithIDGuaranteed, ResolveUser } from "#utils/user"; import { ExpectedErr } from "bliss"; -import { - type GameGroup, - GameToGameGroup, - LEGACY_GameGroupPTToGame, - type LEGACY_Playtype, - type V3Game, -} from "tachi-common"; +import { GameToGameGroup, type V3Game } from "tachi-common"; import { API_V1_ROUTER } from "../router"; @@ -75,8 +70,14 @@ API_V1_ROUTER.add("GET /admin/cron-tasks", withAdmin, async () => { return success("Done.", { executions, tasks }); }); -API_V1_ROUTER.add("POST /admin/resync-pbs", withAdmin, () => { - throw new ExpectedErr(501, "Not implemented."); +API_V1_ROUTER.add("POST /admin/recalc-pbs", withAdmin, async () => { + await UpdateAllPBs(); + await drainPbDirtyAndDownstream(); + + return success( + "Re-queued every distinct (user, chart) from scores into pb_dirty and drained pb/session/game_profile queues until idle.", + {}, + ); }); API_V1_ROUTER.add("POST /admin/delete-score", withAdmin, async ({ input, req }) => { @@ -100,34 +101,28 @@ API_V1_ROUTER.add("POST /admin/delete-session", withAdmin, async ({ input, req } }); API_V1_ROUTER.add("POST /admin/destroy-ugpt", withAdmin, async ({ input }) => { - const gameGroup = input.game as GameGroup; - const playtype = input.playtype as LEGACY_Playtype; - - if (!IsValidPlaytype(gameGroup, playtype)) { - throw new ExpectedErr(400, `Invalid playtype ${playtype} for game ${gameGroup}.`); - } - - const game = LEGACY_GameGroupPTToGame(gameGroup, playtype); - const ugpt = await DB.selectFrom("game_profile") .where("user_id", "=", input.userID) - .where("game", "=", game) + .where("game", "=", input.game) .executeTakeFirst(); if (!ugpt) { - throw new ExpectedErr( - 404, - `No stats for ${input.userID} (${gameGroup} ${playtype}) exist.`, - ); + throw new ExpectedErr(404, `No stats for ${input.userID} (${input.game}) exist.`); } - await DestroyUserGameProfile(input.userID, gameGroup, playtype); + await DestroyUserGameProfile(input.userID, input.game); - return success(`Completely destroyed UGPT for ${input.userID} (${gameGroup} ${playtype}).`, {}); + return success(`Completely destroyed game profile for ${input.userID} (${input.game}).`, {}); }); -API_V1_ROUTER.add("POST /admin/recalc", withAdmin, () => { - throw new ExpectedErr(501, "Not implemented."); +API_V1_ROUTER.add("POST /admin/recalc", withAdmin, async () => { + await RecalcAllScores(); + await drainStatsQueuesFully(); + + return success( + "Enqueued every chart for score re-derivation and drained score/pb/session/game_profile queues until idle.", + {}, + ); }); API_V1_ROUTER.add("POST /admin/announcement", withAdmin, async ({ input }) => { diff --git a/typescript/server/src/server/router/api/v1/games/_gameGroup/_playtype/router.test.ts b/typescript/server/src/server/router/api/v1/games/_gameGroup/_playtype/router.test.ts index 2bf46620d..37ad2ffb7 100644 --- a/typescript/server/src/server/router/api/v1/games/_gameGroup/_playtype/router.test.ts +++ b/typescript/server/src/server/router/api/v1/games/_gameGroup/_playtype/router.test.ts @@ -141,6 +141,26 @@ describe("GET /api/v1/games/:game/leaderboard", () => { expect(res.body.body.gameStats.map((e: { userID: number }) => e.userID)).toEqual([2, 3, 1]); }); + + it("gives the same rank to users tied on the profile algorithm (next rank skips)", async () => { + await DB.updateTable("game_profile") + .set({ ratings: JSON.stringify({ BPI: 100 }) }) + .where("user_id", "=", 3) + .execute(); + + const res = await mockApi.get("/api/v1/games/iidx-sp/leaderboard?alg=BPI&limit=10"); + + expect(res.status).toBe(200); + const stats = res.body.body.gameStats as Array<{ + rank: number; + userID: number; + }>; + const r2 = stats.find((s) => s.userID === 2)!; + const r3 = stats.find((s) => s.userID === 3)!; + expect(r2.rank).toBe(1); + expect(r3.rank).toBe(1); + expect(stats.find((s) => s.userID === 1)!.rank).toBe(3); + }); }); describe("GET /api/v1/games/:game/players", () => { diff --git a/typescript/server/src/server/router/api/v1/games/router.ts b/typescript/server/src/server/router/api/v1/games/router.ts index e1be273d1..cd7f54eae 100644 --- a/typescript/server/src/server/router/api/v1/games/router.ts +++ b/typescript/server/src/server/router/api/v1/games/router.ts @@ -2,7 +2,7 @@ import { GetRecentActivityForMultipleGames } from "#lib/activity/activity"; import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; import { ONE_HOUR } from "#lib/constants/time"; import { LoadFolderDocumentByGameAndSlug, LoadFolderDocumentsByIds } from "#lib/db-formats/folders"; -import { SELECT_GAME_PROFILE, ToGameStatsDocument } from "#lib/db-formats/game-profiles"; +import { ToGameStatsDocument } from "#lib/db-formats/game-profiles"; import { SELECT_GOAL, SELECT_GOAL_SUB_WITH_GOAL_GAME } from "#lib/db-formats/goal"; import { CountPbsOnChart, @@ -77,6 +77,7 @@ import { LEGACY_GameToGameGroupPT, LEGACY_GetGamePTConfig, type UGPTSettingsDocument, + type UserGameStatsWithProfileLeaderboardRank, type V3Game, } from "tachi-common"; @@ -202,12 +203,25 @@ API_V1_ROUTER.add("GET /games/:game/leaderboard", withGame, async ({ input, ctx const ratingCol = sql`coalesce((game_profile.ratings::jsonb->>${sql.lit(alg)})::numeric, 0)`; const gameStats = await DB.selectFrom("game_profile") - .select(SELECT_GAME_PROFILE) + .select([ + "game_profile.user_id", + "game_profile.game", + "game_profile.ratings", + "game_profile.classes", + sql`RANK() OVER (ORDER BY ${ratingCol} DESC)`.as("rank"), + ]) .where("game_profile.game", "=", v3Game) .orderBy(ratingCol, "desc") .limit(limit) .execute() - .then((rows) => rows.map(ToGameStatsDocument)); + .then((rows) => + rows.map( + (row): UserGameStatsWithProfileLeaderboardRank => ({ + ...ToGameStatsDocument(row), + rank: Number(row.rank), + }), + ), + ); const users = await GetUsersWithIDs(gameStats.map((e) => e.userID)); diff --git a/typescript/server/src/server/router/api/v1/import/router.test.ts b/typescript/server/src/server/router/api/v1/import/router.test.ts index 31d54ab8f..8af962d7e 100644 --- a/typescript/server/src/server/router/api/v1/import/router.test.ts +++ b/typescript/server/src/server/router/api/v1/import/router.test.ts @@ -1,3 +1,4 @@ +import { seedApiToken } from "#actions/test-utils/api-tokens"; import DB from "#services/pg/db"; import mockApi, { CloseServerConnection } from "#test-utils/mock-api"; import { seedUser } from "#test-utils/pg-fixtures"; @@ -5,6 +6,53 @@ import { afterAll, describe, expect, it } from "vitest"; afterAll(() => CloseServerConnection()); +describe("import/orphans auth", () => { + it("returns 403 for unauthenticated GET list", async () => { + const res = await mockApi.get("/api/v1/import/orphans"); + expect(res.status).toBe(403); + expect(res.body.success).toBe(false); + }); + + it("returns 403 for unauthenticated POST reprocess", async () => { + const res = await mockApi.post("/api/v1/import/orphans").send({}); + expect(res.status).toBe(403); + expect(res.body.success).toBe(false); + }); + + it("returns 403 for unauthenticated DELETE", async () => { + const res = await mockApi.delete("/api/v1/import/orphans/O_ANY"); + expect(res.status).toBe(403); + expect(res.body.success).toBe(false); + }); + + it("returns 403 for unauthenticated GET detail", async () => { + const res = await mockApi.get("/api/v1/import/orphans/O_ANY"); + expect(res.status).toBe(403); + expect(res.body.success).toBe(false); + }); + + it("returns 403 when API token lacks submit_score", async () => { + const { id: userId } = await seedUser({ + username: "import_orphan_token_user", + withCredential: true, + withSettings: true, + }); + await seedApiToken({ + token: "orphan_no_submit", + userId, + submitScore: false, + }); + + const res = await mockApi + .get("/api/v1/import/orphans") + .set("Authorization", "Bearer orphan_no_submit"); + + expect(res.status).toBe(403); + expect(res.body.success).toBe(false); + expect(String(res.body.description)).toMatch(/submit_score/iu); + }); +}); + async function loginAs(username: string, password = "password123") { const res = await mockApi.post("/api/v1/auth/login").send({ username, @@ -159,7 +207,10 @@ describe("GET /api/v1/import/orphans", () => { }) .execute(); - const first = await mockApi.get("/api/v1/import/orphans").query({ limit: 1 }).set("Cookie", cookie); + const first = await mockApi + .get("/api/v1/import/orphans") + .query({ limit: 1 }) + .set("Cookie", cookie); expect(first.status).toBe(200); expect(first.body.success).toBe(true); @@ -176,9 +227,80 @@ describe("GET /api/v1/import/orphans", () => { .set("Cookie", cookie); expect(second.status).toBe(200); - expect(second.body.body.orphans.some((o: { orphanID: string }) => o.orphanID === "O_LIST_A")).toBe( - true, - ); + expect( + second.body.body.orphans.some((o: { orphanID: string }) => o.orphanID === "O_LIST_A"), + ).toBe(true); + }); + + it("returns one orphan with raw data and context", async () => { + const { id: userId } = await seedUser({ + username: "import_orphan_detail_user", + withCredential: true, + withSettings: true, + }); + const cookie = await loginAs("import_orphan_detail_user"); + + await DB.insertInto("orphan_score") + .values({ + orphan_id: "O_DETAIL_1", + user_id: userId, + import_id: null, + import_type: "ir/direct-manual", + game_group: "iidx", + context: { game: "iidx-sp", version: "27" }, + data: { identifier: "Song X", score: 300 }, + time_inserted: new Date(4_000).toISOString(), + error_message: "not-found", + }) + .execute(); + + const res = await mockApi.get("/api/v1/import/orphans/O_DETAIL_1").set("Cookie", cookie); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.body.orphanID).toBe("O_DETAIL_1"); + expect(res.body.body.importType).toBe("ir/direct-manual"); + expect(res.body.body.gameGroup).toBe("iidx"); + expect(res.body.body.message).toBe("not-found"); + expect(res.body.body.data).toEqual({ identifier: "Song X", score: 300 }); + expect(res.body.body.context).toEqual({ game: "iidx-sp", version: "27" }); + }); + + it("returns 404 for another user’s orphan on GET detail", async () => { + const { id: ownerId } = await seedUser({ + username: "import_orphan_detail_owner", + email: "import_orphan_detail_owner@example.com", + withCredential: true, + withSettings: true, + }); + await seedUser({ + username: "import_orphan_detail_other", + email: "import_orphan_detail_other@example.com", + withCredential: true, + withSettings: true, + }); + const otherCookie = await loginAs("import_orphan_detail_other"); + + await DB.insertInto("orphan_score") + .values({ + orphan_id: "O_DETAIL_OTHER", + user_id: ownerId, + import_id: null, + import_type: "ir/direct-manual", + game_group: "iidx", + context: {}, + data: {}, + time_inserted: new Date().toISOString(), + error_message: "", + }) + .execute(); + + const res = await mockApi + .get("/api/v1/import/orphans/O_DETAIL_OTHER") + .set("Cookie", otherCookie); + + expect(res.status).toBe(404); + expect(res.body.success).toBe(false); }); }); diff --git a/typescript/server/src/server/router/api/v1/import/router.ts b/typescript/server/src/server/router/api/v1/import/router.ts index 2d2511f03..b6eb269b4 100644 --- a/typescript/server/src/server/router/api/v1/import/router.ts +++ b/typescript/server/src/server/router/api/v1/import/router.ts @@ -4,14 +4,16 @@ import type { FileUploadImportTypes } from "tachi-common"; import { SIXTEEN_MEGABTYES } from "#lib/constants/filesize"; import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; import { log } from "#lib/log/log"; +import { withPermission } from "#lib/router/middleware"; import { success } from "#lib/router/typed-router"; import { ExpressWrappedScoreImportMain } from "#lib/score-import/framework/express-wrapper"; import { - DeorphanScores, deleteOrphanScoreForUser, + DeorphanScores, + getOrphanScoreDetailForUser, listOrphanScoresForUser, } from "#lib/score-import/framework/orphans/orphans"; -import { MakeScoreImport } from "#lib/score-import/framework/score-import"; +import { EnqueueScoreImportJob } from "#lib/score-import/worker/enqueue-pg"; import { ServerConfig, TachiConfig } from "#lib/setup/config"; import { RequirePermissions } from "#server/middleware/auth"; import { CreateMulterSingleUploadMiddleware } from "#server/middleware/multer-upload"; @@ -74,7 +76,7 @@ API_V1_ROUTER.rawAdd( }; // Fire the score import, but make no guarantees about its state. - void MakeScoreImport(job); + void EnqueueScoreImportJob(job); return res.status(202).json({ success: true, @@ -112,8 +114,7 @@ API_V1_ROUTER.add("POST /import/from-api", async ({ input, req }) => { const userIntent = req.header("X-User-Intent")?.toLowerCase() === "true"; if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - void (MakeScoreImport as any)({ + void EnqueueScoreImportJob({ importID, importType, parserArguments: [userID], @@ -155,7 +156,7 @@ API_V1_ROUTER.add("POST /import/from-api", async ({ input, req }) => { * * @name POST /api/v1/import/orphans */ -API_V1_ROUTER.add("POST /import/orphans", async ({ req }) => { +API_V1_ROUTER.add("POST /import/orphans", withPermission("submit_score"), async ({ req }) => { const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!); log.info(`User ${FormatUserDoc(userDoc)} forced an orphan sync.`); @@ -180,7 +181,7 @@ API_V1_ROUTER.add("POST /import/orphans", async ({ req }) => { * * @name GET /api/v1/import/orphans */ -API_V1_ROUTER.add("GET /import/orphans", async ({ input, req }) => { +API_V1_ROUTER.add("GET /import/orphans", withPermission("submit_score"), async ({ input, req }) => { const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!); const body = await listOrphanScoresForUser({ @@ -192,19 +193,44 @@ API_V1_ROUTER.add("GET /import/orphans", async ({ input, req }) => { return success(`Returned ${body.orphans.length} orphan scores.`, body); }); +/** + * Return one orphaned score row (including raw data/context) for the current user. + * + * @name GET /api/v1/import/orphans/:orphanID + */ +API_V1_ROUTER.add( + "GET /import/orphans/:orphanID", + withPermission("submit_score"), + async ({ params, req }) => { + const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!); + + const detail = await getOrphanScoreDetailForUser(params.orphanID, userDoc.id); + + if (!detail) { + throw new ExpectedErr(404, "No such orphan score for this user."); + } + + return success("Returned orphan score.", detail); + }, +); + /** * Delete a single orphaned score row for the current user. * * @name DELETE /api/v1/import/orphans/:orphanID */ -API_V1_ROUTER.add("DELETE /import/orphans/:orphanID", async ({ params, req }) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!); +API_V1_ROUTER.add( + "DELETE /import/orphans/:orphanID", + withPermission("submit_score"), + async ({ params, req }) => { + const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!); - const deleted = await deleteOrphanScoreForUser(params.orphanID, userDoc.id); + const deleted = await deleteOrphanScoreForUser(params.orphanID, userDoc.id); - if (!deleted) { - throw new ExpectedErr(404, "No such orphan score for this user."); - } + if (!deleted) { + throw new ExpectedErr(404, "No such orphan score for this user."); + } - return success("Deleted orphan score.", {}); -}); + return success("Deleted orphan score.", {}); + }, +); diff --git a/typescript/server/src/server/router/api/v1/imports/router.ts b/typescript/server/src/server/router/api/v1/imports/router.ts index 48ce3aa27..cd36dbbb2 100644 --- a/typescript/server/src/server/router/api/v1/imports/router.ts +++ b/typescript/server/src/server/router/api/v1/imports/router.ts @@ -1,7 +1,5 @@ -import type { ScoreImportWorkerReturns } from "#lib/score-import/worker/types"; - import { ACTION_DeleteImport } from "#actions/delete-import"; -import { JOB_RETRY_COUNT, SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; +import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; import { GetImportTrackerByImportId, ListFailedImportTrackers, @@ -10,11 +8,17 @@ import { } from "#lib/db-formats/import-document"; import { LoadSessionDocumentById } from "#lib/db-formats/session"; import { GetImportScores } from "#lib/imports/imports"; +import { + JOB_STATUS_DONE, + JOB_STATUS_FAILED, + JOB_STATUS_QUEUED, + JOB_STATUS_RUNNING, +} from "#lib/jobs/job-queue/constants"; import { log } from "#lib/log/log"; import { withImport } from "#lib/router/middleware"; import { success } from "#lib/router/typed-router"; -import ScoreImportQueue, { ScoreImportQueueEvents } from "#lib/score-import/worker/queue"; import { ServerConfig, TachiConfig } from "#lib/setup/config"; +import DB from "#services/pg/db"; import { GetRelevantSongsAndCharts } from "#utils/db"; import { GetUsersWithIDs, GetUserWithID } from "#utils/user"; import { ExpectedErr } from "bliss"; @@ -139,25 +143,12 @@ API_V1_ROUTER.add("POST /imports/:importID/revert", withImport, async ({ params, // ─── Import poll-status ─────────────────────────────────────────────────────── -// Finding jobs is slightly harder than just doing a key lookup, because of retrying. -async function FindImportJob(importID: string) { - const possibleImportIDs = []; - - for (let i = 1; i <= JOB_RETRY_COUNT; i++) { - possibleImportIDs.push(`${importID}:TRY${i}`); - } - - try { - // Note that instead of the cleaner await-inside-for here, we parallelise this - // for performance. - const maybeJob = ( - await Promise.all(possibleImportIDs.map((i) => ScoreImportQueue.getJob(i))) - ).find((k) => k); - - return maybeJob; - } catch (_err) { - return undefined; - } +async function findJobQueueForImport(importID: string) { + return DB.selectFrom("job_queue") + .selectAll() + .where("job_queue.scope", "=", `import:${importID}`) + .orderBy("job_queue.created_at", "desc") + .executeTakeFirst(); } /** @@ -190,7 +181,7 @@ API_V1_ROUTER.add("GET /imports/:importID/poll-status", async ({ params }) => { }); } - const job = await FindImportJob(params.importID); + const job = await findJobQueueForImport(params.importID); if (!job) { const tracker = await GetImportTrackerByImportId(params.importID); @@ -199,9 +190,7 @@ API_V1_ROUTER.add("GET /imports/:importID/poll-status", async ({ params }) => { throw new ExpectedErr(404, "There is no ongoing import here."); } - // The user has requested the status of the import before the job has even - // been sent to redis. This is rare, but prevents a race condition of saying - // that an import is not ongoing when it is. + // The user has requested the status before a job row is visible. Rare race. switch (tracker.type) { case "ONGOING": return success("Import is ongoing.", { importStatus: "ongoing", progress: 0 }); @@ -220,56 +209,39 @@ API_V1_ROUTER.add("GET /imports/:importID/poll-status", async ({ params }) => { } } - let isFailed: boolean; - - try { - isFailed = await job.isFailed(); - } catch (err) { - log.info(`Failed to read job: ${err}`); - isFailed = true; - } - - let isCompleted: boolean; - - try { - isCompleted = await job.isCompleted(); - } catch (err) { - log.info(`Failed to read job: ${err}`); - isCompleted = false; - } - - // job.isFailed() means a critical error has occurred — an unhandled exception was thrown. - if (isFailed) { - log.error({ job }, "Internal Server Error with job?"); + if (job.status === JOB_STATUS_FAILED) { + log.error({ job }, "Postgres job_queue row in failed state."); throw new ExpectedErr(500, "An internal service error has occurred with this import."); } - if (isCompleted) { - const content = (await job.waitUntilFinished( - ScoreImportQueueEvents, - )) as ScoreImportWorkerReturns; + if (job.status === JOB_STATUS_QUEUED || job.status === JOB_STATUS_RUNNING) { + return success("Import is ongoing.", { + importStatus: "ongoing", + progress: { description: "Importing scores." }, + }); + } - // content.success == true means the import finished cleanly. - // Otherwise it was a ScoreImportFatalError (bad user input etc.). - if (content.success) { + // job.status === DONE (2) + if (job.status === JOB_STATUS_DONE) { + const again = await LoadImportDocumentById(params.importID); + if (again) { return success("Import was completed!", { - import: content.ImportDocument, + import: again, importStatus: "completed", }); } - - return { - $status: content.statusCode, - body: {}, - description: content.description, - success: true, - }; + const tracker = await GetImportTrackerByImportId(params.importID); + if (tracker?.type === "FAILED") { + return { + $status: tracker.error.statusCode ?? 500, + body: {}, + description: tracker.error.message, + success: true as const, + }; + } + // small race: job finished, document not yet visible + return success("Import is ongoing.", { importStatus: "ongoing", progress: 0 }); } - const progress = job.progress; - - return success("Import is ongoing.", { - importStatus: "ongoing", - progress: progress === 0 ? { description: "Starting up import." } : progress, - }); + throw new ExpectedErr(500, "Unrecognised job queue state."); }); diff --git a/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts b/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts index 505feb9cb..bc13454f2 100644 --- a/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts +++ b/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts @@ -119,7 +119,51 @@ describe("GET /api/v1/sessions/:sessionID", () => { expect(res.body.body.songs).toHaveLength(1); expect(res.body.body.songs[0].id).toBe(SONG_PG); expect(res.body.body.user.id).toBe(userId); - expect(Array.isArray(res.body.body.scoreInfo)).toBe(true); + expect(res.body.body.scoreInfo).toHaveLength(1); + expect(res.body.body.scoreInfo[0].scoreID).toBe(scoreId); + expect(res.body.body.scoreInfo[0].isNewScore).toBe(true); + }); +}); + +describe("GET /api/v1/sessions/:sessionID/folder-raises", () => { + it("returns folder raise rows when session charts appear in folder_chart_lookup", async () => { + const { sessionId } = await seedSessionFixture(); + const folderId = `F_folder_sess_${sessionId}`; + + await DB.insertInto("folder") + .values({ + id: folderId, + legacy_id: folderId, + game: "iidx-sp", + inactive: false, + title: "Session Folder Raises Test", + slug: folderId, + where: `chart.id = '${CHART_PG}'`, + version_filter: null, + search_terms: [], + }) + .execute(); + + await DB.insertInto("folder_chart_lookup") + .values({ folder_id: folderId, chart_id: CHART_PG }) + .execute(); + + const res = await mockApi.get(`/api/v1/sessions/${sessionId}/folder-raises`); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(Array.isArray(res.body.body)).toBe(true); + expect(res.body.body.length).toBeGreaterThan(0); + + const hit = res.body.body.find( + (r: { folder: { folderID: string } }) => r.folder.folderID === folderId, + ); + + expect(hit).toBeDefined(); + expect(hit.raisedCharts).toContain(CHART_PG); + expect(hit.totalCharts).toBeGreaterThanOrEqual(1); + expect(typeof hit.type).toBe("string"); + expect(typeof hit.value).toBe("string"); }); }); diff --git a/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.ts b/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.ts index 13a694fa9..ec6bc7b6d 100644 --- a/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.ts +++ b/typescript/server/src/server/router/api/v1/sessions/_sessionID/router.ts @@ -1,5 +1,6 @@ import { ACTION_UpdateSession } from "#actions/update-session"; import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; +import { GetSessionFolderRaises } from "#lib/folders/get-session-folder-raises"; import { withSession, withSessionOwner } from "#lib/router/middleware"; import { success } from "#lib/router/typed-router"; import { API_V1_ROUTER } from "#server/router/api/v1/router"; @@ -35,16 +36,13 @@ API_V1_ROUTER.add("GET /sessions/:sessionID", withSession, async ({ ctx }) => { * This allows us to render pretty things in the UI, showing the user what their * best stats were. * - * @warn Folder raise calculation is currently not implemented. - * * @name GET /api/v1/sessions/:sessionID/folder-raises */ -API_V1_ROUTER.add( - "GET /sessions/:sessionID/folder-raises", - withSession, - // Folder raise calculation is currently not implemented. - () => success("Retrieved folder raises.", []), -); +API_V1_ROUTER.add("GET /sessions/:sessionID/folder-raises", withSession, async ({ ctx }) => { + const raises = await GetSessionFolderRaises(ctx.sessionDoc); + + return success("Retrieved folder raises.", raises); +}); /** * Modifies a session. diff --git a/typescript/server/src/server/router/api/v1/spec.ts b/typescript/server/src/server/router/api/v1/spec.ts index 30fa4ecaf..52a6793f9 100644 --- a/typescript/server/src/server/router/api/v1/spec.ts +++ b/typescript/server/src/server/router/api/v1/spec.ts @@ -6,43 +6,45 @@ import type { AnyRouterSpec } from "#lib/router/typed-router"; import type { TachiServerConfig } from "#lib/setup/config"; import type { EvaluateUsersStatsShowcase } from "#lib/showcase/get-stats"; import type { GitCommit } from "#utils/git"; -import type { - APITokenDocument, - CGCardInfo, - ChartDocument, - FervidexSettingsDocument, - FolderDocument, - GameConfig, - GameGroupConfig, - GoalDocument, - GoalSubscriptionDocument, - ImportDocument, - ImportTrackerDocument, - integer, - InviteCodeDocument, - KaiAuthDocument, - KsHookSettingsDocument, - MytCardInfo, - NotificationDocument, - PBScoreDocument, - QuestDocument, - QuestlineDocument, - QuestSubscriptionDocument, - RecentlyViewedFolderDocument, - ScoreDocument, - SessionDocument, - SessionScoreInfo, - SongDocument, - TableDocument, - TachiAPIClientDocument, - UGPTSettingsDocument, - UserDocument, - UserGameStats, - UserGameStatsSnapshotDocument, - UserSettingsDocument, -} from "tachi-common"; import type { CronTask, CronTaskExecution, JobQueue } from "tachi-db"; +import { + ALL_GAMES, + type APITokenDocument, + type CGCardInfo, + type ChartDocument, + type FervidexSettingsDocument, + type FolderDocument, + type GameConfig, + type GameGroupConfig, + type GoalDocument, + type GoalSubscriptionDocument, + type ImportDocument, + type ImportTrackerDocument, + type integer, + type InviteCodeDocument, + type KaiAuthDocument, + type KsHookSettingsDocument, + type MytCardInfo, + type NotificationDocument, + type PBScoreDocument, + type QuestDocument, + type QuestlineDocument, + type QuestSubscriptionDocument, + type RecentlyViewedFolderDocument, + type ScoreDocument, + type SessionDocument, + type SessionScoreInfo, + type SongDocument, + type TableDocument, + type TachiAPIClientDocument, + type UGPTSettingsDocument, + type UserDocument, + type UserGameStats, + type UserGameStatsSnapshotDocument, + type UserGameStatsWithProfileLeaderboardRank, + type UserSettingsDocument, +} from "tachi-common"; import { z } from "zod"; type TachiInstanceConfig = TachiServerConfig["TACHI_CONFIG"]; @@ -250,8 +252,8 @@ export const API_V1_SPEC = { output: doc(), }, - "GET /users/:userID/game-stats": { - description: "All game stats and rankings for a user.", + "GET /users/:userID/game-profiles": { + description: "All per-game profiles (ratings, classes) and rankings for a user.", input: z.object({}), output: docArray(), }, @@ -700,10 +702,10 @@ export const API_V1_SPEC = { description: "Nearby players on the profile leaderboard.", input: z.object({ alg: z.string().optional() }), output: z.strictObject({ - thisUsersStats: doc(), + thisUsersStats: doc(), thisUsersRanking: doc(), - above: docArray(), - below: docArray(), + above: docArray(), + below: docArray(), users: docArray(), }), }, @@ -1325,7 +1327,7 @@ export const API_V1_SPEC = { limit: z.coerce.number().max(500).optional(), }), output: z.strictObject({ - gameStats: docArray(), + gameStats: docArray(), users: docArray(), }), }, @@ -1623,7 +1625,16 @@ export const API_V1_SPEC = { "GET /sessions/:sessionID/folder-raises": { description: "Folder raise summary for a session.", input: z.object({}), - output: z.array(z.unknown()), + output: z.array( + z.strictObject({ + folder: doc(), + previousCount: z.number().int(), + raisedCharts: z.array(z.string()), + totalCharts: z.number().int(), + type: z.string(), + value: z.string(), + }), + ), }, // ──────────────────────────────────────────────── @@ -1728,6 +1739,21 @@ export const API_V1_SPEC = { }), }, + "GET /import/orphans/:orphanID": { + description: + "Return one orphaned score for the authenticated user, including raw `data` and `context` JSON.", + input: z.object({}), + output: z.strictObject({ + orphanID: z.string(), + importType: z.string(), + gameGroup: z.string(), + timeInserted: z.number(), + message: z.string().nullable(), + data: z.unknown(), + context: z.unknown(), + }), + }, + "DELETE /import/orphans/:orphanID": { description: "Delete one orphaned score row belonging to the authenticated user.", input: z.object({}), @@ -1839,12 +1865,10 @@ export const API_V1_SPEC = { }), }, - "POST /admin/resync-pbs": { - description: "Resync PBs (stub; not yet implemented).", - input: z.object({ - userIDs: z.array(z.number()).optional(), - filter: z.unknown().optional(), - }), + "POST /admin/recalc-pbs": { + description: + "Enqueue every distinct (user_id, chart_id) from the score table into pb_dirty, then synchronously drain pb_dirty and downstream session/game_profile queues until idle. No request body.", + input: z.object({}), output: empty, }, @@ -1864,14 +1888,14 @@ export const API_V1_SPEC = { description: "Destroy all data for a user on a game+playtype.", input: z.object({ userID: z.number(), - game: z.string(), - playtype: z.string(), + game: z.enum(ALL_GAMES), }), output: empty, }, "POST /admin/recalc": { - description: "Recalculate ratings (stub; not yet implemented).", + description: + "Enqueue every chart for score re-derivation (derived_data + calculated_data), then synchronously drain score_rederive and downstream pb/session/game_profile queues until idle. No request body.", input: z.object({}), output: empty, }, diff --git a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.test.ts b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.test.ts index b3044e61b..e3ef812a5 100644 --- a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.test.ts +++ b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.test.ts @@ -1,3 +1,4 @@ +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import { mongoScoreDataToPg } from "#lib/v3/migration-tools"; import DB from "#services/pg/db"; import mockApi, { CloseServerConnection } from "#test-utils/mock-api"; @@ -16,6 +17,7 @@ async function seedIidxSpProfile(userId: number) { game: "iidx-sp", ratings: JSON.stringify({}), classes: JSON.stringify({}), + ...newGameProfilePreferenceColumns("iidx-sp"), }) .execute(); } @@ -119,22 +121,6 @@ async function seedIidxChartPb(opts: { userId: number; withComposition?: boolean return { chartPg, chartLegacy, scoreId }; } -async function seedIidxGameSettings(userId: number) { - await DB.insertInto("game_settings") - .values({ - user_id: userId, - game: "iidx-sp", - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), - }) - .execute(); -} - async function insertPbOnChart(opts: { calculatedData: Record; chartPg: string; @@ -214,7 +200,6 @@ describe("GET /api/v1/users/:userID/games/:game/pbs/:chartID/rivals", () => { const { id: mainId } = await seedUser({ username: "ugpt_pb_main" }); const { id: rivalId } = await seedUser({ username: "ugpt_pb_rival" }); await seedIidxSpProfile(mainId); - await seedIidxGameSettings(mainId); await seedIidxSpProfile(rivalId); await DB.insertInto("game_rival") diff --git a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/rivals/router.test.ts b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/rivals/router.test.ts index 997865229..320bb74d0 100644 --- a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/rivals/router.test.ts +++ b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/rivals/router.test.ts @@ -1,3 +1,4 @@ +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import DB from "#services/pg/db"; import mockApi, { CloseServerConnection } from "#test-utils/mock-api"; import { seedUser } from "#test-utils/pg-fixtures"; @@ -22,20 +23,7 @@ async function seedUgpt(userId: number) { game: "iidx-sp", ratings: JSON.stringify({}), classes: JSON.stringify({}), - }) - .execute(); - - await DB.insertInto("game_settings") - .values({ - user_id: userId, - game: "iidx-sp", - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), + ...newGameProfilePreferenceColumns("iidx-sp"), }) .execute(); } diff --git a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts index 09717abfe..47fe92012 100644 --- a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts +++ b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts @@ -26,6 +26,7 @@ import { ISO8601ToUnixMilliseconds, UnixMillisecondsToISO8601 } from "#utils/tim import { FormatUserDoc, GetAllRankings, + GetLeaderboardRanksForUserIds, GetUGPTPlaycount, GetUserPrivateInfo, GetUsersRankingAndOutOf, @@ -39,7 +40,9 @@ import { LEGACY_FormatGameGroupPT, LEGACY_GameToGameGroupPT, type PBScoreDocument, + type UserGameStats, type UserGameStatsSnapshotDocument, + type UserGameStatsWithProfileLeaderboardRank, } from "tachi-common"; /** @@ -309,11 +312,22 @@ API_V1_ROUTER.add( const thisUsersRanking = await GetUsersRankingAndOutOf(thisUsersStats, alg); + const rankByUser = await GetLeaderboardRanksForUserIds(game, alg, [ + user.id, + ...aboveRows.map((r) => r.user_id), + ...belowRows.map((r) => r.user_id), + ]); + + const withRank = (s: UserGameStats): UserGameStatsWithProfileLeaderboardRank => ({ + ...s, + rank: rankByUser.get(s.userID)!, + }); + return success(`Returned ${above.length + below.length} nearby stats.`, { - above: above.reverse(), - below, + above: above.reverse().map(withRank), + below: below.map(withRank), thisUsersRanking, - thisUsersStats, + thisUsersStats: withRank(thisUsersStats), users, }); }, @@ -377,7 +391,7 @@ API_V1_ROUTER.add( throw new ExpectedErr(403, "Invalid password."); } - await DestroyUserGameProfile(user.id, gameGroup, playtype); + await DestroyUserGameProfile(user.id, v3Game); return success("Destroyed profile.", {}); }, diff --git a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts index 979e9deb7..86b30f374 100644 --- a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts +++ b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts @@ -1,4 +1,5 @@ import { seedApiToken } from "#actions/test-utils/api-tokens"; +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import DB from "#services/pg/db"; import mockApi, { CloseServerConnection } from "#test-utils/mock-api"; import { seedUser } from "#test-utils/pg-fixtures"; @@ -13,20 +14,7 @@ async function seedIidxUgpt(userId: number) { game: "iidx-sp", ratings: JSON.stringify({}), classes: JSON.stringify({}), - }) - .execute(); - - await DB.insertInto("game_settings") - .values({ - user_id: userId, - game: "iidx-sp", - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), + ...newGameProfilePreferenceColumns("iidx-sp"), }) .execute(); } diff --git a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts index a815fe60d..c03691adf 100644 --- a/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts +++ b/typescript/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts @@ -1,3 +1,4 @@ +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import DB from "#services/pg/db"; import mockApi, { CloseServerConnection } from "#test-utils/mock-api"; import { seedUser } from "#test-utils/pg-fixtures"; @@ -22,20 +23,7 @@ async function seedIIDXUserProfile(userId: number) { game: "iidx-sp", ratings: JSON.stringify({}), classes: JSON.stringify({}), - }) - .execute(); - - await DB.insertInto("game_settings") - .values({ - user_id: userId, - game: "iidx-sp", - pf_preferred_score_alg: null, - pf_preferred_session_alg: null, - pf_preferred_profile_alg: null, - pf_preferred_default_enum: null, - pf_default_table: null, - pf_preferred_ranking: null, - data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }), + ...newGameProfilePreferenceColumns("iidx-sp"), }) .execute(); } @@ -118,13 +106,13 @@ describe("PUT /api/v1/users/:userID/games/:game/showcase", () => { expect(res.body.success).toBe(true); expect(res.body.body.preferences.stats).toEqual(stats); - const row = await DB.selectFrom("game_settings_showcase") - .selectAll() - .where("user_id", "=", userId) - .where("game", "=", "iidx-sp") + const row = await DB.selectFrom("game_profile") + .select("game_profile.showcase") + .where("game_profile.user_id", "=", userId) + .where("game_profile.game", "=", "iidx-sp") .executeTakeFirst(); expect(row).toBeDefined(); - expect(row?.data).toEqual(stats); + expect(row?.showcase).toEqual(stats); }); }); diff --git a/typescript/server/src/server/router/api/v1/users/_userID/router.ts b/typescript/server/src/server/router/api/v1/users/_userID/router.ts index c8e9bc16f..dab1674bc 100644 --- a/typescript/server/src/server/router/api/v1/users/_userID/router.ts +++ b/typescript/server/src/server/router/api/v1/users/_userID/router.ts @@ -89,14 +89,14 @@ API_V1_ROUTER.add( ); /** - * Returns all of the game-stats this user has. + * Returns all per-game profiles (ratings, classes) for this user. * Additionally, adds a __rankingData property, which contains this users * ranking information. * This endpoint doubles up as a way of checking what games a user has played. * - * @name GET /api/v1/users/:userID/game-stats + * @name GET /api/v1/users/:userID/game-profiles */ -API_V1_ROUTER.add("GET /users/:userID/game-stats", withRequestedUser, async ({ ctx }) => { +API_V1_ROUTER.add("GET /users/:userID/game-profiles", withRequestedUser, async ({ ctx }) => { const { requestedUser: user } = ctx; // a user has played a game if and only if they have stats for it. diff --git a/typescript/server/src/server/router/ir/direct-manual/router.ts b/typescript/server/src/server/router/ir/direct-manual/router.ts index ed3b96b85..f37be3372 100644 --- a/typescript/server/src/server/router/ir/direct-manual/router.ts +++ b/typescript/server/src/server/router/ir/direct-manual/router.ts @@ -2,7 +2,7 @@ import type { ScoreImportJobData } from "#lib/score-import/worker/types"; import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; import { ExpressWrappedScoreImportMain } from "#lib/score-import/framework/express-wrapper"; -import { MakeScoreImport } from "#lib/score-import/framework/score-import"; +import { EnqueueScoreImportJob } from "#lib/score-import/worker/enqueue-pg"; import { ServerConfig } from "#lib/setup/config"; import { RequirePermissions } from "#server/middleware/auth"; import { ScoreImportRateLimiter } from "#server/middleware/rate-limiter"; @@ -35,7 +35,7 @@ router.post( }; // Fire the score import, but make no guarantees about its state. - void MakeScoreImport(job); + void EnqueueScoreImportJob(job); return res.status(202).json({ success: true, diff --git a/typescript/server/src/services/pg/seeds.ts b/typescript/server/src/services/pg/seeds.ts index 8f86d693c..82dadffda 100644 --- a/typescript/server/src/services/pg/seeds.ts +++ b/typescript/server/src/services/pg/seeds.ts @@ -1,9 +1,12 @@ import { + ALL_GAMES, type BMSCourseDocument, type ChartDocument, computeFolderSlug, type FolderDocument, type GoalDocument, + LEGACY_GameGroupPTToGame, + type LEGACY_Playtype, type QuestDocument, type QuestlineDocument, type SeedFolderRow, @@ -65,6 +68,24 @@ type SeedTable = { legacyTableID?: string; } & Omit; +/** + * Maps `goals.json` / `quests.json` / `questlines.json` fields where `game` may be a + * legacy {@link GameGroup} (e.g. "iidx") to the Postgres `game` enum (e.g. "iidx-sp"). + */ +function seedJsonGameToPg(game: string, playtype: string | undefined, label: string): PgGame { + if ((ALL_GAMES as readonly string[]).includes(game)) { + return game as PgGame; + } + + if (playtype === undefined) { + throw new Error( + `[seeds] ${label}: missing playtype for legacy game group ${JSON.stringify(game)}`, + ); + } + + return LEGACY_GameGroupPTToGame(game as GameGroup, playtype as LEGACY_Playtype) as PgGame; +} + const INSERT_CHUNK = 500; /** Resolves chart version filter from seed `versionFilter`. */ @@ -557,13 +578,21 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi console.log("[goal]"); const goals = readCollection("goals.json"); - const goalRows: Array = goals.map((g) => ({ - id: g.goalID, - game: g.game, - name: g.name, - charts: JSON.stringify(g.charts), - criteria: JSON.stringify(g.criteria), - })); + const goalRows: Array = goals.map((g) => { + const gWithPlaytype = g as { playtype?: string } & GoalDocument; + + return { + id: g.goalID, + game: seedJsonGameToPg( + gWithPlaytype.game, + gWithPlaytype.playtype, + `goal ${g.goalID}`, + ), + name: g.name, + charts: JSON.stringify(g.charts), + criteria: JSON.stringify(g.criteria), + }; + }); // Goals are never updated once created — only new ones are inserted. await batchIgnorePg(pg, "goal", goalRows); @@ -575,13 +604,21 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi console.log("[quest]"); const quests = readCollection("quests.json"); - const questRows: Array = quests.map((q) => ({ - id: q.questID, - game: q.game, - name: q.name, - description: q.desc, - quest_data: JSON.stringify(q.questData), - })); + const questRows: Array = quests.map((q) => { + const qWithPlaytype = q as { playtype?: string } & QuestDocument; + + return { + id: q.questID, + game: seedJsonGameToPg( + qWithPlaytype.game, + qWithPlaytype.playtype, + `quest ${q.questID}`, + ), + name: q.name, + description: q.desc, + quest_data: JSON.stringify(q.questData), + }; + }); for (let i = 0; i < questRows.length; i = i + INSERT_CHUNK) { const chunk = questRows.slice(i, i + INSERT_CHUNK); @@ -607,12 +644,20 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi console.log("[questline / questline_quest]"); const questlines = readCollection("questlines.json"); - const qlRows: Array = questlines.map((ql) => ({ - id: ql.questlineID, - game: ql.game, - name: ql.name, - description: ql.desc, - })); + const qlRows: Array = questlines.map((ql) => { + const qlWithPlaytype = ql as { playtype?: string } & QuestlineDocument; + + return { + id: ql.questlineID, + game: seedJsonGameToPg( + qlWithPlaytype.game, + qlWithPlaytype.playtype, + `questline ${ql.questlineID}`, + ), + name: ql.name, + description: ql.desc, + }; + }); for (let i = 0; i < qlRows.length; i = i + INSERT_CHUNK) { const chunk = qlRows.slice(i, i + INSERT_CHUNK); diff --git a/typescript/server/src/test-utils/test-data.ts b/typescript/server/src/test-utils/test-data.ts index 35e5992c0..c3a678eab 100644 --- a/typescript/server/src/test-utils/test-data.ts +++ b/typescript/server/src/test-utils/test-data.ts @@ -70,8 +70,11 @@ export const TestingIIDXSPScorePB: PBScoreDocument<"iidx-sp"> = { chartID: "c2311194e3897ddb5745b1760d2c0141f933e683", userID: 1, calculatedData: { - ktLampRating: 0, BPI: 10.1, + ktLampRating: 0, + ktLampRatingNC: 0, + ktLampRatingHC: 0, + ktLampRatingEXHC: 0, }, composedFrom: [{ name: "Best Score", scoreID: "TESTING_SCORE_ID" }], highlight: false, @@ -197,7 +200,11 @@ export const TestingIIDXSPScore: ScoreDocument<"iidx-sp"> = { }, scoreMeta: {}, calculatedData: { - ktLampRating: 5, + BPI: null, + ktLampRating: 10, + ktLampRatingNC: 10, + ktLampRatingHC: 0, + ktLampRatingEXHC: 0, }, timeAchieved: 1619454485988, songID: "s1", diff --git a/typescript/server/src/utils/class.test.ts b/typescript/server/src/utils/class.test.ts index 8cf42c527..a930ffdcc 100644 --- a/typescript/server/src/utils/class.test.ts +++ b/typescript/server/src/utils/class.test.ts @@ -71,7 +71,7 @@ describe("UpdateClassIfGreater (Postgres)", () => { expect(ach.class_prev_value).toBe("KYU_7"); }); - it("creates game_profile and game_settings when none exist (first class)", async () => { + it("creates game_profile with preference defaults when none exist (first class)", async () => { const { id } = await seedUser({ username: `cls_new_${Date.now()}` }); const result = await UpdateClassIfGreater(id, "iidx-sp", "dan", "DAN_1"); @@ -85,16 +85,11 @@ describe("UpdateClassIfGreater (Postgres)", () => { expect(asClassesJson(profile.classes).dan).toBe("DAN_1"); - const settings = await DB.selectFrom("game_settings") - .selectAll() - .where("user_id", "=", id) - .where("game", "=", "iidx-sp") - .executeTakeFirstOrThrow(); - + const dataRaw = profile.data; const data = - typeof settings.data === "string" - ? (JSON.parse(settings.data) as { bpiTarget?: number; display2DXTra?: boolean }) - : (settings.data as { bpiTarget?: number; display2DXTra?: boolean }); + typeof dataRaw === "string" + ? (JSON.parse(dataRaw) as { bpiTarget?: number; display2DXTra?: boolean }) + : (dataRaw as { bpiTarget?: number; display2DXTra?: boolean }); expect(data.display2DXTra).toBe(false); expect(data.bpiTarget).toBe(0); }); diff --git a/typescript/server/src/utils/class.ts b/typescript/server/src/utils/class.ts index 3ef9b5a90..5be6f0a59 100644 --- a/typescript/server/src/utils/class.ts +++ b/typescript/server/src/utils/class.ts @@ -1,4 +1,4 @@ -import { CreateGameSettings } from "#lib/game-settings/create-game-settings"; +import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings"; import { log } from "#lib/log/log"; import { EmitWebhookEvent } from "#lib/webhooks/webhooks"; import DB from "#services/pg/db"; @@ -150,12 +150,11 @@ export async function UpdateClassIfGreater( game, ratings: JSON.stringify({}), user_id: userID, + ...newGameProfilePreferenceColumns(game), }) .execute(); log.info(`Created new player gamestats for ${userID} (${game})`); - - await CreateGameSettings(userID, game); } const prevForAchievement = diff --git a/typescript/server/src/utils/queries/sessions.ts b/typescript/server/src/utils/queries/sessions.ts index 522260836..6242e2e53 100644 --- a/typescript/server/src/utils/queries/sessions.ts +++ b/typescript/server/src/utils/queries/sessions.ts @@ -1,6 +1,7 @@ import { SELECT_CHART, ToChartDocument } from "#lib/db-formats/chart"; import { SELECT_SCORE_DOCUMENT, ToScoreDocument } from "#lib/db-formats/score"; import { SELECT_SONG_DOCUMENT, ToSongDocument } from "#lib/db-formats/song"; +import { GetSessionScoreInfo } from "#lib/score-import/framework/sessions/sessions"; import DB from "#services/pg/db"; import { GetUserWithIDGuaranteed } from "#utils/user"; import _ from "lodash"; @@ -70,10 +71,7 @@ export async function GetSessionData(session: SessionDocument): Promise<{ let scores = rows.map(ToScoreDocument); scores = _.uniqBy(scores, "scoreID"); - // TODO: Hard to implement efficiently - // need to get the PB for this chart BEFORE the - // given time T - const scoreInfo: Array = []; + const scoreInfo = await GetSessionScoreInfo(session); return { charts, diff --git a/typescript/server/src/utils/reset-state/destroy-user-game-profile.ts b/typescript/server/src/utils/reset-state/destroy-user-game-profile.ts index b15948505..a5774fd39 100644 --- a/typescript/server/src/utils/reset-state/destroy-user-game-profile.ts +++ b/typescript/server/src/utils/reset-state/destroy-user-game-profile.ts @@ -1,38 +1,25 @@ -import type { Game } from "tachi-db"; - import { log } from "#lib/log/log"; import { RecalculatePbsForChartsFromPostgresScores } from "#lib/score-import/framework/pb/process-pbs"; import DB from "#services/pg/db"; import { sql } from "kysely"; -import { - type GameGroup, - type integer, - LEGACY_GameGroupPTToGame, - type LEGACY_Playtype, -} from "tachi-common"; +import { GameToGameGroup, type integer, type V3Game } from "tachi-common"; /** * Completely resets a user's game profile. * * This function is dangerous! Should only be ran by admins. */ -export default async function DestroyUserGameProfile( - userID: integer, - game: GameGroup, - playtype: LEGACY_Playtype, -) { - const v3Game = LEGACY_GameGroupPTToGame(game, playtype) as Game; - +export default async function DestroyUserGameProfile(userID: integer, game: V3Game) { await DB.deleteFrom("game_stats_snapshot") .where("user_id", "=", userID) - .where("game", "=", v3Game) + .where("game", "=", game) .execute(); const chartRows = await DB.selectFrom("pb") .innerJoin("chart", "chart.id", "pb.chart_id") .select("chart.id") .where("pb.user_id", "=", userID) - .where("chart.game", "=", v3Game) + .where("chart.game", "=", game) .where("pb.lens", "is", null) .execute(); @@ -40,7 +27,7 @@ export default async function DestroyUserGameProfile( .innerJoin("chart", "chart.id", "score.chart_id") .select("chart.id") .where("score.user_id", "=", userID) - .where("chart.game", "=", v3Game) + .where("chart.game", "=", game) .execute(); const chartIDs = [ @@ -51,7 +38,7 @@ export default async function DestroyUserGameProfile( .innerJoin("chart", "chart.id", "score.chart_id") .select("score.id") .where("score.user_id", "=", userID) - .where("chart.game", "=", v3Game) + .where("chart.game", "=", game) .execute(); const scoreIds = scoreRows.map((r) => r.id); @@ -66,7 +53,7 @@ export default async function DestroyUserGameProfile( .innerJoin("chart", "chart.id", "pb.chart_id") .select("pb.row_id") .where("pb.user_id", "=", userID) - .where("chart.game", "=", v3Game) + .where("chart.game", "=", game) .execute(); const pbIds = pbRowIds.map((r) => r.row_id); @@ -78,49 +65,36 @@ export default async function DestroyUserGameProfile( } if (chartIDs.length > 0) { - await RecalculatePbsForChartsFromPostgresScores(v3Game, chartIDs, log); + await RecalculatePbsForChartsFromPostgresScores(game, chartIDs, log); } - await DB.deleteFrom("session") - .where("user_id", "=", userID) - .where("game", "=", v3Game) - .execute(); + await DB.deleteFrom("session").where("user_id", "=", userID).where("game", "=", game).execute(); await DB.deleteFrom("import_game") - .where("game", "=", v3Game) + .where("game", "=", game) .where("id", "in", (eb) => eb .selectFrom("import") .select("id") .where("user_id", "=", userID) - .where("game_group", "=", game), + .where("game_group", "=", GameToGameGroup(game)), ) .execute(); await sql` DELETE FROM import AS i WHERE i.user_id = ${userID} - AND i.game_group = ${game} + AND i.game_group = ${GameToGameGroup(game)} AND NOT EXISTS (SELECT 1 FROM import_game ig WHERE ig.id = i.id) `.execute(DB); - await DB.deleteFrom("game_settings_showcase") - .where("user_id", "=", userID) - .where("game", "=", v3Game) - .execute(); - await DB.deleteFrom("game_rival") .where("user_id", "=", userID) - .where("game", "=", v3Game) - .execute(); - - await DB.deleteFrom("game_settings") - .where("user_id", "=", userID) - .where("game", "=", v3Game) + .where("game", "=", game) .execute(); await DB.deleteFrom("game_profile") .where("user_id", "=", userID) - .where("game", "=", v3Game) + .where("game", "=", game) .execute(); } diff --git a/typescript/server/src/utils/user.ts b/typescript/server/src/utils/user.ts index e193753af..581abb093 100644 --- a/typescript/server/src/utils/user.ts +++ b/typescript/server/src/utils/user.ts @@ -212,6 +212,38 @@ export async function GetUsersRankingAndOutOf( }; } +/** + * 1-based leaderboard rank per user for a profile rating algorithm. Ties share the + * same rank; the next rank skips (same as {@link GetUsersRankingAndOutOf}). + */ +export async function GetLeaderboardRanksForUserIds( + game: V3Game, + alg: ProfileRatingAlgorithms[V3Game], + userIds: Array, +): Promise> { + if (userIds.length === 0) { + return new Map(); + } + const uniqueIds = [...new Set(userIds)]; + + const result = await sql<{ rank: string; user_id: number }>` + WITH ranked AS ( + SELECT + game_profile.user_id, + RANK() OVER ( + ORDER BY coalesce((game_profile.ratings::jsonb->>${sql.lit(alg)})::numeric, 0) DESC + ) AS rank + FROM game_profile + WHERE game_profile.game = ${game} + ) + SELECT user_id, rank + FROM ranked + WHERE user_id IN (${sql.join(uniqueIds.map((id) => sql`${id}`))}) + `.execute(DB); + + return new Map(result.rows.map((r) => [r.user_id, Number(r.rank)])); +} + const FIVE_MINUTES = 1000 * 60 * 5; /**