if it hurts to breathe (open a window) (#58)

* if it hurts to breathe (open a window)

* clocked it.

* must sleep tonight, sorry

* feat: leaderboard now shows #1 if you're #1 even in ties

.cursor/skills/db-formats/SKILL.md

@@ -128,5 +128,5 @@ const { clientSecret: _secret, ...publicDoc } = ToAPIClientDocument(row);
 | `api-token.ts` | `priv_api_token` | `APITokenDocument` |
 | `user.ts` | `account` | `UserDocument` |
 | `user-settings.ts` | `account_settings` | `UserSettingsDocument` |
-| `game-stats.ts` | `game_stats` | `UserGameStats` |
+| `game-profiles.ts` | `game_profile` | `UserGameStats` (ratings + classes); UGPT preferences and showcase JSON also live on `game_profile` |
 | `kshook-sv6c-settings.ts` | `svc_kshook_sv6c_settings` | `KsHookSettingsDocument` |

.scripts/ts_autoinherit.js

@@ -10,7 +10,9 @@ process.chdir(cwd);
 const rootPkg = JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8"));
 
 for (const workspacePattern of rootPkg.workspaces) {
-	let glob = new Glob(workspacePattern + "*/package.json");
+	// "typescript/*" + "/package.json" => "typescript/*/package.json" (one workspace dir only).
+	// "typescript/*" + "*/package.json" would merge *+* into ** and match nested paths / node_modules.
+	let glob = new Glob(workspacePattern + "/package.json");
 
 	for (const pkgPath of glob.scanSync(".")) {
 		console.log(pkgPath);

Justfile-apps

@@ -1,9 +1,9 @@
 start:
 	#!/bin/bash
+	set -euo pipefail
 	ARGS=(server client seeds-webui)
 
 	echo "Starting: ${ARGS[@]}"
-	set -e
 	pids=()
 	for target in ${ARGS[@]}; do
 		(just "$target") &
@@ -14,10 +14,33 @@ start:
 
 
 server:
-		cd typescript/server && \
-		VERSION=$(jq -r .version package.json) \
-		COMMIT_HASH=$(git rev-parse --short HEAD) \
-		bun --watch src/main.ts
+	#!/bin/bash
+	set -euo pipefail
+	cd typescript/server
+	VERSION="${VERSION:-"$(jq -r .version package.json)"}"
+	COMMIT_HASH="${COMMIT_HASH:-"$(git rev-parse --short HEAD)"}"
+	export VERSION COMMIT_HASH
+	sidecar_pids=()
+	# Crons: Postgres-backed scheduler. Jobs: `job_queue` (SKIP LOCKED; run multiple for load).
+	if [[ -z "${TACHI_SERVER_NO_SIDECARS:-}" ]]; then
+		bun run cron-worker & sidecar_pids+=($!)
+		n="${TACHI_SERVER_JOB_WORKER_COUNT:-1}"
+		[[ "$n" =~ ^[0-9]+$ ]] && [[ "$n" -ge 1 ]] || n=1
+		for _ in $(seq 1 "$n"); do
+			bun run job-queue-worker & sidecar_pids+=($!)
+		done
+	fi
+	_cleanup_sidecars() {
+		trap - INT TERM EXIT
+		local s=$?
+		if ((${#sidecar_pids[@]})); then
+			kill -TERM "${sidecar_pids[@]}" 2>/dev/null || true
+			wait "${sidecar_pids[@]}" 2>/dev/null || true
+		fi
+		exit "$s"
+	}
+	trap _cleanup_sidecars INT TERM EXIT
+	bun --watch src/main.ts
 
 client:
 	cd typescript/client && bun vite

Justfile-db

@@ -7,6 +7,10 @@ db POSTGRES_DB=DEFAULT_DB:
 
 db-cheap-reset POSTGRES_DB=DEFAULT_DB:
 	#!/bin/bash
+	set -euo pipefail
+	# Disconnect all sessions using this DB so DROP DATABASE can succeed.
+	psql "{{POSTGRES_URL}}/postgres" -v ON_ERROR_STOP=1 -c \
+		"SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '{{POSTGRES_DB}}'"
 	export POSTGRES_URL="{{POSTGRES_URL}}/{{POSTGRES_DB}}"
 	tachidb database drop
 	tachidb database create
@@ -15,6 +19,7 @@ db-cheap-reset POSTGRES_DB=DEFAULT_DB:
 # Drop and recreate the database so migrations can be re-applied from scratch.
 db-reset POSTGRES_DB=DEFAULT_DB:
 	#!/bin/bash
+	set -euo pipefail
 
 	just db-cheap-reset "{{POSTGRES_DB}}"
 	just db-load-seeds "{{POSTGRES_DB}}"

Justfile-test

@@ -85,6 +85,7 @@ test-parity suite="":
 # Workers clone from this template rather than re-running migrations each time.
 bot-db-test-template-reset:
 	#!/bin/bash
+	set -euo pipefail
 	export POSTGRES_URL="{{POSTGRES_URL}}/tachi_bot_test_template"
 	tachidb database drop || true
 	tachidb database create
@@ -94,9 +95,20 @@ bot-db-test-template-reset:
 # Workers clone from this template rather than re-running migrations each time.
 server-db-test-template-reset:
 	#!/bin/bash
+	set -euo pipefail
 	export POSTGRES_URL="{{POSTGRES_URL}}/tachi_server_test_template"
 	tachidb database drop || true
 	tachidb database create
 	tachidb migrate run
 
-# <== End of parallel db test justscripts
+# <== End of parallel db test justscripts
+
+# Score-import HTTP load tests against a live server (multipart /api/v1/import/file).
+# See typescript/server/src/load-tests/README.md
+load-test-score-import *ARGS:
+	cd typescript/server && bun run load-test:score-import -- {{ARGS}}
+
+# Seed N dev users + API tokens (submit_score); writes one token per line to OUTPUT_FILE.
+# Uses the server package env (.env); same Postgres as a local tachi-server.
+load-test-score-import-seed-tokens COUNT OUTPUT_FILE:
+	cd typescript/server && bun run src/load-tests/seed-stress-api-tokens.ts {{COUNT}} {{OUTPUT_FILE}}

bun.lock

@@ -313,13 +313,13 @@
       "name": "tachi-seeds-webui",
       "version": "0.1.0-see-project-root",
       "dependencies": {
-        "@codemirror/lang-sql": "^6.10.0",
-        "@codemirror/language": "^6.12.3",
-        "@lezer/highlight": "^1.2.3",
-        "@sqlite.org/sqlite-wasm": "^3.51.2-build9",
-        "codemirror": "^6.0.2",
-        "comlink": "^4.4.2",
-        "fast-json-patch": "^3.1.1",
+        "@codemirror/lang-sql": "catalog:",
+        "@codemirror/language": "catalog:",
+        "@lezer/highlight": "catalog:",
+        "@sqlite.org/sqlite-wasm": "catalog:",
+        "codemirror": "catalog:",
+        "comlink": "catalog:",
+        "fast-json-patch": "catalog:",
         "nanoid": "catalog:",
         "natural-compare": "catalog:",
         "react": "catalog:",
@@ -371,6 +371,7 @@
         "bunyan": "catalog:",
         "commander": "catalog:",
         "connect-redis": "catalog:",
+        "cron-parser": "catalog:",
         "csv-parse": "catalog:",
         "deepmerge": "catalog:",
         "dotenv": "catalog:",
@@ -452,6 +453,8 @@
   },
   "catalog": {
     "@aws-sdk/client-s3": "3.49.0",
+    "@codemirror/lang-sql": "^6.10.0",
+    "@codemirror/language": "^6.12.3",
     "@discordjs/builders": "0.5.0",
     "@discordjs/rest": "0.1.0-canary.0",
     "@eslint/js": "^9.20.0",
@@ -459,11 +462,13 @@
     "@fullcalendar/core": "^6.0.0",
     "@fullcalendar/daygrid": "^6.0.0",
     "@fullcalendar/react": "^6.0.0",
+    "@lezer/highlight": "^1.2.3",
     "@nivo/bar": "0.79.1",
     "@nivo/core": "0.79.0",
     "@nivo/line": "0.79.1",
     "@octokit/app": "15.1.0",
     "@octokit/webhooks-types": "7.5.1",
+    "@sqlite.org/sqlite-wasm": "^3.51.2-build9",
     "@types/bcryptjs": "^2.4.2",
     "@types/bunyan": "^1.8.8",
     "@types/connect-redis": "0.0.18",
@@ -509,8 +514,11 @@
     "bunyan": "1.8.15",
     "chalk": "4",
     "cheerio": "^1.0.0-rc.12",
+    "codemirror": "^6.0.2",
+    "comlink": "^4.4.2",
     "commander": "^8.2.0",
     "connect-redis": "6.1.1",
+    "cron-parser": "5.0.2",
     "csv-parse": "^5.0.3",
     "deepmerge": "^4.2.2",
     "discord-api-types": "0.22.0",
@@ -531,6 +539,7 @@
     "express-prom-bundle": "^7.0.0",
     "express-rate-limit": "5.5.1",
     "express-session": "1.17.2",
+    "fast-json-patch": "^3.1.1",
     "fast-json-stable-hash": "^1.0.3",
     "fast-xml-parser": "^4.2.5",
     "formik": "2.2.9",
@@ -1688,7 +1697,7 @@
 
     "crelt": ["crelt@1.0.6", "", {}, "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g=="],
 
-    "cron-parser": ["cron-parser@2.18.0", "", { "dependencies": { "is-nan": "^1.3.0", "moment-timezone": "^0.5.31" } }, "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg=="],
+    "cron-parser": ["cron-parser@5.0.2", "", { "dependencies": { "luxon": "^3.5.0" } }, "sha512-RXXr5WuvLInay/DstwRD087/DpLOm24Y3mrkRmOdrMjbdGfQyPiMsx1Ad+SaJF2zN0tN78ckBjcEkIHVd+MX7Q=="],
 
     "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
 
@@ -3258,12 +3267,16 @@
 
     "broadcast-channel/rimraf": ["rimraf@3.0.2", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "bin.js" } }, "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA=="],
 
+    "bullmq/cron-parser": ["cron-parser@2.18.0", "", { "dependencies": { "is-nan": "^1.3.0", "moment-timezone": "^0.5.31" } }, "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg=="],
+
     "bullmq/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
 
     "bullmq/tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="],
 
     "cheerio-select/css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
 
+    "cron-parser/luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="],
+
     "css-select/domhandler": ["domhandler@4.3.1", "", { "dependencies": { "domelementtype": "^2.2.0" } }, "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ=="],
 
     "css-select/domutils": ["domutils@2.8.0", "", { "dependencies": { "dom-serializer": "^1.0.1", "domelementtype": "^2.2.0", "domhandler": "^4.2.0" } }, "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A=="],

db/migrations/20260301154256_genesis.sql

@@ -541,21 +541,6 @@ CREATE TABLE "folder_chart_lookup" (
 CREATE INDEX ON "folder_chart_lookup" (folder_id);
 CREATE INDEX ON "folder_chart_lookup" (chart_id);
 
-CREATE TABLE "game_settings" (
-	user_id BIGINT REFERENCES account(id) NOT NULL,
-	game GAME NOT NULL,
-	PRIMARY KEY (user_id, game),
-
-	pf_preferred_score_alg TEXT,
-	pf_preferred_session_alg TEXT,
-	pf_preferred_profile_alg TEXT,
-	pf_preferred_default_enum TEXT,
-	pf_default_table TEXT,
-	pf_preferred_ranking TEXT CHECK (pf_preferred_ranking IN ('global', 'rival')),
-
-	data JSONB NOT NULL -- game specific payload
-);
-
 CREATE TABLE "game_rival" (
 	user_id BIGINT REFERENCES account(id) NOT NULL,
 	game GAME NOT NULL,
@@ -566,17 +551,7 @@ CREATE TABLE "game_rival" (
 	CHECK (user_id != rival)
 );
 
-CREATE TABLE "game_settings_showcase" (
-	user_id BIGINT REFERENCES account(id) NOT NULL,
-	game GAME NOT NULL,
-
-	PRIMARY KEY (user_id, game),
-
-	-- kinda ridiculous but { mode: "chart", etc. } or
-	-- { mode: "folder" }
-	data JSONB NOT NULL
-);
-
+-- Per-user per-game: ratings/classes (import-derived) + UGPT preferences + showcase JSON (Tachi3 #47).
 CREATE TABLE "game_profile" (
 	user_id BIGINT REFERENCES account(id) NOT NULL,
 	game GAME NOT NULL,
@@ -584,7 +559,20 @@ CREATE TABLE "game_profile" (
 	PRIMARY KEY (user_id, game),
 
 	ratings JSONB NOT NULL,
-	classes JSONB NOT NULL
+	classes JSONB NOT NULL,
+
+	pf_preferred_score_alg TEXT,
+	pf_preferred_session_alg TEXT,
+	pf_preferred_profile_alg TEXT,
+	pf_preferred_default_enum TEXT,
+	pf_default_table TEXT,
+	pf_preferred_ranking TEXT,
+
+	data JSONB NOT NULL DEFAULT '{}'::jsonb,
+	showcase JSONB NOT NULL DEFAULT '[]'::jsonb,
+
+	CONSTRAINT game_profile_pf_preferred_ranking_check
+		CHECK (pf_preferred_ranking IS NULL OR pf_preferred_ranking IN ('global', 'rival'))
 );
 
 CREATE TABLE "game_stats_snapshot" (
@@ -770,6 +758,20 @@ CREATE TABLE "score_rederive" (
 	enqueued_at TIMESTAMPTZ NOT NULL DEFAULT now()
 );
 
+-- Session aggregate `calculated_data` must be refreshed when committed scores in that session change.
+CREATE TABLE session_dirty (
+	session_id TEXT NOT NULL PRIMARY KEY,
+	enqueued_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Per-playtype game stats (`game_profile`) must be refreshed when committed scores change (ratings use PBs).
+CREATE TABLE game_profile_dirty (
+	user_id BIGINT NOT NULL,
+	game GAME NOT NULL,
+	enqueued_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+	PRIMARY KEY (user_id, game)
+);
+
 -- Trigger function: on any score INSERT/UPDATE/DELETE, mark the (user_id, chart_id)
 -- pair as needing PB recalculation.
 CREATE FUNCTION enqueue_pb_dirty() RETURNS trigger AS $$
@@ -791,6 +793,72 @@ CREATE TRIGGER "score_pb_dirty"
 	AFTER INSERT OR UPDATE OR DELETE ON score
 	FOR EACH ROW EXECUTE FUNCTION enqueue_pb_dirty();
 
+-- Staging scores (committed = false) skip these queues until commit.
+CREATE FUNCTION enqueue_session_dirty() RETURNS trigger AS $$
+BEGIN
+	IF TG_OP = 'DELETE' THEN
+		IF OLD.committed AND OLD.session_id IS NOT NULL THEN
+			INSERT INTO session_dirty (session_id)
+			VALUES (OLD.session_id)
+			ON CONFLICT DO NOTHING;
+		END IF;
+	ELSIF TG_OP = 'UPDATE' THEN
+		IF NEW.committed THEN
+			IF OLD.session_id IS NOT NULL AND OLD.session_id IS DISTINCT FROM NEW.session_id THEN
+				INSERT INTO session_dirty (session_id)
+				VALUES (OLD.session_id)
+				ON CONFLICT DO NOTHING;
+			END IF;
+			IF NEW.session_id IS NOT NULL THEN
+				INSERT INTO session_dirty (session_id)
+				VALUES (NEW.session_id)
+				ON CONFLICT DO NOTHING;
+			END IF;
+		END IF;
+	ELSE
+		IF NEW.committed AND NEW.session_id IS NOT NULL THEN
+			INSERT INTO session_dirty (session_id)
+			VALUES (NEW.session_id)
+			ON CONFLICT DO NOTHING;
+		END IF;
+	END IF;
+	RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER "score_session_dirty"
+	AFTER INSERT OR UPDATE OR DELETE ON score
+	FOR EACH ROW EXECUTE FUNCTION enqueue_session_dirty();
+
+CREATE FUNCTION enqueue_game_profile_dirty() RETURNS trigger AS $$
+BEGIN
+	IF TG_OP = 'DELETE' THEN
+		IF OLD.committed THEN
+			INSERT INTO game_profile_dirty (user_id, game)
+			VALUES (OLD.user_id, OLD.game)
+			ON CONFLICT DO NOTHING;
+		END IF;
+	ELSIF TG_OP = 'UPDATE' THEN
+		IF NEW.committed THEN
+			INSERT INTO game_profile_dirty (user_id, game)
+			VALUES (NEW.user_id, NEW.game)
+			ON CONFLICT DO NOTHING;
+		END IF;
+	ELSE
+		IF NEW.committed THEN
+			INSERT INTO game_profile_dirty (user_id, game)
+			VALUES (NEW.user_id, NEW.game)
+			ON CONFLICT DO NOTHING;
+		END IF;
+	END IF;
+	RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER "score_game_profile_dirty"
+	AFTER INSERT OR UPDATE OR DELETE ON score
+	FOR EACH ROW EXECUTE FUNCTION enqueue_game_profile_dirty();
+
 -- Trigger function: on chart UPDATE, if derivation_checksum changed, enqueue the chart
 -- for score re-derivation.
 CREATE FUNCTION enqueue_score_rederive() RETURNS trigger AS $$

db/migrations/20260422130000_grant_grafana_ro_public_read.sql

@@ -0,0 +1,17 @@
+-- Grafana local-dev role (see dev/postgres-init.sql): init runs before migrations, so
+-- GRANT SELECT ON ALL TABLES saw no tables; default privileges can also miss edge cases.
+-- Re-apply read access to all existing objects; no-op when grafana_ro is absent.
+DO $grant$
+BEGIN
+	IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'grafana_ro') THEN
+		EXECUTE 'GRANT USAGE ON SCHEMA public TO grafana_ro';
+		EXECUTE 'GRANT SELECT ON ALL TABLES IN SCHEMA public TO grafana_ro';
+		EXECUTE 'GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO grafana_ro';
+	END IF;
+	IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'grafana_ro')
+		AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tachi') THEN
+		EXECUTE 'ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON TABLES TO grafana_ro';
+		EXECUTE 'ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON SEQUENCES TO grafana_ro';
+	END IF;
+END
+$grant$;

db/migrations/20260423120000_job_queue_dequeue_index.sql

@@ -0,0 +1,4 @@
+-- Fast dequeue of queued (status = 0) jobs ordered by when they are due.
+CREATE INDEX job_queue_dequeue_idx
+ON job_queue (scheduled_for ASC, created_at ASC)
+WHERE status = 0;

dev/functions.fish

@@ -54,9 +54,15 @@ function fish_greeting
 		echo $(rgb "This terminal is set up inside a Linux Machine!" ffffff 000000)
 		echo $(rgb "This machine comes pre-installed with Tachi and helpful tools." ffffff 000000)
 		echo ""
-		echo "Type $(cmd "just start") to start up a frontend and backend."
-		echo "    $(rgb "The server will start on http://localhost:3000." ffff00 000000)"
-		echo "    $(rgb "Use Ctrl+C to stop the server." ffff00 000000)"
+		echo "Type $(cmd "just start") to start up tachi."
+		echo "    $(rgb "The site will start on http://localhost:3000." ffff00 000000)"
+		echo "    $(rgb "The seeds web UI will start on http://localhost:3100." ffff00 000000)"
+		echo "    $(rgb "Use Ctrl+C to stop Tachi." ffff00 000000)"
+		echo ""
+		echo "You can also run:"
+		echo "    $(cmd "just grafana") to view the Grafana dashboard."
+		echo "    $(cmd "just mailpit") to view emails that have been sent by Tachi."
+		echo "    $(cmd "just seeds-webui") to view the seeds web UI."
 		echo ""
 		echo "Type $(cmd "seeds") to run seeds scripts."
 		echo "    $(rgb "Create new script files in typescript/seeds-scripts." ffff00 000000)"

dev/observability/grafana/provisioning/datasources/datasources.yml

@@ -12,9 +12,9 @@ datasources:
     type: postgres
     access: proxy
     url: tachi-postgres:5432
-    user: tachi
+    user: grafana_ro
     secureJsonData:
-      password: tachi
+      password: grafana_ro
     jsonData:
       database: tachi_dev
       sslmode: disable

dev/observability/setup-datasources.sh

@@ -2,8 +2,8 @@
 set -euo pipefail
 
 GRAFANA_URL="${GRAFANA_URL:-http://tachi-grafana:3000}"
-GRAFANA_USER="${GRAFANA_USER:-admin}"
-GRAFANA_PASS="${GRAFANA_PASS:-admin}"
+GRAFANA_USER="${GRAFANA_USER:-tachi}"
+GRAFANA_PASS="${GRAFANA_PASS:-tachi}"
 
 PROM_URL="${PROM_URL:-http://tachi-prometheus:9090}"
 ALLOY_URL="${ALLOY_URL:-http://tachi-alloy:12345}"

dev/postgres-init.sql

@@ -15,4 +15,6 @@ END$$;
 GRANT CONNECT ON DATABASE tachi_dev TO grafana_ro;
 GRANT USAGE ON SCHEMA public TO grafana_ro;
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO grafana_ro;
+GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO grafana_ro;
 ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON TABLES TO grafana_ro;
+ALTER DEFAULT PRIVILEGES FOR ROLE tachi IN SCHEMA public GRANT SELECT ON SEQUENCES TO grafana_ro;

docker-compose-dev.yml

@@ -106,8 +106,8 @@ services:
     ports:
       - "3005:3000"
     environment:
-      GF_SECURITY_ADMIN_USER: admin
-      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_SECURITY_ADMIN_USER: tachi
+      GF_SECURITY_ADMIN_PASSWORD: tachi
       GF_USERS_ALLOW_SIGN_UP: "false"
       GF_SERVER_HTTP_ADDR: 0.0.0.0
       GF_SERVER_HTTP_PORT: "3000"

docs/docs/api/routes/admin.md

@@ -65,14 +65,11 @@ Empty Object.
 
 *****
 
-## Resynchronise all PBs that match the given query or users.
+## Re-run PB processing for every scored user+chart (synchronous).
 
-`POST /api/v1/admin/resync-pbs`
+`POST /api/v1/admin/recalc-pbs`
 
-!!! info
-	This is intended to be used in the case that PBs fall out of
-	sync with what they should be. This could be due to a
-	database migration going awry, or anything else.
+Inserts every distinct `(user_id, chart_id)` from the **`score`** table into **`pb_dirty`**, then **drains** `pb_dirty` and downstream **`session_dirty`** / **`game_profile_dirty`** queues until nothing remains (same batching as the background worker, but the HTTP request waits until idle). Intended when PBs may be out of sync (e.g. after a bad migration). There is **no request body** and no filter—always all distinct pairs that appear on scores.
 
 ### Permissions
 
@@ -80,34 +77,29 @@ Empty Object.
 
 ### Parameters
 
-| Property | Type | Description |
-| :: | :: | :: |
-| `userIDs` | Array<Integer> (Optional) | The list of userIDs to resynchronise PBs for. |
-| `filters` | Mongo Query for the PBs Collection. (Optional) | A query to reduce the amount of PBs that get reprocessed. |
+None (send `{}` if your client requires a body).
 
 ### Response
 
-Empty Object.
+Empty object (standard success wrapper with `body`).
 
 ### Example
 
 #### Request
+
 ```
-POST /api/v1/admin/resync-pbs
+POST /api/v1/admin/recalc-pbs
 ```
 
 ```js
-{
-	"filter": {
-		"game": {$in: ["iidx","sdvx"]}
-	},
-	"userIDs": [1,2,3,4]
-}
+{}
 ```
 
 #### Response
 
-Nothing.
+```js
+{}
+```
 
 *****
 
@@ -195,43 +187,38 @@ Empty Object.
 
 *****
 
-## Perform a site recalc on this set of scores.
+## Re-derive all scores site-wide (synchronous).
 
 `POST /api/v1/admin/recalc`
 
+Enqueues **every chart** into **`score_rederive`**, then **drains** `score_rederive` and downstream **`pb_dirty`**, **`session_dirty`**, and **`game_profile_dirty`** queues until nothing remains (the request waits until idle). Re-runs `scoreDeriver` and `scoreCalcs` for every score. There is **no request body** and no filter—always all charts.
+
 ### Permissions
 
 - Admin
 
 ### Parameters
 
-| Property | Type | Description |
-| :: | :: | :: |
-| `<body>` | Mongo Query for scores | Filters the amount of scores recalced. If not provided, defaults to every score on the site. |
+None (send an empty JSON object `{}` if your client requires a body).
 
 ### Response
 
-| Property | Type | Description |
-| :: | :: | :: |
-| `scoresRecalced` | Integer | The amount of scores recalced. |
+Empty object (standard success wrapper with `body`).
 
 ### Example
 
 #### Request
+
 ```
 POST /api/v1/admin/recalc
 ```
+
 ```js
-{
-	"game": "iidx",
-	"scoreData.percent": {$gt: 90},
-}
+{}
 ```
 
 #### Response
 
 ```js
-{
-	"scoresRecalced": 174
-}
+{}
 ```

docs/docs/api/routes/import.md

@@ -242,6 +242,36 @@ Returns rows from `orphan_score` for the authenticated user (scores that failed
 
 *****
 
+## Get one orphaned score (full payload)
+
+`GET /api/v1/import/orphans/:orphanID`
+
+Returns a single `orphan_score` row for the authenticated user, including raw `data` and `context` JSON (for debugging unmatched imports). `404` if the row does not exist or belongs to another user.
+
+### Permissions
+
+- submit_score
+
+### Parameters
+
+| Parameter | Type | Description |
+| :: | :: | :: |
+| `orphanID` | Path | The orphan’s `orphanID` (same as in `GET /import/orphans` or import errors). |
+
+### Response
+
+| Property | Type | Description |
+| :: | :: | :: |
+| `orphanID` | String | Stable orphan identifier. |
+| `importType` | String | Import type that produced the orphan. |
+| `gameGroup` | String | Game group. |
+| `timeInserted` | Number | Unix time in ms when the row was stored. |
+| `message` | String or null | Stored error / context message. |
+| `data` | Object | Raw import datapoint (shape depends on `importType`). |
+| `context` | Object | Raw import context. |
+
+*****
+
 ## Delete one orphaned score
 
 `DELETE /api/v1/import/orphans/:orphanID`

docs/docs/api/routes/users.md

@@ -158,9 +158,9 @@ GET /api/v1/users/me IF authenticated as userID 1.
 
 ---
 
-## Retrieve user's statistics on all games.
+## Retrieve per-game profiles for a user.
 
-`GET /api/v1/users/:userID/game-stats`
+`GET /api/v1/users/:userID/game-profiles`
 
 ### Parameters
 
@@ -170,7 +170,7 @@ None.
 
 | Property |                         Type                         |                 Description                 |
 | :------: | :--------------------------------------------------: | :-----------------------------------------: |
-| `<body>` | Array&lt;UserGameStatsDocument & \_\_rankingData&gt; | The array of User Game Stats this user has. |
+| `<body>` | Array&lt;UserGameStatsDocument & \_\_rankingData&gt; | The array of per-game profile documents (ratings and classes) this user has. |
 
 !!! info
 For UI reasons, the UserGameStatsDocuments here have an additional `__rankingData` property, which contains leaderboard ranking information for this user.
@@ -182,7 +182,7 @@ For UI reasons, the UserGameStatsDocuments here have an additional `__rankingDat
 ```
 GET /api/v1/users/zkldime-stats
 OR
-GET /api/v1/users/1/game-stats
+GET /api/v1/users/1/game-profiles
 ```
 
 #### Response

package.json

@@ -198,7 +198,15 @@
 		"@types/tap": "^15.0.5",
 		"tap": "^15.1.6",
 		"@types/istanbul-lib-coverage": "^2.0.6",
-		"istanbul-lib-coverage": "^3.2.2"
+		"istanbul-lib-coverage": "^3.2.2",
+		"@codemirror/lang-sql": "^6.10.0",
+		"@codemirror/language": "^6.12.3",
+		"@lezer/highlight": "^1.2.3",
+		"@sqlite.org/sqlite-wasm": "^3.51.2-build9",
+		"codemirror": "^6.0.2",
+		"comlink": "^4.4.2",
+		"fast-json-patch": "^3.1.1",
+		"cron-parser": "5.0.2"
 	},
 	"overrides": {
 		"@types/mongodb": "3.6.20",

typescript/client/src/app/pages/admin/AdminOperationsPage.tsx

@@ -19,9 +19,6 @@ export default function AdminOperationsPage() {
 
 	const [folderId, setFolderId] = useState("");
 
-	const [resyncBody, setResyncBody] = useState("{}");
-	const [recalcBody, setRecalcBody] = useState("{}");
-
 	const [supporterUser, setSupporterUser] = useState("");
 
 	const announcementGameConfig = announcementGame ? GetGameGroupConfig(announcementGame) : null;
@@ -226,34 +223,22 @@ export default function AdminOperationsPage() {
 
 			<Col lg={6}>
 				<Card>
-					<Card.Header>Resync PBs</Card.Header>
+					<Card.Header>Recalc PBs</Card.Header>
 					<Card.Body>
-						<Form.Group className="mb-3" controlId="resync-json">
-							<Form.Label>JSON body</Form.Label>
-							<Form.Control
-								as="textarea"
-								onChange={(e) => setResyncBody(e.target.value)}
-								placeholder='{} or { "userIDs": [1, 2], "filter": { ... } }'
-								rows={5}
-								style={{ fontFamily: "monospace", fontSize: "0.85rem" }}
-								value={resyncBody}
-							/>
-						</Form.Group>
+						<p className="text-muted small mb-3">
+							Enqueues every distinct user+chart that has at least one score into
+							<code className="mx-1">pb_dirty</code>, then drains that queue and
+							downstream session/profile queues until idle (all games). This request
+							waits until processing finishes.
+						</p>
 						<Button
 							onClick={() => {
-								let parsed: unknown = {};
-								try {
-									parsed = JSON.parse(resyncBody) as unknown;
-								} catch {
-									alert("Invalid JSON.");
-									return;
-								}
 								void APIFetchV1(
-									`/admin/resync-pbs`,
+									`/admin/recalc-pbs`,
 									{
 										method: "POST",
 										headers: { "Content-Type": "application/json" },
-										body: JSON.stringify(parsed),
+										body: JSON.stringify({}),
 									},
 									true,
 									true,
@@ -261,7 +246,7 @@ export default function AdminOperationsPage() {
 							}}
 							variant="primary"
 						>
-							Resync PBs
+							Recalc all PBs
 						</Button>
 					</Card.Body>
 				</Card>
@@ -271,32 +256,19 @@ export default function AdminOperationsPage() {
 				<Card>
 					<Card.Header>Recalc scores</Card.Header>
 					<Card.Body>
-						<Form.Group className="mb-3" controlId="recalc-json">
-							<Form.Label>Mongo filter (JSON object)</Form.Label>
-							<Form.Control
-								as="textarea"
-								onChange={(e) => setRecalcBody(e.target.value)}
-								placeholder="{}"
-								rows={5}
-								style={{ fontFamily: "monospace", fontSize: "0.85rem" }}
-								value={recalcBody}
-							/>
-						</Form.Group>
+						<p className="text-muted small mb-3">
+							Enqueues every chart for full score re-derivation (all games), then
+							drains score and downstream queues until idle. This request waits until
+							processing finishes; can take a long time on large databases.
+						</p>
 						<Button
 							onClick={() => {
-								let parsed: unknown = {};
-								try {
-									parsed = JSON.parse(recalcBody) as unknown;
-								} catch {
-									alert("Invalid JSON.");
-									return;
-								}
 								void APIFetchV1(
 									`/admin/recalc`,
 									{
 										method: "POST",
 										headers: { "Content-Type": "application/json" },
-										body: JSON.stringify(parsed),
+										body: JSON.stringify({}),
 									},
 									true,
 									true,
@@ -304,7 +276,7 @@ export default function AdminOperationsPage() {
 							}}
 							variant="primary"
 						>
-							Recalc
+							Recalc all scores
 						</Button>
 					</Card.Body>
 				</Card>

typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTLeaderboardsPage.tsx

@@ -16,7 +16,11 @@ import { type UserLeaderboardReturns } from "#types/api-returns";
 import { type GamePT } from "#types/react";
 import { type UGSDataset } from "#types/tables";
 import { CreateUserMap } from "#util/data";
-import { FormatGPTProfileRating, FormatGPTProfileRatingName } from "#util/misc";
+import {
+	FormatGPTProfileRating,
+	FormatGPTProfileRatingName,
+	getProfileRatingAlgKeysInDisplayOrder,
+} from "#util/misc";
 import { NumericSOV, StrSOV } from "#util/sorts";
 import React, { useState } from "react";
 import { Col, Form, Row } from "react-bootstrap";
@@ -25,7 +29,6 @@ import {
 	type Classes,
 	FormatGame,
 	GameToGameGroup,
-	GetGameConfig,
 	GetGameGroupConfig,
 	type V3Game,
 } from "tachi-common";
@@ -64,16 +67,18 @@ export default function GPTLeaderboardsPage({ game }: GamePT) {
 }
 
 function ProfileLeaderboard({ game }: GamePT) {
-	const gameConfig = GetGameConfig(game);
-
 	const defaultAlg = useProfileRatingAlg(game);
 
 	const [alg, setAlg] = useState(defaultAlg);
 
+	const profileAlgKeys = getProfileRatingAlgKeysInDisplayOrder(
+		game,
+	) as Array<AnyProfileRatingAlg>;
+
 	const SelectComponent =
-		Object.keys(gameConfig.profileRatingAlgs).length > 1 ? (
+		profileAlgKeys.length > 1 ? (
 			<Form.Select onChange={(e) => setAlg(e.target.value as any)} value={alg}>
-				{Object.keys(gameConfig.profileRatingAlgs).map((e) => (
+				{profileAlgKeys.map((e) => (
 					<option key={e} value={e}>
 						{FormatGPTProfileRatingName(game, e)}
 					</option>
@@ -107,12 +112,12 @@ function ProfileLeaderboard({ game }: GamePT) {
 
 	const userDataset: UGSDataset = [];
 
-	for (const [index, gs] of data.gameStats.entries()) {
+	for (const gs of data.gameStats) {
 		userDataset.push({
 			...gs,
 			__related: {
 				user: userMap.get(gs.userID)!,
-				index,
+				index: gs.rank - 1,
 			},
 		});
 	}
@@ -125,11 +130,9 @@ function ProfileLeaderboard({ game }: GamePT) {
 				dataset={userDataset}
 				entryName="Rankers"
 				headers={[
-					["Ranking", "Rank", NumericSOV((x) => x.__related.index)],
+					["Ranking", "Rank", NumericSOV((x) => x.rank)],
 					["User", "User", StrSOV((x) => x.__related.user.username)],
-					...(
-						Object.keys(gameConfig.profileRatingAlgs) as Array<AnyProfileRatingAlg>
-					).map(
+					...profileAlgKeys.map(
 						(e) =>
 							[
 								FormatGPTProfileRatingName(game, e),
@@ -143,9 +146,7 @@ function ProfileLeaderboard({ game }: GamePT) {
 					<tr>
 						<IndexCell index={r.__related.index} />
 						<UserCell game={game} user={r.__related.user} />
-						{(
-							Object.keys(gameConfig.profileRatingAlgs) as Array<AnyProfileRatingAlg>
-						).map((e) => (
+						{profileAlgKeys.map((e) => (
 							<td key={e}>
 								{r.ratings[e]
 									? FormatGPTProfileRating(game, e, r.ratings[e]!)

typescript/client/src/app/pages/dashboard/users/UserOrphansPage.tsx

@@ -3,28 +3,38 @@ import TachiTable from "#components/tables/components/TachiTable";
 import Loading from "#components/util/Loading";
 import { APIFetchV1 } from "#util/api";
 import React, { useCallback, useEffect, useState } from "react";
-import { Alert, Button } from "react-bootstrap";
-import { GetGameGroupConfig, type GameGroup, type UserDocument } from "tachi-common";
+import { Alert, Button, Modal } from "react-bootstrap";
+import { type GameGroup, GetGameGroupConfig, type UserDocument } from "tachi-common";
 
 type OrphanListItem = {
+	gameGroup: string;
+	importType: string;
+	message: string | null;
 	orphanID: string;
 	rowID: string;
-	importType: string;
-	gameGroup: string;
-	timeInserted: number;
-	message: string | null;
 	summary: string | null;
+	timeInserted: number;
 };
 
-type ListBody = { orphans: OrphanListItem[]; hasMore: boolean };
+type ListBody = { hasMore: boolean; orphans: OrphanListItem[] };
 
 type ReprocessBody = {
+	failed: number;
 	processed: number;
 	removed: number;
-	failed: number;
 	success: number;
 };
 
+type OrphanDetailBody = {
+	context: unknown;
+	data: unknown;
+	gameGroup: string;
+	importType: string;
+	message: string | null;
+	orphanID: string;
+	timeInserted: number;
+};
+
 export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument }) {
 	useSetSubheader(
 		["Users", reqUser.username, "Orphan scores"],
@@ -38,6 +48,10 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument })
 	const [loadingMore, setLoadingMore] = useState(false);
 	const [reprocessBusy, setReprocessBusy] = useState(false);
 	const [lastMessage, setLastMessage] = useState<string | null>(null);
+	const [detailOpen, setDetailOpen] = useState(false);
+	const [detailLoading, setDetailLoading] = useState(false);
+	const [detailJson, setDetailJson] = useState<string | null>(null);
+	const [detailTitle, setDetailTitle] = useState<string>("");
 
 	const fetchPage = useCallback(async (afterRowID: string | undefined, append: boolean) => {
 		const params = new URLSearchParams({ limit: "50" });
@@ -107,6 +121,26 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument })
 		}
 	};
 
+	const onOpenDetail = async (orphanID: string) => {
+		setDetailTitle(orphanID);
+		setDetailJson(null);
+		setDetailOpen(true);
+		setDetailLoading(true);
+		const res = await APIFetchV1<OrphanDetailBody>(
+			`/import/orphans/${encodeURIComponent(orphanID)}`,
+			undefined,
+			false,
+			true,
+		);
+		setDetailLoading(false);
+		if (!res.success) {
+			setDetailJson(null);
+			return;
+		}
+		const { data, context, ...meta } = res.body;
+		setDetailJson(JSON.stringify({ ...meta, data, context }, null, 2));
+	};
+
 	const onDelete = async (orphanID: string) => {
 		if (!window.confirm(`Delete orphan ${orphanID}? This cannot be undone.`)) {
 			return;
@@ -128,24 +162,46 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument })
 			<div>
 				<h4>Orphan scores</h4>
 				<p className="mb-2">
-					When an import cannot match a song or chart (SongOrChartNotFound), Tachi still stores
-					that datapoint as an <strong>orphan</strong>. Orphans are retried automatically around{" "}
-					<strong>00:01 UTC</strong> each day, or you can run a full reprocess below.
+					When an import cannot match a song or chart (SongOrChartNotFound), Tachi still
+					stores that datapoint as an <strong>orphan</strong>. Orphans are retried
+					automatically around <strong>00:01 UTC</strong> each day, or you can run a full
+					reprocess below.
 				</p>
 				<p className="mb-0">
-					Deleting an orphan only removes that queued datapoint; it does not revert an entire
-					import.
+					Deleting an orphan only removes that queued datapoint; it does not revert an
+					entire import.
 				</p>
 			</div>
 
 			<div className="d-flex flex-wrap gap-2 align-items-center">
-				<Button disabled={reprocessBusy} onClick={() => void onReprocess()} variant="primary">
+				<Button
+					disabled={reprocessBusy}
+					onClick={() => void onReprocess()}
+					variant="primary"
+				>
 					{reprocessBusy ? "Reprocessing…" : "Reprocess all my orphans now"}
 				</Button>
 			</div>
 
 			{lastMessage && <Alert variant="info">{lastMessage}</Alert>}
 
+			<Modal onHide={() => setDetailOpen(false)} show={detailOpen} size="lg">
+				<Modal.Header closeButton>
+					<Modal.Title>Orphan details — {detailTitle}</Modal.Title>
+				</Modal.Header>
+				<Modal.Body>
+					{detailLoading ? (
+						<Loading />
+					) : detailJson ? (
+						<pre className="small mb-0" style={{ maxHeight: "70vh", overflow: "auto" }}>
+							{detailJson}
+						</pre>
+					) : (
+						<p className="text-muted mb-0">Could not load details.</p>
+					)}
+				</Modal.Body>
+			</Modal>
+
 			{loading ? (
 				<Loading />
 			) : orphans.length === 0 ? (
@@ -170,18 +226,28 @@ export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument })
 									<code className="small">{o.importType}</code>
 								</td>
 								<td>
-									{GetGameGroupConfig(o.gameGroup as GameGroup)?.name ?? o.gameGroup}
+									{GetGameGroupConfig(o.gameGroup as GameGroup)?.name ??
+										o.gameGroup}
 								</td>
 								<td>{new Date(o.timeInserted).toLocaleString()}</td>
 								<td className="small text-muted">{o.message ?? "—"}</td>
 								<td>
-									<Button
-										onClick={() => void onDelete(o.orphanID)}
-										size="sm"
-										variant="outline-danger"
-									>
-										Delete
-									</Button>
+									<div className="d-flex flex-wrap gap-1">
+										<Button
+											onClick={() => void onOpenDetail(o.orphanID)}
+											size="sm"
+											variant="outline-secondary"
+										>
+											Details
+										</Button>
+										<Button
+											onClick={() => void onDelete(o.orphanID)}
+											size="sm"
+											variant="outline-danger"
+										>
+											Delete
+										</Button>
+									</div>
 								</td>
 							</tr>
 						)}

typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/LeaderboardsPage.tsx

@@ -24,7 +24,7 @@ import {
 	type integer,
 	type ProfileRatingAlgorithms,
 	type UserDocument,
-	type UserGameStats,
+	type UserGameStatsWithProfileLeaderboardRank,
 	type V3Game,
 } from "tachi-common";
 
@@ -111,7 +111,7 @@ function LeaderboardsPageContent({
 
 	const bestNearbyUser = stats.thisUsersRanking.ranking - stats.above.length - 1;
 
-	function LeaderboardRow({ s, i }: { i: integer; s: UserGameStats }) {
+	function LeaderboardRow({ s }: { s: UserGameStatsWithProfileLeaderboardRank }) {
 		return (
 			<tr
 				style={{
@@ -121,7 +121,7 @@ function LeaderboardsPageContent({
 				}}
 			>
 				<td>
-					<strong>#{i}</strong>
+					<strong>#{s.rank}</strong>
 					{reqUser.id === s.userID && (
 						<small className="text-body-secondary">
 							/{stats.thisUsersRanking.outOf}
@@ -177,18 +177,14 @@ function LeaderboardsPageContent({
 					{bestNearbyUser >= 1 &&
 						leaderboard.gameStats
 							.slice(0, bestNearbyUser)
-							.map((s, i) => <LeaderboardRow i={i + 1} key={s.userID} s={s} />)}
+							.map((s) => <LeaderboardRow key={s.userID} s={s} />)}
 					{bestNearbyUser > 4 && (
 						<tr style={{ lineHeight: "0.5rem" }}>
 							<td colSpan={4}>...</td>
 						</tr>
 					)}
-					{[...stats.above, stats.thisUsersStats, ...stats.below].map((s, i) => (
-						<LeaderboardRow
-							i={stats.thisUsersRanking.ranking - stats.above.length + i}
-							key={s.userID}
-							s={s}
-						/>
+					{[...stats.above, stats.thisUsersStats, ...stats.below].map((s) => (
+						<LeaderboardRow key={s.userID} s={s} />
 					))}
 				</>
 			</MiniTable>

typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/OverviewPage.tsx

@@ -15,7 +15,12 @@ import SelectButton from "#components/util/SelectButton";
 import { useProfileRatingAlg } from "#components/util/useScoreRatingAlg";
 import { type UGPTHistory } from "#types/api-returns";
 import { type GamePT, type SetState, type UGPT } from "#types/react";
-import { FormatGPTProfileRating, FormatGPTProfileRatingName, UppercaseFirst } from "#util/misc";
+import {
+	FormatGPTProfileRating,
+	FormatGPTProfileRatingName,
+	getProfileRatingAlgKeysInDisplayOrder,
+	UppercaseFirst,
+} from "#util/misc";
 import { FormatDate, MillisToSince } from "#util/time";
 import { DateTime } from "luxon";
 import React, { useMemo, useState } from "react";
@@ -158,7 +163,7 @@ function UserHistory({
 			<Divider className="mt-6 mb-2" />
 			{mode === "ranking" ? (
 				<>
-					{Object.keys(gameConfig.profileRatingAlgs).length > 1 && (
+					{getProfileRatingAlgKeysInDisplayOrder(game).length > 1 && (
 						<div className="col-12 offset-md-4 col-md-4 mt-4">
 							<FormSelect
 								onChange={(e) =>
@@ -166,7 +171,7 @@ function UserHistory({
 								}
 								value={rating}
 							>
-								{Object.keys(gameConfig.profileRatingAlgs).map((e) => (
+								{getProfileRatingAlgKeysInDisplayOrder(game).map((e) => (
 									<option key={e} value={e}>
 										{FormatGPTProfileRatingName(game, e)}
 									</option>
@@ -211,7 +216,7 @@ function UserHistory({
 				/>
 			) : (
 				<>
-					{Object.keys(gameConfig.profileRatingAlgs).length > 1 && (
+					{getProfileRatingAlgKeysInDisplayOrder(game).length > 1 && (
 						<div className="col-12 offset-md-4 col-md-4 mt-4">
 							<FormSelect
 								onChange={(e) =>
@@ -219,7 +224,7 @@ function UserHistory({
 								}
 								value={rating}
 							>
-								{Object.keys(gameConfig.profileRatingAlgs).map((e) => (
+								{getProfileRatingAlgKeysInDisplayOrder(game).map((e) => (
 									<option key={e} value={e}>
 										{FormatGPTProfileRatingName(game, e)}
 									</option>

typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/SpecificSessionPage.tsx

@@ -73,7 +73,7 @@ function SessionPage({ data, game }: { data: SessionReturns } & UGPT) {
 	const scoreDataset = useMemo(() => {
 		const d = [];
 
-		for (const sci of data.scoreInfo) {
+		for (const sci of sessionData.scoreInfo) {
 			const score = scoreMap.get(sci.scoreID);
 
 			if (!score) {

typescript/client/src/app/pages/dashboard/users/games/_game/_playtype/UGPTSettingsPage.tsx

@@ -20,6 +20,7 @@ import {
 	FormatGPTProfileRatingName,
 	FormatGPTScoreRatingName,
 	FormatGPTSessionRatingName,
+	getProfileRatingAlgKeysInDisplayOrder,
 	ToFixedFloor,
 	UppercaseFirst,
 } from "#util/misc";
@@ -254,7 +255,7 @@ function PreferencesForm({
 					</Form.Text>
 				</Form.Group>
 			)}
-			{Object.keys(gameConfig.profileRatingAlgs).length > 1 && (
+			{getProfileRatingAlgKeysInDisplayOrder(game).length > 1 && (
 				<Form.Group className={formGroupClassNames}>
 					<Form.Label>Preferred Profile Algorithm</Form.Label>
 					<Form.Select
@@ -262,7 +263,7 @@ function PreferencesForm({
 						onChange={formik.handleChange}
 						value={formik.values.preferredProfileAlg}
 					>
-						{Object.keys(gameConfig.profileRatingAlgs).map((e) => (
+						{getProfileRatingAlgKeysInDisplayOrder(game).map((e) => (
 							<option key={e} value={e}>
 								{FormatGPTProfileRatingName(game, e)}
 							</option>

typescript/client/src/app/routes/UserRoutes.tsx

@@ -9,9 +9,9 @@ import UGPTSettingsPage from "#app/pages/dashboard/users/games/_game/_playtype/U
 import UGPTUtilsPage from "#app/pages/dashboard/users/games/_game/_playtype/utils/UGPTUtilsPage";
 import UserGamesPage from "#app/pages/dashboard/users/UserGamesPage";
 import UserImportsPage from "#app/pages/dashboard/users/UserImportsPage";
-import UserOrphansPage from "#app/pages/dashboard/users/UserOrphansPage";
 import UserIntegrationsPage from "#app/pages/dashboard/users/UserIntegrationsPage";
 import UserInvitesPage from "#app/pages/dashboard/users/UserInvitesPage";
+import UserOrphansPage from "#app/pages/dashboard/users/UserOrphansPage";
 import UserSettingsPage from "#app/pages/dashboard/users/UserSettingsPage";
 import { ErrorPage } from "#app/pages/ErrorPage";
 import RequireAuthAsUserParam from "#components/auth/RequireAuthAsUserParam";

typescript/client/src/components/imports/ImportInfo.tsx

@@ -53,7 +53,7 @@ export default function ImportInfo({
 			return;
 		}
 
-		APIFetchV1<UserGameStats[]>(`/users/${user!.id}/game-stats`).then((r) => {
+		APIFetchV1<UserGameStats[]>(`/users/${user!.id}/game-profiles`).then((r) => {
 			if (!r.success) {
 				console.warn(`Can't update user stats post-import. ${r.description}`);
 				return;
@@ -130,8 +130,8 @@ export default function ImportInfo({
 							are matched with data, all we have to display might be a hash.
 							<br />
 							<strong>SongOrChartNotFound</strong> means the score was still{" "}
-							<strong>saved as an orphan</strong> for nightly matching (around 00:01 UTC) or
-							manual reprocess — see{" "}
+							<strong>saved as an orphan</strong> for nightly matching (around 00:01
+							UTC) or manual reprocess — see{" "}
 							{user ? (
 								<Link to={`/u/${user.username}/orphans`}>Orphan scores</Link>
 							) : (
@@ -157,7 +157,9 @@ export default function ImportInfo({
 											<div className="mt-2 small text-muted">
 												This may be stored as an orphan.{" "}
 												{user ? (
-													<Link to={`/u/${user.username}/orphans`}>Open orphan queue</Link>
+													<Link to={`/u/${user.username}/orphans`}>
+														Open orphan queue
+													</Link>
 												) : (
 													"Open orphan queue"
 												)}

typescript/client/src/components/layout/header/HeaderMenu.tsx

@@ -28,7 +28,7 @@ export function HeaderMenu({
 
 	const { data, error } = useApiQuery<UserGameStats[]>(
 		// We should generate a valid url just in case the skip somehow fails
-		`/users/${user?.id ?? "me"}/game-stats`,
+		`/users/${user?.id ?? "me"}/game-profiles`,
 		undefined,
 		undefined,
 		// We should skip if a user isn't logged in.

typescript/client/src/components/tables/cells/IndexCell.tsx

@@ -16,7 +16,7 @@ export default function IndexCell({ index }: { index: integer }) {
 			</span>
 			<span
 				style={{
-					fontWeight: index < 10 ? "bold" : undefined,
+					fontWeight: index + 1 < 10 ? "bold" : undefined,
 				}}
 			>
 				{index + 1}

typescript/client/src/components/tables/components/SelectableRating.tsx

@@ -1,6 +1,11 @@
 import Icon from "#components/util/Icon";
 import { type SetState } from "#types/react";
-import { FormatGPTScoreRatingName, FormatGPTSessionRatingName } from "#util/misc";
+import {
+	FormatGPTProfileRatingName,
+	FormatGPTScoreRatingName,
+	FormatGPTSessionRatingName,
+	getProfileRatingAlgKeysInDisplayOrder,
+} from "#util/misc";
 import React from "react";
 import { GetGameConfig, type V3Game } from "tachi-common";
 
@@ -33,6 +38,11 @@ export default function SelectableRating({
 		key = "sessionRatingAlgs";
 	}
 
+	const ratingKeys =
+		mode === "profile"
+			? getProfileRatingAlgKeysInDisplayOrder(game)
+			: (Object.keys(gameConfig[key]) as string[]);
+
 	return (
 		<th>
 			<div className="vstack gap-1 align-items-center justify-content-center">
@@ -41,11 +51,13 @@ export default function SelectableRating({
 					onChange={(v) => setRating(v.target.value as AllRatings)}
 					value={rating}
 				>
-					{Object.keys(gameConfig[key]).map((s) => (
+					{ratingKeys.map((s) => (
 						<option key={s} value={s}>
 							{mode === "session"
 								? FormatGPTSessionRatingName(game, s)
-								: FormatGPTScoreRatingName(game, s)}
+								: mode === "profile"
+									? FormatGPTProfileRatingName(game, s)
+									: FormatGPTScoreRatingName(game, s)}
 						</option>
 					))}
 				</select>

typescript/client/src/components/user/UGPTProfiles.tsx

@@ -48,7 +48,7 @@ const ContextualGamesInfo = memo(({ user }: { user: UserDocument }) => {
 
 function QueryGamesInfo({ reqUser }: { reqUser: UserDocument }) {
 	const { data, error } = useApiQuery<UserGameStats[]>(
-		`/users/${reqUser.id}/game-stats`,
+		`/users/${reqUser.id}/game-profiles`,
 		undefined,
 		undefined,
 		!reqUser,

typescript/client/src/components/user/UGPTRankingData.tsx

@@ -1,6 +1,6 @@
 import { useProfileRatingAlg } from "#components/util/useScoreRatingAlg";
 import { type GamePT } from "#types/react";
-import { FormatGPTProfileRatingName } from "#util/misc";
+import { FormatGPTProfileRatingName, getProfileRatingAlgKeysInDisplayOrder } from "#util/misc";
 import React from "react";
 import { Link } from "react-router-dom";
 import { type integer, type ProfileRatingAlgorithms, type V3Game } from "tachi-common";
@@ -22,19 +22,21 @@ export default function RankingData({
 
 	const extendData = [];
 
-	for (const k in rankingData) {
+	for (const k of getProfileRatingAlgKeysInDisplayOrder(game)) {
 		const key = k as ProfileRatingAlgorithms[V3Game];
 
-		if (key !== alg) {
-			extendData.push(
-				<div className="col-12" key={key}>
-					<small className="text-body-secondary">
-						{FormatGPTProfileRatingName(game, key)}: #{rankingData[key].ranking}/
-						{rankingData[key].outOf}
-					</small>
-				</div>,
-			);
+		if (!(key in rankingData) || key === alg) {
+			continue;
 		}
+
+		extendData.push(
+			<div className="col-12" key={key}>
+				<small className="text-body-secondary">
+					{FormatGPTProfileRatingName(game, key)}: #{rankingData[key].ranking}/
+					{rankingData[key].outOf}
+				</small>
+			</div>,
+		);
 	}
 
 	return (

typescript/client/src/components/user/UGPTStatsOverview.tsx

@@ -6,23 +6,22 @@ import {
 	FormatGPTProfileRating,
 	FormatGPTProfileRatingName,
 	FormatGPTScoreRatingName,
+	getProfileRatingAlgRowStyle,
+	sortProfileRatingEntries,
 	UppercaseFirst,
 } from "#util/misc";
 import { StrSOV } from "#util/sorts";
 import React from "react";
-import {
-	type Classes,
-	GetGameConfig,
-	type ProfileRatingAlgorithms,
-	type UserGameStats,
-	type V3Game,
-} from "tachi-common";
+import { type Classes, GetGameConfig, type UserGameStats, type V3Game } from "tachi-common";
 
 export default function UGPTRatingsTable({ ugs }: { ugs: UserGameStats }) {
 	const game = ugs.game;
 	const gameConfig = GetGameConfig(game);
 
-	const ratings = Object.entries(ugs.ratings) as [ProfileRatingAlgorithms[V3Game], number][];
+	const ratings = sortProfileRatingEntries(
+		game,
+		Object.entries(ugs.ratings) as [string, number][],
+	);
 
 	return (
 		<MiniTable className="table-sm text-center" colSpan={2} headers={["Player Stats"]}>
@@ -79,7 +78,9 @@ export default function UGPTRatingsTable({ ugs }: { ugs: UserGameStats }) {
 								</div>
 							</QuickTooltip>
 						</td>
-						<td>{FormatGPTProfileRating(game, k as any, v)}</td>
+						<td style={getProfileRatingAlgRowStyle(game, k)}>
+							{FormatGPTProfileRating(game, k as any, v)}
+						</td>
 					</tr>
 				))}
 			</>

typescript/client/src/components/util/UpdateUserGameStats.tsx

@@ -3,7 +3,7 @@ import { APIFetchV1 } from "#util/api";
 import { type UserGameStats } from "tachi-common";
 
 export default async function UpdateUserGameStats(setUGS: SetState<UserGameStats[] | null>) {
-	const res = await APIFetchV1<UserGameStats[]>("/users/me/game-stats");
+	const res = await APIFetchV1<UserGameStats[]>("/users/me/game-profiles");
 
 	if (!res.success) {
 		setUGS(null);

typescript/client/src/lib/games/iidx.tsx

@@ -5,6 +5,7 @@ import RatingCell from "#components/tables/cells/RatingCell";
 import ScoreCell from "#components/tables/cells/ScoreCell";
 import { GetEnumColour } from "#lib/game-implementations";
 import { type GPTClientImplementation } from "#lib/types";
+import { ChangeOpacity } from "#util/color-opacity";
 import { NumericSOV } from "#util/sorts";
 import React from "react";
 import { COLOUR_SET, type GamesForGroup, IIDX_LAMPS, IIDXLIKE_GBOUNDARIES } from "tachi-common";
@@ -114,6 +115,17 @@ const IIDXCoreCells: GPTClientImplementation<GamesForGroup["iidx"]>["scoreCoreCe
 	</>
 );
 
+/** Applied to the profile stats table value cell only (see UGPTRatingsTable). */
+const IIDX_PROFILE_RATING_VALUE_CELL_STYLE: NonNullable<
+	GPTClientImplementation<GamesForGroup["iidx"]>["profileRatingAlgRowStyle"]
+> = {
+	ktLampRating: { backgroundColor: ChangeOpacity(COLOUR_SET.purple, 0.12) },
+	ktLampRatingNC: { backgroundColor: ChangeOpacity(COLOUR_SET.blue, 0.14) },
+	ktLampRatingHC: { backgroundColor: ChangeOpacity(COLOUR_SET.orange, 0.14) },
+	ktLampRatingEXHC: { backgroundColor: ChangeOpacity(COLOUR_SET.gold, 0.12) },
+	BPI: { backgroundColor: ChangeOpacity(COLOUR_SET.paleBlue, 0.12) },
+};
+
 const IIDXRatingCell: GPTClientImplementation<GamesForGroup["iidx"]>["ratingCell"] = ({
 	sc,
 	chart,
@@ -169,6 +181,7 @@ export const IIDX_SP_IMPL: GPTClientImplementation<"iidx-sp"> = {
 	classColours: IIDX_COLOURS,
 	scoreCoreCells: IIDXCoreCells,
 	ratingCell: IIDXRatingCell,
+	profileRatingAlgRowStyle: IIDX_PROFILE_RATING_VALUE_CELL_STYLE,
 };
 
 export const IIDX_DP_IMPL: GPTClientImplementation<"iidx-dp"> = {
@@ -194,4 +207,5 @@ export const IIDX_DP_IMPL: GPTClientImplementation<"iidx-dp"> = {
 	classColours: IIDX_COLOURS,
 	scoreCoreCells: IIDXCoreCells,
 	ratingCell: IIDXRatingCell,
+	profileRatingAlgRowStyle: IIDX_PROFILE_RATING_VALUE_CELL_STYLE,
 };

typescript/client/src/lib/types.ts

@@ -106,6 +106,12 @@ export interface GPTClientImplementation<GPT extends V3Game = V3Game> {
 		session?: Record<string, string>;
 	};
 
+	/**
+	 * Optional styles for the **value** cell (not the label) in the profile stats mini-table,
+	 * keyed by profile rating algorithm id. Typically a tinted background.
+	 */
+	profileRatingAlgRowStyle?: Partial<Record<string, CSSProperties>>;
+
 	/**
 	 * What headers should be used when rendering scores in a table for this game?
 	 */

typescript/client/src/types/api-returns.ts

@@ -26,6 +26,7 @@ import {
 	type UserDocument,
 	type UserGameStats,
 	type UserGameStatsSnapshotDocument,
+	type UserGameStatsWithProfileLeaderboardRank,
 	type V3Game,
 } from "tachi-common";
 
@@ -45,10 +46,10 @@ export interface UGPTStatsReturn<GPT extends V3Game = V3Game> {
 }
 
 export interface UGPTLeaderboardAdjacent {
-	above: UserGameStats[];
-	below: UserGameStats[];
+	above: UserGameStatsWithProfileLeaderboardRank[];
+	below: UserGameStatsWithProfileLeaderboardRank[];
 	users: UserDocument[];
-	thisUsersStats: UserGameStats;
+	thisUsersStats: UserGameStatsWithProfileLeaderboardRank;
 	thisUsersRanking: {
 		outOf: integer;
 		ranking: integer;
@@ -56,7 +57,7 @@ export interface UGPTLeaderboardAdjacent {
 }
 
 export interface GPTLeaderboard {
-	gameStats: UserGameStats[];
+	gameStats: UserGameStatsWithProfileLeaderboardRank[];
 	users: UserDocument[];
 }
 
@@ -179,7 +180,7 @@ export interface ScoreLeaderboardReturns<GPT extends V3Game = V3Game> {
 
 export interface UserLeaderboardReturns {
 	users: UserDocument[];
-	gameStats: UserGameStats[];
+	gameStats: UserGameStatsWithProfileLeaderboardRank[];
 }
 
 export interface UserRecentSummary {

typescript/client/src/types/tables.ts

@@ -11,7 +11,7 @@ import {
 	type ScoreDocument,
 	type SongDocument,
 	type UserDocument,
-	type UserGameStats,
+	type UserGameStatsWithProfileLeaderboardRank,
 	type V3Game,
 } from "tachi-common";
 
@@ -53,7 +53,7 @@ export type UGSDataset = ({
 		index: integer;
 		user: UserDocument;
 	};
-} & UserGameStats)[];
+} & UserGameStatsWithProfileLeaderboardRank)[];
 
 export type RivalChartDataset<GPT extends V3Game = V3Game> = ({
 	__related: {

typescript/client/src/util/data.ts

@@ -2,11 +2,10 @@ import { type GoalsOnChartReturn, type GoalsOnFolderReturn } from "#types/api-re
 import {
 	type ChartDocument,
 	type GameGroup,
-	type GoalDocument,
 	type integer,
 	type ScoreDocument,
-	type SessionScoreInfo,
 	type SEEDS_SongDocument,
+	type SessionScoreInfo,
 	type SongDocument,
 	type UserDocument,
 	type V3Game,
@@ -29,7 +28,7 @@ export function GetPBs(scoreInfo: SessionScoreInfo[]) {
 }
 
 export function CreateSongMap<G extends GameGroup = GameGroup>(
-	songs: Array<SongDocument<G> | SEEDS_SongDocument<G>>,
+	songs: Array<SEEDS_SongDocument<G> | SongDocument<G>>,
 ) {
 	const songMap = new Map<string, SongDocument<G>>();
 

typescript/client/src/util/misc.ts

@@ -1,5 +1,6 @@
 import { GPT_CLIENT_IMPLEMENTATIONS } from "#lib/game-implementations";
 import fjsh from "fast-json-stable-hash";
+import { type CSSProperties } from "react";
 import toast from "react-hot-toast";
 import { type useHistory } from "react-router-dom";
 import {
@@ -10,7 +11,6 @@ import {
 	type GameConfig,
 	GetGameConfig,
 	type integer,
-	type QuestDocument,
 	type QuestSubscriptionDocument,
 	type ScoreDocument,
 	type V3Game,
@@ -105,6 +105,39 @@ export function FormatGPTScoreRatingName(game: V3Game, key: string) {
 	return gameConfig.ratingAlgNameOverrides?.score?.[key] ?? UppercaseFirst(key);
 }
 
+/** Lower `displayOrder` first; missing `displayOrder` sorts after set values, then by key. */
+export function compareProfileRatingAlgKeys(game: V3Game, a: string, b: string): number {
+	const cfg = GetGameConfig(game);
+	const oa = cfg.profileRatingAlgs[a as keyof typeof cfg.profileRatingAlgs]?.displayOrder;
+	const ob = cfg.profileRatingAlgs[b as keyof typeof cfg.profileRatingAlgs]?.displayOrder;
+	const va = typeof oa === "number" ? oa : 10_000;
+	const vb = typeof ob === "number" ? ob : 10_000;
+	if (va !== vb) {
+		return va - vb;
+	}
+
+	return a.localeCompare(b);
+}
+
+export function getProfileRatingAlgKeysInDisplayOrder(game: V3Game): string[] {
+	const cfg = GetGameConfig(game);
+	return (Object.keys(cfg.profileRatingAlgs) as string[]).sort((x, y) =>
+		compareProfileRatingAlgKeys(game, x, y),
+	);
+}
+
+export function sortProfileRatingEntries<K extends string, V>(
+	game: V3Game,
+	entries: Array<[K, V]>,
+): Array<[K, V]> {
+	return [...entries].sort(([a], [b]) => compareProfileRatingAlgKeys(game, a, b));
+}
+
+/** Styles for the profile rating value cell (right column), not the label. */
+export function getProfileRatingAlgRowStyle(game: V3Game, key: string): CSSProperties | undefined {
+	return GPT_CLIENT_IMPLEMENTATIONS[game]?.profileRatingAlgRowStyle?.[key];
+}
+
 export function ReverseStr(str: string) {
 	return str.split("").reverse().join("");
 }

typescript/client/src/util/seeds.ts

@@ -17,6 +17,7 @@ import {
 	CreateSongMap,
 	DatabaseSeedNames,
 	type GameGroup,
+	type SeedFolderRow,
 	type SEEDS_BMSCourseDocument,
 	type SEEDS_ChartDocument,
 	type SEEDS_FolderDocument,
@@ -24,7 +25,6 @@ import {
 	type SEEDS_QuestDocument,
 	type SEEDS_QuestlineDocument,
 	type SEEDS_TableDocument,
-	type SeedFolderRow,
 	type SongDocument,
 } from "tachi-common";
 

typescript/common/src/config/game-support/iidx.ts

@@ -172,6 +172,17 @@ export const GAME_IIDX_SP_CONF = {
 			description:
 				"A rating system that values your clear lamps on charts. Tierlist information is taken into account.",
 		},
+		ktLampRatingNC: {
+			description:
+				"Your normal clear performance. Tierlist information is taken into account.",
+		},
+		ktLampRatingHC: {
+			description: "Your hard clear performance. Tierlist information is taken into account.",
+		},
+		ktLampRatingEXHC: {
+			description:
+				"Your EX-hard clear performance. Tierlist information is taken into account.",
+		},
 		BPI: {
 			description:
 				"A rating system for Kaiden level play. Only applies to 11s and 12s. A BPI of 0 states the score is equal to the Kaiden Average for that chart. A BPI of 100 is equal to the world record.",
@@ -182,11 +193,36 @@ export const GAME_IIDX_SP_CONF = {
 		ktLampRating: {
 			description: `An average of your best 20 ktLampRatings.`,
 			associatedScoreAlgs: ["ktLampRating"],
+			displayOrder: 0,
+		},
+		ktLampRatingNC: {
+			description: `An average of your best 20 ktLampRatingNCs.`,
+			associatedScoreAlgs: ["ktLampRatingNC"],
+			displayOrder: 1,
+		},
+		ktLampRatingHC: {
+			description: `An average of your best 20 ktLampRatingHCs.`,
+			associatedScoreAlgs: ["ktLampRatingHC"],
+			displayOrder: 2,
+		},
+		ktLampRatingEXHC: {
+			description: `An average of your best 20 ktLampRatingEXHCs.`,
+			associatedScoreAlgs: ["ktLampRatingEXHC"],
+			displayOrder: 3,
+		},
+		BPI: {
+			description: `An average of your best 20 BPIs.`,
+			associatedScoreAlgs: ["BPI"],
+			displayOrder: 4,
 		},
-		BPI: { description: `An average of your best 20 BPIs.`, associatedScoreAlgs: ["BPI"] },
 	},
 	sessionRatingAlgs: {
 		ktLampRating: { description: `An average of the best 10 ktLampRatings this session.` },
+		ktLampRatingNC: { description: `An average of the best 10 ktLampRatingNCs this session.` },
+		ktLampRatingHC: { description: `An average of the best 10 ktLampRatingHCs this session.` },
+		ktLampRatingEXHC: {
+			description: `An average of the best 10 ktLampRatingEXHCs this session.`,
+		},
 		BPI: { description: `An average of the best 10 BPIs this session.` },
 	},
 

typescript/common/src/types/documents.ts

@@ -338,6 +338,11 @@ export interface UserGameStats {
 	classes: AnyClasses;
 }
 
+/** `GET /games/:game/leaderboard` and `.../leaderboard-adjacent` (tie-aware profile rating rank). */
+export type UserGameStatsWithProfileLeaderboardRank = {
+	rank: integer;
+} & UserGameStats;
+
 export interface ChartTierlistInfo {
 	text: string;
 	value: number;

typescript/common/src/types/game-config-utils.ts

@@ -21,6 +21,13 @@ export interface ProfileRatingAlgorithmConfig extends RatingAlgorithmConfig {
 	 * of this algorithm's description?
 	 */
 	associatedScoreAlgs: ReadonlyArray<string>;
+
+	/**
+	 * Sort order in the UI when multiple profile ratings are listed (e.g. profile header
+	 * table, settings). Lower numbers appear first; algorithms without this field sort
+	 * after those that have it, then by key name.
+	 */
+	displayOrder?: number;
 }
 
 export interface ClassInfo<ClassID extends string> {
@@ -143,10 +150,10 @@ export interface FixedDifficulties<Difficulty extends string> {
  * Originally, I called this "SEMI_FIXED", which is the sort of ZK quirkiness you've came to know at
  * this point, but now there's no point, this is for chugekimai, and this makes it clear.
  */
-export interface ChuGekiMaiDifficulties<Difficulty extends string> {
+export interface ChuGekiMaiDifficulties<TDifficulty extends string> {
 	type: "CHUGEKIMAI_STYLE";
 
-	order: ReadonlyArray<Difficulty>;
+	order: ReadonlyArray<TDifficulty>;
 
 	/**
 	 * How should we format these difficulty names?
@@ -154,9 +161,9 @@ export interface ChuGekiMaiDifficulties<Difficulty extends string> {
 	 * Dynamic ones (i.e. ones not in the Order set) are not formatted - they are printed
 	 * exactly as-is.
 	 */
-	format: Partial<Record<Difficulty, string>>;
+	format: Partial<Record<TDifficulty, string>>;
 
-	default: Difficulty;
+	default: TDifficulty;
 }
 
 export type DifficultyConfig<D extends string = string> =

typescript/db/src/generated/index.ts

@@ -11,7 +11,6 @@ export { type default as InviteLockTable, type InviteLock, type NewInviteLock, t
 export { type orphan_chart_id, type default as OrphanChartTable, type OrphanChart, type NewOrphanChart, type OrphanChartUpdate } from './public/OrphanChart';
 export { type game_rival_game, type default as GameRivalTable, type GameRival, type NewGameRival, type GameRivalUpdate } from './public/GameRival';
 export { type goal_id, type default as GoalTable, type Goal, type NewGoal, type GoalUpdate } from './public/Goal';
-export { type game_settings_game, type default as GameSettingsTable, type GameSettings, type NewGameSettings, type GameSettingsUpdate } from './public/GameSettings';
 export { type import_quest_row_id, type default as ImportQuestTable, type ImportQuest, type NewImportQuest, type ImportQuestUpdate } from './public/ImportQuest';
 export { type cron_task_execution_id, type default as CronTaskExecutionTable, type CronTaskExecution, type NewCronTaskExecution, type CronTaskExecutionUpdate } from './public/CronTaskExecution';
 export { type notification_row_id, type default as NotificationTable, type Notification, type NewNotification, type NotificationUpdate } from './public/Notification';
@@ -39,7 +38,7 @@ export { type default as MigrationTable, type Migration, type NewMigration, type
 export { type action_row_id, type default as ActionTable, type Action, type NewAction, type ActionUpdate } from './public/Action';
 export { type priv_svc_kai_auth_token_service, type default as PrivSvcKaiAuthTokenTable, type PrivSvcKaiAuthToken, type NewPrivSvcKaiAuthToken, type PrivSvcKaiAuthTokenUpdate } from './public/PrivSvcKaiAuthToken';
 export { type chart_id, type default as ChartTable, type Chart, type NewChart, type ChartUpdate } from './public/Chart';
-export { type game_settings_showcase_game, type default as GameSettingsShowcaseTable, type GameSettingsShowcase, type NewGameSettingsShowcase, type GameSettingsShowcaseUpdate } from './public/GameSettingsShowcase';
+export { type game_profile_dirty_user_id, type game_profile_dirty_game, type default as GameProfileDirtyTable, type GameProfileDirty, type NewGameProfileDirty, type GameProfileDirtyUpdate } from './public/GameProfileDirty';
 export { type default as AccountFollowingTable, type AccountFollowing, type NewAccountFollowing, type AccountFollowingUpdate } from './public/AccountFollowing';
 export { type import_id, type default as ImportTable, type Import, type NewImport, type ImportUpdate } from './public/Import';
 export { type folder_id, type default as FolderTable, type Folder, type NewFolder, type FolderUpdate } from './public/Folder';
@@ -56,6 +55,7 @@ export { type priv_api_client_client_id, type default as PrivApiClientTable, typ
 export { type priv_api_token_token, type default as PrivApiTokenTable, type PrivApiToken, type NewPrivApiToken, type PrivApiTokenUpdate } from './public/PrivApiToken';
 export { type default as SvcKshookSv6cSettingsTable, type SvcKshookSv6cSettings, type NewSvcKshookSv6cSettings, type SvcKshookSv6cSettingsUpdate } from './public/SvcKshookSv6cSettings';
 export { type import_error_row_id, type default as ImportErrorTable, type ImportError, type NewImportError, type ImportErrorUpdate } from './public/ImportError';
+export { type session_dirty_session_id, type default as SessionDirtyTable, type SessionDirty, type NewSessionDirty, type SessionDirtyUpdate } from './public/SessionDirty';
 export { type default as FolderChartLookupTable, type FolderChartLookup, type NewFolderChartLookup, type FolderChartLookupUpdate } from './public/FolderChartLookup';
 export { type quest_id, type default as QuestTable, type Quest, type NewQuest, type QuestUpdate } from './public/Quest';
 export { type default as AccountSettingsTable, type AccountSettings, type NewAccountSettings, type AccountSettingsUpdate } from './public/AccountSettings';
@@ -74,6 +74,8 @@ export { type default as ActionResult } from './public/ActionResult';
 export { type default as GameGroup } from './public/GameGroup';
 export { type default as ImportType } from './public/ImportType';
 export { type enqueue_pb_dirty_params } from './public/enqueue_pb_dirty';
+export { type enqueue_game_profile_dirty_params } from './public/enqueue_game_profile_dirty';
 export { type enqueue_score_rederive_params } from './public/enqueue_score_rederive';
+export { type enqueue_session_dirty_params } from './public/enqueue_session_dirty';
 export { type default as PublicSchema } from './public/PublicSchema';
 export { type default as Database } from './Database';

typescript/db/src/generated/public/GameProfile.ts

@@ -17,6 +17,22 @@ export default interface GameProfileTable {
   ratings: ColumnType<unknown, unknown, unknown>;
 
   classes: ColumnType<unknown, unknown, unknown>;
+
+  pf_preferred_score_alg: ColumnType<string | null, string | null, string | null>;
+
+  pf_preferred_session_alg: ColumnType<string | null, string | null, string | null>;
+
+  pf_preferred_profile_alg: ColumnType<string | null, string | null, string | null>;
+
+  pf_preferred_default_enum: ColumnType<string | null, string | null, string | null>;
+
+  pf_default_table: ColumnType<string | null, string | null, string | null>;
+
+  pf_preferred_ranking: ColumnType<string | null, string | null, string | null>;
+
+  data: ColumnType<unknown, unknown | undefined, unknown>;
+
+  showcase: ColumnType<unknown, unknown | undefined, unknown>;
 }
 
 export type GameProfile = Selectable<GameProfileTable>;

typescript/db/src/generated/public/GameProfileDirty.ts

@@ -0,0 +1,26 @@
+// @generated
+// This file is automatically generated by Kanel. Do not modify manually.
+
+import type { default as Game } from './Game';
+import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
+
+/** Identifier type for public.game_profile_dirty */
+export type game_profile_dirty_user_id = number;
+
+/** Identifier type for public.game_profile_dirty */
+export type game_profile_dirty_game = Game;
+
+/** Represents the table public.game_profile_dirty */
+export default interface GameProfileDirtyTable {
+  user_id: ColumnType<game_profile_dirty_user_id, game_profile_dirty_user_id, game_profile_dirty_user_id>;
+
+  game: ColumnType<game_profile_dirty_game, game_profile_dirty_game, game_profile_dirty_game>;
+
+  enqueued_at: ColumnType<string, string | undefined, string>;
+}
+
+export type GameProfileDirty = Selectable<GameProfileDirtyTable>;
+
+export type NewGameProfileDirty = Insertable<GameProfileDirtyTable>;
+
+export type GameProfileDirtyUpdate = Updateable<GameProfileDirtyTable>;

typescript/db/src/generated/public/GameSettings.ts

@@ -1,36 +0,0 @@
-// @generated
-// This file is automatically generated by Kanel. Do not modify manually.
-
-import type { default as Game } from './Game';
-import type { account_id } from './Account';
-import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
-
-/** Identifier type for public.game_settings */
-export type game_settings_game = Game;
-
-/** Represents the table public.game_settings */
-export default interface GameSettingsTable {
-  user_id: ColumnType<account_id, account_id, account_id>;
-
-  game: ColumnType<game_settings_game, game_settings_game, game_settings_game>;
-
-  pf_preferred_score_alg: ColumnType<string | null, string | null, string | null>;
-
-  pf_preferred_session_alg: ColumnType<string | null, string | null, string | null>;
-
-  pf_preferred_profile_alg: ColumnType<string | null, string | null, string | null>;
-
-  pf_preferred_default_enum: ColumnType<string | null, string | null, string | null>;
-
-  pf_default_table: ColumnType<string | null, string | null, string | null>;
-
-  pf_preferred_ranking: ColumnType<string | null, string | null, string | null>;
-
-  data: ColumnType<unknown, unknown, unknown>;
-}
-
-export type GameSettings = Selectable<GameSettingsTable>;
-
-export type NewGameSettings = Insertable<GameSettingsTable>;
-
-export type GameSettingsUpdate = Updateable<GameSettingsTable>;

typescript/db/src/generated/public/GameSettingsShowcase.ts

@@ -1,24 +0,0 @@
-// @generated
-// This file is automatically generated by Kanel. Do not modify manually.
-
-import type { default as Game } from './Game';
-import type { account_id } from './Account';
-import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
-
-/** Identifier type for public.game_settings_showcase */
-export type game_settings_showcase_game = Game;
-
-/** Represents the table public.game_settings_showcase */
-export default interface GameSettingsShowcaseTable {
-  user_id: ColumnType<account_id, account_id, account_id>;
-
-  game: ColumnType<game_settings_showcase_game, game_settings_showcase_game, game_settings_showcase_game>;
-
-  data: ColumnType<unknown, unknown, unknown>;
-}
-
-export type GameSettingsShowcase = Selectable<GameSettingsShowcaseTable>;
-
-export type NewGameSettingsShowcase = Insertable<GameSettingsShowcaseTable>;
-
-export type GameSettingsShowcaseUpdate = Updateable<GameSettingsShowcaseTable>;

typescript/db/src/generated/public/PublicSchema.ts

@@ -11,7 +11,6 @@ import type { default as InviteLockTable } from './InviteLock';
 import type { default as OrphanChartTable } from './OrphanChart';
 import type { default as GameRivalTable } from './GameRival';
 import type { default as GoalTable } from './Goal';
-import type { default as GameSettingsTable } from './GameSettings';
 import type { default as ImportQuestTable } from './ImportQuest';
 import type { default as CronTaskExecutionTable } from './CronTaskExecution';
 import type { default as NotificationTable } from './Notification';
@@ -39,7 +38,7 @@ import type { default as MigrationTable } from './Migration';
 import type { default as ActionTable } from './Action';
 import type { default as PrivSvcKaiAuthTokenTable } from './PrivSvcKaiAuthToken';
 import type { default as ChartTable } from './Chart';
-import type { default as GameSettingsShowcaseTable } from './GameSettingsShowcase';
+import type { default as GameProfileDirtyTable } from './GameProfileDirty';
 import type { default as AccountFollowingTable } from './AccountFollowing';
 import type { default as ImportTable } from './Import';
 import type { default as FolderTable } from './Folder';
@@ -56,6 +55,7 @@ import type { default as PrivApiClientTable } from './PrivApiClient';
 import type { default as PrivApiTokenTable } from './PrivApiToken';
 import type { default as SvcKshookSv6cSettingsTable } from './SvcKshookSv6cSettings';
 import type { default as ImportErrorTable } from './ImportError';
+import type { default as SessionDirtyTable } from './SessionDirty';
 import type { default as FolderChartLookupTable } from './FolderChartLookup';
 import type { default as QuestTable } from './Quest';
 import type { default as AccountSettingsTable } from './AccountSettings';
@@ -89,8 +89,6 @@ export default interface PublicSchema {
 
   goal: GoalTable;
 
-  game_settings: GameSettingsTable;
-
   import_quest: ImportQuestTable;
 
   cron_task_execution: CronTaskExecutionTable;
@@ -145,7 +143,7 @@ export default interface PublicSchema {
 
   chart: ChartTable;
 
-  game_settings_showcase: GameSettingsShowcaseTable;
+  game_profile_dirty: GameProfileDirtyTable;
 
   account_following: AccountFollowingTable;
 
@@ -179,6 +177,8 @@ export default interface PublicSchema {
 
   import_error: ImportErrorTable;
 
+  session_dirty: SessionDirtyTable;
+
   folder_chart_lookup: FolderChartLookupTable;
 
   quest: QuestTable;

typescript/db/src/generated/public/SessionDirty.ts

@@ -0,0 +1,20 @@
+// @generated
+// This file is automatically generated by Kanel. Do not modify manually.
+
+import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
+
+/** Identifier type for public.session_dirty */
+export type session_dirty_session_id = string;
+
+/** Represents the table public.session_dirty */
+export default interface SessionDirtyTable {
+  session_id: ColumnType<session_dirty_session_id, session_dirty_session_id, session_dirty_session_id>;
+
+  enqueued_at: ColumnType<string, string | undefined, string>;
+}
+
+export type SessionDirty = Selectable<SessionDirtyTable>;
+
+export type NewSessionDirty = Insertable<SessionDirtyTable>;
+
+export type SessionDirtyUpdate = Updateable<SessionDirtyTable>;

typescript/db/src/generated/public/enqueue_game_profile_dirty.ts

@@ -0,0 +1,5 @@
+// @generated
+// This file is automatically generated by Kanel. Do not modify manually.
+
+export interface enqueue_game_profile_dirty_params {
+}

typescript/db/src/generated/public/enqueue_session_dirty.ts

@@ -0,0 +1,5 @@
+// @generated
+// This file is automatically generated by Kanel. Do not modify manually.
+
+export interface enqueue_session_dirty_params {
+}

typescript/seeds-scripts/eslint.config.mjs

@@ -4,6 +4,13 @@ export default [
 	...configTachi.base,
 	configTachi.node,
 	{
-		ignores: ["../../db/seeds/**", "node_modules/**", "js/**", "**/*.js"],
+		ignores: [
+			"../../db/seeds/**",
+			"node_modules/**",
+			"js/**",
+			"**/*.js",
+			// Excluded from tsconfig.json; type-aware eslint would error on every file here.
+			"rerunners/**",
+		],
 	},
 ];

typescript/seeds-scripts/test/schemas.ts

@@ -1,6 +1,7 @@
 import {
 	ALL_GAMES,
 	allSupportedGameGroups,
+	type GameGroup,
 	SEEDS_BMS_COURSE_DOCUMENT_SCHEMA,
 	SEEDS_CHART_DOCUMENT_SCHEMAS,
 	SEEDS_FOLDER_DOCUMENT_SCHEMA,
@@ -9,7 +10,6 @@ import {
 	SEEDS_QUESTLINE_DOCUMENT_SCHEMA,
 	SEEDS_SONG_DOCUMENT_SCHEMAS,
 	SEEDS_TABLE_DOCUMENT_SCHEMA,
-	type GameGroup,
 	type V3Game,
 } from "tachi-common";
 import { type ZodType } from "zod";

typescript/seeds-webui/package.json

@@ -15,13 +15,13 @@
 		"bench": "node -e \"process.exit(0)\""
 	},
 	"dependencies": {
-		"@codemirror/lang-sql": "^6.10.0",
-		"@codemirror/language": "^6.12.3",
-		"@lezer/highlight": "^1.2.3",
-		"@sqlite.org/sqlite-wasm": "^3.51.2-build9",
-		"codemirror": "^6.0.2",
-		"comlink": "^4.4.2",
-		"fast-json-patch": "^3.1.1",
+		"@codemirror/lang-sql": "catalog:",
+		"@codemirror/language": "catalog:",
+		"@lezer/highlight": "catalog:",
+		"@sqlite.org/sqlite-wasm": "catalog:",
+		"codemirror": "catalog:",
+		"comlink": "catalog:",
+		"fast-json-patch": "catalog:",
 		"nanoid": "catalog:",
 		"natural-compare": "catalog:",
 		"react": "catalog:",

typescript/seeds-webui/src/components/SqlEditor.tsx

@@ -2,8 +2,8 @@ import { sql, SQLite } from "@codemirror/lang-sql";
 import { HighlightStyle, syntaxHighlighting } from "@codemirror/language";
 import { Compartment, EditorState } from "@codemirror/state";
 import { keymap } from "@codemirror/view";
-import { basicSetup, EditorView } from "codemirror";
 import { tags } from "@lezer/highlight";
+import { basicSetup, EditorView } from "codemirror";
 import { useCallback, useEffect, useRef, useState } from "react";
 
 /**

typescript/seeds-webui/src/components/SqliteWorkspaceGate.tsx

@@ -1,6 +1,7 @@
-import { useIngest } from "#lib/ingest/IngestProvider";
 import type { ReactNode } from "react";
 
+import { useIngest } from "#lib/ingest/IngestProvider";
+
 type SqliteWorkspaceGateProps = { children: ReactNode };
 
 /**
@@ -55,12 +56,12 @@ export function SqliteWorkspaceGate({ children }: SqliteWorkspaceGateProps) {
 							</div>
 							{progress.total > 0 ? (
 								<div
+									aria-label="Ingest progress"
+									aria-valuemax={100}
+									aria-valuemin={0}
+									aria-valuenow={pct}
 									className="ingest-bar sql-workspace-gate-bar"
 									role="progressbar"
-									aria-valuenow={pct}
-									aria-valuemin={0}
-									aria-valuemax={100}
-									aria-label="Ingest progress"
 								>
 									<div className="ingest-bar-fill" style={{ width: `${pct}%` }} />
 								</div>

typescript/server/.env

@@ -39,7 +39,7 @@ TACHI_OUR_URL=http://localhost:3000
 ; TACHI_INVITE_CODE_BATCH_SIZE=2
 ; TACHI_INVITE_CODE_INVITE_CAP=100
 ; TACHI_INVITE_CODE_BETA_USER_BONUS=5
-TACHI_CDN_WEB_LOCATION=http://tachi-s3:9000/tachi-public
+TACHI_CDN_WEB_LOCATION=http://127.0.0.1:9000/tachi-public
 TACHI_CDN_SAVE_LOCATION_ENDPOINT=http://tachi-s3:9000
 TACHI_CDN_SAVE_LOCATION_ACCESS_KEY_ID=minio
 TACHI_CDN_SAVE_LOCATION_SECRET_ACCESS_KEY=password

typescript/server/package.json

@@ -11,7 +11,10 @@
 		"build": "tsgo -b tsconfig.build.json -v",
 		"typecheck": "tsgo --noEmit",
 		"lint": "eslint ./src",
-		"lint-fix": "eslint ./src --fix"
+		"lint-fix": "eslint ./src --fix",
+		"cron-worker": "bun run src/cron-worker.ts",
+		"job-queue-worker": "bun run src/job-queue-worker.ts",
+		"load-test:score-import": "bun run src/load-tests/score-import-load-cli.ts"
 	},
 	"author": "zk",
 	"license": "AGPL3",
@@ -81,7 +84,8 @@
 		"tachi-db-migration-engine": "workspace:*",
 		"ts-node": "catalog:",
 		"typescript": "catalog:",
-		"zod": "catalog:"
+		"zod": "catalog:",
+		"cron-parser": "catalog:"
 	},
 	"nyc": {
 		"reporter": [

typescript/server/src/actions/customise-score.test.ts

@@ -1,5 +1,5 @@
 import { LoadScoreDocumentById } from "#lib/db-formats/score";
-import { mongoScoreDataToPg, pgScoreDataToMongo } from "#lib/v3/migration-tools";
+import { mongoScoreDataToPg, pgScoreDataToAPI } from "#lib/v3/migration-tools";
 import DB from "#services/pg/db";
 import { seedUser } from "#test-utils/pg-fixtures";
 import { type ScoreData } from "tachi-common";
@@ -20,7 +20,7 @@ describe("mergeScoreDataFromPg", () => {
 		} as ScoreData<"iidx-sp">;
 
 		const pg = mongoScoreDataToPg("iidx-sp", { ...original, judgements: {} });
-		const back = pgScoreDataToMongo("iidx-sp", pg);
+		const back = pgScoreDataToAPI("iidx-sp", pg);
 
 		expect(back).toMatchObject({
 			grade: "F",

typescript/server/src/actions/patch-ugpt-settings.test.ts

@@ -1,5 +1,9 @@
 import { SELECT_ACTION } from "#lib/db-formats/action";
-import { GetUGPTSettingsDocument, SELECT_GAME_SETTINGS } from "#lib/db-formats/ugpt-settings";
+import {
+	GetUGPTSettingsDocument,
+	SELECT_GAME_PROFILE_SETTINGS,
+} from "#lib/db-formats/ugpt-settings";
+import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings";
 import DB from "#services/pg/db";
 import { seedUser } from "#test-utils/pg-fixtures";
 import { beforeEach, describe, expect, it } from "vitest";
@@ -13,17 +17,13 @@ describe("ACTION_PatchUGPTSettings", () => {
 	beforeEach(async () => {
 		({ id: userId, username } = await seedUser({ username: `ugpt_set_${Date.now()}` }));
 
-		await DB.insertInto("game_settings")
+		await DB.insertInto("game_profile")
 			.values({
 				user_id: userId,
 				game: "iidx-sp",
-				pf_preferred_score_alg: null,
-				pf_preferred_session_alg: null,
-				pf_preferred_profile_alg: null,
-				pf_preferred_default_enum: null,
-				pf_default_table: null,
-				pf_preferred_ranking: null,
-				data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }),
+				ratings: JSON.stringify({}),
+				classes: JSON.stringify({}),
+				...newGameProfilePreferenceColumns("iidx-sp"),
 			})
 			.execute();
 	});
@@ -41,10 +41,10 @@ describe("ACTION_PatchUGPTSettings", () => {
 
 		expect(settings?.preferences.preferredScoreAlg).toBe("ktLampRating");
 
-		const row = await DB.selectFrom("game_settings")
-			.select(SELECT_GAME_SETTINGS)
-			.where("game_settings.user_id", "=", userId)
-			.where("game_settings.game", "=", "iidx-sp")
+		const row = await DB.selectFrom("game_profile")
+			.select(SELECT_GAME_PROFILE_SETTINGS)
+			.where("game_profile.user_id", "=", userId)
+			.where("game_profile.game", "=", "iidx-sp")
 			.executeTakeFirstOrThrow();
 
 		expect(row.pf_preferred_score_alg).toBe("ktLampRating");

typescript/server/src/actions/patch-ugpt-settings.ts

@@ -1,10 +1,13 @@
 import { MakeAction } from "#lib/actions/actions";
-import { GetUGPTSettingsDocument, SELECT_GAME_SETTINGS } from "#lib/db-formats/ugpt-settings";
+import {
+	GetUGPTSettingsDocument,
+	SELECT_GAME_PROFILE_SETTINGS,
+} from "#lib/db-formats/ugpt-settings";
 import DB from "#services/pg/db";
 import { IsUserAdmin } from "#utils/user";
 import { ExpectedErr } from "bliss";
 import { type UGPTSettingsDocument } from "tachi-common";
-import { type GameSettingsUpdate } from "tachi-db";
+import { type GameProfileUpdate } from "tachi-db";
 
 export const ACTION_PatchUGPTSettings = MakeAction("PATCH_UGPT_SETTINGS", async (taker, input) => {
 	const { userID, game, preferences } = input;
@@ -52,17 +55,17 @@ export const ACTION_PatchUGPTSettings = MakeAction("PATCH_UGPT_SETTINGS", async
 		return { settings };
 	}
 
-	const row = await DB.selectFrom("game_settings")
-		.select(SELECT_GAME_SETTINGS)
-		.where("game_settings.user_id", "=", userID)
-		.where("game_settings.game", "=", game)
+	const row = await DB.selectFrom("game_profile")
+		.select(SELECT_GAME_PROFILE_SETTINGS)
+		.where("game_profile.user_id", "=", userID)
+		.where("game_profile.game", "=", game)
 		.executeTakeFirst();
 
 	if (!row) {
 		throw new ExpectedErr(404, "You do not have an account for this game.");
 	}
 
-	const set: GameSettingsUpdate = {};
+	const set: GameProfileUpdate = {};
 
 	if (body.preferredScoreAlg !== undefined) {
 		set.pf_preferred_score_alg = body.preferredScoreAlg;
@@ -99,10 +102,10 @@ export const ACTION_PatchUGPTSettings = MakeAction("PATCH_UGPT_SETTINGS", async
 		return { settings };
 	}
 
-	await DB.updateTable("game_settings")
+	await DB.updateTable("game_profile")
 		.set(set)
-		.where("user_id", "=", userID)
-		.where("game", "=", game)
+		.where("game_profile.user_id", "=", userID)
+		.where("game_profile.game", "=", game)
 		.execute();
 
 	const settings = await GetUGPTSettingsDocument(userID, game);

typescript/server/src/actions/score-import.ts

@@ -0,0 +1,57 @@
+import type { ScoreImportJobData } from "#lib/score-import/worker/types";
+
+import { MakeAction } from "#lib/actions/actions";
+import { GetInputParser } from "#lib/score-import/framework/common/get-input-parser";
+import ScoreImportFatalError from "#lib/score-import/framework/score-importing/score-import-error";
+import ScoreImportMain from "#lib/score-import/framework/score-importing/score-import-main";
+import {
+	EndTrackingImport,
+	MarkImportAsFailed,
+	StartTrackingImport,
+} from "#lib/score-import/framework/status-tracking/import-status-tracking";
+import { ExpectedErr } from "bliss";
+import { type ImportTypes } from "tachi-common";
+
+/**
+ * Authoritative score-import mutation: tracking (unless worker already did), parse + convert,
+ * `EndTrackingImport` or `MarkImportAsFailed`, and `action` table audit. User-facing
+ * `ScoreImportFatalError` is turned into `ExpectedErr` for correct audit (`BAD` / not `THROW`).
+ */
+export const ACTION_ScoreImport = MakeAction("SCORE_IMPORT", async (taker, input) => {
+	const { importID, importType, userIntent, skipStartTracking } = input;
+	const parserArguments = input[
+		"!parserArguments"
+	] as ScoreImportJobData<ImportTypes>["parserArguments"];
+
+	const jobData: ScoreImportJobData<ImportTypes> = {
+		importID,
+		importType: importType as ImportTypes,
+		userID: taker.acct.id,
+		userIntent,
+		parserArguments,
+	};
+
+	if (!skipStartTracking) {
+		await StartTrackingImport(jobData);
+	}
+
+	try {
+		const InputParser = GetInputParser(jobData);
+		await ScoreImportMain(
+			taker.acct.id,
+			userIntent,
+			importType as ImportTypes,
+			InputParser,
+			importID,
+		);
+		await EndTrackingImport(importID);
+		return { importID };
+	} catch (e) {
+		const err = e as Error | ScoreImportFatalError;
+		await MarkImportAsFailed(importID, err);
+		if (err instanceof ScoreImportFatalError) {
+			throw new ExpectedErr(err.statusCode, err.message);
+		}
+		throw e;
+	}
+});

typescript/server/src/actions/set-rivals.test.ts

@@ -1,3 +1,4 @@
+import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings";
 import { ServerConfig } from "#lib/setup/config";
 import DB from "#services/pg/db";
 import { seedUser } from "#test-utils/pg-fixtures";
@@ -14,21 +15,23 @@ describe("ACTION_SetRivals", () => {
 		({ id: userId, username } = await seedUser({ username: `rival_set_${Date.now()}` }));
 		rivalId = (await seedUser({ username: `rival_target_${Date.now()}` })).id;
 
-		const ugptRow = {
-			game: "iidx-sp" as const,
-			pf_preferred_score_alg: null,
-			pf_preferred_session_alg: null,
-			pf_preferred_profile_alg: null,
-			pf_preferred_default_enum: null,
-			pf_default_table: null,
-			pf_preferred_ranking: null,
-			data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }),
-		};
-
-		await DB.insertInto("game_settings")
+		const prefs = newGameProfilePreferenceColumns("iidx-sp");
+		await DB.insertInto("game_profile")
 			.values([
-				{ user_id: userId, ...ugptRow },
-				{ user_id: rivalId, ...ugptRow },
+				{
+					user_id: userId,
+					game: "iidx-sp",
+					ratings: JSON.stringify({}),
+					classes: JSON.stringify({}),
+					...prefs,
+				},
+				{
+					user_id: rivalId,
+					game: "iidx-sp",
+					ratings: JSON.stringify({}),
+					classes: JSON.stringify({}),
+					...prefs,
+				},
 			])
 			.execute();
 	});
@@ -102,21 +105,23 @@ describe("ACTION_SetRivals", () => {
 			),
 		);
 
-		const ugptRow = {
-			game: "iidx-sp" as const,
-			pf_preferred_score_alg: null,
-			pf_preferred_session_alg: null,
-			pf_preferred_profile_alg: null,
-			pf_preferred_default_enum: null,
-			pf_default_table: null,
-			pf_preferred_ranking: null,
-			data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }),
-		};
-
-		await DB.insertInto("game_settings")
+		const prefs = newGameProfilePreferenceColumns("iidx-sp");
+		await DB.insertInto("game_profile")
 			.values([
-				{ user_id: main.id, ...ugptRow },
-				...rivalUsers.map((u) => ({ user_id: u.id, ...ugptRow })),
+				{
+					user_id: main.id,
+					game: "iidx-sp",
+					ratings: JSON.stringify({}),
+					classes: JSON.stringify({}),
+					...prefs,
+				},
+				...rivalUsers.map((u) => ({
+					user_id: u.id,
+					game: "iidx-sp" as const,
+					ratings: JSON.stringify({}),
+					classes: JSON.stringify({}),
+					...prefs,
+				})),
 			])
 			.execute();
 

typescript/server/src/actions/update-ugpt-showcase.ts

@@ -14,27 +14,22 @@ export const ACTION_UpdateUgptShowcase = MakeAction(
 			throw new ExpectedErr(403, "You are not authorised to modify this user's showcase.");
 		}
 
-		const settingsRow = await DB.selectFrom("game_settings")
-			.select("user_id")
-			.where("user_id", "=", userID)
-			.where("game", "=", game)
+		const profileRow = await DB.selectFrom("game_profile")
+			.select("game_profile.user_id")
+			.where("game_profile.user_id", "=", userID)
+			.where("game_profile.game", "=", game)
 			.executeTakeFirst();
 
-		if (!settingsRow) {
+		if (!profileRow) {
 			throw new ExpectedErr(404, "You do not have a profile for this game.");
 		}
 
 		const payload = stats as Array<ShowcaseStatDetails>;
 
-		await DB.insertInto("game_settings_showcase")
-			.values({
-				user_id: userID,
-				game,
-				data: JSON.stringify(payload),
-			})
-			.onConflict((oc) =>
-				oc.columns(["user_id", "game"]).doUpdateSet({ data: JSON.stringify(payload) }),
-			)
+		await DB.updateTable("game_profile")
+			.set({ showcase: JSON.stringify(payload) })
+			.where("game_profile.user_id", "=", userID)
+			.where("game_profile.game", "=", game)
 			.execute();
 
 		const newSettings = await GetUGPTSettingsDocument(userID, game);

typescript/server/src/cron-worker.ts

@@ -0,0 +1,47 @@
+import { loadServerEnvFile } from "#lib/setup/load-server-env";
+loadServerEnvFile(process.env.NODE_ENV === "test" ? ".env.test" : ".env");
+
+import { runCronTickOnce } from "#lib/jobs/cron/cron-service";
+import { log } from "#lib/log/log";
+import { Env } from "#lib/setup/config";
+import { ClosePgConnection } from "#services/pg/db";
+import { applyMigrations } from "tachi-db-migration-engine";
+
+const TICK_MS = 5_000;
+
+void bootstrap();
+
+/**
+ * Often started alongside the API by `just server` (or `bun run cron-worker` alone).
+ * Single active scheduler (Postgres `cron_task`); extra processes no-op when the advisory lock is held.
+ */
+async function bootstrap() {
+	await applyMigrations(Env.POSTGRES_URL, Env.MIGRATIONS_DIR);
+	log.info({ bootInfo: true }, "tachi cron worker starting.");
+	let stopping = false;
+	const shutdown = () => {
+		stopping = true;
+	};
+	process.on("SIGINT", shutdown);
+	process.on("SIGTERM", shutdown);
+
+	// eslint-disable-next-line no-unmodified-loop-condition
+	while (!stopping) {
+		try {
+			// eslint-disable-next-line no-await-in-loop
+			await runCronTickOnce();
+		} catch (e) {
+			log.error(e, "Cron tick error.");
+		}
+		if (stopping) {
+			break;
+		}
+		// eslint-disable-next-line no-await-in-loop
+		await new Promise((r) => {
+			setTimeout(r, TICK_MS);
+		});
+	}
+	log.info("Cron worker stopped.");
+	await ClosePgConnection();
+	process.exit(0);
+}

typescript/server/src/game-implementations/games/iidx.impl.test.ts

@@ -236,6 +236,110 @@ describe("IIDX_IMPL (unit)", () => {
 		});
 	});
 
+	describe("scoreCalcs ktLampRatingNC / ktLampRatingHC / ktLampRatingEXHC", () => {
+		it("IIDX SP", () => {
+			const run = (
+				scoreData: Partial<ScoreData<"iidx-sp">>,
+				chartData: Partial<ChartDocumentData["iidx-sp"]>,
+			) =>
+				IIDX_SP_IMPL.scoreCalcs(
+					dmf(TestingIIDXSPScore.scoreData, scoreData),
+					IIDX_SP_IMPL.scoreDeriver(
+						dmf(TestingIIDXSPScore.scoreData, scoreData),
+						dmf(Testing511SPA, { data: chartData as never }),
+					),
+					dmf(Testing511SPA, { data: chartData as never }),
+				);
+
+			function mkTier(v: number) {
+				return { value: v, text: "whatever", individualDifference: false };
+			}
+
+			const tiered = { ncTier: mkTier(15), hcTier: mkTier(16), exhcTier: mkTier(17) };
+
+			expect(run({ lamp: "FAILED" }, {})).toMatchObject({
+				ktLampRatingNC: 0,
+				ktLampRatingHC: 0,
+				ktLampRatingEXHC: 0,
+			});
+			expect(run({ lamp: "EASY CLEAR" }, {})).toMatchObject({
+				ktLampRatingNC: 0,
+				ktLampRatingHC: 0,
+				ktLampRatingEXHC: 0,
+			});
+			expect(run({ lamp: "CLEAR" }, tiered)).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 0,
+				ktLampRatingEXHC: 0,
+			});
+			expect(run({ lamp: "HARD CLEAR" }, tiered)).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 16,
+				ktLampRatingEXHC: 0,
+			});
+			expect(run({ lamp: "EX HARD CLEAR" }, tiered)).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 16,
+				ktLampRatingEXHC: 17,
+			});
+			expect(run({ lamp: "FULL COMBO" }, tiered)).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 16,
+				ktLampRatingEXHC: 17,
+			});
+			expect(run({ lamp: "HARD CLEAR" }, { ncTier: mkTier(15) })).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 15,
+				ktLampRatingEXHC: 0,
+			});
+		});
+
+		it("IIDX DP", () => {
+			const run = (
+				scoreData: Partial<ScoreData<"iidx-dp">>,
+				chartData: Partial<ChartDocumentData["iidx-dp"]>,
+			) =>
+				IIDX_DP_IMPL.scoreCalcs(
+					dmf(TestingIIDXSPScore.scoreData, scoreData),
+					IIDX_DP_IMPL.scoreDeriver(
+						dmf(TestingIIDXSPScore.scoreData, scoreData),
+						dmf(Testing511SPA, { data: chartData as never }) as never,
+					),
+					dmf(Testing511SPA, { data: chartData as never }) as never,
+				);
+
+			function mkTier(v: number) {
+				return { value: v, text: "whatever", individualDifference: false };
+			}
+
+			expect(run({ lamp: "EASY CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({
+				ktLampRatingNC: 0,
+				ktLampRatingHC: 0,
+				ktLampRatingEXHC: 0,
+			});
+			expect(run({ lamp: "CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 0,
+				ktLampRatingEXHC: 0,
+			});
+			expect(run({ lamp: "HARD CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 15,
+				ktLampRatingEXHC: 0,
+			});
+			expect(run({ lamp: "EX HARD CLEAR" }, { dpTier: mkTier(15) })).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 15,
+				ktLampRatingEXHC: 15,
+			});
+			expect(run({ lamp: "FULL COMBO" }, { dpTier: mkTier(15) })).toMatchObject({
+				ktLampRatingNC: 15,
+				ktLampRatingHC: 15,
+				ktLampRatingEXHC: 15,
+			});
+		});
+	});
+
 	describe("goal formatters", () => {
 		describe.each([IIDX_SP_IMPL, IIDX_DP_IMPL] as const)("impl", (impl) => {
 			it("criteria", () => {

typescript/server/src/game-implementations/games/iidx.ts

@@ -29,15 +29,23 @@ type IIDXGames = "iidx-dp" | "iidx-sp";
 const IIDX_SESSION_CALCS: GPTSessionCalcs<IIDXGames> = (arr) => ({
 	BPI: SessionAvgBest10For("BPI")(arr),
 	ktLampRating: SessionAvgBest10For("ktLampRating")(arr),
+	ktLampRatingNC: SessionAvgBest10For("ktLampRatingNC")(arr),
+	ktLampRatingHC: SessionAvgBest10For("ktLampRatingHC")(arr),
+	ktLampRatingEXHC: SessionAvgBest10For("ktLampRatingEXHC")(arr),
 });
 
 const IIDX_PROFILE_CALCS: GPTProfileCalcs<IIDXGames> = async (game, userID) => {
-	const [BPI, ktLampRating] = await Promise.all([
-		ProfileAvgBestN("BPI", 20, true)(game, userID),
-		ProfileAvgBestN("ktLampRating", 20)(game, userID),
-	]);
+	const [BPI, ktLampRating, ktLampRatingNC, ktLampRatingHC, ktLampRatingEXHC] = await Promise.all(
+		[
+			ProfileAvgBestN("BPI", 20, true)(game, userID),
+			ProfileAvgBestN("ktLampRating", 20)(game, userID),
+			ProfileAvgBestN("ktLampRatingNC", 20)(game, userID),
+			ProfileAvgBestN("ktLampRatingHC", 20)(game, userID),
+			ProfileAvgBestN("ktLampRatingEXHC", 20)(game, userID),
+		],
+	);
 
-	return { BPI, ktLampRating };
+	return { BPI, ktLampRating, ktLampRatingNC, ktLampRatingHC, ktLampRatingEXHC };
 };
 
 const IIDX_MERGERS: Array<PBMergeFunction<IIDXGames>> = [
@@ -148,7 +156,29 @@ export const IIDX_SP_IMPL: GameImplementation<"iidx-sp"> = {
 				ktLampRating = 0;
 		}
 
-		return { BPI: bpi, ktLampRating };
+		const atLeastNcClear =
+			scoreData.lamp === "CLEAR" ||
+			scoreData.lamp === "HARD CLEAR" ||
+			scoreData.lamp === "EX HARD CLEAR" ||
+			scoreData.lamp === "FULL COMBO";
+		const atLeastHcClear =
+			scoreData.lamp === "HARD CLEAR" ||
+			scoreData.lamp === "EX HARD CLEAR" ||
+			scoreData.lamp === "FULL COMBO";
+		const atLeastExhcClear =
+			scoreData.lamp === "EX HARD CLEAR" || scoreData.lamp === "FULL COMBO";
+
+		const ktLampRatingNC = atLeastNcClear ? ncValue : 0;
+		const ktLampRatingHC = atLeastHcClear ? hcValue : 0;
+		const ktLampRatingEXHC = atLeastExhcClear ? exhcValue : 0;
+
+		return {
+			BPI: bpi,
+			ktLampRating,
+			ktLampRatingNC,
+			ktLampRatingHC,
+			ktLampRatingEXHC,
+		};
 	},
 	sessionCalcs: IIDX_SESSION_CALCS,
 	profileCalcs: IIDX_PROFILE_CALCS,
@@ -205,7 +235,29 @@ export const IIDX_DP_IMPL: GameImplementation<"iidx-dp"> = {
 				ktLampRating = 0;
 		}
 
-		return { BPI: bpi, ktLampRating };
+		const atLeastNcClear =
+			scoreData.lamp === "CLEAR" ||
+			scoreData.lamp === "HARD CLEAR" ||
+			scoreData.lamp === "EX HARD CLEAR" ||
+			scoreData.lamp === "FULL COMBO";
+		const atLeastHcClear =
+			scoreData.lamp === "HARD CLEAR" ||
+			scoreData.lamp === "EX HARD CLEAR" ||
+			scoreData.lamp === "FULL COMBO";
+		const atLeastExhcClear =
+			scoreData.lamp === "EX HARD CLEAR" || scoreData.lamp === "FULL COMBO";
+
+		const ktLampRatingNC = atLeastNcClear ? ecValue : 0;
+		const ktLampRatingHC = atLeastHcClear ? ecValue : 0;
+		const ktLampRatingEXHC = atLeastExhcClear ? ecValue : 0;
+
+		return {
+			BPI: bpi,
+			ktLampRating,
+			ktLampRatingNC,
+			ktLampRatingHC,
+			ktLampRatingEXHC,
+		};
 	},
 	sessionCalcs: IIDX_SESSION_CALCS,
 	profileCalcs: IIDX_PROFILE_CALCS,

typescript/server/src/job-queue-worker.ts

@@ -0,0 +1,64 @@
+/* eslint-disable no-await-in-loop */
+import { loadServerEnvFile } from "#lib/setup/load-server-env";
+loadServerEnvFile(process.env.NODE_ENV === "test" ? ".env.test" : ".env");
+
+import { JOB_KIND_SCORE_IMPORT } from "#lib/jobs/job-queue/constants";
+import { ClaimNextJob, MarkJobDone, MarkJobFailed } from "#lib/jobs/job-queue/queue-ops";
+import { log } from "#lib/log/log";
+import { CloseScoreImportQueue } from "#lib/score-import/worker/queue";
+import { processScoreImportJobFromPayload } from "#lib/score-import/worker/score-import-job-processor";
+import { Env } from "#lib/setup/config";
+import { ClosePgConnection } from "#services/pg/db";
+import { CloseRedisConnection } from "#services/redis/redis";
+import { applyMigrations } from "tachi-db-migration-engine";
+
+const POLL_MS = 250;
+
+void bootstrap();
+
+/**
+ * Often run by `just server` (one or more via `TACHI_SERVER_JOB_WORKER_COUNT`); each process claims
+ * with `FOR UPDATE SKIP LOCKED`.
+ */
+async function bootstrap() {
+	await applyMigrations(Env.POSTGRES_URL, Env.MIGRATIONS_DIR);
+	log.info({ bootInfo: true }, "tachi job-queue worker starting (Postgres job_queue).");
+	let stopping = false;
+	const shutdown = () => {
+		stopping = true;
+	};
+	process.on("SIGINT", shutdown);
+	process.on("SIGTERM", shutdown);
+	// eslint-disable-next-line no-unmodified-loop-condition
+	while (!stopping) {
+		const job = await ClaimNextJob();
+		if (!job) {
+			if (stopping) {
+				break;
+			}
+			await new Promise((r) => {
+				setTimeout(r, POLL_MS);
+			});
+			continue;
+		}
+		try {
+			switch (job.job_kind) {
+				case JOB_KIND_SCORE_IMPORT:
+					await processScoreImportJobFromPayload(job.payload);
+					break;
+				default:
+					log.error({ job_kind: job.job_kind, row_id: job.row_id }, "Unknown job_kind.");
+					throw new Error(`Unknown job_kind ${String(job.job_kind)}`);
+			}
+			await MarkJobDone(job.row_id);
+		} catch (e) {
+			log.error(e, `Job ${job.row_id} failed.`);
+			await MarkJobFailed(job.row_id);
+		}
+	}
+	log.info("Job worker loop stopped, closing resources.");
+	await CloseScoreImportQueue();
+	await CloseRedisConnection();
+	await ClosePgConnection();
+	process.exit(0);
+}

typescript/server/src/lib/actions/actions.ts

@@ -378,6 +378,18 @@ export const ActionSignatures = {
 		}),
 		output: z.object({}),
 	},
+	SCORE_IMPORT: {
+		input: z.object({
+			importID: z.string(),
+			importType: z.string(),
+			userIntent: z.boolean(),
+			"!parserArguments": z.array(z.unknown()),
+			skipStartTracking: z.boolean().optional(),
+		}),
+		output: z.object({
+			importID: z.string(),
+		}),
+	},
 	BMS_TABLE_SYNC: {
 		input: z.object({}),
 		output: z.object({}),

typescript/server/src/lib/db-formats/pb.ts

@@ -1,6 +1,6 @@
 import type { Game } from "tachi-db";
 
-import { pgScoreDataToMongo } from "#lib/v3/migration-tools";
+import { pgScoreDataToAPI } from "#lib/v3/migration-tools";
 import DB from "#services/pg/db";
 import { EscapeForILIKE } from "#utils/misc";
 import { ISO8601ToUnixMilliseconds } from "#utils/time";
@@ -103,13 +103,13 @@ export async function ToPbScoreDocument(row: PbDocumentJoinRow): Promise<PBScore
 				})) as [PBReference, ...PBReference[]])
 			: [{ name: "Primary", scoreID: "unknown" }];
 
-	const scoreData = pgScoreDataToMongo(
+	const scoreData = pgScoreDataToAPI(
 		row.chart_game as V3Game,
 		{
 			data: row.data,
 			derived: row.derived_data,
 			judgements: {},
-		} as Parameters<typeof pgScoreDataToMongo>[1],
+		} as Parameters<typeof pgScoreDataToAPI>[1],
 	);
 
 	const rawCd = row.calculated_data;

typescript/server/src/lib/db-formats/score.ts

@@ -1,4 +1,4 @@
-import { pgScoreDataToMongo } from "#lib/v3/migration-tools";
+import { pgScoreDataToAPI } from "#lib/v3/migration-tools";
 import DB from "#services/pg/db";
 import { ISO8601ToUnixMilliseconds } from "#utils/time";
 import { type GameGroup, type ImportTypes, type ScoreDocument } from "tachi-common";
@@ -47,7 +47,7 @@ export interface ScoreDocumentJoinRow {
 }
 
 export function ToScoreDocument(row: ScoreDocumentJoinRow): ScoreDocument {
-	const scoreData = pgScoreDataToMongo(row.score_game, {
+	const scoreData = pgScoreDataToAPI(row.score_game, {
 		data: row.score_data as any,
 		derived: row.score_derived_data as any,
 		judgements: row.score_judgements as any,

typescript/server/src/lib/db-formats/ugpt-settings.ts

@@ -9,30 +9,32 @@ import {
 } from "tachi-common";
 import { type Database } from "tachi-db";
 
-export const SELECT_GAME_SETTINGS = [
-	"game_settings.user_id",
-	"game_settings.game",
-	"game_settings.pf_preferred_score_alg",
-	"game_settings.pf_preferred_session_alg",
-	"game_settings.pf_preferred_profile_alg",
-	"game_settings.pf_preferred_default_enum",
-	"game_settings.pf_default_table",
-	"game_settings.pf_preferred_ranking",
-	"game_settings.data",
+export const SELECT_GAME_PROFILE_SETTINGS = [
+	"game_profile.user_id",
+	"game_profile.game",
+	"game_profile.pf_preferred_score_alg",
+	"game_profile.pf_preferred_session_alg",
+	"game_profile.pf_preferred_profile_alg",
+	"game_profile.pf_preferred_default_enum",
+	"game_profile.pf_default_table",
+	"game_profile.pf_preferred_ranking",
+	"game_profile.data",
+	"game_profile.showcase",
 ] as const;
 
-export type GameSettingsRow = Selection<
+export type GameProfilePreferenceRow = Selection<
 	Database,
-	"game_settings",
-	(typeof SELECT_GAME_SETTINGS)[number]
+	"game_profile",
+	(typeof SELECT_GAME_PROFILE_SETTINGS)[number]
 >;
 
 export function ToUGPTSettingsDocument(
-	row: GameSettingsRow,
+	row: GameProfilePreferenceRow,
 	rivals: Array<integer>,
-	stats: Array<ShowcaseStatDetails>,
 ): UGPTSettingsDocument {
 	const gameSpecific = row.data as UGPTSettingsDocument["preferences"]["gameSpecific"];
+	const rawShowcase = row.showcase as Array<ShowcaseStatDetails>;
+	const stats = normalizeShowcaseStats(rawShowcase);
 
 	return {
 		userID: row.user_id,
@@ -62,10 +64,10 @@ export async function GetUGPTSettingsDocument(
 	userID: integer,
 	game: V3Game,
 ): Promise<UGPTSettingsDocument | null> {
-	const row = await DB.selectFrom("game_settings")
-		.select(SELECT_GAME_SETTINGS)
-		.where("game_settings.user_id", "=", userID)
-		.where("game_settings.game", "=", game)
+	const row = await DB.selectFrom("game_profile")
+		.select(SELECT_GAME_PROFILE_SETTINGS)
+		.where("game_profile.user_id", "=", userID)
+		.where("game_profile.game", "=", game)
 		.executeTakeFirst();
 
 	if (!row) {
@@ -80,18 +82,12 @@ export async function GetUGPTSettingsDocument(
 
 	const rivals = rivalRows.map((r) => r.rival);
 
-	const showcaseRow = await DB.selectFrom("game_settings_showcase")
-		.select("data")
-		.where("user_id", "=", userID)
-		.where("game", "=", game)
-		.executeTakeFirst();
-
-	const stats = showcaseRow ? (showcaseRow.data as Array<ShowcaseStatDetails>) : [];
-
-	return ToUGPTSettingsDocument(row, rivals, normalizeShowcaseStats(stats));
+	return ToUGPTSettingsDocument(row, rivals);
 }
 
 /** Strips legacy `metric` from chart entries stored before chart showcase was PB+playcount-only. */
+// TODO(zk): nonsense, lets just remove this
+// with a migration?
 function normalizeShowcaseStats(raw: Array<ShowcaseStatDetails>): Array<ShowcaseStatDetails> {
 	return raw.map((stat) => {
 		if (stat.mode === "chart") {

typescript/server/src/lib/folders/folders.ts

@@ -10,7 +10,7 @@ import { LoadPbsForUserOnChartsByPgIds } from "#lib/db-formats/pb";
 import { GetSongsByIDs } from "#lib/db-formats/song";
 import { LoadTableDocumentByLegacyId } from "#lib/db-formats/table";
 import { log } from "#lib/log/log";
-import { pgScoreDataToMongo } from "#lib/v3/migration-tools";
+import { pgScoreDataToAPI } from "#lib/v3/migration-tools";
 import DB from "#services/pg/db";
 import { GetFolderForIDGuaranteed } from "#utils/db";
 import { ISO8601ToUnixMilliseconds, UnixMillisecondsToISO8601 } from "#utils/time";
@@ -293,7 +293,7 @@ export async function GetEnumDistForFolderAsOf(
 		const maxByChart = new Map<string, Record<string, integer>>();
 
 		for (const row of rows) {
-			const mongoData = pgScoreDataToMongo(v3Game, {
+			const mongoData = pgScoreDataToAPI(v3Game, {
 				data: row.data as any,
 				derived: row.derived_data as any,
 				judgements: row.judgements as any,

typescript/server/src/lib/folders/get-session-folder-raises.ts

@@ -0,0 +1,159 @@
+import { LoadFolderDocumentsByIds } from "#lib/db-formats/folders";
+import { GetEnumDistForFolderAsOf, GetFolderIDsForChartId } from "#lib/folders/folders";
+import { GetSessionData } from "#utils/queries/sessions";
+import {
+	type FolderDocument,
+	GetGameConfig,
+	GetScoreEnumConfs,
+	type integer,
+	type SessionDocument,
+	type V3Game,
+} from "tachi-common";
+
+export type SessionFolderRaisesPayload = {
+	folder: FolderDocument;
+	previousCount: integer;
+	raisedCharts: Array<string>;
+	totalCharts: integer;
+	type: string;
+	value: string;
+};
+
+function bucketKey(folderId: string, metric: string, value: string): string {
+	return JSON.stringify([folderId, metric, value]);
+}
+
+function parseBucketKey(key: string): [string, string, string] {
+	const parsed = JSON.parse(key) as [string, string, string];
+	return parsed;
+}
+
+/**
+ * Per-chart folder ids (lookup is identical for all scores on the same chart).
+ */
+async function folderIdsForChartCached(
+	chartId: string,
+	cache: Map<string, Array<string>>,
+): Promise<Array<string>> {
+	let ids = cache.get(chartId);
+
+	if (!ids) {
+		ids = await GetFolderIDsForChartId(chartId);
+		cache.set(chartId, ids);
+	}
+
+	return ids;
+}
+
+/**
+ * Folder raise rows for the session view: which folders gained which enum
+ * values on which charts, and the user's exact folder distribution before
+ * {@link SessionDocument.timeStarted}.
+ */
+export async function GetSessionFolderRaises(
+	session: SessionDocument,
+): Promise<Array<SessionFolderRaisesPayload>> {
+	const { scores, scoreInfo } = await GetSessionData(session);
+	const scoreMap = new Map(scores.map((s) => [s.scoreID, s]));
+	const gameConfig = GetGameConfig(session.game as V3Game);
+	const enumMetrics = GetScoreEnumConfs(gameConfig);
+
+	const chartFolderCache = new Map<string, Array<string>>();
+	const bucket = new Map<string, Set<string>>();
+
+	for (const info of scoreInfo) {
+		const score = scoreMap.get(info.scoreID);
+
+		if (!score) {
+			continue;
+		}
+
+		for (const [metric, conf] of Object.entries(enumMetrics)) {
+			if (!info.isNewScore) {
+				const delta = info.deltas[metric];
+
+				if (delta === undefined || delta <= 0) {
+					continue;
+				}
+			}
+
+			const enumIndexes = score.scoreData.enumIndexes;
+			const idx = enumIndexes?.[metric as keyof typeof enumIndexes];
+
+			if (idx === undefined) {
+				continue;
+			}
+
+			if (idx <= conf.values.indexOf(conf.minimumRelevantValue)) {
+				continue;
+			}
+
+			const valueRaw = (score.scoreData as Record<string, unknown>)[metric];
+
+			if (typeof valueRaw !== "string") {
+				continue;
+			}
+
+			const folderIds = await folderIdsForChartCached(score.chartID, chartFolderCache);
+
+			for (const folderId of folderIds) {
+				const key = bucketKey(folderId, metric, valueRaw);
+				let set = bucket.get(key);
+
+				if (!set) {
+					set = new Set();
+					bucket.set(key, set);
+				}
+
+				set.add(score.chartID);
+			}
+		}
+	}
+
+	if (bucket.size === 0) {
+		return [];
+	}
+
+	const folderIds = [...new Set([...bucket.keys()].map((k) => parseBucketKey(k)[0]))];
+	const folderDocs = await LoadFolderDocumentsByIds(folderIds);
+
+	const distCache = new Map<string, Awaited<ReturnType<typeof GetEnumDistForFolderAsOf>>>();
+
+	await Promise.all(
+		folderIds.map(async (fid) => {
+			const dist = await GetEnumDistForFolderAsOf(session.userID, fid, session.timeStarted);
+
+			distCache.set(fid, dist);
+		}),
+	);
+
+	const out: Array<SessionFolderRaisesPayload> = [];
+
+	for (const [key, chartSet] of bucket) {
+		const [folderId, metric, value] = parseBucketKey(key);
+		const folder = folderDocs.get(folderId);
+
+		if (!folder || folder.game !== session.game) {
+			continue;
+		}
+
+		const dist = distCache.get(folderId);
+
+		if (!dist) {
+			continue;
+		}
+
+		const previousCount = dist.enumDist[metric]?.[value] ?? 0;
+
+		out.push({
+			folder,
+			previousCount,
+			raisedCharts: [...chartSet],
+			totalCharts: dist.chartIDs.length,
+			type: metric,
+			value,
+		});
+	}
+
+	return out;
+}

typescript/server/src/lib/game-settings/create-game-settings.test.ts

@@ -1,37 +1,18 @@
-import DB from "#services/pg/db";
-import { seedUser } from "#test-utils/pg-fixtures";
 import { describe, expect, it } from "vitest";
 
-import { CreateGameSettings } from "./create-game-settings";
+import { newGameProfilePreferenceColumns } from "./create-game-settings";
 
-describe("CreateGameSettings", () => {
-	it("creates settings for a new user and game", async () => {
-		const { id: userId } = await seedUser();
-
-		await CreateGameSettings(userId, "bms-7k");
-
-		const row = await DB.selectFrom("game_settings")
-			.selectAll()
-			.where("user_id", "=", userId)
-			.where("game", "=", "bms-7k")
-			.executeTakeFirst();
-
-		expect(row).not.toBeUndefined();
-		expect(row?.game).toBe("bms-7k");
+describe("newGameProfilePreferenceColumns", () => {
+	it("returns IIDX-specific defaults for iidx-sp", () => {
+		const cols = newGameProfilePreferenceColumns("iidx-sp");
+		expect(JSON.parse(cols.data)).toEqual({ display2DXTra: false, bpiTarget: 0 });
+		expect(JSON.parse(cols.showcase)).toEqual([]);
+		expect(cols.pf_preferred_score_alg).toBeNull();
 	});
 
-	it("throws if settings already exist", async () => {
-		const { id } = await seedUser();
-		const localUserId = id;
-
-		try {
-			await CreateGameSettings(localUserId, "popn");
-			await expect(CreateGameSettings(localUserId, "popn")).rejects.toThrow(
-				/Cannot create .* game-settings as one already exists/u,
-			);
-		} finally {
-			await DB.deleteFrom("game_settings").where("user_id", "=", localUserId).execute();
-			await DB.deleteFrom("account").where("id", "=", localUserId).execute();
-		}
+	it("returns empty gameSpecific JSON for non-IIDX games", () => {
+		const cols = newGameProfilePreferenceColumns("bms-7k");
+		expect(JSON.parse(cols.data)).toEqual({});
+		expect(JSON.parse(cols.showcase)).toEqual([]);
 	});
 });

typescript/server/src/lib/game-settings/create-game-settings.ts

@@ -1,23 +1,9 @@
-import { log } from "#lib/log/log";
-import DB from "#services/pg/db";
-import { type integer, type V3Game } from "tachi-common";
+import { type V3Game } from "tachi-common";
 
 /**
- * Create GameSettings for a UGPT (which contains their preferences).
+ * Default preference columns for a new `game_profile` row (ratings/classes live alongside these).
  */
-export async function CreateGameSettings(userID: integer, game: V3Game) {
-	const exists = await DB.selectFrom("game_settings")
-		.select("user_id")
-		.where("user_id", "=", userID)
-		.where("game", "=", game)
-		.executeTakeFirst();
-
-	if (exists) {
-		log.error(`Cannot create ${userID} ${game} game-settings as one already exists?`);
-
-		throw new Error(`Cannot create ${userID} ${game} game-settings as one already exists?`);
-	}
-
+export function newGameProfilePreferenceColumns(game: V3Game) {
 	const gameSpecific =
 		game === "iidx-sp" || game === "iidx-dp"
 			? {
@@ -26,19 +12,14 @@ export async function CreateGameSettings(userID: integer, game: V3Game) {
 				}
 			: {};
 
-	await DB.insertInto("game_settings")
-		.values({
-			data: JSON.stringify(gameSpecific),
-			game,
-			pf_default_table: null,
-			pf_preferred_default_enum: null,
-			pf_preferred_profile_alg: null,
-			pf_preferred_ranking: null,
-			pf_preferred_score_alg: null,
-			pf_preferred_session_alg: null,
-			user_id: userID,
-		})
-		.execute();
-
-	log.info(`Created game settings for ${userID} (${game}).`);
+	return {
+		pf_preferred_score_alg: null as string | null,
+		pf_preferred_session_alg: null as string | null,
+		pf_preferred_profile_alg: null as string | null,
+		pf_preferred_default_enum: null as string | null,
+		pf_default_table: null as string | null,
+		pf_preferred_ranking: null as string | null,
+		data: JSON.stringify(gameSpecific),
+		showcase: JSON.stringify([]),
+	};
 }

typescript/server/src/lib/jobs/cron/cron-registry.ts

@@ -0,0 +1,125 @@
+import { ACTION_BacksyncBmsPmsSeeds } from "#actions/backsync-bms-pms-seeds";
+import { ACTION_BMSTableSync } from "#actions/bms-table-sync";
+import { ACTION_UGSSnapshot } from "#actions/ugs-snapshot";
+import { ACTION_UpdateBpiData } from "#actions/update-bpi-data";
+import { ACTION_UpdateDpTiers } from "#actions/update-dp-tiers";
+import { UpdateAILevels } from "#lib/jobs/bms-ai-table-sync";
+import { DefaultAdminUser } from "#lib/jobs/default-admin-user";
+import { DeorphanScoresMain } from "#lib/jobs/deorphan-scores";
+import { drainStatsQueuesInOrder } from "#lib/jobs/drain-dirty-queues";
+import { RebuildFolderChartLookupJob } from "#lib/jobs/rebuild-folder-chart-lookup";
+import { TachiConfig } from "#lib/setup/config";
+import { DedupeArr } from "#utils/misc";
+
+export interface CronTaskDef {
+	/** `cron_task.id` primary key. */
+	id: string;
+	/** 5-field cron in UTC. */
+	schedule: string;
+	/** Shown in admin and synced to `cron_task.description`. */
+	description: string;
+	run: () => Promise<void>;
+}
+
+function buildList(): Array<CronTaskDef> {
+	const out: Array<CronTaskDef> = [
+		{
+			id: "rebuild_folder_chart_lookup",
+			schedule: "5 0 * * *",
+			description: "Rebuild folder chart lookup",
+			run: RebuildFolderChartLookupJob,
+		},
+		{
+			id: "ugs_snapshot",
+			schedule: "0 0 * * *",
+			description: "Snapshot User Game Stats",
+			run: async () => {
+				const taker = await DefaultAdminUser.actionTaker();
+				await ACTION_UGSSnapshot(taker, {});
+			},
+		},
+		{
+			id: "deorphan_scores",
+			schedule: "1 0 * * *",
+			description: "De-Orphan Scores",
+			run: DeorphanScoresMain,
+		},
+		{
+			id: "drain_stats_queues",
+			schedule: "* * * * *",
+			description:
+				"Drain score_rederive, pb_dirty, session_dirty, game_profile_dirty (ordered)",
+			run: drainStatsQueuesInOrder,
+		},
+	];
+
+	if (TachiConfig.TYPE !== "boku") {
+		out.push(
+			{
+				id: "update_bpi",
+				schedule: "2 0 * * *",
+				description: "Update BPI",
+				run: async () => {
+					const taker = await DefaultAdminUser.actionTaker();
+					await ACTION_UpdateBpiData(taker, {});
+				},
+			},
+			{
+				id: "update_dp_tiers",
+				schedule: "3 0 * * *",
+				description: "Update DP Tiers",
+				run: async () => {
+					const taker = await DefaultAdminUser.actionTaker();
+					await ACTION_UpdateDpTiers(taker, {});
+				},
+			},
+		);
+	}
+
+	if (TachiConfig.TYPE !== "kamai") {
+		out.push(
+			{
+				id: "update_ai_table",
+				schedule: "2 0 * * *",
+				description: "Update AI Table",
+				run: UpdateAILevels,
+			},
+			{
+				id: "update_bms_tables",
+				schedule: "3 0 * * *",
+				description: "Update Tables (BMS)",
+				run: async () => {
+					const taker = await DefaultAdminUser.actionTaker();
+					await ACTION_BMSTableSync(taker, {});
+				},
+			},
+			{
+				id: "backsync_bms_pms",
+				schedule: "4 0 * * *",
+				description: "Backsync BMS + PMS",
+				run: async () => {
+					const taker = await DefaultAdminUser.actionTaker();
+					await ACTION_BacksyncBmsPmsSeeds(taker, {});
+				},
+			},
+		);
+	}
+
+	const names = out.map((e) => e.id);
+	if (DedupeArr(names).length !== names.length) {
+		throw new Error("cron task registry has duplicate id fields");
+	}
+	return out;
+}
+
+let cached: Array<CronTaskDef> | undefined;
+
+/**
+ * In-memory cron definitions (code is authoritative for schedules; rows are upserted to Postgres).
+ */
+export function getCronTaskDefinitions(): Array<CronTaskDef> {
+	if (!cached) {
+		cached = buildList();
+	}
+	return cached;
+}

typescript/server/src/lib/jobs/cron/cron-service.ts

@@ -0,0 +1,146 @@
+import { getCronTaskDefinitions } from "#lib/jobs/cron/cron-registry";
+import { log } from "#lib/log/log";
+import DB from "#services/pg/db";
+import CronExpressionParser from "cron-parser";
+import { sql } from "kysely";
+
+/** Namespaces `pg_try_advisory_lock` for the cron scheduler. */
+const CRON_ADVISORY_KEY1 = 0x54_61_63_68; // "Tach"
+const CRON_ADVISORY_KEY2 = 0x63_72_6f_6e; // "cron"
+
+/**
+ * Next cron fire strictly after `last`, that is still <= `now`.
+ * If none, `null` (not due). "Skip missed" to the latest such fire time.
+ */
+export function getDueFireTime(schedule: string, last: Date | null, now: Date): Date | null {
+	const it = CronExpressionParser.parse(schedule, {
+		currentDate: last ? new Date(last.getTime() + 1) : new Date(0),
+	});
+	const first = it.next().toDate();
+	if (first.getTime() > now.getTime()) {
+		return null;
+	}
+	let lastDue = first;
+	for (;;) {
+		const n = it.next().toDate();
+		if (n.getTime() > now.getTime()) {
+			return lastDue;
+		}
+		lastDue = n;
+	}
+}
+
+export async function syncCronTasksFromRegistry(): Promise<void> {
+	const defs = getCronTaskDefinitions();
+	const now = new Date().toISOString();
+	for (const d of defs) {
+		await DB.insertInto("cron_task")
+			.values({
+				id: d.id,
+				schedule: d.schedule,
+				description: d.description,
+				created_at: now,
+				updated_at: now,
+				last_scheduled_at: null,
+			})
+			.onConflict((oc) =>
+				oc.column("id").doUpdateSet({
+					schedule: d.schedule,
+					description: d.description,
+					updated_at: now,
+				}),
+			)
+			.execute();
+	}
+}
+
+async function tryAcquireCronTickLock(): Promise<boolean> {
+	const r = await sql<{ acquired: boolean }>`
+		SELECT pg_try_advisory_lock(${CRON_ADVISORY_KEY1}, ${CRON_ADVISORY_KEY2}) AS acquired
+	`.execute(DB);
+	const row = r.rows[0] as { acquired: boolean } | undefined;
+	return row?.acquired === true;
+}
+
+async function releaseCronTickLock(): Promise<void> {
+	await sql`SELECT pg_advisory_unlock(${CRON_ADVISORY_KEY1}, ${CRON_ADVISORY_KEY2})`.execute(DB);
+}
+
+export async function runCronTickOnce(): Promise<void> {
+	const got = await tryAcquireCronTickLock();
+	if (!got) {
+		return;
+	}
+	try {
+		await syncCronTasksFromRegistry();
+		const now = new Date();
+		const rows = await DB.selectFrom("cron_task")
+			.select(["cron_task.id", "cron_task.schedule", "cron_task.last_scheduled_at"])
+			.execute();
+		const byId = new Map(rows.map((r) => [r.id, r]));
+		for (const def of getCronTaskDefinitions()) {
+			const row = byId.get(def.id);
+			if (!row) {
+				continue;
+			}
+			const last = row.last_scheduled_at ? new Date(row.last_scheduled_at) : null;
+			const due = getDueFireTime(def.schedule, last, now);
+			if (!due) {
+				continue;
+			}
+			const scheduledAtIso = due.toISOString();
+			const execRow = await DB.insertInto("cron_task_execution")
+				.values({
+					task_id: def.id,
+					scheduled_at: scheduledAtIso,
+					status: "running",
+					completed_at: null,
+					output: null,
+					error: null,
+				})
+				.returning("cron_task_execution.id")
+				.executeTakeFirstOrThrow();
+
+			try {
+				await def.run();
+				await DB.updateTable("cron_task")
+					.set({
+						last_scheduled_at: scheduledAtIso,
+						updated_at: new Date().toISOString(),
+					})
+					.where("cron_task.id", "=", def.id)
+					.execute();
+				await DB.updateTable("cron_task_execution")
+					.set({
+						status: "success",
+						completed_at: new Date().toISOString(),
+						output: null,
+						error: null,
+					})
+					.where("cron_task_execution.id", "=", execRow.id)
+					.execute();
+				log.info(`Cron task ${def.id} completed for fire ${scheduledAtIso}.`);
+			} catch (err) {
+				const message = err instanceof Error ? err.message : String(err);
+				log.error({ err }, `Cron task ${def.id} failed.`);
+				await DB.updateTable("cron_task")
+					.set({
+						last_scheduled_at: scheduledAtIso,
+						updated_at: new Date().toISOString(),
+					})
+					.where("cron_task.id", "=", def.id)
+					.execute();
+				await DB.updateTable("cron_task_execution")
+					.set({
+						status: "failure",
+						completed_at: new Date().toISOString(),
+						error: message,
+					})
+					.where("cron_task_execution.id", "=", execRow.id)
+					.execute();
+			}
+		}
+	} finally {
+		await releaseCronTickLock();
+	}
+}

typescript/server/src/lib/jobs/drain-dirty-queues.ts

@@ -1,6 +1,15 @@
+import {
+	type ScoreDocumentJoinRow,
+	SELECT_SCORE_DOCUMENT,
+	ToScoreDocument,
+} from "#lib/db-formats/score";
+import { SELECT_SESSION_DOCUMENT } from "#lib/db-formats/session";
 import { log } from "#lib/log/log";
+import { CreateSessionCalcData } from "#lib/score-import/framework/calculated-data/session";
 import { ProcessPBs } from "#lib/score-import/framework/pb/process-pbs";
 import { rederiveScoresForChart } from "#lib/score-import/framework/pb/rederive-scores";
+import { scoreVisibleSql } from "#lib/score-import/framework/pg/score-visibility";
+import { UpdateUsersGamePlaytypeStats } from "#lib/score-import/framework/ugpt-stats/update-ugpt-stats";
 import DB from "#services/pg/db";
 import {
 	type GameGroup,
@@ -11,8 +20,12 @@ import {
 	type V3Game,
 } from "tachi-common";
 
-const PB_DIRTY_BATCH = 1000;
-const SCORE_REDERIVE_BATCH = 50;
+const PB_DIRTY_BATCH = 5000;
+const SCORE_REDERIVE_BATCH = 5000;
+const SESSION_DIRTY_BATCH = 5000;
+const GAME_PROFILE_DIRTY_BATCH = 500;
+/** Safety cap for one cron tick across score_rederive + pb_dirty + session + game_profile drains. */
+const STATS_QUEUE_DRAIN_CAP = 100_000;
 
 /**
  * Drain the `pb_dirty` queue: group entries by (game, playtype, user_id),
@@ -100,13 +113,280 @@ export async function drainScoreRederive(): Promise<number> {
 			.execute();
 	}
 
-	log.info(
+	log.debug(
 		`Drained ${rows.length} score_rederive entries, re-derived ${totalScores} total scores.`,
 	);
 
 	return rows.length;
 }
 
+/**
+ * Drain `session_dirty`: recompute `session.calculated_data` from visible scores in that session.
+ */
+export async function drainSessionDirty(): Promise<number> {
+	const rows = await DB.selectFrom("session_dirty")
+		.select(["session_dirty.session_id"])
+		.orderBy("session_dirty.enqueued_at", "asc")
+		.limit(SESSION_DIRTY_BATCH)
+		.execute();
+
+	if (rows.length === 0) {
+		return 0;
+	}
+
+	for (const row of rows) {
+		const sessionId = row.session_id;
+
+		// eslint-disable-next-line no-await-in-loop
+		const scoreRows = await DB.selectFrom("score")
+			.innerJoin("chart", "chart.id", "score.chart_id")
+			.innerJoin("song", "song.id", "chart.song_id")
+			.leftJoin("import", "import.id", "score.import_id")
+			.select(SELECT_SCORE_DOCUMENT)
+			.where("score.session_id", "=", sessionId)
+			.where(scoreVisibleSql())
+			.execute();
+
+		if (scoreRows.length === 0) {
+			// eslint-disable-next-line no-await-in-loop
+			await DB.deleteFrom("session_dirty")
+				.where("session_dirty.session_id", "=", sessionId)
+				.execute();
+			continue;
+		}
+
+		// eslint-disable-next-line no-await-in-loop
+		const sessionRow = await DB.selectFrom("session")
+			.select(SELECT_SESSION_DOCUMENT)
+			.where("session.id", "=", sessionId)
+			.executeTakeFirst();
+
+		if (!sessionRow) {
+			// eslint-disable-next-line no-await-in-loop
+			await DB.deleteFrom("session_dirty")
+				.where("session_dirty.session_id", "=", sessionId)
+				.execute();
+			continue;
+		}
+
+		const scoreDocs = scoreRows.map((r) => ToScoreDocument(r as ScoreDocumentJoinRow));
+		const calculatedData = CreateSessionCalcData(sessionRow.game as V3Game, scoreDocs);
+
+		// eslint-disable-next-line no-await-in-loop
+		await DB.updateTable("session")
+			.set({
+				calculated_data: JSON.stringify(calculatedData),
+			})
+			.where("session.id", "=", sessionId)
+			.execute();
+
+		// eslint-disable-next-line no-await-in-loop
+		await DB.deleteFrom("session_dirty")
+			.where("session_dirty.session_id", "=", sessionId)
+			.execute();
+	}
+
+	log.info(`Drained ${rows.length} session_dirty entries.`);
+
+	return rows.length;
+}
+
+/**
+ * Drain `game_profile_dirty`: recompute `game_profile` ratings/classes for each (user, playtype).
+ */
+export async function drainGameProfileDirty(): Promise<number> {
+	const rows = await DB.selectFrom("game_profile_dirty")
+		.select(["game_profile_dirty.user_id", "game_profile_dirty.game"])
+		.orderBy("game_profile_dirty.enqueued_at", "asc")
+		.limit(GAME_PROFILE_DIRTY_BATCH)
+		.execute();
+
+	if (rows.length === 0) {
+		return 0;
+	}
+
+	for (const row of rows) {
+		const userId = row.user_id;
+
+		// eslint-disable-next-line no-await-in-loop
+		await UpdateUsersGamePlaytypeStats(row.game as V3Game, userId, null, log);
+
+		// eslint-disable-next-line no-await-in-loop
+		await DB.deleteFrom("game_profile_dirty")
+			.where("game_profile_dirty.user_id", "=", userId)
+			.where("game_profile_dirty.game", "=", row.game)
+			.execute();
+	}
+
+	log.info(`Drained ${rows.length} game_profile_dirty entries.`);
+
+	return rows.length;
+}
+
+/**
+ * Drain `score_rederive`, then `pb_dirty`, then `session_dirty`, then `game_profile_dirty`,
+ * repeating until a full pass does nothing or `STATS_QUEUE_DRAIN_CAP` row-processings is reached.
+ * PBs must run before game profiles (ratings read from `pb`).
+ */
+export async function drainStatsQueuesInOrder(): Promise<void> {
+	let totalProcessed = 0;
+
+	while (totalProcessed < STATS_QUEUE_DRAIN_CAP) {
+		let cycleMoved = 0;
+
+		while (totalProcessed < STATS_QUEUE_DRAIN_CAP) {
+			// eslint-disable-next-line no-await-in-loop
+			const n = await drainScoreRederive();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+			totalProcessed += n;
+		}
+
+		while (totalProcessed < STATS_QUEUE_DRAIN_CAP) {
+			// eslint-disable-next-line no-await-in-loop
+			const n = await drainPbDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+			totalProcessed += n;
+		}
+
+		while (totalProcessed < STATS_QUEUE_DRAIN_CAP) {
+			// eslint-disable-next-line no-await-in-loop
+			const n = await drainSessionDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+			totalProcessed += n;
+		}
+
+		while (totalProcessed < STATS_QUEUE_DRAIN_CAP) {
+			// eslint-disable-next-line no-await-in-loop
+			const n = await drainGameProfileDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+			totalProcessed += n;
+		}
+
+		if (cycleMoved === 0) {
+			break;
+		}
+	}
+}
+
+/**
+ * Drain `score_rederive`, then `pb_dirty`, `session_dirty`, and `game_profile_dirty`,
+ * repeating until a full pass moves nothing. No per-tick row cap (unlike the cron
+ * drain) — intended for admin synchronous recalc.
+ */
+export async function drainStatsQueuesFully(): Promise<void> {
+	for (;;) {
+		let cycleMoved = 0;
+
+		for (;;) {
+			const n = await drainScoreRederive();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+		}
+
+		for (;;) {
+			const n = await drainPbDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+		}
+
+		for (;;) {
+			const n = await drainSessionDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+		}
+
+		for (;;) {
+			const n = await drainGameProfileDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+		}
+
+		if (cycleMoved === 0) {
+			break;
+		}
+	}
+}
+
+/**
+ * Drain `pb_dirty` then `session_dirty` and `game_profile_dirty`, repeating until
+ * idle. No per-tick row cap — intended for admin synchronous PB recalc.
+ */
+export async function drainPbDirtyAndDownstream(): Promise<void> {
+	for (;;) {
+		let cycleMoved = 0;
+
+		for (;;) {
+			const n = await drainPbDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+		}
+
+		for (;;) {
+			const n = await drainSessionDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+		}
+
+		for (;;) {
+			const n = await drainGameProfileDirty();
+
+			if (n === 0) {
+				break;
+			}
+
+			cycleMoved += n;
+		}
+
+		if (cycleMoved === 0) {
+			break;
+		}
+	}
+}
+
 /**
  * Delete `pb_dirty` rows for the given user + chart IDs. Call this after
  * a synchronous `ProcessPBs` to prevent the async worker from redundantly

typescript/server/src/lib/jobs/inline-job-runner/job-runner.ts

@@ -1,181 +0,0 @@
-import { ACTION_BacksyncBmsPmsSeeds } from "#actions/backsync-bms-pms-seeds";
-import { ACTION_BMSTableSync } from "#actions/bms-table-sync";
-import { ACTION_UGSSnapshot } from "#actions/ugs-snapshot";
-import { ACTION_UpdateBpiData } from "#actions/update-bpi-data";
-import { ACTION_UpdateDpTiers } from "#actions/update-dp-tiers";
-import { DefaultAdminUser } from "#lib/jobs/default-admin-user";
-import { log } from "#lib/log/log";
-import { TachiConfig } from "#lib/setup/config";
-import { DedupeArr } from "#utils/misc";
-import { Queue, Worker } from "bullmq";
-
-import { UpdateAILevels } from "../bms-ai-table-sync";
-import { DeorphanScoresMain } from "../deorphan-scores";
-import { drainPbDirty, drainScoreRederive } from "../drain-dirty-queues";
-import { RebuildFolderChartLookupJob } from "../rebuild-folder-chart-lookup";
-
-interface Job {
-	name: string;
-	cronFormat: string;
-	run: () => Promise<void>;
-}
-
-const jobs: Array<Job> = [
-	{
-		name: "Rebuild folder chart lookup",
-		cronFormat: "5 0 * * *",
-		run: RebuildFolderChartLookupJob,
-	},
-	{
-		name: "Snapshot User Game Stats",
-		cronFormat: "0 0 * * *",
-		run: async () => {
-			const taker = await DefaultAdminUser.actionTaker();
-			await ACTION_UGSSnapshot(taker, {});
-		},
-	},
-	{
-		name: "De-Orphan Scores",
-
-		// We run an hour after snapshotting UGS
-		// just to spread load out a bit.
-		cronFormat: "1 0 * * *",
-		run: DeorphanScoresMain,
-	},
-	{
-		name: "Drain pb_dirty",
-		cronFormat: "* * * * *",
-		run: async () => {
-			let drained = 0;
-
-			// Keep draining until the queue is empty or we hit a safety cap.
-
-			while (true) {
-				// eslint-disable-next-line no-await-in-loop
-				const n = await drainPbDirty();
-
-				if (n === 0) {
-					break;
-				}
-
-				drained += n;
-
-				if (drained >= 10_000) {
-					break;
-				}
-			}
-		},
-	},
-	{
-		name: "Drain score_rederive",
-		cronFormat: "*/5 * * * *",
-		run: async () => {
-			while (true) {
-				// eslint-disable-next-line no-await-in-loop
-				const n = await drainScoreRederive();
-
-				if (n === 0) {
-					break;
-				}
-			}
-		},
-	},
-];
-
-// if kamaitachi or omnitachi
-if (TachiConfig.TYPE !== "boku") {
-	jobs.push({
-		name: "Update BPI",
-		cronFormat: "2 0 * * *",
-		run: async () => {
-			const taker = await DefaultAdminUser.actionTaker();
-			await ACTION_UpdateBpiData(taker, {});
-		},
-	});
-
-	jobs.push({
-		name: "Update DP Tiers",
-		cronFormat: "3 0 * * *",
-		run: async () => {
-			const taker = await DefaultAdminUser.actionTaker();
-			await ACTION_UpdateDpTiers(taker, {});
-		},
-	});
-}
-
-// if bokutachi or omnitachi
-if (TachiConfig.TYPE !== "kamai") {
-	jobs.push({
-		name: "Update AI Table",
-		cronFormat: "2 0 * * *",
-		run: UpdateAILevels,
-	});
-
-	jobs.push({
-		name: "Update Tables",
-		cronFormat: "3 0 * * *",
-		run: async () => {
-			const taker = await DefaultAdminUser.actionTaker();
-			await ACTION_BMSTableSync(taker, {});
-		},
-	});
-
-	jobs.push({
-		name: "Backsync BMS + PMS",
-		cronFormat: "4 0 * * *",
-		run: async () => {
-			const taker = await DefaultAdminUser.actionTaker();
-			await ACTION_BacksyncBmsPmsSeeds(taker, {});
-		},
-	});
-}
-
-/**
- * Initalises a tachi-server job runner.
- * This runs the list of jobs defined in jobConfig.jobs.
- */
-export function InitialiseJobRunner() {
-	log.info(`Booting up Job Runner.`);
-
-	const names = jobs.map((e) => e.name);
-
-	if (DedupeArr(names).length !== names.length) {
-		log.fatal(() => {
-			process.exit(1);
-		}, `Jobs has duplicate name fields, refusing to run.`);
-	}
-
-	const JobQueue = new Queue("Job Runner");
-
-	const jobNameMap = new Map<string, Job>();
-
-	for (const job of jobs) {
-		void JobQueue.add(job.name, { jobName: job.name }, { repeat: { cron: job.cronFormat } });
-		jobNameMap.set(job.name, job);
-	}
-
-	const worker = new Worker("Job Runner", async (j) => {
-		const { jobName } = j.data as { jobName: string };
-
-		log.info(`Running job ${jobName}.`);
-
-		const jobInfo = jobNameMap.get(jobName);
-
-		if (!jobInfo) {
-			log.error(`Unknown job name ${jobName}, couldn't find a run function?`);
-			return false;
-		}
-
-		await jobInfo.run();
-
-		return true;
-	});
-
-	log.info(`Initialised ${jobs.length} jobs (${jobs.map((e) => e.name).join(", ")}).`);
-
-	return worker;
-}
-
-if (require.main === module) {
-	InitialiseJobRunner();
-}

typescript/server/src/lib/jobs/job-queue/constants.ts

@@ -0,0 +1,8 @@
+/** Canonical `job_queue.job_kind` for score import jobs. */
+export const JOB_KIND_SCORE_IMPORT = "score_import" as const;
+
+/** Matches client adminConstants and Zenith `job_queue.status` values. */
+export const JOB_STATUS_QUEUED = 0;
+export const JOB_STATUS_RUNNING = 1;
+export const JOB_STATUS_DONE = 2;
+export const JOB_STATUS_FAILED = 3;

typescript/server/src/lib/jobs/job-queue/queue-ops.ts

@@ -0,0 +1,78 @@
+import type { JobQueue, NewJobQueue } from "tachi-db";
+
+type EnqueueInput = Omit<NewJobQueue, "failed_attempts" | "status">;
+import {
+	JOB_STATUS_DONE,
+	JOB_STATUS_FAILED,
+	JOB_STATUS_QUEUED,
+	JOB_STATUS_RUNNING,
+} from "#lib/jobs/job-queue/constants";
+import DB from "#services/pg/db";
+import { sql } from "kysely";
+
+/**
+ * Enqueue a job (queued, due on or before `scheduled_for`).
+ */
+export async function EnqueueJob(row: EnqueueInput): Promise<string> {
+	const r = await DB.insertInto("job_queue")
+		.values({
+			...row,
+			status: JOB_STATUS_QUEUED,
+			failed_attempts: 0,
+		})
+		.returning("job_queue.row_id")
+		.executeTakeFirstOrThrow();
+	return r.row_id;
+}
+
+/**
+ * Claim the next job using `FOR UPDATE SKIP LOCKED` (fair multi-worker).
+ */
+export async function ClaimNextJob(): Promise<JobQueue | undefined> {
+	const r = await sql<JobQueue>`
+		WITH picked AS (
+			SELECT "job_queue"."row_id"
+			FROM "job_queue"
+			WHERE "job_queue"."status" = ${JOB_STATUS_QUEUED}
+				AND "job_queue"."scheduled_for" <= NOW()
+			ORDER BY "job_queue"."scheduled_for" ASC, "job_queue"."created_at" ASC
+			FOR UPDATE OF "job_queue" SKIP LOCKED
+			LIMIT 1
+		)
+		UPDATE "job_queue"
+		SET
+			"status" = ${JOB_STATUS_RUNNING},
+			"updated_at" = NOW()
+		FROM picked
+		WHERE "job_queue"."row_id" = "picked"."row_id"
+		RETURNING
+			"job_queue"."row_id",
+			"job_queue"."created_at",
+			"job_queue"."updated_at",
+			"job_queue"."scheduled_for",
+			"job_queue"."failed_attempts",
+			"job_queue"."status",
+			"job_queue"."scope",
+			"job_queue"."job_kind",
+			"job_queue"."payload"
+	`.execute(DB);
+
+	if (r.rows.length === 0) {
+		return undefined;
+	}
+	return r.rows[0] as unknown as JobQueue;
+}
+
+export async function MarkJobDone(rowId: string): Promise<void> {
+	await DB.updateTable("job_queue")
+		.set({ status: JOB_STATUS_DONE, updated_at: new Date().toISOString() })
+		.where("job_queue.row_id", "=", rowId)
+		.execute();
+}
+
+export async function MarkJobFailed(rowId: string): Promise<void> {
+	await DB.updateTable("job_queue")
+		.set({ status: JOB_STATUS_FAILED, updated_at: new Date().toISOString() })
+		.where("job_queue.row_id", "=", rowId)
+		.execute();
+}

typescript/server/src/lib/rivals/rivals.ts

@@ -4,7 +4,7 @@ import { SetRivalsFailReasons } from "#lib/constants/err-codes";
 import { log } from "#lib/log/log";
 import { SendSetRivalNotification } from "#lib/notifications/notification-wrappers";
 import { ServerConfig } from "#lib/setup/config";
-import { pgScoreDataToMongo } from "#lib/v3/migration-tools";
+import { pgScoreDataToAPI } from "#lib/v3/migration-tools";
 import DB from "#services/pg/db";
 import { ArrayDiff } from "#utils/misc";
 import { GetUsersWithIDs, GetUserWithIDGuaranteed } from "#utils/user";
@@ -15,13 +15,13 @@ import { GetGameConfig, type integer, type PgScoreData, type V3Game } from "tach
  * Throws if the user hasn't played the GPT in question.
  */
 export async function GetRivalIDs(userID: integer, game: V3Game) {
-	const settings = await DB.selectFrom("game_settings")
-		.select("user_id")
-		.where("user_id", "=", userID)
-		.where("game", "=", game)
+	const profile = await DB.selectFrom("game_profile")
+		.select("game_profile.user_id")
+		.where("game_profile.user_id", "=", userID)
+		.where("game_profile.game", "=", game)
 		.executeTakeFirst();
 
-	if (!settings) {
+	if (!profile) {
 		throw new Error(`User ${userID} has not played ${game}. Cannot retrieve rivals.`);
 	}
 
@@ -73,7 +73,7 @@ function metricValueFromPbRow(
 	derivedData: unknown,
 	metricKey: string,
 ): number | null {
-	const scoreData = pgScoreDataToMongo(v3Game, {
+	const scoreData = pgScoreDataToAPI(v3Game, {
 		data,
 		derived: derivedData,
 		judgements: {},
@@ -102,10 +102,10 @@ export async function setRivalsWithResult(
 		return SetRivalsFailReasons.RIVALED_SELF;
 	}
 
-	const { count } = await DB.selectFrom("game_settings")
+	const { count } = await DB.selectFrom("game_profile")
 		.select(DB.fn.countAll().as("count"))
-		.where("game", "=", game)
-		.where("user_id", "in", newRivals)
+		.where("game_profile.game", "=", game)
+		.where("game_profile.user_id", "in", newRivals)
 		.executeTakeFirstOrThrow();
 
 	const playedGPTCount = Number(count);
@@ -114,19 +114,19 @@ export async function setRivalsWithResult(
 		return SetRivalsFailReasons.RIVALS_HAVENT_PLAYED_GPT;
 	}
 
-	const currentGameSettings = await DB.selectFrom("game_settings")
-		.select("user_id")
-		.where("user_id", "=", userID)
-		.where("game", "=", game)
+	const currentGameProfile = await DB.selectFrom("game_profile")
+		.select("game_profile.user_id")
+		.where("game_profile.user_id", "=", userID)
+		.where("game_profile.game", "=", game)
 		.executeTakeFirst();
 
-	if (!currentGameSettings) {
+	if (!currentGameProfile) {
 		log.error(
-			`User ${userID} attempted to set rivals for ${game}, but doesn't have game settings. Was their account deleted in midair?`,
+			`User ${userID} attempted to set rivals for ${game}, but doesn't have a game profile. Was their account deleted in midair?`,
 		);
 
 		throw new Error(
-			`User ${userID} attempted to set rivals for ${game}, but doesn't have game settings. Was their account deleted in midair?`,
+			`User ${userID} attempted to set rivals for ${game}, but doesn't have a game profile. Was their account deleted in midair?`,
 		);
 	}
 

typescript/server/src/lib/score-import/framework/express-wrapper.ts

@@ -8,6 +8,7 @@ import type {
 
 import { log } from "#lib/log/log";
 import { Random20Hex } from "#utils/misc";
+import { ExpectedErr } from "bliss";
 
 import type { ParserArguments } from "../worker/types";
 
@@ -52,14 +53,17 @@ export async function ExpressWrappedScoreImportMain<I extends ImportTypes>(
 			},
 		};
 	} catch (err) {
-		// this is definitely fine, as the errors are emitted from the same place.
-		if (err instanceof ScoreImportFatalError) {
-			log.info(err.message);
+		// `ACTION_ScoreImport` throws `ExpectedErr` (mapped from `ScoreImportFatalError`); the
+		// external-worker guard in `MakeScoreImport` still throws `ScoreImportFatalError`.
+		if (ExpectedErr.is(err) || err instanceof ScoreImportFatalError) {
+			const description = ExpectedErr.is(err) ? err.reason : err.message;
+			const statusCode = ExpectedErr.is(err) ? err.code : err.statusCode;
+			log.info(description);
 			return {
-				statusCode: err.statusCode,
+				statusCode,
 				body: {
 					success: false,
-					description: err.message,
+					description,
 				},
 			};
 		}

typescript/server/src/lib/score-import/framework/orphans/orphans.ts

@@ -48,13 +48,24 @@ async function deleteOrphanByOrphanId(orphanID: string): Promise<void> {
 
 /** API-facing row for listing a user’s orphan_score entries. */
 export type OrphanScoreListItem = {
+	gameGroup: string;
+	importType: string;
+	message: string | null;
 	orphanID: string;
 	rowID: string;
-	importType: string;
-	gameGroup: string;
-	timeInserted: number;
-	message: string | null;
 	summary: string | null;
+	timeInserted: number;
+};
+
+/** API-facing detail for one orphan_score row (includes raw import payload). */
+export type OrphanScoreDetail = {
+	context: unknown;
+	data: unknown;
+	gameGroup: string;
+	importType: string;
+	message: string | null;
+	orphanID: string;
+	timeInserted: number;
 };
 
 function summarizeOrphanRow(row: PgOrphanScoreRow): string | null {
@@ -102,8 +113,38 @@ function orphanRowToListItem(row: PgOrphanScoreRow): OrphanScoreListItem {
 	};
 }
 
+/** Loads one orphan_score row for the user, or null if none. */
+export async function getOrphanScoreDetailForUser(
+	orphanID: string,
+	userID: integer,
+): Promise<OrphanScoreDetail | null> {
+	const row = await DB.selectFrom("orphan_score")
+		.select(SELECT_ORPHAN_SCORE)
+		.where("orphan_score.orphan_id", "=", orphanID)
+		.where("orphan_score.user_id", "=", userID)
+		.executeTakeFirst();
+
+	if (!row) {
+		return null;
+	}
+
+	const msg = row.error_message.trim();
+	return {
+		orphanID: row.orphan_id,
+		importType: row.import_type,
+		gameGroup: row.game_group,
+		timeInserted: new Date(row.time_inserted).getTime(),
+		message: msg.length > 0 ? msg : null,
+		data: row.data,
+		context: row.context,
+	};
+}
+
 /** Deletes one orphan_score row if it belongs to the given user. Returns whether a row was removed. */
-export async function deleteOrphanScoreForUser(orphanID: string, userID: integer): Promise<boolean> {
+export async function deleteOrphanScoreForUser(
+	orphanID: string,
+	userID: integer,
+): Promise<boolean> {
 	const result = await DB.deleteFrom("orphan_score")
 		.where("orphan_score.orphan_id", "=", orphanID)
 		.where("orphan_score.user_id", "=", userID)
@@ -117,12 +158,12 @@ export async function deleteOrphanScoreForUser(orphanID: string, userID: integer
  * @param afterRowID — `row_id` of the last item from the previous page (omit on first page).
  */
 export async function listOrphanScoresForUser(opts: {
-	userID: integer;
-	limit: number;
 	afterRowID?: string;
-}): Promise<{ orphans: OrphanScoreListItem[]; hasMore: boolean }> {
+	limit: number;
+	userID: integer;
+}): Promise<{ hasMore: boolean; orphans: OrphanScoreListItem[] }> {
 	const cap = Math.min(Math.max(opts.limit, 1), 100);
-	let anchor: { time_inserted: string; row_id: string } | undefined;
+	let anchor: { row_id: string; time_inserted: string } | undefined;
 
 	if (opts.afterRowID !== undefined && opts.afterRowID.length > 0) {
 		anchor = await DB.selectFrom("orphan_score")

typescript/server/src/lib/score-import/framework/pb/rederive-scores.recalc.test.ts

@@ -0,0 +1,501 @@
+import { ComputeChartStabilityChecksum } from "#game-implementations/utils/derivation-checksum";
+import {
+	type ScoreDocumentJoinRow,
+	SELECT_SCORE_DOCUMENT,
+	ToScoreDocument,
+} from "#lib/db-formats/score";
+import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings";
+import {
+	drainGameProfileDirty,
+	drainPbDirty,
+	drainScoreRederive,
+	drainSessionDirty,
+	drainStatsQueuesInOrder,
+} from "#lib/jobs/drain-dirty-queues";
+import { log } from "#lib/log/log";
+import { CreateSessionCalcData } from "#lib/score-import/framework/calculated-data/session";
+import { rederiveScoresForChart } from "#lib/score-import/framework/pb/rederive-scores";
+import { scoreVisibleSql } from "#lib/score-import/framework/pg/score-visibility";
+import { mongoScoreDataToPg, pgScoreDataToAPI } from "#lib/v3/migration-tools";
+import DB from "#services/pg/db";
+import { seedUser } from "#test-utils/pg-fixtures";
+import { Testing511Song, Testing511SPA } from "#test-utils/test-data";
+import { type ChartDocument, type PgScoreData, type ScoreData } from "tachi-common";
+import { describe, expect, it } from "vitest";
+
+let recalcSeedCounter = 0;
+
+function buildIidxSpChartDoc(
+	chartId: string,
+	songId: string,
+	levelNum: number,
+	notecount: number,
+): ChartDocument<"iidx-sp"> {
+	return {
+		...Testing511SPA,
+		chartID: chartId,
+		song: { ...Testing511Song, id: songId },
+		levelNum,
+		level: String(levelNum),
+		data: {
+			...Testing511SPA.data,
+			notecount,
+		},
+	};
+}
+
+async function seedIidxSpGameProfile(userId: number) {
+	await DB.insertInto("game_profile")
+		.values({
+			user_id: userId,
+			game: "iidx-sp",
+			ratings: JSON.stringify({}),
+			classes: JSON.stringify({}),
+			...newGameProfilePreferenceColumns("iidx-sp"),
+		})
+		.execute();
+}
+
+async function insertSongAndChart(chartDoc: ChartDocument<"iidx-sp">) {
+	const songId = chartDoc.song.id;
+	const chartId = chartDoc.chartID;
+	const n = ++recalcSeedCounter;
+
+	await DB.insertInto("song")
+		.values({
+			id: songId,
+			legacy_id: 91_000 + n,
+			game_group: "iidx",
+			title: chartDoc.song.title,
+			artist: chartDoc.song.artist,
+			search_terms: chartDoc.song.searchTerms,
+			alt_titles: chartDoc.song.altTitles,
+			data: chartDoc.song.data as object,
+			fts_document: "",
+		})
+		.execute();
+
+	await DB.insertInto("chart")
+		.values({
+			id: chartId,
+			legacy_id: `legacy_recalc_${chartId}`,
+			game: "iidx-sp",
+			song_id: songId,
+			difficulty: chartDoc.difficulty,
+			level: chartDoc.level,
+			level_num: chartDoc.levelNum,
+			is_primary: chartDoc.isPrimary,
+			versions: chartDoc.versions,
+			data: chartDoc.data as object,
+			derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", chartDoc),
+		})
+		.execute();
+
+	return { chartId, songId };
+}
+
+async function insertIidxScore(opts: {
+	chartId: string;
+	scoreData: ScoreData<"iidx-sp">;
+	sessionId?: string | null;
+	userId: number;
+}) {
+	const now = new Date().toISOString();
+	const { data, derived, judgements } = mongoScoreDataToPg("iidx-sp", opts.scoreData);
+	const scoreId = `sc-recalc-${opts.chartId}`;
+
+	await DB.insertInto("score")
+		.values({
+			id: scoreId,
+			user_id: opts.userId,
+			chart_id: opts.chartId,
+			game: "iidx-sp",
+			session_id: opts.sessionId ?? null,
+			import_id: null,
+			data: JSON.stringify(data),
+			derived_data: JSON.stringify(derived),
+			judgements: JSON.stringify(judgements),
+			calculated_data: JSON.stringify({}),
+			meta: JSON.stringify({}),
+			time_achieved: now,
+			time_added: now,
+			highlight: false,
+			comment: null,
+		})
+		.execute();
+
+	return scoreId;
+}
+
+function parseJsonb<T>(v: unknown): T {
+	if (typeof v === "string") {
+		return JSON.parse(v) as T;
+	}
+
+	return v as T;
+}
+
+async function loadScoresForSession(sessionId: string) {
+	const scoreRows = await DB.selectFrom("score")
+		.innerJoin("chart", "chart.id", "score.chart_id")
+		.innerJoin("song", "song.id", "chart.song_id")
+		.leftJoin("import", "import.id", "score.import_id")
+		.select(SELECT_SCORE_DOCUMENT)
+		.where("score.session_id", "=", sessionId)
+		.where(scoreVisibleSql())
+		.execute();
+
+	return scoreRows.map((r) => ToScoreDocument(r as ScoreDocumentJoinRow));
+}
+
+async function loadScorePayload(chartId: string) {
+	const row = await DB.selectFrom("score")
+		.select(["score.data", "score.derived_data", "score.judgements", "score.calculated_data"])
+		.where("score.chart_id", "=", chartId)
+		.executeTakeFirstOrThrow();
+
+	return {
+		calculatedData: parseJsonb<{ BPI: number | null; ktLampRating: number }>(
+			row.calculated_data,
+		),
+		scoreData: pgScoreDataToAPI("iidx-sp", {
+			data: parseJsonb(row.data),
+			derived: parseJsonb(row.derived_data),
+			judgements: parseJsonb(row.judgements),
+		} as PgScoreData<"iidx-sp">),
+	};
+}
+
+describe("rederiveScoresForChart / chart checksum recalc (Postgres)", () => {
+	it("enqueues score_rederive when chart derivation_checksum changes", async () => {
+		const { id: userId } = await seedUser();
+		await seedIidxSpGameProfile(userId);
+
+		const chartId = `C_RECALC_TRIG_${++recalcSeedCounter}`;
+		const songId = `S_RECALC_TRIG_${recalcSeedCounter}`;
+		const docV1 = buildIidxSpChartDoc(chartId, songId, 10, 786);
+		await insertSongAndChart(docV1);
+
+		const docV2 = buildIidxSpChartDoc(chartId, songId, 11, 786);
+		const checksum2 = ComputeChartStabilityChecksum("iidx-sp", docV2);
+
+		await DB.updateTable("chart")
+			.set({
+				level: docV2.level,
+				level_num: docV2.levelNum,
+				derivation_checksum: checksum2,
+			})
+			.where("chart.id", "=", chartId)
+			.execute();
+
+		const queued = await DB.selectFrom("score_rederive")
+			.select(["score_rederive.chart_id"])
+			.where("score_rederive.chart_id", "=", chartId)
+			.executeTakeFirst();
+
+		expect(queued?.chart_id).toBe(chartId);
+	});
+
+	it("does not enqueue score_rederive when derivation_checksum is unchanged", async () => {
+		const chartId = `C_RECALC_NOTRIG_${++recalcSeedCounter}`;
+		const songId = `S_RECALC_NOTRIG_${recalcSeedCounter}`;
+		const doc = buildIidxSpChartDoc(chartId, songId, 10, 786);
+		await insertSongAndChart(doc);
+
+		await DB.updateTable("chart")
+			.set({ legacy_id: `legacy_recalc_updated_${chartId}` })
+			.where("chart.id", "=", chartId)
+			.execute();
+
+		const queued = await DB.selectFrom("score_rederive")
+			.select(["score_rederive.chart_id"])
+			.where("score_rederive.chart_id", "=", chartId)
+			.executeTakeFirst();
+
+		expect(queued).toBeUndefined();
+	});
+
+	it("updates calculated_data.ktLampRating when chart level_num changes (CLEAR lamp)", async () => {
+		const { id: userId } = await seedUser();
+		await seedIidxSpGameProfile(userId);
+
+		const chartId = `C_RECALC_LVL_${++recalcSeedCounter}`;
+		const songId = `S_RECALC_LVL_${recalcSeedCounter}`;
+		const doc10 = buildIidxSpChartDoc(chartId, songId, 10, 786);
+		await insertSongAndChart(doc10);
+
+		await insertIidxScore({
+			chartId,
+			userId,
+			scoreData: {
+				lamp: "CLEAR",
+				score: 1000,
+				grade: "AAA",
+				percent: 90,
+				optional: {},
+				judgements: { pgreat: 500, great: 0 },
+			} as ScoreData<"iidx-sp">,
+		});
+
+		await rederiveScoresForChart(chartId, log);
+		let { calculatedData } = await loadScorePayload(chartId);
+		expect(calculatedData.ktLampRating).toBe(10);
+
+		const doc12 = buildIidxSpChartDoc(chartId, songId, 12, 786);
+		await DB.updateTable("chart")
+			.set({
+				level: doc12.level,
+				level_num: doc12.levelNum,
+				derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc12),
+			})
+			.where("chart.id", "=", chartId)
+			.execute();
+
+		await rederiveScoresForChart(chartId, log);
+		({ calculatedData } = await loadScorePayload(chartId));
+		expect(calculatedData.ktLampRating).toBe(12);
+	});
+
+	it("updates derived percent and grade when chart notecount changes (fixed EX score)", async () => {
+		const { id: userId } = await seedUser();
+		await seedIidxSpGameProfile(userId);
+
+		const chartId = `C_RECALC_NC_${++recalcSeedCounter}`;
+		const songId = `S_RECALC_NC_${recalcSeedCounter}`;
+		const doc1k = buildIidxSpChartDoc(chartId, songId, 10, 1000);
+		await insertSongAndChart(doc1k);
+
+		await insertIidxScore({
+			chartId,
+			userId,
+			scoreData: {
+				lamp: "CLEAR",
+				score: 1000,
+				grade: "AAA",
+				percent: 50,
+				optional: {},
+				judgements: { pgreat: 500, great: 0 },
+			} as ScoreData<"iidx-sp">,
+		});
+
+		await rederiveScoresForChart(chartId, log);
+		let { scoreData } = await loadScorePayload(chartId);
+		expect(scoreData.percent).toBeCloseTo(50, 5);
+		expect(scoreData.grade).toBe("C");
+
+		const doc500 = buildIidxSpChartDoc(chartId, songId, 10, 500);
+		await DB.updateTable("chart")
+			.set({
+				data: doc500.data as object,
+				derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc500),
+			})
+			.where("chart.id", "=", chartId)
+			.execute();
+
+		await rederiveScoresForChart(chartId, log);
+		({ scoreData } = await loadScorePayload(chartId));
+		expect(scoreData.percent).toBeCloseTo(100, 5);
+		expect(scoreData.grade).toBe("MAX");
+	});
+
+	it("refreshes pb.calculated_data after rederive and drainPbDirty", async () => {
+		const { id: userId } = await seedUser();
+		await seedIidxSpGameProfile(userId);
+
+		const chartId = `C_RECALC_PB_${++recalcSeedCounter}`;
+		const songId = `S_RECALC_PB_${recalcSeedCounter}`;
+		const doc10 = buildIidxSpChartDoc(chartId, songId, 10, 786);
+		await insertSongAndChart(doc10);
+
+		await insertIidxScore({
+			chartId,
+			userId,
+			scoreData: {
+				lamp: "CLEAR",
+				score: 1000,
+				grade: "AAA",
+				percent: 50,
+				optional: {},
+				judgements: { pgreat: 500, great: 0 },
+			} as ScoreData<"iidx-sp">,
+		});
+
+		await rederiveScoresForChart(chartId, log);
+		await drainPbDirty();
+
+		const doc14 = buildIidxSpChartDoc(chartId, songId, 14, 786);
+		await DB.updateTable("chart")
+			.set({
+				level: doc14.level,
+				level_num: doc14.levelNum,
+				derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc14),
+			})
+			.where("chart.id", "=", chartId)
+			.execute();
+
+		await rederiveScoresForChart(chartId, log);
+		await drainPbDirty();
+
+		const scoreRow = await loadScorePayload(chartId);
+		const pbRow = await DB.selectFrom("pb")
+			.select(["pb.calculated_data"])
+			.where("pb.user_id", "=", userId)
+			.where("pb.chart_id", "=", chartId)
+			.where("pb.lens", "is", null)
+			.executeTakeFirstOrThrow();
+
+		const pbCalc = parseJsonb<{ ktLampRating: number }>(pbRow.calculated_data);
+		expect(scoreRow.calculatedData.ktLampRating).toBe(14);
+		expect(pbCalc.ktLampRating).toBe(14);
+	});
+
+	it("drainScoreRederive runs rederive and clears score_rederive for the chart", async () => {
+		const { id: userId } = await seedUser();
+		await seedIidxSpGameProfile(userId);
+
+		const chartId = `C_RECALC_DRAIN_${++recalcSeedCounter}`;
+		const songId = `S_RECALC_DRAIN_${recalcSeedCounter}`;
+		const docA = buildIidxSpChartDoc(chartId, songId, 9, 786);
+		await insertSongAndChart(docA);
+
+		await insertIidxScore({
+			chartId,
+			userId,
+			scoreData: {
+				lamp: "CLEAR",
+				score: 1000,
+				grade: "AAA",
+				percent: 50,
+				optional: {},
+				judgements: { pgreat: 500, great: 0 },
+			} as ScoreData<"iidx-sp">,
+		});
+
+		const docB = buildIidxSpChartDoc(chartId, songId, 10, 786);
+		await DB.updateTable("chart")
+			.set({
+				level: docB.level,
+				level_num: docB.levelNum,
+				derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", docB),
+			})
+			.where("chart.id", "=", chartId)
+			.execute();
+
+		const n = await drainScoreRederive();
+		expect(n).toBeGreaterThanOrEqual(1);
+
+		const stillQueued = await DB.selectFrom("score_rederive")
+			.select(["score_rederive.chart_id"])
+			.where("score_rederive.chart_id", "=", chartId)
+			.executeTakeFirst();
+
+		expect(stillQueued).toBeUndefined();
+
+		const { calculatedData } = await loadScorePayload(chartId);
+		expect(calculatedData.ktLampRating).toBe(10);
+	});
+
+	it("updates session.calculated_data and game_profile.ratings after rederive + pb + session + profile drains", async () => {
+		const { id: userId } = await seedUser();
+		await seedIidxSpGameProfile(userId);
+
+		const chartId = `C_RECALC_STATS_${++recalcSeedCounter}`;
+		const songId = `S_RECALC_STATS_${recalcSeedCounter}`;
+		const sessionId = `sess-recalc-stats-${chartId}`;
+		const doc10 = buildIidxSpChartDoc(chartId, songId, 10, 786);
+		await insertSongAndChart(doc10);
+
+		const now = new Date().toISOString();
+
+		await DB.insertInto("session")
+			.values({
+				id: sessionId,
+				user_id: userId,
+				game: "iidx-sp",
+				name: "Recalc stats test",
+				description: null,
+				time_inserted: now,
+				time_started: now,
+				time_ended: now,
+				calculated_data: JSON.stringify({}),
+				highlight: false,
+			})
+			.execute();
+
+		await insertIidxScore({
+			chartId,
+			userId,
+			sessionId,
+			scoreData: {
+				lamp: "CLEAR",
+				score: 1000,
+				grade: "AAA",
+				percent: 50,
+				optional: {},
+				judgements: { pgreat: 500, great: 0 },
+			} as ScoreData<"iidx-sp">,
+		});
+
+		await rederiveScoresForChart(chartId, log);
+		await drainPbDirty();
+		await drainSessionDirty();
+		await drainGameProfileDirty();
+
+		let sessRow = await DB.selectFrom("session")
+			.select(["session.calculated_data"])
+			.where("session.id", "=", sessionId)
+			.executeTakeFirstOrThrow();
+
+		let scoreDocs = await loadScoresForSession(sessionId);
+		let expectedSession = CreateSessionCalcData("iidx-sp", scoreDocs);
+		expect(parseJsonb(sessRow.calculated_data)).toEqual(expectedSession);
+
+		let gpRow = await DB.selectFrom("game_profile")
+			.select(["game_profile.ratings"])
+			.where("game_profile.user_id", "=", userId)
+			.where("game_profile.game", "=", "iidx-sp")
+			.executeTakeFirstOrThrow();
+
+		let ratings = parseJsonb<{ ktLampRating: number | null }>(gpRow.ratings);
+		// ProfileAvgBestN(..., 20, returnMean): one PB => ktLampRating / 20.
+		expect(ratings.ktLampRating).toBeCloseTo(0.5, 5);
+
+		const doc14 = buildIidxSpChartDoc(chartId, songId, 14, 786);
+
+		await DB.updateTable("chart")
+			.set({
+				level: doc14.level,
+				level_num: doc14.levelNum,
+				derivation_checksum: ComputeChartStabilityChecksum("iidx-sp", doc14),
+			})
+			.where("chart.id", "=", chartId)
+			.execute();
+
+		await rederiveScoresForChart(chartId, log);
+		await drainPbDirty();
+		await drainSessionDirty();
+		await drainGameProfileDirty();
+
+		sessRow = await DB.selectFrom("session")
+			.select(["session.calculated_data"])
+			.where("session.id", "=", sessionId)
+			.executeTakeFirstOrThrow();
+
+		scoreDocs = await loadScoresForSession(sessionId);
+		expectedSession = CreateSessionCalcData("iidx-sp", scoreDocs);
+		expect(parseJsonb(sessRow.calculated_data)).toEqual(expectedSession);
+
+		gpRow = await DB.selectFrom("game_profile")
+			.select(["game_profile.ratings"])
+			.where("game_profile.user_id", "=", userId)
+			.where("game_profile.game", "=", "iidx-sp")
+			.executeTakeFirstOrThrow();
+
+		ratings = parseJsonb<{ ktLampRating: number | null }>(gpRow.ratings);
+		expect(ratings.ktLampRating).toBeCloseTo(0.7, 5);
+	});
+
+	it("drainStatsQueuesInOrder completes with empty queues", async () => {
+		await drainStatsQueuesInOrder();
+	});
+});

typescript/server/src/lib/score-import/framework/pb/rederive-scores.ts

@@ -10,7 +10,7 @@ import {
 import { mongoScoreDataToPg } from "#lib/v3/migration-tools";
 import DB from "#services/pg/db";
 
-const BATCH_SIZE = 500;
+const BATCH_SIZE = 5000;
 
 /**
  * Re-derive `derived_data` and `calculated_data` for every score on the
@@ -92,7 +92,9 @@ export async function rederiveScoresForChart(chartId: string, log: KtLogger): Pr
 		}
 	}
 
-	log.info({ chartId, totalUpdated }, `Re-derived ${totalUpdated} score(s) for chart.`);
+	if (totalUpdated > 0) {
+		log.debug({ chartId, totalUpdated }, `Re-derived ${totalUpdated} score(s) for chart.`);
+	}
 
 	return totalUpdated;
 }

typescript/server/src/lib/score-import/framework/score-import.ts

@@ -1,139 +1,47 @@
-import type { ImportDocument, ImportTypes, integer } from "tachi-common";
+import type { ImportDocument, ImportTypes } from "tachi-common";
 
-import { JOB_RETRY_COUNT } from "#lib/constants/tachi";
-import { log } from "#lib/log/log";
+import { ACTION_ScoreImport } from "#actions/score-import";
+import { LoadImportDocumentById } from "#lib/db-formats/import-document";
 import { ServerConfig } from "#lib/setup/config";
-import { Sleep } from "#utils/misc";
+import { GetUserWithIDGuaranteed } from "#utils/user";
 
-import type { ScoreImportJobData, ScoreImportWorkerReturns } from "../worker/types";
+import type { ScoreImportJobData } from "../worker/types";
 
-import ScoreImportQueue, { ScoreImportQueueEvents } from "../worker/queue";
-/* eslint-disable no-await-in-loop */
-import { GetInputParser } from "./common/get-input-parser";
-import { UnsetOngoingImportLock } from "./import-locks/lock";
 import ScoreImportFatalError from "./score-importing/score-import-error";
-import ScoreImportMain from "./score-importing/score-import-main";
-import {
-	EndTrackingImport,
-	MarkImportAsFailed,
-	StartTrackingImport,
-} from "./status-tracking/import-status-tracking";
 
 /**
- * Makes a score import given ScoreImportJobData.
- * If USE_EXTERNAL_SCORE_IMPORT_WORKER is set, then this will instead
- * place this on the score import queue, and the worker will process it.
+ * Makes a score import given ScoreImportJobData (same process as the API).
  *
- * Otherwise, it will just perform score importing on the same process.
- * @returns An import document if awaited, however, you should not
- * await this if you don't need the import document! Import Documents
- * may take multiple minutes to generate for large imports. If you control
- * the client, make it poll /api/v1/ongoing-imports/:importID.
+ * When `USE_EXTERNAL_SCORE_IMPORT_WORKER` is true, HTTP routes call
+ * {@link EnqueueScoreImportJob} instead — this function is only used for the
+ * inline (non-queued) path.
  */
 export async function MakeScoreImport<I extends ImportTypes>(
 	jobData: ScoreImportJobData<I>,
 ): Promise<ImportDocument> {
-	await StartTrackingImport(jobData);
-
-	try {
-		const ImportDocument = await MakeScoreImportInner(jobData);
-
-		await EndTrackingImport(jobData.importID);
-
-		return ImportDocument;
-	} catch (e) {
-		const err = e as Error | ScoreImportFatalError;
-
-		await MarkImportAsFailed(jobData.importID, err);
-
-		throw err;
-	}
-}
-
-/**
- * Inner function that actually makes the score import. This is intended to be wrapped
- * by the import-tracking code, as it's useful for errors.
- */
-async function MakeScoreImportInner<I extends ImportTypes>(
-	jobData: ScoreImportJobData<I>,
-): Promise<ImportDocument> {
-	if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER && process.env.IS_JOB === undefined) {
-		let timesAttempted = 1;
-
-		// There's no chance this thing goes on 7 times.
-		// if it does, this import has been trying for the past 6 hours or so.
-		while (timesAttempted <= JOB_RETRY_COUNT) {
-			const job = await ScoreImportQueue.add(
-				`Import ${jobData.importID}${timesAttempted > 0 ? ` (TRY${timesAttempted})` : ""}`,
-				jobData,
-				{
-					jobId: `${jobData.importID}:TRY${timesAttempted}`,
-				},
-			);
-
-			const data = (await job.waitUntilFinished(
-				ScoreImportQueueEvents,
-			)) as ScoreImportWorkerReturns;
-
-			if (data.success) {
-				await EndTrackingImport(jobData.importID);
-				return data.ImportDocument;
-			} else if (data.statusCode !== 409) {
-				throw new ScoreImportFatalError(data.statusCode, data.description);
-			}
-
-			const backoff = ExponentialBackoff(timesAttempted - 1);
-
-			log.info(
-				`User ${jobData.userID} already had an import ongoing. (${
-					jobData.importID
-				}) Backing off for ${(backoff / 1_000).toFixed(2)} seconds.`,
-			);
-
-			// If we get here, we were 409'd and the user already has an ongoing
-			// import.
-			// In the interest of not just throwing scores away, we'll back off a bit
-			// and then restart the job.
-			await Sleep(backoff);
-
-			timesAttempted++;
-		}
-
-		log.error(
-			jobData,
-			`User ${jobData.userID} didn't get an import through in around 6 hours. Has their lock gotten stuck?`,
-		);
-
-		await UnsetOngoingImportLock(jobData.userID);
-
-		log.error(`Forcing off ${jobData.userID}'s import lock. Sketchy.`);
-
+	if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
 		throw new ScoreImportFatalError(
-			409,
-			"Couldn't get an import through in the past 6 hours, at all.",
-		);
-	} else {
-		const InputParser = GetInputParser(jobData);
-
-		return ScoreImportMain(
-			jobData.userID,
-			jobData.userIntent,
-			jobData.importType,
-			InputParser,
-			jobData.importID,
+			500,
+			"MakeScoreImport may not be used when an external score-import worker is enabled; use EnqueueScoreImportJob instead.",
 		);
 	}
-}
 
-function ExponentialBackoff(exponent: integer) {
-	// n | backoff
-	// 0 | 4 Seconds
-	// 1 | 16 Seconds
-	// 2 | 64 Seconds
-	// 3 | 256 Seconds
-	// 4 | 1024 Seconds
-	// ...
-	// ends at 7, which is around 4 hours.
-
-	return Math.random() * 1000 * 4 ** (exponent + 1);
+	const user = await GetUserWithIDGuaranteed(jobData.userID);
+	await ACTION_ScoreImport(
+		{ ip: null, acct: { id: user.id, username: user.username } },
+		{
+			importID: jobData.importID,
+			importType: jobData.importType,
+			userIntent: jobData.userIntent,
+			"!parserArguments": jobData.parserArguments as Array<unknown>,
+		},
+	);
+	const importDocument = await LoadImportDocumentById(jobData.importID);
+	if (!importDocument) {
+		throw new ScoreImportFatalError(
+			500,
+			"Import completed but the import document could not be loaded.",
+		);
+	}
+	return importDocument;
 }

typescript/server/src/lib/score-import/framework/sessions/sessions.ts

@@ -87,6 +87,9 @@ function ScoreToSessionScoreInfo(
 export async function GetSessionScoreInfo(
 	session: SessionDocument,
 ): Promise<Array<SessionScoreInfo>> {
+	// TODO: Hard to implement efficiently
+	// need to get the PB for this chart BEFORE the
+	// given time T
 	const scores = await DB.selectFrom("score")
 		.innerJoin("chart", "chart.id", "score.chart_id")
 		.innerJoin("song", "song.id", "chart.song_id")

typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.test.ts

@@ -112,7 +112,7 @@ describe("UpdateUsersGamePlaytypeStats (ported from update-ugpt-stats.oldtest.ts
 		await insertPbFromTesting({ userId: 1, pb: TestingIIDXSPScorePB });
 	});
 
-	it("creates game_profile and game_settings when the user has none", async () => {
+	it("creates game_profile with preference defaults when the user has none", async () => {
 		const res = await UpdateUsersGamePlaytypeStats("iidx-sp", 1, null, log);
 
 		expect(res).toEqual([]);
@@ -124,14 +124,15 @@ describe("UpdateUsersGamePlaytypeStats (ported from update-ugpt-stats.oldtest.ts
 			.executeTakeFirstOrThrow();
 
 		const ratings = typeof gp.ratings === "string" ? JSON.parse(gp.ratings) : gp.ratings;
-		expect(ratings).toMatchObject({ ktLampRating: expect.any(Number) });
+		expect(ratings).toMatchObject({
+			ktLampRating: expect.any(Number),
+			ktLampRatingNC: expect.any(Number),
+			ktLampRatingHC: expect.any(Number),
+			ktLampRatingEXHC: expect.any(Number),
+		});
 
-		const settings = await DB.selectFrom("game_settings")
-			.selectAll()
-			.where("user_id", "=", 1)
-			.where("game", "=", "iidx-sp")
-			.executeTakeFirst();
-		expect(settings).toBeDefined();
+		const dataRaw = gp.data;
+		expect(dataRaw).toBeDefined();
 	});
 
 	it("updates ratings when game_profile already exists", async () => {
@@ -172,8 +173,11 @@ describe("UpdateUsersGamePlaytypeStats (ported from update-ugpt-stats.oldtest.ts
 				pb: deepmerge(TestingIIDXSPScorePB, {
 					chartID: fakeChart,
 					calculatedData: {
-						ktLampRating: e,
 						BPI: 10.1,
+						ktLampRating: e,
+						ktLampRatingNC: e,
+						ktLampRatingHC: e,
+						ktLampRatingEXHC: e,
 					},
 				}),
 			});

typescript/server/src/lib/score-import/framework/ugpt-stats/update-ugpt-stats.ts

@@ -1,6 +1,6 @@
 import type { KtLogger } from "#lib/log/log";
 
-import { CreateGameSettings } from "#lib/game-settings/create-game-settings";
+import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings";
 import DB from "#services/pg/db";
 import { loadUserGameStats } from "#utils/class";
 import { type ClassDelta, type integer, type V3Game } from "tachi-common";
@@ -80,9 +80,9 @@ export async function UpdateUsersGamePlaytypeStats(
 				game,
 				ratings: JSON.stringify(ratings),
 				classes: JSON.stringify(classes),
+				...newGameProfilePreferenceColumns(game),
 			})
 			.execute();
-		await CreateGameSettings(userID, game);
 	}
 
 	return deltas;

typescript/server/src/lib/score-import/import-types/api/myt-wacca/class-handler.ts

@@ -15,7 +15,7 @@ export default async function CreateMytWACCAClassHandler(
 	const req = create(DataRequestSchema, { apiId: titleApiId });
 	const data = await client.getData(req);
 
-	return (_gptString, _userID, _ratings, _logger) => {
+	return (_game, _userID, _ratings, _logger) => {
 		// Currently (May 2025) Reverse and Plus are supported on Myt.
 		// We look for both Reverse and PLUS version data, PLUS being prioritized if exists.
 		// If / when custom dans are added, this will need to change.

typescript/server/src/lib/score-import/worker/enqueue-pg.ts

@@ -0,0 +1,23 @@
+import type { ScoreImportJobData } from "#lib/score-import/worker/types";
+import type { ImportTypes } from "tachi-common";
+
+import { JOB_KIND_SCORE_IMPORT } from "#lib/jobs/job-queue/constants";
+import { EnqueueJob } from "#lib/jobs/job-queue/queue-ops";
+import { StartTrackingImport } from "#lib/score-import/framework/status-tracking/import-status-tracking";
+import { jsonSerializeWithBuffers } from "#lib/score-import/worker/score-import-job-processor";
+
+/**
+ * Enqueue a score import on the Postgres `job_queue` and begin tracking. Returns `job_queue.row_id`.
+ */
+export async function EnqueueScoreImportJob(
+	jobData: ScoreImportJobData<ImportTypes>,
+): Promise<string> {
+	await StartTrackingImport(jobData);
+	const payload = jsonSerializeWithBuffers(jobData);
+	return EnqueueJob({
+		scheduled_for: new Date().toISOString(),
+		scope: `import:${jobData.importID}`,
+		job_kind: JOB_KIND_SCORE_IMPORT,
+		payload: JSON.parse(payload) as unknown,
+	});
+}

typescript/server/src/lib/score-import/worker/queue.ts

@@ -1,21 +1,11 @@
-import { Env, TachiConfig } from "#lib/setup/config";
-import { Queue, QueueEvents } from "bullmq";
+/**
+ * BullMQ score import queue (legacy). Replaced by Postgres `job_queue`.
+ * The API process no longer instantiates a Redis queue; keep a no-op close for shutdown paths.
+ */
+export default null;
 
-const ScoreImportQueue = new Queue(`${TachiConfig.NAME} Score Import Queue`, {
-	connection: { host: Env.REDIS_URL, port: 6379 },
-	defaultJobOptions: {
-		removeOnComplete: true,
-		removeOnFail: 10, // keep the last 10 failed jobs, but start pruning beyond that.
-	},
-});
+export const ScoreImportQueueEvents = null;
 
-export default ScoreImportQueue;
-
-export const ScoreImportQueueEvents = new QueueEvents(ScoreImportQueue.name, {
-	connection: { host: Env.REDIS_URL, port: 6379 },
-});
-
-export async function CloseScoreImportQueue() {
-	await ScoreImportQueueEvents.close();
-	return ScoreImportQueue.close();
+export async function CloseScoreImportQueue(): Promise<void> {
+	// no-op (Postgres `job_queue` is used for score imports)
 }

typescript/server/src/lib/score-import/worker/score-import-job-processor.ts

@@ -0,0 +1,86 @@
+import type { ScoreImportJobData, ScoreImportWorkerReturns } from "#lib/score-import/worker/types";
+import type { ImportTypes } from "tachi-common";
+
+import { ACTION_ScoreImport } from "#actions/score-import";
+import { LoadImportDocumentById } from "#lib/db-formats/import-document";
+import { log } from "#lib/log/log";
+import { GetUserWithID } from "#utils/user";
+import { ExpectedErr } from "bliss";
+
+export function jsonSerializeWithBuffers<T>(data: T): string {
+	return JSON.stringify(data, (_k, v) => {
+		if (Buffer.isBuffer(v)) {
+			return { type: "Buffer" as const, data: Array.from(v as ArrayLike<number> & Buffer) };
+		}
+		return v;
+	});
+}
+
+function decodeParserArguments(raw: Array<unknown>): Array<unknown> {
+	const out: Array<unknown> = [];
+
+	for (const arg of raw) {
+		if (
+			arg &&
+			typeof arg === "object" &&
+			(arg as { buffer?: { data?: number[]; type?: string } }).buffer?.type === "Buffer"
+		) {
+			const a = arg as { buffer: { data: number[] } };
+			out.push({ ...a, buffer: Buffer.from(a.buffer.data) });
+		} else {
+			out.push(arg);
+		}
+	}
+	return out;
+}
+
+/**
+ * `payload` is JSONB from `job_queue` (object) or a JSON string.
+ */
+export async function processScoreImportJobFromPayload(
+	payload: unknown,
+): Promise<ScoreImportWorkerReturns> {
+	const parsed: unknown =
+		typeof payload === "string" ? (JSON.parse(payload) as unknown) : payload;
+	const data = parsed as ScoreImportJobData<ImportTypes>;
+	data.parserArguments = decodeParserArguments(
+		data.parserArguments as Array<unknown>,
+	) as ScoreImportJobData<ImportTypes>["parserArguments"];
+
+	const user = await GetUserWithID(data.userID);
+	if (!user) {
+		log.error(`Couldn't find user with ID ${data.userID} for import ${data.importID}.`);
+		throw new Error(`Couldn't find user with ID ${data.userID} for import ${data.importID}.`);
+	}
+
+	log.debug(`Starting import ${data.importID}.`);
+	try {
+		await ACTION_ScoreImport(
+			{ ip: null, acct: { id: user.id, username: user.username } },
+			{
+				importID: data.importID,
+				importType: data.importType,
+				userIntent: data.userIntent,
+				"!parserArguments": data.parserArguments as Array<unknown>,
+				skipStartTracking: true,
+			},
+		);
+		const importDocument = await LoadImportDocumentById(data.importID);
+		if (!importDocument) {
+			throw new Error(
+				`Import ${data.importID} completed but the import document could not be loaded.`,
+			);
+		}
+		return { success: true, ImportDocument: importDocument };
+	} catch (e) {
+		if (ExpectedErr.is(e)) {
+			log.info(
+				{ err: e, importID: data.importID },
+				`Import ${data.importID} hit ExpectedErr (user fault): ${e.reason}`,
+			);
+			return { success: false, statusCode: e.code, description: e.reason };
+		}
+		log.error(e, `Import ${data.importID} failed unexpectedly.`);
+		throw e;
+	}
+}

typescript/server/src/lib/score-import/worker/worker.ts

@@ -1,139 +0,0 @@
-import type { ImportTypes } from "tachi-common";
-
-import { HandleSIGTERMGracefully } from "#lib/handlers/sigterm";
-import { log } from "#lib/log/log";
-import { Env, ServerConfig } from "#lib/setup/config";
-import { GetUserWithID } from "#utils/user";
-import { Worker } from "bullmq";
-import { EventEmitter } from "events";
-
-import type ScoreImportFatalError from "../framework/score-importing/score-import-error";
-import type { ScoreImportJob, ScoreImportJobData } from "./types";
-
-import { GetInputParser } from "../framework/common/get-input-parser";
-import ScoreImportMain from "../framework/score-importing/score-import-main";
-import ScoreImportQueue from "./queue";
-
-EventEmitter.defaultMaxListeners = 20;
-
-// For scaling performance, running score-importing in a separate worker is preferable
-// as that way, other API calls don't get halted by particularly expensive imports on
-// all cores. Parallelism can only get us so far in the same process.
-
-// You don't have to run this. If it is being ran, you need to set USE_EXTERNAL_SCORE_IMPORT_WORKER
-// in env (TACHI_USE_EXTERNAL_SCORE_IMPORT_WORKER). That will ensure all score import jobs are thrown at redis and eventually
-// end up here.
-
-// If you don't, score importing will happen on the same thread as your router. That's probably
-// fine for lower throughputs, but hey. We're aiming a bit higher.
-
-// Explicitly set this before importing anything!
-process.env.IS_SCORE_WORKER_SERVER = "true";
-
-// Exit if we're not called with node. Think of this like if __name__ != "__main__" in python.
-if (require.main !== module) {
-	log.fatal(
-		"The Score Import Worker was imported, instead of ran directly with node. This is a fatal error. Exiting.",
-	);
-	process.exit(1);
-}
-
-/**
- * When a job is fired, this code will actually process the given data
- * and import it into the codebase.
- */
-export const worker = new Worker(
-	ScoreImportQueue.name,
-	async <I extends ImportTypes>(job: ScoreImportJob<I>) => {
-		const user = await GetUserWithID(job.data.userID);
-
-		if (!user) {
-			log.error(
-				`Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).`,
-			);
-			throw new Error(
-				`Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).`,
-			);
-		}
-
-		// Here's a hack. You cant pass buffers to bull workers, as content
-		// **must** be JSON serialisable. As such, all of our buffers get
-		// turned into nonsense objects. We need to "deJSONify" these buffers
-		// so lets do that now.
-
-		const processedArgs: Array<any> = [];
-
-		// eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
-
-		for (const arg of job.data.parserArguments as Array<any>) {
-			if (arg?.buffer?.type === "Buffer") {
-				processedArgs.push({ ...arg, buffer: Buffer.from(arg.buffer.data) });
-			} else {
-				processedArgs.push(arg);
-			}
-		}
-
-		job.data.parserArguments = processedArgs as ScoreImportJobData<I>["parserArguments"];
-
-		log.debug({ job }, `Received score import job ${job.id}`);
-
-		const InputParser = GetInputParser(job.data);
-
-		log.debug(`Starting import.`);
-
-		void job.updateProgress({
-			description: "Importing Scores.",
-		});
-
-		try {
-			const ImportDocument = await ScoreImportMain(
-				user.id,
-				job.data.userIntent,
-				job.data.importType,
-				InputParser,
-				job.data.importID,
-				undefined,
-				job,
-			);
-
-			log.debug(`Finished import.`);
-
-			return { success: true, ImportDocument };
-		} catch (e) {
-			const err = e as Error | ScoreImportFatalError;
-
-			// originally, we did `err instanceof ScoreImportFatalError`, but something in our toolchain
-			// has suddenly made all `instanceof` calls faulty. I still - to this day - have absolutely
-			// no idea why or how this broke, but now `instanceof` is considered a footgun so, great.
-			if ("statusCode" in err) {
-				log.info(
-					{ id: job.id, err, jobData: job.data },
-					`Job hit ScoreImportFatalError (User Fault) with message: ${err.message}`,
-				);
-				return { success: false, statusCode: err.statusCode, description: err.message };
-			}
-
-			throw err;
-		}
-	},
-	{
-		concurrency: ServerConfig.EXTERNAL_SCORE_IMPORT_WORKER_CONCURRENCY ?? 10,
-		connection: {
-			port: 6379,
-			host: Env.REDIS_URL,
-		},
-	},
-);
-
-worker.on("failed", (job, err) => {
-	// any errors that escalate this far are unexpected, as they haven't been caught by previous calls.
-	log.error(err, `Job ${job.id} failed unexpectedly with message: ${err.message}`);
-});
-
-worker.on("completed", (job, result) => {
-	log.debug(result, `Job ${job.id} finished successfully.`);
-});
-
-process.on("SIGTERM", () => {
-	void HandleSIGTERMGracefully();
-});

typescript/server/src/lib/setup/config.ts

@@ -60,6 +60,8 @@ const configSchema = z.object({
 	USE_EXTERNAL_SCORE_IMPORT_WORKER: z.boolean().default(false),
 	EXTERNAL_SCORE_IMPORT_WORKER_CONCURRENCY: z.number().int().positive().optional(),
 	ALLOW_RUNNING_OFFLINE: z.boolean().optional(),
+	/** Dev/stress: when true, score import HTTP routes use an unlimited score-import rate limiter. */
+	DISABLE_SCORE_IMPORT_RATE_LIMIT: z.boolean().default(false),
 	ENABLE_METRICS: z.boolean().default(true),
 	EMAIL_CONFIG: z.object({
 		FROM: z.string(),
@@ -425,6 +427,8 @@ const configFromEnv: unknown = {
 		10,
 	),
 	ALLOW_RUNNING_OFFLINE: parseBool("TACHI_ALLOW_RUNNING_OFFLINE"),
+	DISABLE_SCORE_IMPORT_RATE_LIMIT:
+		parseBool("TACHI_DISABLE_SCORE_IMPORT_RATE_LIMIT", false) ?? false,
 	ENABLE_METRICS: parseBool("TACHI_ENABLE_METRICS", true) ?? true,
 	EMAIL_CONFIG: emailCfg,
 	USC_QUEUE_SIZE: parseIntEnv("TACHI_USC_QUEUE_SIZE", 3),

typescript/server/src/lib/showcase/get-stats.test.ts

@@ -1,4 +1,5 @@
 import { seedUser } from "#actions/test-utils/api-tokens";
+import { newGameProfilePreferenceColumns } from "#lib/game-settings/create-game-settings";
 import { mongoScoreDataToPg } from "#lib/v3/migration-tools";
 import DB from "#services/pg/db";
 import {
@@ -63,20 +64,6 @@ async function seedFixtures() {
 		.values({ folder_id: TestingIIDXFolderSP10.folderID, chart_id: Testing511SPA.chartID })
 		.execute();
 
-	await DB.insertInto("game_settings")
-		.values({
-			user_id: 1,
-			game: "iidx-sp",
-			pf_preferred_score_alg: null,
-			pf_preferred_session_alg: null,
-			pf_preferred_profile_alg: null,
-			pf_preferred_default_enum: null,
-			pf_default_table: null,
-			pf_preferred_ranking: null,
-			data: JSON.stringify({ display2DXTra: false, bpiTarget: 0 }),
-		})
-		.execute();
-
 	const stats = [
 		{
 			mode: "folder" as const,
@@ -90,11 +77,14 @@ async function seedFixtures() {
 		},
 	];
 
-	await DB.insertInto("game_settings_showcase")
+	await DB.insertInto("game_profile")
 		.values({
 			user_id: 1,
 			game: "iidx-sp",
-			data: JSON.stringify(stats),
+			ratings: JSON.stringify({}),
+			classes: JSON.stringify({}),
+			...newGameProfilePreferenceColumns("iidx-sp"),
+			showcase: JSON.stringify(stats),
 		})
 		.execute();
 
@@ -142,9 +132,8 @@ describe("EvaluateUsersStatsShowcase (ported from get-stats.oldtest.ts)", () =>
 		expect(res[1]?.stat).toMatchObject({ mode: "chart", chartID: Testing511SPA.chartID });
 	});
 
-	it("throws when the user has no game_settings row", async () => {
-		await DB.deleteFrom("game_settings").where("user_id", "=", 1).execute();
-		await DB.deleteFrom("game_settings_showcase").where("user_id", "=", 1).execute();
+	it("throws when the user has no game_profile row", async () => {
+		await DB.deleteFrom("game_profile").where("game_profile.user_id", "=", 1).execute();
 
 		await expect(EvaluateUsersStatsShowcase(1, "iidx-sp")).rejects.toThrow();
 	});