diff --git a/.gitignore b/.gitignore index 9ae9b0011..0ac38b581 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ build coverage test-results +# Vitest bench --outputJson (see Justfile-bench bench-json) +typescript/server/bench-results.json + *.tsbuildinfo # legacy bootstrap flag diff --git a/Justfile b/Justfile index b3639a012..84da88f43 100644 --- a/Justfile +++ b/Justfile @@ -4,6 +4,7 @@ import "Justfile-db" import "Justfile-misc" import "Justfile-migrate" import "Justfile-test" +import "Justfile-bench" import "Justfile-dataset" import "Justfile-repo" diff --git a/Justfile-bench b/Justfile-bench new file mode 100644 index 000000000..31f309cdd --- /dev/null +++ b/Justfile-bench @@ -0,0 +1,20 @@ +# Vitest benchmarks (*.bench.ts). +# +# Why `just bench` looked bad: +# - `bun run --filter` defaults to --elide-lines=10, so each workspace’s log is cut to ~10 lines. +# - The default Foreman-style runner interleaves packages unless you pass --sequential. +# +# Recipes: +# - bench — all workspaces, full lines, one package after another (readable). +# - bench-server — only tachi-server (skip no-op packages; best for DB/search benches). +# - bench-json — server benches only; writes Vitest’s JSON to typescript/server/bench-results.json +# (open it or jq it; use with --compare later for regressions). + +bench: + bun run --filter '*' --sequential --elide-lines=0 bench + +bench-server: + bun run --filter tachi-server --elide-lines=0 bench + +bench-json: + bun run --filter tachi-server bench -- --outputJson bench-results.json diff --git a/bun.lock b/bun.lock index 82f8ab648..e3246427b 100644 --- a/bun.lock +++ b/bun.lock @@ -177,6 +177,7 @@ "@typescript/native-preview": "catalog:", "eslint-config-tachi": "workspace:*", "typescript": "catalog:", + "vitest": "catalog:", }, }, "typescript/db": { diff --git a/db/migrations/20260301154256_genesis.sql b/db/migrations/20260301154256_genesis.sql index 24a3b7a54..1f12e80bc 100644 --- a/db/migrations/20260301154256_genesis.sql +++ b/db/migrations/20260301154256_genesis.sql @@ -1,5 +1,6 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS pg_trgm; LOAD 'auto_explain'; -- Tables starting with "priv_" are private and should never ever be exposed @@ -474,6 +475,13 @@ CREATE TABLE "song" ( title TEXT NOT NULL, artist TEXT NOT NULL, + -- Denormalized search_term + alt_title text for FTS (kept in sync with seeds / triggers). + fts_document TEXT NOT NULL DEFAULT '', + textsearch tsvector NOT NULL GENERATED ALWAYS AS ( + setweight(to_tsvector('simple', coalesce(title, '')), 'A') || + setweight(to_tsvector('simple', coalesce(artist, '')), 'B') || + setweight(to_tsvector('simple', coalesce(fts_document, '')), 'C') + ) STORED, data JSONB NOT NULL -- game specific payload ); @@ -491,6 +499,32 @@ CREATE TABLE "song_alt_title" ( PRIMARY KEY (song_id, alt_title) ); +-- Populate fts_document from child tables (no-op on empty DB; seeds also set this column). +UPDATE song AS s +SET fts_document = trim( + both ' ' FROM concat_ws( + ' ', + ( + SELECT coalesce(string_agg(DISTINCT st.search_term, ' '), '') + FROM song_search_term AS st + WHERE st.song_id = s.id + ), + ( + SELECT coalesce(string_agg(DISTINCT at.alt_title, ' '), '') + FROM song_alt_title AS at + WHERE at.song_id = s.id + ) + ) +); + +CREATE INDEX song_textsearch_gin ON song USING GIN (textsearch); + +CREATE INDEX song_title_trgm ON song USING GIN (title gin_trgm_ops); + +CREATE INDEX song_artist_trgm ON song USING GIN (artist gin_trgm_ops); + +CREATE INDEX song_fts_document_trgm ON song USING GIN (fts_document gin_trgm_ops); + CREATE TABLE "chart" ( id TEXT PRIMARY KEY NOT NULL, legacy_id TEXT UNIQUE NOT NULL, diff --git a/typescript/api-parity/package.json b/typescript/api-parity/package.json index 76dde7247..f33854bba 100644 --- a/typescript/api-parity/package.json +++ b/typescript/api-parity/package.json @@ -6,6 +6,7 @@ "license": "MIT", "scripts": { "test": "vitest", + "bench": "vitest bench --passWithNoTests", "test:watch": "vitest", "typecheck": "tsc --noEmit" }, diff --git a/typescript/bliss/package.json b/typescript/bliss/package.json index 5e936f88e..568062fd5 100644 --- a/typescript/bliss/package.json +++ b/typescript/bliss/package.json @@ -16,7 +16,8 @@ "prepublishOnly": "tsgo -p tsconfig.build.json", "lint": "eslint .", "lint-fix": "eslint . --fix", - "typecheck": "tsgo --noEmit" + "typecheck": "tsgo --noEmit", + "bench": "node -e \"process.exit(0)\"" }, "publishConfig": { "main": "./build/index.js", diff --git a/typescript/bot/package.json b/typescript/bot/package.json index 864b70ba6..da16193e1 100644 --- a/typescript/bot/package.json +++ b/typescript/bot/package.json @@ -12,7 +12,8 @@ "lint": "eslint .", "lint-fix": "eslint . --fix", "typecheck": "tsgo --noEmit", - "test": "vitest" + "test": "vitest", + "bench": "vitest bench --passWithNoTests" }, "devDependencies": { "@types/express": "catalog:", diff --git a/typescript/client/package.json b/typescript/client/package.json index a2f6e8ce8..fdd35b67b 100644 --- a/typescript/client/package.json +++ b/typescript/client/package.json @@ -10,7 +10,8 @@ "lint": "eslint src", "lint-fix": "eslint src --fix", "preview": "vite build && vite preview", - "typecheck": "tsgo --noEmit" + "typecheck": "tsgo --noEmit", + "bench": "node -e \"process.exit(0)\"" }, "browserslist": { "production": [ diff --git a/typescript/common/package.json b/typescript/common/package.json index 267c00a8f..a5b89c6e3 100644 --- a/typescript/common/package.json +++ b/typescript/common/package.json @@ -13,6 +13,7 @@ }, "scripts": { "test": "vitest", + "bench": "vitest bench --passWithNoTests", "build": "tsgo -p tsconfig.build.json", "prepublishOnly": "tsgo -p tsconfig.build.json", "lint": "eslint .", @@ -44,7 +45,8 @@ "devDependencies": { "@typescript/native-preview": "catalog:", "eslint-config-tachi": "workspace:*", - "typescript": "catalog:" + "typescript": "catalog:", + "vitest": "catalog:" }, "nyc": { "reporter": [ diff --git a/typescript/db-cli/package.json b/typescript/db-cli/package.json index 746bcd128..b16e3be3f 100644 --- a/typescript/db-cli/package.json +++ b/typescript/db-cli/package.json @@ -10,7 +10,8 @@ "scripts": { "lint": "eslint .", "lint-fix": "eslint . --fix", - "typecheck": "tsgo --noEmit" + "typecheck": "tsgo --noEmit", + "bench": "node -e \"process.exit(0)\"" }, "dependencies": { "commander": "catalog:", diff --git a/typescript/db-migration-engine/package.json b/typescript/db-migration-engine/package.json index cf3d3b632..b68499786 100644 --- a/typescript/db-migration-engine/package.json +++ b/typescript/db-migration-engine/package.json @@ -17,7 +17,8 @@ "lint": "eslint .", "lint-fix": "eslint . --fix", "typecheck": "tsgo --noEmit", - "build": "tsgo -p tsconfig.build.json" + "build": "tsgo -p tsconfig.build.json", + "bench": "node -e \"process.exit(0)\"" }, "dependencies": { "kysely": "catalog:", diff --git a/typescript/db/package.json b/typescript/db/package.json index f590f428c..fb63c621c 100644 --- a/typescript/db/package.json +++ b/typescript/db/package.json @@ -8,7 +8,8 @@ "scripts": { "build": "tsgo -b", "start": "tsgo -b && node js/main.js", - "typecheck": "tsgo --noEmit" + "typecheck": "tsgo --noEmit", + "bench": "vitest bench --passWithNoTests" }, "dependencies": { "kysely": "catalog:", diff --git a/typescript/db/src/generated/public/Song.ts b/typescript/db/src/generated/public/Song.ts index 66b69c8db..3ff7ff455 100644 --- a/typescript/db/src/generated/public/Song.ts +++ b/typescript/db/src/generated/public/Song.ts @@ -19,6 +19,10 @@ export default interface SongTable { artist: ColumnType; + fts_document: ColumnType; + + textsearch: ColumnType; + data: ColumnType; } diff --git a/typescript/docs-autogen-scripts/package.json b/typescript/docs-autogen-scripts/package.json index 1155ef405..75ea60b7a 100644 --- a/typescript/docs-autogen-scripts/package.json +++ b/typescript/docs-autogen-scripts/package.json @@ -8,7 +8,8 @@ "lint": "eslint .", "lint-fix": "eslint . --fix", "run": "bun run src/main.ts", - "typecheck": "tsgo --noEmit" + "typecheck": "tsgo --noEmit", + "bench": "node -e \"process.exit(0)\"" }, "keywords": [], "author": "zk", diff --git a/typescript/eslint-config/package.json b/typescript/eslint-config/package.json index ad6678263..a70e684a3 100644 --- a/typescript/eslint-config/package.json +++ b/typescript/eslint-config/package.json @@ -2,6 +2,9 @@ "name": "eslint-config-tachi", "version": "1.0.0", "type": "module", + "scripts": { + "bench": "node -e \"process.exit(0)\"" + }, "exports": { ".": { "default": "./index.js" diff --git a/typescript/github-bot/package.json b/typescript/github-bot/package.json index 409cb6fa9..a3ea09701 100644 --- a/typescript/github-bot/package.json +++ b/typescript/github-bot/package.json @@ -8,7 +8,8 @@ "scripts": { "build": "tsgo -b", "start": "tsgo -b && node js/main.js", - "typecheck": "tsgo --noEmit" + "typecheck": "tsgo --noEmit", + "bench": "node -e \"process.exit(0)\"" }, "dependencies": { "@octokit/app": "catalog:", diff --git a/typescript/seeds-scripts/package.json b/typescript/seeds-scripts/package.json index d7b57c5ad..b82206108 100644 --- a/typescript/seeds-scripts/package.json +++ b/typescript/seeds-scripts/package.json @@ -10,7 +10,8 @@ "lint-fix": "eslint . --fix", "typecheck": "tsgo --noEmit", "sort": "node sort-seeds.js", - "test": "./run-tests.sh" + "test": "./run-tests.sh", + "bench": "node -e \"process.exit(0)\"" }, "author": "zk", "license": "MIT", diff --git a/typescript/server/package.json b/typescript/server/package.json index 84da4d4e1..42e1e6141 100644 --- a/typescript/server/package.json +++ b/typescript/server/package.json @@ -7,6 +7,7 @@ "scripts": { "dev": "bun --watch src/main.ts", "test": "vitest", + "bench": "vitest bench --passWithNoTests", "build": "tsgo -b tsconfig.build.json -v", "typecheck": "tsgo --noEmit", "lint": "eslint ./src", diff --git a/typescript/server/src/actions/customise-score.test.ts b/typescript/server/src/actions/customise-score.test.ts index 093d5ed3a..6649f4106 100644 --- a/typescript/server/src/actions/customise-score.test.ts +++ b/typescript/server/src/actions/customise-score.test.ts @@ -18,8 +18,8 @@ describe("mergeScoreDataFromPg", () => { optional: {}, } as MongoScoreData<"iidx:SP">; - const { data, derived, judgements } = mongoScoreDataToPg("iidx:SP", original); - const back = pgScoreDataToMongo("iidx:SP", data, derived, judgements); + const pg = mongoScoreDataToPg("iidx:SP", { ...original, judgements: {} }); + const back = pgScoreDataToMongo("iidx-sp", pg); expect(back).toMatchObject({ grade: "F", @@ -50,6 +50,7 @@ describe("ACTION_CustomiseScore", () => { title: "S", artist: "A", data: JSON.stringify({}), + fts_document: "", }) .execute(); @@ -73,6 +74,7 @@ describe("ACTION_CustomiseScore", () => { percent: 0, score: 100, optional: {}, + judgements: {}, } as MongoScoreData<"iidx:SP">); await DB.insertInto("score") diff --git a/typescript/server/src/lib/db-formats/session.full.test.ts b/typescript/server/src/lib/db-formats/session.full.test.ts index e53ad536c..f6c043d1a 100644 --- a/typescript/server/src/lib/db-formats/session.full.test.ts +++ b/typescript/server/src/lib/db-formats/session.full.test.ts @@ -40,6 +40,7 @@ describe("LoadSessionDocumentById", () => { title: "T", artist: "A", data: JSON.stringify({}), + fts_document: "", }) .execute(); diff --git a/typescript/server/src/lib/db-formats/song-search.bench.ts b/typescript/server/src/lib/db-formats/song-search.bench.ts new file mode 100644 index 000000000..cbdad5c95 --- /dev/null +++ b/typescript/server/src/lib/db-formats/song-search.bench.ts @@ -0,0 +1,57 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { importSeeds } from "#services/pg/seeds"; +import DB from "#services/pg/db"; +import type { GameGroup } from "tachi-common"; +import { beforeAll, bench, describe } from "vitest"; + +import { LoadSongChildrenForPgIds, SearchSongsForGameFtsAndTrgm } from "./song-search"; + +/** Default: repo `db/seeds` (override with `SEEDS_DIR` for custom trees). */ +const SEEDS_DIR = + process.env.SEEDS_DIR ?? + path.resolve(fileURLToPath(new URL(".", import.meta.url)), "../../../../../db/seeds"); + +const GAME_IIDX = "iidx" as const satisfies GameGroup; +const GAME_BMS = "bms" as const satisfies GameGroup; + +async function searchWithSongChildren(game: GameGroup, query: string, limit: number) { + const rows = await SearchSongsForGameFtsAndTrgm(game, query, limit); + + if (rows.length === 0) { + return; + } + + await LoadSongChildrenForPgIds(rows.map((r) => r.id)); +} + +describe("Postgres song search (full seeds)", () => { + beforeAll(async () => { + await importSeeds(DB, SEEDS_DIR); + }, 600_000); + + bench("iidx FTS — gradius (title)", async () => { + await SearchSongsForGameFtsAndTrgm(GAME_IIDX, "gradius", 50); + }); + + bench("iidx FTS — taka (artist)", async () => { + await SearchSongsForGameFtsAndTrgm(GAME_IIDX, "taka", 50); + }); + + bench("iidx short query — ab (FTS + trgm)", async () => { + await SearchSongsForGameFtsAndTrgm(GAME_IIDX, "ab", 50); + }); + + bench("iidx sparse — xyzunlikely (mostly trgm / empty FTS)", async () => { + await SearchSongsForGameFtsAndTrgm(GAME_IIDX, "xyzunlikely", 50); + }); + + bench("bms FTS — fezike (artist)", async () => { + await SearchSongsForGameFtsAndTrgm(GAME_BMS, "fezike", 50); + }); + + bench("iidx search + song_search_term / song_alt_title children", async () => { + await searchWithSongChildren(GAME_IIDX, "gradius", 50); + }); +}); diff --git a/typescript/server/src/lib/db-formats/song-search.test.ts b/typescript/server/src/lib/db-formats/song-search.test.ts new file mode 100644 index 000000000..7ad8e26f4 --- /dev/null +++ b/typescript/server/src/lib/db-formats/song-search.test.ts @@ -0,0 +1,298 @@ +import { SearchSpecificGameSongs } from "#lib/search/search"; +import DB from "#services/pg/db"; +import { importSeedsSubset } from "#services/pg/seeds"; +import { resolveSeedsDir, seedsJsonAvailable } from "#test-utils/seed-paths"; +import { sql } from "kysely"; +import { describe, expect, it } from "vitest"; + +import { + LoadSongChildrenForPgIds, + MAX_SONG_SEARCH_RESULTS_PER_GAME, + SearchSongsForGameFtsAndTrgm, +} from "./song-search"; + +function makeSongId(n: number): string { + return `S${n.toString(16).padStart(20, "0")}`; +} + +async function countSongRows(): Promise { + const { rows } = await sql<{ c: bigint }>` + SELECT count(*)::bigint AS c FROM song + `.execute(DB); + + return Number(rows[0]?.c ?? 0); +} + +/** + * Strings that would be dangerous if concatenated into SQL as raw text. + * Kysely `sql`…`${value}` binds values as parameters, so these must not execute as SQL. + */ +const HOSTILE_SEARCH_PAYLOADS = [ + "'; DROP TABLE song; --", + "1' OR '1'='1", + "1; DELETE FROM song WHERE 1=1;--", + "' OR 1=1--", + "') OR ('a'='a", + "\\'; SELECT pg_sleep(10);--", + '" UNION SELECT * FROM account--', +]; + +describe("SearchSongsForGameFtsAndTrgm (synthetic rows)", () => { + it("returns no rows for empty or whitespace search", async () => { + await DB.insertInto("song") + .values({ + id: makeSongId(1), + legacy_id: 9_000_001, + game_group: "iidx", + title: "Empty Query Test", + artist: "X", + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + expect(await SearchSongsForGameFtsAndTrgm("iidx", "", 10)).toEqual([]); + expect(await SearchSongsForGameFtsAndTrgm("iidx", " ", 10)).toEqual([]); + }); + + it("matches FTS on title and respects game_group", async () => { + await DB.insertInto("song") + .values([ + { + id: makeSongId(2), + legacy_id: 9_000_002, + game_group: "iidx", + title: "UniqueAlphaToken", + artist: "Artist A", + fts_document: "", + data: JSON.stringify({}), + }, + { + id: makeSongId(3), + legacy_id: 9_000_003, + game_group: "sdvx", + title: "UniqueAlphaToken Other Game", + artist: "Artist B", + fts_document: "", + data: JSON.stringify({}), + }, + ]) + .execute(); + + const iidx = await SearchSongsForGameFtsAndTrgm("iidx", "UniqueAlphaToken", 10); + + expect(iidx).toHaveLength(1); + expect(iidx[0]?.legacy_id).toBe(9_000_002); + }); + + it("loads search terms and alt titles via LoadSongChildrenForPgIds", async () => { + const sid = makeSongId(4); + + await DB.insertInto("song") + .values({ + id: sid, + legacy_id: 9_000_004, + game_group: "iidx", + title: "Child Row Test", + artist: "Z", + fts_document: "synonym extra", + data: JSON.stringify({}), + }) + .execute(); + + await DB.insertInto("song_search_term") + .values({ song_id: sid, search_term: "synonym" }) + .execute(); + await DB.insertInto("song_alt_title") + .values({ song_id: sid, alt_title: "Extra JP" }) + .execute(); + + const rows = await SearchSongsForGameFtsAndTrgm("iidx", "Child", 10); + const children = await LoadSongChildrenForPgIds(rows.map((r) => r.id)); + + expect(children.get(sid)).toEqual({ + searchTerms: ["synonym"], + altTitles: ["Extra JP"], + }); + }); + + it("caps results at MAX_SONG_SEARCH_RESULTS_PER_GAME even when limit is higher", async () => { + const rows = Array.from({ length: 150 }, (_, i) => ({ + id: makeSongId(100 + i), + legacy_id: 9_100_000 + i, + game_group: "iidx" as const, + title: `CapBulk ${i}`, + artist: "Cap Artist", + fts_document: "", + data: JSON.stringify({}), + })); + + await DB.insertInto("song").values(rows).execute(); + + const res = await SearchSongsForGameFtsAndTrgm("iidx", "CapBulk", 500); + + expect(res.length).toBe(MAX_SONG_SEARCH_RESULTS_PER_GAME); + }); + + it("uses trgm / short-query path for very short queries", async () => { + await DB.insertInto("song") + .values({ + id: makeSongId(300), + legacy_id: 9_000_300, + game_group: "iidx", + title: "Qx", + artist: "ShortQ Artist", + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + const res = await SearchSongsForGameFtsAndTrgm("iidx", "Qx", 10); + + expect(res.some((r) => r.legacy_id === 9_000_300)).toBe(true); + }); +}); + +describe("SearchSongsForGameFtsAndTrgm (hostile / injection-shaped input)", () => { + it("does not change song row count when search strings look like SQL injection", async () => { + await DB.insertInto("song") + .values([ + { + id: makeSongId(400), + legacy_id: 9_000_400, + game_group: "iidx", + title: "Bait Song", + artist: "Bait", + fts_document: "", + data: JSON.stringify({}), + }, + { + id: makeSongId(401), + legacy_id: 9_000_401, + game_group: "iidx", + title: "Other Bait", + artist: "Bait", + fts_document: "", + data: JSON.stringify({}), + }, + ]) + .execute(); + + const before = await countSongRows(); + + await Promise.all( + HOSTILE_SEARCH_PAYLOADS.map(async (payload) => { + await SearchSongsForGameFtsAndTrgm("iidx", payload, 10); + expect(await countSongRows()).toBe(before); + }), + ); + }); + + it("does not change row count via SearchSpecificGameSongs with the same payloads", async () => { + const before = await countSongRows(); + + await Promise.all( + HOSTILE_SEARCH_PAYLOADS.map(async (payload) => { + await SearchSpecificGameSongs("iidx", payload, 10); + expect(await countSongRows()).toBe(before); + }), + ); + }); + + it("rejects NUL in search string (PostgreSQL UTF-8 text; not SQL injection)", async () => { + await DB.insertInto("song") + .values({ + id: makeSongId(600), + legacy_id: 9_000_600, + game_group: "iidx", + title: "NulProbe", + artist: "X", + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + const before = await countSongRows(); + + // TODO(zk): Throwing on nulbytes in strings is spicy + // but we can't fix this, so whatever. + await expect(SearchSongsForGameFtsAndTrgm("iidx", "a\x00b", 10)).rejects.toThrow( + /UTF8|invalid byte sequence|0x00/iu, + ); + + expect(await countSongRows()).toBe(before); + }); + + it("treats ILIKE metacharacters in the search string as literals (no broad % / _ wildcard match)", async () => { + await DB.insertInto("song") + .values([ + { + id: makeSongId(500), + legacy_id: 9_000_500, + game_group: "iidx", + title: "ExactPercent", + artist: "NoWildcard", + fts_document: "", + data: JSON.stringify({}), + }, + { + id: makeSongId(501), + legacy_id: 9_000_501, + game_group: "iidx", + title: "Something Else Entirely", + artist: "NoWildcard", + fts_document: "", + data: JSON.stringify({}), + }, + ]) + .execute(); + + const pct = await SearchSongsForGameFtsAndTrgm("iidx", "%", 10); + const titles = pct.map((r) => r.title); + + expect(titles).not.toContain("Something Else Entirely"); + }); +}); + +describe("SearchSpecificGameSongs", () => { + it("returns __textScore and song document fields", async () => { + await DB.insertInto("song") + .values({ + id: makeSongId(5), + legacy_id: 9_000_005, + game_group: "iidx", + title: "ScoreFieldTest", + artist: "Z", + fts_document: "", + data: JSON.stringify({ displayVersion: "1" }), + }) + .execute(); + + const songs = await SearchSpecificGameSongs("iidx", "ScoreFieldTest", 10); + + expect(songs).toHaveLength(1); + expect(songs[0]?.title).toBe("ScoreFieldTest"); + expect(typeof songs[0]?.__textScore).toBe("number"); + }); +}); + +describe("SearchSongsForGameFtsAndTrgm (real seed subset)", () => { + it.skipIf(!seedsJsonAvailable())( + "finds known IIDX titles from a small songs-iidx slice", + async () => { + await importSeedsSubset(DB, resolveSeedsDir(), { + gameGroups: ["iidx"], + maxSongsPerGame: 80, + includeCharts: false, + }); + + const gradius = await SearchSongsForGameFtsAndTrgm("iidx", "GRADIUSIC CYBER", 20); + + expect(gradius.some((r) => r.title === "GRADIUSIC CYBER")).toBe(true); + + const prince = await SearchSongsForGameFtsAndTrgm("iidx", "Prince on a star", 20); + + expect(prince.some((r) => r.title === "Prince on a star")).toBe(true); + }, + ); +}); diff --git a/typescript/server/src/lib/db-formats/song-search.ts b/typescript/server/src/lib/db-formats/song-search.ts new file mode 100644 index 000000000..34c450645 --- /dev/null +++ b/typescript/server/src/lib/db-formats/song-search.ts @@ -0,0 +1,187 @@ +import type { GameGroup } from "tachi-common"; + +import DB from "#services/pg/db"; +import { EscapeForILIKE } from "#utils/misc"; +import { sql } from "kysely"; + +/** Hard cap on song hits per `game_group` (FTS + trgm combined). */ +export const MAX_SONG_SEARCH_RESULTS_PER_GAME = 100; + +/** Use trigram / ILIKE supplement when the query is this short or FTS returns nothing. */ +const SHORT_QUERY_LEN = 3; + +export type SongSearchRow = { + artist: string; + data: unknown; + id: string; + legacy_id: number; + rank: number; + title: string; +}; + +/** + * Indexed song search: PostgreSQL FTS (tsvector) plus optional pg_trgm / ILIKE fallback + * (Zenith-style — no full-table load, no huge IN lists). + */ +export async function SearchSongsForGameFtsAndTrgm( + game: GameGroup, + search: string, + limit: number, +): Promise> { + const q = search.trim(); + + if (q.length === 0) { + return []; + } + + const cap = Math.min(Math.max(1, limit), MAX_SONG_SEARCH_RESULTS_PER_GAME); + + const { rows: ftsRows } = await sql` + SELECT + id, + legacy_id, + title, + artist, + data, + (ts_rank_cd(textsearch, websearch_to_tsquery('simple', ${q})))::float8 AS rank + FROM song + WHERE game_group = ${game} + AND textsearch @@ websearch_to_tsquery('simple', ${q}) + ORDER BY rank DESC + LIMIT ${cap} + `.execute(DB); + + const needTrgm = + ftsRows.length < cap && (q.length <= SHORT_QUERY_LEN || ftsRows.length === 0); + + if (!needTrgm) { + return ftsRows; + } + + const excludeIds = ftsRows.map((r) => r.id); + const trgmLimit = cap - ftsRows.length; + const likeEsc = EscapeForILIKE(q.toLowerCase()); + const pat = `%${likeEsc}%`; + + const { rows: trgmRows } = + excludeIds.length === 0 + ? await sql` + SELECT + id, + legacy_id, + title, + artist, + data, + GREATEST( + similarity(lower(title), lower(${q})), + similarity(lower(artist), lower(${q})), + similarity(lower(fts_document), lower(${q})) + )::float8 AS rank + FROM song + WHERE game_group = ${game} + AND ( + title ILIKE ${pat} + OR artist ILIKE ${pat} + OR fts_document ILIKE ${pat} + ) + ORDER BY rank DESC + LIMIT ${trgmLimit} + `.execute(DB) + : await sql` + SELECT + id, + legacy_id, + title, + artist, + data, + GREATEST( + similarity(lower(title), lower(${q})), + similarity(lower(artist), lower(${q})), + similarity(lower(fts_document), lower(${q})) + )::float8 AS rank + FROM song + WHERE game_group = ${game} + AND id NOT IN (${sql.join(excludeIds)}) + AND ( + title ILIKE ${pat} + OR artist ILIKE ${pat} + OR fts_document ILIKE ${pat} + ) + ORDER BY rank DESC + LIMIT ${trgmLimit} + `.execute(DB); + + const byId = new Map(); + + for (const r of ftsRows) { + byId.set(r.id, r); + } + + for (const r of trgmRows) { + const existing = byId.get(r.id); + + if (!existing || r.rank > existing.rank) { + byId.set(r.id, r); + } + } + + return [...byId.values()].sort((a, b) => b.rank - a.rank).slice(0, cap); +} + +/** + * Loads search_term and alt_title rows for a bounded set of song PKs (e.g. search hits). + */ +export async function LoadSongChildrenForPgIds( + songIds: string[], +): Promise> { + if (songIds.length === 0) { + return new Map(); + } + + const [searchTermRows, altTitleRows] = await Promise.all([ + DB.selectFrom("song_search_term") + .select(["song_id", "search_term"]) + .where("song_id", "in", songIds) + .execute(), + DB.selectFrom("song_alt_title") + .select(["song_id", "alt_title"]) + .where("song_id", "in", songIds) + .execute(), + ]); + + const termsBySong = new Map(); + const altsBySong = new Map(); + + for (const r of searchTermRows) { + let list = termsBySong.get(r.song_id); + + if (!list) { + list = []; + termsBySong.set(r.song_id, list); + } + + list.push(r.search_term); + } + + for (const r of altTitleRows) { + let list = altsBySong.get(r.song_id); + + if (!list) { + list = []; + altsBySong.set(r.song_id, list); + } + + list.push(r.alt_title); + } + + const out = new Map(); + + for (const id of songIds) { + out.set(id, { + searchTerms: termsBySong.get(id) ?? [], + altTitles: altsBySong.get(id) ?? [], + }); + } + + return out; +} diff --git a/typescript/server/src/lib/db-formats/song.ts b/typescript/server/src/lib/db-formats/song.ts index 77e7914f1..5b576f4a0 100644 --- a/typescript/server/src/lib/db-formats/song.ts +++ b/typescript/server/src/lib/db-formats/song.ts @@ -1,5 +1,6 @@ +import type { GameGroup, integer, SongDocument, SongDocumentData } from "tachi-common"; + import DB from "#services/pg/db"; -import type { GameGroup, SongDocument, SongDocumentData } from "tachi-common"; /** * Fetches a song by its legacy numeric ID (from the URL / Mongo era), together @@ -39,3 +40,90 @@ export async function GetSongByLegacyID( return { doc, pgId: row.id }; } + +/** + * Batch-loads song documents by legacy numeric IDs (order follows first occurrence in `legacyIds`). + */ +export async function GetSongsByLegacyIDs( + game: GameGroup, + legacyIds: Array, +): Promise> { + if (legacyIds.length === 0) { + return []; + } + + const unique = [...new Set(legacyIds)]; + + const songRows = await DB.selectFrom("song") + .select(["id", "legacy_id", "title", "artist", "data"]) + .where("game_group", "=", game) + .where("legacy_id", "in", unique) + .execute(); + + if (songRows.length === 0) { + return []; + } + + const ids = songRows.map((s) => s.id); + + const [searchTermRows, altTitleRows] = await Promise.all([ + DB.selectFrom("song_search_term") + .select(["song_id", "search_term"]) + .where("song_id", "in", ids) + .execute(), + DB.selectFrom("song_alt_title") + .select(["song_id", "alt_title"]) + .where("song_id", "in", ids) + .execute(), + ]); + + const termsBySong = new Map(); + const altsBySong = new Map(); + + for (const r of searchTermRows) { + let list = termsBySong.get(r.song_id); + + if (!list) { + list = []; + termsBySong.set(r.song_id, list); + } + + list.push(r.search_term); + } + + for (const r of altTitleRows) { + let list = altsBySong.get(r.song_id); + + if (!list) { + list = []; + altsBySong.set(r.song_id, list); + } + + list.push(r.alt_title); + } + + const byLegacy = new Map(); + + for (const row of songRows) { + byLegacy.set(row.legacy_id, { + id: row.legacy_id, + title: row.title, + artist: row.artist, + searchTerms: termsBySong.get(row.id) ?? [], + altTitles: altsBySong.get(row.id) ?? [], + data: row.data as SongDocumentData[typeof game], + }); + } + + const out: Array = []; + + for (const id of legacyIds) { + const doc = byLegacy.get(id); + + if (doc) { + out.push(doc); + } + } + + return out; +} diff --git a/typescript/server/src/lib/search/search.oldtest.ts b/typescript/server/src/lib/search/search.oldtest.ts index 57f25a926..910375238 100644 --- a/typescript/server/src/lib/search/search.oldtest.ts +++ b/typescript/server/src/lib/search/search.oldtest.ts @@ -5,7 +5,7 @@ import t from "tap"; import { SearchSpecificGameSongs, SearchUsersRegExp } from "./search"; -t.test("#SearchSpecificGameSongs", (t) => { +t.test("#SearchSpecificGameSongs", { skip: true }, (t) => { t.beforeEach(ResetDBState); t.beforeEach(LoadTachiIIDXData); t.beforeEach(async () => { diff --git a/typescript/server/src/lib/search/search.ts b/typescript/server/src/lib/search/search.ts index 6b297307b..fb4e60313 100644 --- a/typescript/server/src/lib/search/search.ts +++ b/typescript/server/src/lib/search/search.ts @@ -1,6 +1,8 @@ import type { FilterQuery } from "mongodb"; import type { ICollection } from "monk"; +import { GetChartsBySongPgId } from "#lib/db-formats/chart"; +import { LoadSongChildrenForPgIds, SearchSongsForGameFtsAndTrgm } from "#lib/db-formats/song-search"; import { SELECT_USER, ToUserDocument } from "#lib/db-formats/user"; import { log } from "#lib/log/log"; import { TachiConfig } from "#lib/setup/config"; @@ -10,23 +12,27 @@ import { GetSongForIDGuaranteed } from "#utils/db"; import { EscapeForILIKE } from "#utils/misc"; import { UnixMillisecondsToISO8601 } from "#utils/time"; import { GetOnlineCutoff } from "#utils/user"; +import { sql } from "kysely"; import { type ChartDocument, CreateSongMap, type FolderDocument, type GameGroup, + GamePTToV3, type GPTString, type GPTStrings, type integer, type Playtype, type SessionDocument, type SongDocument, + type SongDocumentData, SplitGPT, type UserDocument, } from "tachi-common"; import { AsyncFzf } from "./fzf/main"; + interface SearchControls { keys: Array; primary: string; @@ -157,12 +163,55 @@ export type SongSearchReturn = { __textScore: number; } & SongDocument; -export function SearchSpecificGameSongs( +/** + * Fuzzy song search over Postgres `song` metadata (same behaviour as legacy Mongo SearchCollection). + */ +export async function searchSpecificGameSongsWithPgIds( + game: GameGroup, + search: string, + limit = 100, +): Promise<{ + pgIdByLegacyId: Map; + songs: Array; +}> { + const rows = await SearchSongsForGameFtsAndTrgm(game, search, limit); + + if (rows.length === 0) { + return { songs: [], pgIdByLegacyId: new Map() }; + } + + const children = await LoadSongChildrenForPgIds(rows.map((r) => r.id)); + + const pgIdByLegacyId = new Map(); + const songs: Array = []; + + for (const row of rows) { + const ch = children.get(row.id); + + pgIdByLegacyId.set(row.legacy_id, row.id); + + songs.push({ + id: row.legacy_id, + title: row.title, + artist: row.artist, + searchTerms: ch?.searchTerms ?? [], + altTitles: ch?.altTitles ?? [], + data: row.data as SongDocumentData[typeof game], + __textScore: Math.round(1000 * row.rank), + }); + } + + return { songs, pgIdByLegacyId }; +} + +export async function SearchSpecificGameSongs( game: GameGroup, search: string, limit = 100, ): Promise> { - return SearchCollection(MONGODB_KILL.anySongs[game], search, "songs", {}, limit); + const { songs } = await searchSpecificGameSongsWithPgIds(game, search, limit); + + return songs; } export async function SearchSpecificGameSongsAndCharts( @@ -171,17 +220,27 @@ export async function SearchSpecificGameSongsAndCharts( playtype?: Playtype, limit = 100, ) { - const songs = await SearchSpecificGameSongs(game, search, limit); + const { songs, pgIdByLegacyId } = await searchSpecificGameSongsWithPgIds(game, search, limit); - const chartQuery: FilterQuery = { - songID: { $in: songs.map((e) => e.id) }, - }; - - if (playtype) { - chartQuery.playtype = playtype; + if (!playtype) { + throw new Error("SearchSpecificGameSongsAndCharts requires playtype"); } - const charts = (await MONGODB_KILL.anyCharts[game].find(chartQuery)) as Array; + const v3Game = GamePTToV3(game, playtype); + + const chartLists = await Promise.all( + songs.map((song) => { + const pgId = pgIdByLegacyId.get(song.id); + + if (!pgId) { + return Promise.resolve([] as Array); + } + + return GetChartsBySongPgId(v3Game, pgId, song.id); + }), + ); + + const charts = chartLists.flat(); return { songs, charts }; } @@ -195,36 +254,43 @@ export async function SearchGlobalGameSongsAndCharts( playtype?: Playtype, limit = 100, ): Promise> { - const songs = await SearchSpecificGameSongs(game, search, limit); + const { songs, pgIdByLegacyId } = await searchSpecificGameSongsWithPgIds(game, search, limit); - const chartQuery: FilterQuery = { - songID: { $in: songs.map((e) => e.id) }, - }; - - if (playtype) { - chartQuery.playtype = playtype; + if (!playtype) { + throw new Error("SearchGlobalGameSongsAndCharts requires playtype"); } - const charts = (await MONGODB_KILL.anyCharts[game].find(chartQuery)) as unknown as Array< - { __playcount: integer } & ChartDocument - >; + const v3Game = GamePTToV3(game, playtype); - const playcounts: Array<{ _id: string; playcount: integer }> = - await MONGODB_KILL.scores.aggregate([ - { - $match: { - chartID: { $in: charts.map((e) => e.chartID) }, - }, - }, - { - $group: { - _id: "$chartID", - playcount: { $sum: 1 }, - }, - }, - ]); + const chartLists = await Promise.all( + songs.map((song) => { + const pgId = pgIdByLegacyId.get(song.id); + + if (!pgId) { + return Promise.resolve([] as Array); + } + + return GetChartsBySongPgId(v3Game, pgId, song.id); + }), + ); + + const charts = chartLists.flat(); + + if (charts.length === 0) { + return []; + } + + const chartLegacyIds = charts.map((c) => c.chartID); + + const playcountRows = await DB.selectFrom("score") + .innerJoin("chart", "chart.id", "score.chart_id") + .select(["chart.legacy_id", sql`count(score.id)::int`.as("playcount")]) + .where("chart.legacy_id", "in", chartLegacyIds) + .groupBy("chart.legacy_id") + .execute(); + + const playcountLookup = Object.fromEntries(playcountRows.map((r) => [r.legacy_id, r.playcount])); - const playcountLookup = Object.fromEntries(playcounts.map((e) => [e._id, e.playcount])); const songMap = CreateSongMap(songs); const output = []; @@ -234,6 +300,7 @@ export async function SearchGlobalGameSongsAndCharts( if (!song) { log.warn(`Failed to find parent song for ${chart.songID} (${game})? Skipping.`); + continue; } diff --git a/typescript/server/src/scripts/load-seeds-pg.ts b/typescript/server/src/scripts/load-seeds-pg.ts index 790f5d772..4be6a312e 100644 --- a/typescript/server/src/scripts/load-seeds-pg.ts +++ b/typescript/server/src/scripts/load-seeds-pg.ts @@ -20,9 +20,9 @@ import { Kysely, PostgresDialect } from "kysely"; import path from "path"; import { Pool } from "pg"; -import { buildChartIdMap, importSeeds, toPgGame } from "../services/pg/seeds"; +import { buildChartIdMap, importSeeds, importSeedsSubset, toPgGame } from "../services/pg/seeds"; -export { buildChartIdMap, importSeeds, toPgGame }; +export { buildChartIdMap, importSeeds, importSeedsSubset, toPgGame }; // ── Standalone entrypoint ────────────────────────────────────────────────── diff --git a/typescript/server/src/scripts/migrate-to-postgres.ts b/typescript/server/src/scripts/migrate-to-postgres.ts index 41d69b357..bd002bf8d 100644 --- a/typescript/server/src/scripts/migrate-to-postgres.ts +++ b/typescript/server/src/scripts/migrate-to-postgres.ts @@ -27,7 +27,7 @@ import type { NewGameRival, NewGameSettings, NewGameSettingsShowcase, - NewGameStats, + NewGameProfile, NewGameStatsSnapshot, NewGoalSub, NewImport, @@ -743,20 +743,20 @@ async function main(): Promise { ); } - // ── game_stats ──────────────────────────────────────────────────────────── + // ── game_profile ─────────────────────────────────────────────────────────── { - console.log("\n[game_stats]"); + console.log("\n[game_profile]"); const gameStats = await mongoDB.get("game-stats").find({}); - const statsRows: Array = gameStats.map((gs) => ({ + const statsRows: Array = gameStats.map((gs) => ({ user_id: gs.userID, game: toGame(gs.game, gs.playtype), ratings: JSON.stringify(gs.ratings), classes: JSON.stringify(gs.classes), })); - await batchInsert("game_stats", statsRows); - console.log(` ${gameStats.length} game stats.`); + await batchInsert("game_profile", statsRows); + console.log(` ${gameStats.length} game profiles.`); } // ── game_stats_snapshot ─────────────────────────────────────────────────── diff --git a/typescript/server/src/server/router/api/v1/games/_game/_playtype/charts/router.ts b/typescript/server/src/server/router/api/v1/games/_game/_playtype/charts/router.ts index e3e730cd0..b38c3663c 100644 --- a/typescript/server/src/server/router/api/v1/games/_game/_playtype/charts/router.ts +++ b/typescript/server/src/server/router/api/v1/games/_game/_playtype/charts/router.ts @@ -1,4 +1,5 @@ import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; +import { GetSongsByLegacyIDs } from "#lib/db-formats/song"; import { log } from "#lib/log/log"; import { ResolveSongAndChart } from "#lib/score-import/import-types/common/batch-manual/converter"; import { SearchSpecificGameSongs } from "#lib/search/search"; @@ -85,9 +86,7 @@ router.get("/", async (req, res) => { // @optimisable // could use songIDs from above instead of refetching // but this is not very expensive. - const songs = await MONGODB_KILL.anySongs[game].find({ - id: { $in: charts.map((e) => e.songID) }, - }); + const songs = await GetSongsByLegacyIDs(game, charts.map((e) => e.songID)); // Edge case. // If the game is IIDX and the player does not want diff --git a/typescript/server/src/server/router/api/v1/games/_game/_playtype/songs/_songID/router.test.ts b/typescript/server/src/server/router/api/v1/games/_game/_playtype/songs/_songID/router.test.ts index de16ddd2d..738b489d2 100644 --- a/typescript/server/src/server/router/api/v1/games/_game/_playtype/songs/_songID/router.test.ts +++ b/typescript/server/src/server/router/api/v1/games/_game/_playtype/songs/_songID/router.test.ts @@ -31,6 +31,7 @@ async function seedSong({ title, artist, data: { displayVersion: "1", genre: "PIANO AMBIENT" }, + fts_document: [...searchTerms, ...altTitles].filter(Boolean).join(" "), }) .execute(); diff --git a/typescript/server/src/server/router/api/v1/users/_userID/router.test.ts b/typescript/server/src/server/router/api/v1/users/_userID/router.test.ts index 76686030e..4c39a1cf7 100644 --- a/typescript/server/src/server/router/api/v1/users/_userID/router.test.ts +++ b/typescript/server/src/server/router/api/v1/users/_userID/router.test.ts @@ -20,6 +20,7 @@ async function seedSongAndChart() { title: "Test Song", artist: "Test Artist", data: JSON.stringify({}), + fts_document: "", }) .execute(); diff --git a/typescript/server/src/services/pg/seeds.ts b/typescript/server/src/services/pg/seeds.ts index 17e7c193e..627387489 100644 --- a/typescript/server/src/services/pg/seeds.ts +++ b/typescript/server/src/services/pg/seeds.ts @@ -57,6 +57,235 @@ type SeedTable = { legacyTableID?: string; } & Omit; +const INSERT_CHUNK = 500; + +function readJsonSeed(seedsDir: string, filename: string): Array { + return JSON.parse(fs.readFileSync(path.join(seedsDir, filename), "utf-8")) as Array; +} + +async function chunkedDeletePg( + pg: Kysely, + table: keyof Database, + column: string, + ids: ReadonlyArray, +): Promise { + for (let i = 0; i < ids.length; i = i + INSERT_CHUNK) { + const chunk = ids.slice(i, i + INSERT_CHUNK); + + await (pg.deleteFrom(table as never) as any).where(column, "in", chunk).execute(); + } +} + +async function batchIgnorePg( + pg: Kysely, + table: T, + rows: ReadonlyArray>, +): Promise { + if (rows.length === 0) { + return; + } + + for (let i = 0; i < rows.length; i = i + INSERT_CHUNK) { + const chunk = rows.slice(i, i + INSERT_CHUNK); + + await pg + .insertInto(table) + .values(chunk as never) + .onConflict((oc) => oc.doNothing()) + .execute(); + } +} + +async function upsertSongsForGameGroup( + pg: Kysely, + gameGroup: GameGroup, + songs: Array, +): Promise { + const songRows: Array = []; + const searchTermRows: Array = []; + const altTitleRows: Array = []; + + for (const s of songs) { + if (!s.id) { + throw new Error( + `Song ${gameGroup}:${s.legacySongID} is missing an id. Run 1-migrate-to-pg-style.ts first.`, + ); + } + + songRows.push({ + id: s.id, + legacy_id: + s.legacySongID ?? + (() => { + throw new Error(`Song ${gameGroup}:${s.id} is missing legacySongID.`); + })(), + game_group: gameGroup, + title: s.title, + artist: s.artist, + data: JSON.stringify(s.data), + fts_document: [...s.searchTerms, ...s.altTitles].filter(Boolean).join(" "), + }); + + for (const term of s.searchTerms) { + searchTermRows.push({ song_id: s.id, search_term: term }); + } + + for (const alt of s.altTitles) { + altTitleRows.push({ song_id: s.id, alt_title: alt }); + } + } + + const songIds = songs.map((s) => s.id); + + for (let i = 0; i < songRows.length; i = i + INSERT_CHUNK) { + const chunk = songRows.slice(i, i + INSERT_CHUNK); + + await pg + .insertInto("song") + .values(chunk) + .onConflict((oc) => + oc.column("id").doUpdateSet({ + title: sql`excluded.title`, + artist: sql`excluded.artist`, + data: sql`excluded.data`, + fts_document: sql`excluded.fts_document`, + }), + ) + .execute(); + } + + await chunkedDeletePg(pg, "song_search_term", "song_id", songIds); + await chunkedDeletePg(pg, "song_alt_title", "song_id", songIds); + + await batchIgnorePg(pg, "song_search_term", searchTermRows); + await batchIgnorePg(pg, "song_alt_title", altTitleRows); +} + +async function upsertChartsForPgGame( + pg: Kysely, + pgGame: PgGame, + charts: Array, +): Promise { + if (charts.length === 0) { + return; + } + + const chartRows: Array = []; + const versionRows: Array = []; + + for (const c of charts) { + if (!c.id) { + throw new Error( + `Chart ${c.legacyChartID ?? "(unknown)"}` + + ` (${pgGame}) is missing an id. Run 1-migrate-to-pg-style.ts first.`, + ); + } + + if (!c.legacyChartID) { + throw new Error( + `Chart ${c.id} (${pgGame}) is missing legacyChartID. Run 1-migrate-to-pg-style.ts first.`, + ); + } + + chartRows.push({ + id: c.id, + legacy_id: c.legacyChartID, + game: pgGame, + song_id: c.songID, + level: c.level, + level_num: c.levelNum, + is_primary: c.isPrimary, + difficulty: c.difficulty, + data: JSON.stringify(c.data), + }); + + for (const version of c.versions) { + versionRows.push({ chart_id: c.id, version: version as string }); + } + } + + const chartSids = charts.map((c) => c.id); + + for (let i = 0; i < chartRows.length; i = i + INSERT_CHUNK) { + const chunk = chartRows.slice(i, i + INSERT_CHUNK); + + await pg + .insertInto("chart") + .values(chunk) + .onConflict((oc) => + oc.column("id").doUpdateSet({ + level: sql`excluded.level`, + level_num: sql`excluded.level_num`, + is_primary: sql`excluded.is_primary`, + difficulty: sql`excluded.difficulty`, + data: sql`excluded.data`, + }), + ) + .execute(); + } + + await chunkedDeletePg(pg, "chart_version", "chart_id", chartSids); + await batchIgnorePg(pg, "chart_version", versionRows); +} + +export type ImportSeedsSubsetOptions = { + gameGroups: GameGroup[]; + /** When true (default), load chart rows whose `songID` is in the loaded song set. */ + includeCharts?: boolean; + maxSongsPerGame: number; +}; + +/** + * Loads a bounded slice of real seed JSON (songs + optional charts) for tests or tooling. + */ +export async function importSeedsSubset( + pg: Kysely, + seedsDir: string, + options: ImportSeedsSubsetOptions, +): Promise { + const { maxSongsPerGame, gameGroups, includeCharts = true } = options; + const loadedSongIds = new Set(); + + for (const gg of gameGroups) { + const filename = `songs-${gg}.json`; + const filepath = path.join(seedsDir, filename); + + if (!fs.existsSync(filepath)) { + throw new Error(`seed file not found: ${filepath}`); + } + + const all = readJsonSeed(seedsDir, filename); + const songs = all.slice(0, maxSongsPerGame); + + await upsertSongsForGameGroup(pg, gg, songs); + + for (const s of songs) { + loadedSongIds.add(s.id); + } + } + + if (!includeCharts) { + return; + } + + const chartFiles = fs + .readdirSync(seedsDir) + .filter((f) => f.startsWith("charts-") && f.endsWith(".json")); + + for (const gg of gameGroups) { + const filesForGame = chartFiles.filter((f) => f.startsWith(`charts-${gg}`)); + + for (const file of filesForGame) { + const pgGame = file.replace(/^charts-/u, "").replace(/\.json$/u, "") as PgGame; + const charts = readJsonSeed(seedsDir, file).filter((c) => + loadedSongIds.has(c.songID), + ); + + await upsertChartsForPgGame(pg, pgGame, charts); + } + } +} + // ── Game helpers ─────────────────────────────────────────────────────────── const SINGLE_PT_GAMES = new Set([ @@ -113,43 +342,7 @@ export function buildChartIdMap(seedsDir: string): Map { // ── Core import logic ────────────────────────────────────────────────────── export async function importSeeds(pg: Kysely, seedsDir: string): Promise { - const INSERT_CHUNK = 500; - - async function batchIgnore( - table: T, - rows: ReadonlyArray>, - ): Promise { - if (rows.length === 0) { - return; - } - - for (let i = 0; i < rows.length; i = i + INSERT_CHUNK) { - const chunk = rows.slice(i, i + INSERT_CHUNK); - - await pg - .insertInto(table) - .values(chunk as never) - .onConflict((oc) => oc.doNothing()) - .execute(); - } - } - - // Deletes rows matching a large list of IDs by chunking the IN clause. - async function chunkedDelete( - table: keyof Database, - column: string, - ids: ReadonlyArray, - ): Promise { - for (let i = 0; i < ids.length; i = i + INSERT_CHUNK) { - const chunk = ids.slice(i, i + INSERT_CHUNK); - - await (pg.deleteFrom(table as never) as any).where(column, "in", chunk).execute(); - } - } - - function readCollection(filename: string): Array { - return JSON.parse(fs.readFileSync(path.join(seedsDir, filename), "utf-8")) as Array; - } + const readCollection = (filename: string) => readJsonSeed(seedsDir, filename); const files = new Set(fs.readdirSync(seedsDir)); const songFiles = [...files].filter((f) => f.startsWith("songs-") && f.endsWith(".json")); @@ -166,63 +359,7 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi const gameGroup = file.replace(/^songs-/u, "").replace(/\.json$/u, "") as GameGroup; const songs = readCollection(file); - const songRows: Array = []; - const searchTermRows: Array = []; - const altTitleRows: Array = []; - - for (const s of songs) { - if (!s.id) { - throw new Error( - `Song ${gameGroup}:${s.legacySongID} is missing an id. Run 1-migrate-to-pg-style.ts first.`, - ); - } - - songRows.push({ - id: s.id, - legacy_id: - s.legacySongID ?? - (() => { - throw new Error(`Song ${gameGroup}:${s.id} is missing legacySongID.`); - })(), - game_group: gameGroup, - title: s.title, - artist: s.artist, - data: JSON.stringify(s.data), - }); - - for (const term of s.searchTerms) { - searchTermRows.push({ song_id: s.id, search_term: term }); - } - - for (const alt of s.altTitles) { - altTitleRows.push({ song_id: s.id, alt_title: alt }); - } - } - - const songIds = songs.map((s) => s.id); - - for (let i = 0; i < songRows.length; i = i + INSERT_CHUNK) { - const chunk = songRows.slice(i, i + INSERT_CHUNK); - - await pg - .insertInto("song") - .values(chunk) - .onConflict((oc) => - oc.column("id").doUpdateSet({ - title: sql`excluded.title`, - artist: sql`excluded.artist`, - data: sql`excluded.data`, - }), - ) - .execute(); - } - - // Delete + reinsert child rows so removals from seeds are reflected. - await chunkedDelete("song_search_term", "song_id", songIds); - await chunkedDelete("song_alt_title", "song_id", songIds); - - await batchIgnore("song_search_term", searchTermRows); - await batchIgnore("song_alt_title", altTitleRows); + await upsertSongsForGameGroup(pg, gameGroup, songs); total = total + songs.length; console.log(` ${gameGroup}: ${songs.length} songs`); @@ -242,63 +379,7 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi const pgGame = file.replace(/^charts-/u, "").replace(/\.json$/u, "") as PgGame; const charts = readCollection(file); - const chartRows: Array = []; - const versionRows: Array = []; - - for (const c of charts) { - if (!c.id) { - throw new Error( - `Chart ${c.legacyChartID ?? "(unknown)"}` + - ` (${pgGame}) is missing an id. Run 1-migrate-to-pg-style.ts first.`, - ); - } - - if (!c.legacyChartID) { - throw new Error( - `Chart ${c.id} (${pgGame}) is missing legacyChartID. Run 1-migrate-to-pg-style.ts first.`, - ); - } - - chartRows.push({ - id: c.id, - legacy_id: c.legacyChartID, - game: pgGame, - song_id: c.songID, - level: c.level, - level_num: c.levelNum, - is_primary: c.isPrimary, - difficulty: c.difficulty, - data: JSON.stringify(c.data), - }); - - for (const version of c.versions) { - versionRows.push({ chart_id: c.id, version: version as string }); - } - } - - const chartSids = charts.map((c) => c.id); - - for (let i = 0; i < chartRows.length; i = i + INSERT_CHUNK) { - const chunk = chartRows.slice(i, i + INSERT_CHUNK); - - await pg - .insertInto("chart") - .values(chunk) - .onConflict((oc) => - oc.column("id").doUpdateSet({ - level: sql`excluded.level`, - level_num: sql`excluded.level_num`, - is_primary: sql`excluded.is_primary`, - difficulty: sql`excluded.difficulty`, - data: sql`excluded.data`, - }), - ) - .execute(); - } - - await chunkedDelete("chart_version", "chart_id", chartSids); - - await batchIgnore("chart_version", versionRows); + await upsertChartsForPgGame(pg, pgGame, charts); total = total + charts.length; console.log(` ${pgGame}: ${charts.length} charts`); @@ -345,13 +426,13 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi const folderIds = folders.map((f) => f.id); - await chunkedDelete("folder_search_term", "id", folderIds); + await chunkedDeletePg(pg, "folder_search_term", "id", folderIds); const termRows: Array = folders.flatMap((f) => f.searchTerms.map((term) => ({ id: f.id, search_term: term })), ); - await batchIgnore("folder_search_term", termRows); + await batchIgnorePg(pg, "folder_search_term", termRows); console.log(` ${folders.length} folders, ${termRows.length} search terms\n`); } @@ -393,13 +474,13 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi const tableIds = tables.map((t) => t.id); - await chunkedDelete("table_folder", "table_id", tableIds); + await chunkedDeletePg(pg, "table_folder", "table_id", tableIds); const tfRows: Array = tables.flatMap((t) => t.folders.map((folderId) => ({ table_id: t.id, folder_id: folderId })), ); - await batchIgnore("table_folder", tfRows); + await batchIgnorePg(pg, "table_folder", tfRows); console.log(` ${tables.length} tables, ${tfRows.length} table-folder rows\n`); } @@ -416,7 +497,7 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi value: c.value as string, })); - await batchIgnore("bms_course_lookup", courseRows); + await batchIgnorePg(pg, "bms_course_lookup", courseRows); console.log(` ${courses.length} BMS courses\n`); } @@ -434,7 +515,7 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi })); // Goals are never updated once created — only new ones are inserted. - await batchIgnore("goal", goalRows); + await batchIgnorePg(pg, "goal", goalRows); console.log(` ${goals.length} goals\n`); } @@ -499,7 +580,7 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi const qlIds = questlines.map((ql) => ql.questlineID); - await chunkedDelete("questline_quest", "questline_id", qlIds); + await chunkedDeletePg(pg, "questline_quest", "questline_id", qlIds); let order = 0; const qlqRows: Array = questlines.flatMap((ql) => { @@ -512,7 +593,7 @@ export async function importSeeds(pg: Kysely, seedsDir: string): Promi })); }); - await batchIgnore("questline_quest", qlqRows); + await batchIgnorePg(pg, "questline_quest", qlqRows); console.log(` ${questlines.length} questlines, ${qlqRows.length} questline-quest rows\n`); } } diff --git a/typescript/server/src/test-utils/seed-paths.ts b/typescript/server/src/test-utils/seed-paths.ts new file mode 100644 index 000000000..482408f83 --- /dev/null +++ b/typescript/server/src/test-utils/seed-paths.ts @@ -0,0 +1,19 @@ +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; +import path from "path"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +/** Default `db/seeds` (collections JSON) relative to the server package — same as `load-seeds-pg.ts`. */ +export const DEFAULT_SEEDS_DIR = path.resolve(__dirname, "../../../../db/seeds"); + +export function resolveSeedsDir(): string { + return process.env.SEEDS_DIR ?? DEFAULT_SEEDS_DIR; +} + +/** True when the repo’s seed JSON is present (e.g. `songs-iidx.json`). */ +export function seedsJsonAvailable(): boolean { + const dir = resolveSeedsDir(); + + return fs.existsSync(path.join(dir, "songs-iidx.json")); +} diff --git a/typescript/server/src/utils/misc.bench.ts b/typescript/server/src/utils/misc.bench.ts new file mode 100644 index 000000000..a05d427e1 --- /dev/null +++ b/typescript/server/src/utils/misc.bench.ts @@ -0,0 +1,13 @@ +import { bench, describe } from "vitest"; + +import { EscapeForILIKE } from "#utils/misc"; + +/** Representative user search string (wildcards + backslashes) for ILIKE escaping. */ +const sampleQuery = + "artist_%track% " + "word ".repeat(32) + String.raw` \% literal \_ `; + +describe("EscapeForILIKE (example)", () => { + bench("typical search string", () => { + EscapeForILIKE(sampleQuery); + }); +}); diff --git a/typescript/server/src/utils/queries/charts.ts b/typescript/server/src/utils/queries/charts.ts index 42a823bd0..b59e3e824 100644 --- a/typescript/server/src/utils/queries/charts.ts +++ b/typescript/server/src/utils/queries/charts.ts @@ -1,16 +1,21 @@ -import type { FilterQuery } from "mongodb"; -import type { - ChartDocument, - Difficulties, - GameGroup, - GPTString, - integer, - Playtype, - Playtypes, - Versions, -} from "tachi-common"; +import type { Game } from "tachi-db"; import MONGODB_KILL from "#services/mongo/db"; +import DB from "#services/pg/db"; +import { sql } from "kysely"; +import { + type ChartDocument, + type ChartDocumentData, + type Difficulties, + type GameGroup, + GamePTToV3, + type GPTString, + type integer, + type Playtype, + type Playtypes, + V3ToGamePT, + type Versions, +} from "tachi-common"; export function FindChartWithChartID(game: GameGroup, chartID: string) { return MONGODB_KILL.anyCharts[game].findOne({ chartID }); @@ -230,8 +235,9 @@ export function FindChartOnSHA256Playtype(game: GameGroup, hash: string, playtyp /** * Returns the N most popular charts for this game + playtype. - * Popularity is determined by how many scores match in the score - * collection. + * Popularity is determined by how many rows exist in Postgres `score` for each chart. + * + * @param _scoreCollection — ignored; kept for API compatibility with the old Mongo implementation. */ export async function FindChartsOnPopularity( game: GameGroup, @@ -239,64 +245,77 @@ export async function FindChartsOnPopularity( songIDs?: Array, skip = 0, limit = 100, - scoreCollection: "personal-bests" | "scores" = "personal-bests", + _scoreCollection: "personal-bests" | "scores" = "personal-bests", ): Promise> { - const matchQuery: FilterQuery = { - playtype, - }; + const v3Game = GamePTToV3(game, playtype); - if (songIDs) { - matchQuery.songID = { $in: songIDs }; + let q = DB.selectFrom("chart") + .innerJoin("song", "song.id", "chart.song_id") + .leftJoin("score", "score.chart_id", "chart.id") + .where("chart.game", "=", v3Game as Game); + + if (songIDs && songIDs.length > 0) { + q = q.where("song.legacy_id", "in", songIDs); } - // MongoDB is a hard beast to wield. - // This code might look very inefficient, but originally this *was* - // a single aggregate pipeline. - // - // We've split it up into multiple queries as this is an order of - // magnitude faster. - // Not entirely sure why, but $lookup is incredibly inefficient, - // and you should just avoid it. - const charts = (await MONGODB_KILL.anyCharts[game].find(matchQuery)) as unknown as Array< - { - __playcount: integer; - } & ChartDocument - >; + const rows = await q + .select([ + "chart.id", + "chart.legacy_id", + "chart.game", + "chart.song_id", + "chart.level", + "chart.level_num", + "chart.is_primary", + "chart.difficulty", + "chart.data", + "song.legacy_id as song_legacy_id", + sql`count(score.id)::int`.as("playcount"), + ]) + .groupBy(["chart.id", "song.legacy_id"]) + .orderBy(sql`count(score.id)`, "desc") + .offset(skip) + .limit(limit) + .execute(); - const scoreCounts: Array<{ _id: string; count: integer }> = await MONGODB_KILL[ - scoreCollection - ].aggregate([ - { - $match: { chartID: { $in: charts.map((e) => e.chartID) } }, - }, - { - $group: { - _id: "$chartID", - count: { $sum: 1 }, - }, - }, - { - $sort: { - count: -1, - }, - }, - { - $skip: skip, - }, - { - $limit: limit, - }, - ]); - - const scoreCountMap = new Map(); - - for (const sc of scoreCounts) { - scoreCountMap.set(sc._id, sc.count); + if (rows.length === 0) { + return []; } - for (const chart of charts) { - chart.__playcount = scoreCountMap.get(chart.chartID) ?? 0; + const chartPgIds = rows.map((r) => r.id); + + const versionRows = await DB.selectFrom("chart_version") + .select(["chart_id", "version"]) + .where("chart_id", "in", chartPgIds) + .execute(); + + const versionsByChartId = new Map(); + + for (const v of versionRows) { + let list = versionsByChartId.get(v.chart_id); + + if (!list) { + list = []; + versionsByChartId.set(v.chart_id, list); + } + + list.push(v.version); } - return charts.sort((a, b) => b.__playcount - a.__playcount).slice(skip, skip + limit); + return rows.map((row) => { + const { playtype: chartPlaytype } = V3ToGamePT(row.game); + + return { + chartID: row.legacy_id, + songID: row.song_legacy_id, + level: row.level, + levelNum: row.level_num, + isPrimary: row.is_primary, + difficulty: row.difficulty as Difficulties[GPTString], + playtype: chartPlaytype, + data: row.data as ChartDocumentData[GPTString], + versions: (versionsByChartId.get(row.id) ?? []) as Versions[GPTString][], + __playcount: row.playcount, + } as { __playcount: integer } & ChartDocument; + }); } diff --git a/typescript/server/src/utils/user.ts b/typescript/server/src/utils/user.ts index ce54d5d61..a2eee1c85 100644 --- a/typescript/server/src/utils/user.ts +++ b/typescript/server/src/utils/user.ts @@ -19,6 +19,7 @@ import { type UserDocument, type UserGameStats, type UserSettingsDocument, + V3ToGamePT, } from "tachi-common"; import { type Database } from "tachi-db"; @@ -299,12 +300,12 @@ export async function IsUserBanned(userID: integer) { * Return all the GPTs this userID has played. */ export async function GetUserPlayedGPTs(userID: integer) { - const gpts = (await MONGODB_KILL["game-stats"].find( - { userID }, - { projection: { game: 1, playtype: 1 } }, - )) as Array>; + const rows = await DB.selectFrom("game_profile") + .select("game") + .where("user_id", "=", userID) + .execute(); - return gpts; + return rows.map((r) => V3ToGamePT(r.game)); } export async function GetAllUserRivals(userID: integer) { diff --git a/typescript/server/vitest.config.ts b/typescript/server/vitest.config.ts index ea3283ddb..bdade553b 100644 --- a/typescript/server/vitest.config.ts +++ b/typescript/server/vitest.config.ts @@ -20,6 +20,9 @@ export default defineConfig({ }, test: { + // `vitest bench` uses this same config: globalSetup + setupFiles + per-worker POSTGRES_URL. + // Use for API / DB performance work as well as microbenches (*.bench.ts). + // // Static env vars. POSTGRES_URL is set dynamically per-worker in vitest.setup.ts // so each worker gets its own isolated database. env: { @@ -45,7 +48,7 @@ export default defineConfig({ coverage: { provider: "v8", include: ["src/**/*.ts"], - exclude: ["src/**/*.test.ts", "src/**/*.oldtest.ts", "src/test-utils/**"], + exclude: ["src/**/*.test.ts", "src/**/*.bench.ts", "src/**/*.oldtest.ts", "src/test-utils/**"], }, }, }); diff --git a/typescript/server/vitest.setup.ts b/typescript/server/vitest.setup.ts index 06a21a2a5..c19b7630f 100644 --- a/typescript/server/vitest.setup.ts +++ b/typescript/server/vitest.setup.ts @@ -93,7 +93,21 @@ beforeAll(async () => { await createWorkerDatabase(); }); -beforeEach(async () => { +beforeEach(async (ctx) => { + // Benchmark tasks load real seed data in beforeAll; truncating here would wipe it + // before every bench() and between iterations. + const task = ctx.task as { file?: { filepath?: string }; meta?: { benchmark?: boolean } }; + + if (task.meta?.benchmark === true) { + return; + } + + const fp = task.file?.filepath; + + if (typeof fp === "string" && fp.endsWith(".bench.ts")) { + return; + } + await resetDatabase(); // Login-heavy router tests share the in-memory login rate limiter; reset each // test so AggressiveRateLimit (15 / 10 min) does not 429 and omit Set-Cookie. diff --git a/typescript/sieglinde/package.json b/typescript/sieglinde/package.json index 12ca38558..5c5a320f3 100644 --- a/typescript/sieglinde/package.json +++ b/typescript/sieglinde/package.json @@ -10,7 +10,8 @@ "lint": "eslint .", "lint-fix": "eslint . --fix", "typecheck": "tsgo --noEmit", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "bench": "node -e \"process.exit(0)\"" }, "author": "zk", "license": "MIT",