From 9bf3ad972de36def2dd1b22dc7540458ad4b79a8 Mon Sep 17 00:00:00 2001 From: zk Date: Sun, 24 May 2026 01:04:25 +0100 Subject: [PATCH] feat: highly optimise playcount-based queries (#1572) --- .../20260524120000_chart_playcount.sql | 139 ++++++++ docs/src/api/routes/gpt.md | 20 -- old-docs/docs/api/routes/gpt.md | 20 -- .../games/_game/_playtype/GPTChartPage.tsx | 12 +- typescript/db/src/generated/index.ts | 5 + .../db/src/generated/public/ChartPlaycount.ts | 21 ++ .../db/src/generated/public/PublicSchema.ts | 3 + .../public/apply_chart_playcount_delta.ts | 8 + ...fresh_playcount_stmt_after_score_delete.ts | 5 + ...fresh_playcount_stmt_after_score_insert.ts | 5 + ...fresh_playcount_stmt_after_score_update.ts | 5 + typescript/server/src/lib/db-formats/pb.ts | 12 - .../src/lib/jobs/chart-playcount.test.ts | 314 ++++++++++++++++++ .../server/src/lib/jobs/cron/cron-registry.ts | 7 + .../src/lib/jobs/reconcile-chart-playcount.ts | 54 +++ .../server/src/lib/search/song-charts.ts | 9 +- .../src/server/router/api/v1/games/router.ts | 18 - .../server/src/server/router/api/v1/spec.ts | 6 - typescript/server/src/utils/queries/charts.ts | 36 +- 19 files changed, 575 insertions(+), 124 deletions(-) create mode 100644 db/migrations/20260524120000_chart_playcount.sql create mode 100644 typescript/db/src/generated/public/ChartPlaycount.ts create mode 100644 typescript/db/src/generated/public/apply_chart_playcount_delta.ts create mode 100644 typescript/db/src/generated/public/refresh_playcount_stmt_after_score_delete.ts create mode 100644 typescript/db/src/generated/public/refresh_playcount_stmt_after_score_insert.ts create mode 100644 typescript/db/src/generated/public/refresh_playcount_stmt_after_score_update.ts create mode 100644 typescript/server/src/lib/jobs/chart-playcount.test.ts create mode 100644 typescript/server/src/lib/jobs/reconcile-chart-playcount.ts diff --git a/db/migrations/20260524120000_chart_playcount.sql b/db/migrations/20260524120000_chart_playcount.sql new file mode 100644 index 000000000..c8916a688 --- /dev/null +++ b/db/migrations/20260524120000_chart_playcount.sql @@ -0,0 +1,139 @@ +-- Cached total score count per chart. Maintained by AFTER STATEMENT triggers on score +-- (batch increment/decrement per statement, same pattern as chart_leaderboard on pb). +-- After migrate: regenerate DB types from your checkout (see db/kanel_config.js). + +CREATE TABLE chart_playcount ( + chart_id TEXT PRIMARY KEY REFERENCES chart (id) ON DELETE CASCADE, + playcount INT NOT NULL DEFAULT 0 CHECK (playcount >= 0) +); + +CREATE INDEX chart_playcount_playcount_idx ON chart_playcount USING btree (playcount DESC); + +COMMENT ON TABLE chart_playcount IS + 'Cached total score rows per chart. Maintained by AFTER STATEMENT triggers on score.'; + +INSERT INTO chart_playcount (chart_id, playcount) +SELECT + score.chart_id, + COUNT(*)::int AS playcount +FROM score +GROUP BY score.chart_id; + +CREATE FUNCTION apply_chart_playcount_delta ( + p_chart_id TEXT, + p_delta INT +) +RETURNS VOID +LANGUAGE plpgsql +SET search_path = public +AS $$ +BEGIN + IF p_delta = 0 THEN + RETURN; + END IF; + + IF p_delta > 0 THEN + INSERT INTO chart_playcount AS cp (chart_id, playcount) + VALUES (p_chart_id, p_delta) + ON CONFLICT (chart_id) DO UPDATE + SET playcount = cp.playcount + EXCLUDED.playcount; + ELSE + UPDATE chart_playcount + SET playcount = playcount + p_delta + WHERE chart_id = p_chart_id; + + DELETE FROM chart_playcount + WHERE chart_id = p_chart_id AND playcount <= 0; + END IF; +END; +$$; + +CREATE FUNCTION refresh_playcount_stmt_after_score_insert() +RETURNS TRIGGER +LANGUAGE plpgsql +SET search_path = public +AS $$ +BEGIN + PERFORM apply_chart_playcount_delta(d.chart_id, d.cnt) + FROM ( + SELECT + score_inserted.chart_id, + COUNT(*)::int AS cnt + FROM score_inserted + GROUP BY score_inserted.chart_id + ) AS d; + + RETURN NULL; +END; +$$; + +CREATE FUNCTION refresh_playcount_stmt_after_score_delete() +RETURNS TRIGGER +LANGUAGE plpgsql +SET search_path = public +AS $$ +BEGIN + PERFORM apply_chart_playcount_delta(d.chart_id, -d.cnt) + FROM ( + SELECT + score_deleted.chart_id, + COUNT(*)::int AS cnt + FROM score_deleted + GROUP BY score_deleted.chart_id + ) AS d; + + RETURN NULL; +END; +$$; + +CREATE FUNCTION refresh_playcount_stmt_after_score_update() +RETURNS TRIGGER +LANGUAGE plpgsql +SET search_path = public +AS $$ +BEGIN + PERFORM apply_chart_playcount_delta(d.chart_id, -d.cnt) + FROM ( + SELECT + score_updated_old.chart_id, + COUNT(*)::int AS cnt + FROM score_updated_old + INNER JOIN score_updated_new ON score_updated_old.id = score_updated_new.id + WHERE score_updated_old.chart_id IS DISTINCT FROM score_updated_new.chart_id + GROUP BY score_updated_old.chart_id + ) AS d; + + PERFORM apply_chart_playcount_delta(d.chart_id, d.cnt) + FROM ( + SELECT + score_updated_new.chart_id, + COUNT(*)::int AS cnt + FROM score_updated_old + INNER JOIN score_updated_new ON score_updated_old.id = score_updated_new.id + WHERE score_updated_old.chart_id IS DISTINCT FROM score_updated_new.chart_id + GROUP BY score_updated_new.chart_id + ) AS d; + + RETURN NULL; +END; +$$; + +CREATE TRIGGER score_playcount_ai + AFTER INSERT ON score + REFERENCING NEW TABLE AS score_inserted + FOR EACH STATEMENT + EXECUTE FUNCTION refresh_playcount_stmt_after_score_insert(); + +CREATE TRIGGER score_playcount_ad + AFTER DELETE ON score + REFERENCING OLD TABLE AS score_deleted + FOR EACH STATEMENT + EXECUTE FUNCTION refresh_playcount_stmt_after_score_delete(); + +CREATE TRIGGER score_playcount_au + AFTER UPDATE ON score + REFERENCING OLD TABLE AS score_updated_old NEW TABLE AS score_updated_new + FOR EACH STATEMENT + EXECUTE FUNCTION refresh_playcount_stmt_after_score_update(); + +ANALYZE chart_playcount; diff --git a/docs/src/api/routes/gpt.md b/docs/src/api/routes/gpt.md index 6f49888c4..df5e50f46 100644 --- a/docs/src/api/routes/gpt.md +++ b/docs/src/api/routes/gpt.md @@ -268,26 +268,6 @@ GET /api/v1/games/iidx/SP/charts/some_chart_id --- -## Retrieve playcount for this chart. - -`GET /api/v1/games/:gameGroup/:playtype/charts/:chartID/playcount` - -### Parameters - -None. - -### Response - -| Property | Type | Description | -| :------: | :-----: | :--------------------------------: | -| `count` | Integer | The amount of plays on this chart. | - -### Example - -Self-explanatory. - ---- - ## Retrieve leaderboards for this chart. `GET /api/v1/games/:gameGroup/:playtype/charts/:chartID/pbs` diff --git a/old-docs/docs/api/routes/gpt.md b/old-docs/docs/api/routes/gpt.md index 6f49888c4..df5e50f46 100644 --- a/old-docs/docs/api/routes/gpt.md +++ b/old-docs/docs/api/routes/gpt.md @@ -268,26 +268,6 @@ GET /api/v1/games/iidx/SP/charts/some_chart_id --- -## Retrieve playcount for this chart. - -`GET /api/v1/games/:gameGroup/:playtype/charts/:chartID/playcount` - -### Parameters - -None. - -### Response - -| Property | Type | Description | -| :------: | :-----: | :--------------------------------: | -| `count` | Integer | The amount of plays on this chart. | - -### Example - -Self-explanatory. - ---- - ## Retrieve leaderboards for this chart. `GET /api/v1/games/:gameGroup/:playtype/charts/:chartID/pbs` diff --git a/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTChartPage.tsx b/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTChartPage.tsx index 3e80e98c9..976851bd8 100644 --- a/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTChartPage.tsx +++ b/typescript/client/src/app/pages/dashboard/games/_game/_playtype/GPTChartPage.tsx @@ -79,7 +79,6 @@ interface ChartPBData { leaderboard: ChartPBLeaderboardReturn; adjacent?: UGPTChartLeaderboardAdjacent; rivals?: ChartRivalsReturn; - playcount: integer; } function InternalGPTChartPage({ @@ -103,14 +102,6 @@ function InternalGPTChartPage({ throw lRes; } - const pRes = await APIFetchV1<{ count: integer }>( - `/games/${game}/charts/${chart.chartID}/playcount`, - ); - - if (!pRes.success) { - throw pRes; - } - if (user) { const nRes = await APIFetchV1( `/users/${user.id}/games/${game}/pbs/${chart.chartID}/leaderboard-adjacent`, @@ -122,7 +113,6 @@ function InternalGPTChartPage({ const returnValue: ChartPBData = { leaderboard: lRes.body, - playcount: pRes.body.count, }; if (nRes.success) { @@ -136,7 +126,7 @@ function InternalGPTChartPage({ return returnValue; } - return { leaderboard: lRes.body, playcount: pRes.body.count }; + return { leaderboard: lRes.body }; }, ); diff --git a/typescript/db/src/generated/index.ts b/typescript/db/src/generated/index.ts index e3a505d6a..e6d6296f9 100644 --- a/typescript/db/src/generated/index.ts +++ b/typescript/db/src/generated/index.ts @@ -10,6 +10,7 @@ export { type default as SvcFerSettingsTable, type SvcFerSettings, type NewSvcFe export { type default as InviteLockTable, type InviteLock, type NewInviteLock, type InviteLockUpdate } from './public/InviteLock'; export { type orphan_chart_id, type default as OrphanChartTable, type OrphanChart, type NewOrphanChart, type OrphanChartUpdate } from './public/OrphanChart'; export { type game_rival_game, type default as GameRivalTable, type GameRival, type NewGameRival, type GameRivalUpdate } from './public/GameRival'; +export { type default as ChartPlaycountTable, type ChartPlaycount, type NewChartPlaycount, type ChartPlaycountUpdate } from './public/ChartPlaycount'; export { type goal_id, type default as GoalTable, type Goal, type NewGoal, type GoalUpdate } from './public/Goal'; export { type quest_proposal_row_id, type default as QuestProposalTable, type QuestProposal, type NewQuestProposal, type QuestProposalUpdate } from './public/QuestProposal'; export { type import_quest_row_id, type default as ImportQuestTable, type ImportQuest, type NewImportQuest, type ImportQuestUpdate } from './public/ImportQuest'; @@ -76,12 +77,16 @@ export { type default as ActionResult } from './public/ActionResult'; export { type default as GameGroup } from './public/GameGroup'; export { type default as ImportType } from './public/ImportType'; export { type refresh_leaderboard_stmt_after_pb_insert_params } from './public/refresh_leaderboard_stmt_after_pb_insert'; +export { type refresh_playcount_stmt_after_score_delete_params } from './public/refresh_playcount_stmt_after_score_delete'; export { type enqueue_pb_dirty_params } from './public/enqueue_pb_dirty'; +export { type refresh_playcount_stmt_after_score_update_params } from './public/refresh_playcount_stmt_after_score_update'; export { type enqueue_game_profile_dirty_params } from './public/enqueue_game_profile_dirty'; export { type enqueue_score_rederive_params } from './public/enqueue_score_rederive'; export { type refresh_leaderboard_stmt_after_pb_delete_params } from './public/refresh_leaderboard_stmt_after_pb_delete'; +export { type apply_chart_playcount_delta_params } from './public/apply_chart_playcount_delta'; export { type enqueue_session_dirty_params } from './public/enqueue_session_dirty'; export { type refresh_leaderboard_stmt_after_pb_update_params } from './public/refresh_leaderboard_stmt_after_pb_update'; +export { type refresh_playcount_stmt_after_score_insert_params } from './public/refresh_playcount_stmt_after_score_insert'; export { type refresh_chart_leaderboard_partition_params } from './public/refresh_chart_leaderboard_partition'; export { type default as PublicSchema } from './public/PublicSchema'; export { type default as Database } from './Database'; diff --git a/typescript/db/src/generated/public/ChartPlaycount.ts b/typescript/db/src/generated/public/ChartPlaycount.ts new file mode 100644 index 000000000..e8bf4437c --- /dev/null +++ b/typescript/db/src/generated/public/ChartPlaycount.ts @@ -0,0 +1,21 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +import type { chart_id } from './Chart'; +import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely'; + +/** + * Represents the table public.chart_playcount + * Cached total score rows per chart. Maintained by AFTER STATEMENT triggers on score. + */ +export default interface ChartPlaycountTable { + chart_id: ColumnType; + + playcount: ColumnType; +} + +export type ChartPlaycount = Selectable; + +export type NewChartPlaycount = Insertable; + +export type ChartPlaycountUpdate = Updateable; diff --git a/typescript/db/src/generated/public/PublicSchema.ts b/typescript/db/src/generated/public/PublicSchema.ts index 8d8b02cb9..6e40bfd69 100644 --- a/typescript/db/src/generated/public/PublicSchema.ts +++ b/typescript/db/src/generated/public/PublicSchema.ts @@ -10,6 +10,7 @@ import type { default as SvcFerSettingsTable } from './SvcFerSettings'; import type { default as InviteLockTable } from './InviteLock'; import type { default as OrphanChartTable } from './OrphanChart'; import type { default as GameRivalTable } from './GameRival'; +import type { default as ChartPlaycountTable } from './ChartPlaycount'; import type { default as GoalTable } from './Goal'; import type { default as QuestProposalTable } from './QuestProposal'; import type { default as ImportQuestTable } from './ImportQuest'; @@ -88,6 +89,8 @@ export default interface PublicSchema { game_rival: GameRivalTable; + chart_playcount: ChartPlaycountTable; + goal: GoalTable; quest_proposal: QuestProposalTable; diff --git a/typescript/db/src/generated/public/apply_chart_playcount_delta.ts b/typescript/db/src/generated/public/apply_chart_playcount_delta.ts new file mode 100644 index 000000000..fb0ef32de --- /dev/null +++ b/typescript/db/src/generated/public/apply_chart_playcount_delta.ts @@ -0,0 +1,8 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +export interface apply_chart_playcount_delta_params { + p_chart_id: string; + + p_delta: number; +} diff --git a/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_delete.ts b/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_delete.ts new file mode 100644 index 000000000..8353662a4 --- /dev/null +++ b/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_delete.ts @@ -0,0 +1,5 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +export interface refresh_playcount_stmt_after_score_delete_params { +} diff --git a/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_insert.ts b/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_insert.ts new file mode 100644 index 000000000..97b881a29 --- /dev/null +++ b/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_insert.ts @@ -0,0 +1,5 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +export interface refresh_playcount_stmt_after_score_insert_params { +} diff --git a/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_update.ts b/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_update.ts new file mode 100644 index 000000000..0d7c4d6b4 --- /dev/null +++ b/typescript/db/src/generated/public/refresh_playcount_stmt_after_score_update.ts @@ -0,0 +1,5 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +export interface refresh_playcount_stmt_after_score_update_params { +} diff --git a/typescript/server/src/lib/db-formats/pb.ts b/typescript/server/src/lib/db-formats/pb.ts index 7ebb010db..27017ef5f 100644 --- a/typescript/server/src/lib/db-formats/pb.ts +++ b/typescript/server/src/lib/db-formats/pb.ts @@ -424,18 +424,6 @@ export async function LoadPbsOnChartByRankingValueDesc( return Promise.all(rows.map((r) => ToPbScoreDocument(r as PbDocumentJoinRow))); } - -/** Number of PB rows on a chart (legacy Mongo `personal-bests` count by `chartID`). */ -export async function CountPbsOnChart(chartLegacyId: string): Promise { - const row = await DB.selectFrom("pb") - .innerJoin("chart", "chart.id", "pb.chart_id") - .where("chart.id", "=", chartLegacyId) - .select((eb) => eb.fn.countAll().as("count")) - .executeTakeFirst(); - - return Number(row?.count ?? 0); -} - /** * PBs on a chart with leaderboard rank >= `startRanking`, sorted by rank ascending * (legacy GET …/charts/:chartID/pbs). diff --git a/typescript/server/src/lib/jobs/chart-playcount.test.ts b/typescript/server/src/lib/jobs/chart-playcount.test.ts new file mode 100644 index 000000000..30bda24a1 --- /dev/null +++ b/typescript/server/src/lib/jobs/chart-playcount.test.ts @@ -0,0 +1,314 @@ +import { ReconcileChartPlaycountJob } from "#lib/jobs/reconcile-chart-playcount"; +import DB from "#services/pg/db"; +import { seedUser } from "#test-utils/pg-fixtures"; +import { FindChartsOnPopularity } from "#utils/queries/charts"; +import { describe, expect, it } from "vitest"; + +function makeSongId(n: number): string { + return `S${n.toString(16).padStart(20, "0")}`; +} + +function makeChartId(n: number): string { + return `C${n.toString(16).padStart(20, "0")}`; +} + +function makeLegacyId(n: number): string { + return n.toString(16).padStart(40, "0"); +} + +function makeScoreId(n: number): string { + return `SC${n.toString(16).padStart(20, "0")}`; +} + +async function getCachedPlaycount(chartId: string): Promise { + const row = await DB.selectFrom("chart_playcount") + .select("chart_playcount.playcount") + .where("chart_playcount.chart_id", "=", chartId) + .executeTakeFirst(); + + return row?.playcount ?? 0; +} + +function scoreRow(id: string, userId: number, chartId: string, game: "iidx-sp" = "iidx-sp") { + return { + id, + user_id: userId, + chart_id: chartId, + game, + session_id: null, + import_id: null, + data: JSON.stringify({}), + derived_data: JSON.stringify({}), + judgements: JSON.stringify({}), + calculated_data: JSON.stringify({}), + meta: JSON.stringify({}), + time_achieved: new Date().toISOString(), + time_added: new Date().toISOString(), + highlight: false, + comment: null, + }; +} + +describe("chart_playcount triggers", () => { + it("increments on score insert and decrements on delete", async () => { + const songID = makeSongId(10_001); + const chartID = makeChartId(10_001); + + await DB.insertInto("song") + .values({ + id: songID, + legacy_id: 9_010_001, + game_group: "iidx", + title: "Playcount Trigger Song", + artist: "X", + search_terms: [], + alt_titles: [], + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + await DB.insertInto("chart") + .values({ + id: chartID, + legacy_id: makeLegacyId(10_001), + game: "iidx-sp", + song_id: songID, + level: "10", + level_num: 10, + is_primary: true, + difficulty: "ANOTHER", + versions: [], + data: JSON.stringify({ notecount: 100 }), + }) + .execute(); + + expect(await getCachedPlaycount(chartID)).toBe(0); + + const { id: userID } = await seedUser({ username: "playcount_trigger_user" }); + const scoreID = makeScoreId(10_001); + + await DB.insertInto("score") + .values(scoreRow(scoreID, userID, chartID)) + .execute(); + + expect(await getCachedPlaycount(chartID)).toBe(1); + + await DB.deleteFrom("score").where("score.id", "=", scoreID).execute(); + + expect(await getCachedPlaycount(chartID)).toBe(0); + }); + + it("batch-inserts increment once per chart", async () => { + const songID = makeSongId(10_002); + const chartID = makeChartId(10_002); + + await DB.insertInto("song") + .values({ + id: songID, + legacy_id: 9_010_002, + game_group: "iidx", + title: "Playcount Batch Song", + artist: "X", + search_terms: [], + alt_titles: [], + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + await DB.insertInto("chart") + .values({ + id: chartID, + legacy_id: makeLegacyId(10_002), + game: "iidx-sp", + song_id: songID, + level: "10", + level_num: 10, + is_primary: true, + difficulty: "ANOTHER", + versions: [], + data: JSON.stringify({ notecount: 100 }), + }) + .execute(); + + const { id: userID } = await seedUser({ username: "playcount_batch_user" }); + + await DB.insertInto("score") + .values([ + scoreRow(makeScoreId(10_002), userID, chartID), + scoreRow(makeScoreId(10_003), userID, chartID), + scoreRow(makeScoreId(10_004), userID, chartID), + ]) + .execute(); + + expect(await getCachedPlaycount(chartID)).toBe(3); + }); + + it("moves count when score chart_id changes on update", async () => { + const songID = makeSongId(10_003); + const chartA = makeChartId(10_003); + const chartB = makeChartId(10_004); + + await DB.insertInto("song") + .values({ + id: songID, + legacy_id: 9_010_003, + game_group: "iidx", + title: "Playcount Move Song", + artist: "X", + search_terms: [], + alt_titles: [], + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + await DB.insertInto("chart") + .values([ + { + id: chartA, + legacy_id: makeLegacyId(10_003), + game: "iidx-sp", + song_id: songID, + level: "10", + level_num: 10, + is_primary: true, + difficulty: "ANOTHER", + versions: [], + data: JSON.stringify({ notecount: 100 }), + }, + { + id: chartB, + legacy_id: makeLegacyId(10_004), + game: "iidx-sp", + song_id: songID, + level: "11", + level_num: 11, + is_primary: false, + difficulty: "HYPER", + versions: [], + data: JSON.stringify({ notecount: 80 }), + }, + ]) + .execute(); + + const { id: userID } = await seedUser({ username: "playcount_move_user" }); + const scoreID = makeScoreId(10_005); + + await DB.insertInto("score") + .values(scoreRow(scoreID, userID, chartA)) + .execute(); + + expect(await getCachedPlaycount(chartA)).toBe(1); + expect(await getCachedPlaycount(chartB)).toBe(0); + + await DB.updateTable("score") + .set({ chart_id: chartB }) + .where("score.id", "=", scoreID) + .execute(); + + expect(await getCachedPlaycount(chartA)).toBe(0); + expect(await getCachedPlaycount(chartB)).toBe(1); + }); + + it("FindChartsOnPopularity reads cached playcounts", async () => { + const songID = makeSongId(10_005); + const chartID = makeChartId(10_005); + + await DB.insertInto("song") + .values({ + id: songID, + legacy_id: 9_010_005, + game_group: "iidx", + title: "Popularity Cache Song", + artist: "X", + search_terms: [], + alt_titles: [], + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + await DB.insertInto("chart") + .values({ + id: chartID, + legacy_id: makeLegacyId(10_005), + game: "iidx-sp", + song_id: songID, + level: "10", + level_num: 10, + is_primary: true, + difficulty: "ANOTHER", + versions: [], + data: JSON.stringify({ notecount: 100 }), + }) + .execute(); + + const { id: userID } = await seedUser({ username: "popularity_cache_user" }); + + await DB.insertInto("score") + .values(scoreRow(makeScoreId(10_006), userID, chartID)) + .execute(); + + const charts = await FindChartsOnPopularity( + "iidx-sp", + { songIDs: [songID], chartIDs: undefined }, + 0, + 100, + ); + + expect(charts).toHaveLength(1); + expect(charts[0]?.chartID).toBe(chartID); + expect(charts[0]?.__playcount).toBe(1); + }); + + it("ReconcileChartPlaycountJob fixes drift", async () => { + const songID = makeSongId(10_006); + const chartID = makeChartId(10_006); + + await DB.insertInto("song") + .values({ + id: songID, + legacy_id: 9_010_006, + game_group: "iidx", + title: "Reconcile Song", + artist: "X", + search_terms: [], + alt_titles: [], + fts_document: "", + data: JSON.stringify({}), + }) + .execute(); + + await DB.insertInto("chart") + .values({ + id: chartID, + legacy_id: makeLegacyId(10_006), + game: "iidx-sp", + song_id: songID, + level: "10", + level_num: 10, + is_primary: true, + difficulty: "ANOTHER", + versions: [], + data: JSON.stringify({ notecount: 100 }), + }) + .execute(); + + const { id: userID } = await seedUser({ username: "reconcile_user" }); + + await DB.insertInto("score") + .values(scoreRow(makeScoreId(10_007), userID, chartID)) + .execute(); + + await DB.updateTable("chart_playcount") + .set({ playcount: 99 }) + .where("chart_playcount.chart_id", "=", chartID) + .execute(); + + await ReconcileChartPlaycountJob(); + + expect(await getCachedPlaycount(chartID)).toBe(1); + }); +}); diff --git a/typescript/server/src/lib/jobs/cron/cron-registry.ts b/typescript/server/src/lib/jobs/cron/cron-registry.ts index 2b925f949..f084fdd04 100644 --- a/typescript/server/src/lib/jobs/cron/cron-registry.ts +++ b/typescript/server/src/lib/jobs/cron/cron-registry.ts @@ -8,6 +8,7 @@ import { DefaultAdminUser } from "#lib/jobs/default-admin-user"; import { DeorphanScoresMain } from "#lib/jobs/deorphan-scores"; import { drainStatsQueuesInOrder } from "#lib/jobs/drain-dirty-queues"; import { RebuildFolderChartLookupJob } from "#lib/jobs/rebuild-folder-chart-lookup"; +import { ReconcileChartPlaycountJob } from "#lib/jobs/reconcile-chart-playcount"; import { Env, TachiConfig } from "#lib/setup/config"; import { DedupeArr } from "#utils/misc"; @@ -47,6 +48,12 @@ function buildList(): Array { description: "De-Orphan Scores", run: DeorphanScoresMain, }, + { + id: "reconcile_chart_playcount", + schedule: "0 4 * * *", + description: "Reconcile cached chart playcounts", + run: ReconcileChartPlaycountJob, + }, { id: "drain_stats_queues", schedule: "* * * * *", // TODO(zk): really? this is so lazy diff --git a/typescript/server/src/lib/jobs/reconcile-chart-playcount.ts b/typescript/server/src/lib/jobs/reconcile-chart-playcount.ts new file mode 100644 index 000000000..395df697a --- /dev/null +++ b/typescript/server/src/lib/jobs/reconcile-chart-playcount.ts @@ -0,0 +1,54 @@ +import { log } from "#lib/log/log"; +import DB from "#services/pg/db"; +import { sql } from "kysely"; + +/** Recompute chart_playcount rows that drifted from live score counts. */ +export async function ReconcileChartPlaycountJob() { + log.info("Starting ReconcileChartPlaycount job."); + + const driftRows = await sql<{ chart_id: string; live_count: number }>` + SELECT + score.chart_id, + COUNT(*)::int AS live_count + FROM score + LEFT JOIN chart_playcount AS cp ON cp.chart_id = score.chart_id + GROUP BY score.chart_id, cp.playcount + HAVING COUNT(*)::int IS DISTINCT FROM COALESCE(cp.playcount, 0) + `.execute(DB); + + let fixed = 0; + + for (const row of driftRows.rows) { + await DB.insertInto("chart_playcount") + .values({ + chart_id: row.chart_id, + playcount: row.live_count, + }) + .onConflict((oc) => + oc.column("chart_id").doUpdateSet({ + playcount: row.live_count, + }), + ) + .execute(); + fixed++; + } + + const staleRows = await sql<{ chart_id: string }>` + SELECT cp.chart_id + FROM chart_playcount AS cp + LEFT JOIN score ON score.chart_id = cp.chart_id + GROUP BY cp.chart_id + HAVING COUNT(score.id) = 0 + `.execute(DB); + + for (const row of staleRows.rows) { + await DB.deleteFrom("chart_playcount") + .where("chart_playcount.chart_id", "=", row.chart_id) + .execute(); + fixed++; + } + + log.info( + `ReconcileChartPlaycount done: ${fixed} row(s) corrected (${driftRows.rows.length} drift, ${staleRows.rows.length} stale).`, + ); +} diff --git a/typescript/server/src/lib/search/song-charts.ts b/typescript/server/src/lib/search/song-charts.ts index d61c12bbe..8a1b21d62 100644 --- a/typescript/server/src/lib/search/song-charts.ts +++ b/typescript/server/src/lib/search/song-charts.ts @@ -1,7 +1,6 @@ import { GetChartsBySongId } from "#lib/db-formats/chart"; import { AllEnabledGames } from "#lib/setup/config"; import DB from "#services/pg/db"; -import { sql } from "kysely"; import { type ChartDocument, GameToGameGroup, @@ -77,11 +76,9 @@ export async function SearchGlobalGameSongsAndCharts( const chartIDs = output.map((o) => o.chart.chartID); - const playcountRows = await DB.selectFrom("score") - .innerJoin("chart", "chart.id", "score.chart_id") - .select(["chart.id", sql`count(score.id)::int`.as("playcount")]) - .where("chart.id", "in", chartIDs) - .groupBy("chart.id") + const playcountRows = await DB.selectFrom("chart_playcount") + .select(["chart_playcount.chart_id as id", "chart_playcount.playcount"]) + .where("chart_playcount.chart_id", "in", chartIDs) .execute(); const playcountLookup = Object.fromEntries(playcountRows.map((r) => [r.id, r.playcount])); diff --git a/typescript/server/src/server/router/api/v1/games/router.ts b/typescript/server/src/server/router/api/v1/games/router.ts index 21664ed6b..aa2a3fc5a 100644 --- a/typescript/server/src/server/router/api/v1/games/router.ts +++ b/typescript/server/src/server/router/api/v1/games/router.ts @@ -5,7 +5,6 @@ import { LoadFolderDocumentByGameAndSlug, LoadFolderDocumentsByIds } from "#lib/ import { ToGameStatsDocument } from "#lib/db-formats/game-profiles"; import { SELECT_GOAL, SELECT_GOAL_SUB_WITH_GOAL_GAME } from "#lib/db-formats/goal"; import { - CountPbsOnChart, LoadPbDocumentsForGameSortedByCalculatedAlg, LoadPbsOnChartByRankAsc, LoadPbsOnChartForUserSearch, @@ -466,23 +465,6 @@ API_V1_ROUTER.add( }, ); -/** - * Returns the total amount of unique players that have played this chart. - * - * @name GET /api/v1/games/:game/charts/:chartID/playcount - */ -API_V1_ROUTER.add( - "GET /games/:game/charts/:chartID/playcount", - withGame, - withChart, - async ({ ctx }) => { - const { chartDoc: chart } = ctx; - const count = await CountPbsOnChart(chart.chartID); - - return success("Counted scores for chart.", { count }); - }, -); - /** * Returns the personal bests for this chart sorted by ranking. * diff --git a/typescript/server/src/server/router/api/v1/spec.ts b/typescript/server/src/server/router/api/v1/spec.ts index e618c9d69..5280ebf01 100644 --- a/typescript/server/src/server/router/api/v1/spec.ts +++ b/typescript/server/src/server/router/api/v1/spec.ts @@ -1508,12 +1508,6 @@ export const API_V1_SPEC = { output: docArray(), }, - "GET /games/:game/charts/:chartID/playcount": { - description: "Number of unique players on a chart.", - input: z.object({}), - output: z.strictObject({ count: z.number() }), - }, - "GET /games/:game/charts/:chartID/pbs": { description: "Top PBs on a chart by ranking page.", input: z.object({ startRanking: z.coerce.number().optional() }), diff --git a/typescript/server/src/utils/queries/charts.ts b/typescript/server/src/utils/queries/charts.ts index 925389625..232a762bd 100644 --- a/typescript/server/src/utils/queries/charts.ts +++ b/typescript/server/src/utils/queries/charts.ts @@ -746,10 +746,7 @@ export async function FindUSCChartsByHashSHA1(hash: string): Promise { - let sq = db - .selectFrom("score") - .where("score.game", "=", game) - .select(["score.chart_id", sql`count(*)::int`.as("playcount")]) - .groupBy("score.chart_id"); - - if (chartIdFilter !== undefined) { - sq = sq.where("score.chart_id", "in", chartIdFilter); - } - - // When searching by song title, only aggregate scores for matching charts. - // Without this, every search scans and groups every score row for the game - // (e.g. 3.7M rows for bms-7k on boku) before filtering to ~100 charts. - if (songIdFilter !== undefined) { - sq = sq - .innerJoin("chart as score_count_chart", "score_count_chart.id", "score.chart_id") - .where("score_count_chart.song_id", "in", songIdFilter); - } - - return sq; - }) - .selectFrom("chart") + let q = DB.selectFrom("chart") .innerJoin("song", "song.id", "chart.song_id") - .leftJoin("score_counts", "score_counts.chart_id", "chart.id") + .leftJoin("chart_playcount", "chart_playcount.chart_id", "chart.id") .where("chart.game", "=", game) .where(sql`(chart.data->>'2dxtraSet') IS NULL`); @@ -807,9 +781,9 @@ export async function FindChartsOnPopularity( const rows = await q .select([ ...SELECT_CHART, // format-bearing comment - sql`coalesce(score_counts.playcount, 0)::int`.as("playcount"), + sql`coalesce(chart_playcount.playcount, 0)::int`.as("playcount"), ]) - .orderBy(sql`coalesce(score_counts.playcount, 0)`, "desc") + .orderBy(sql`coalesce(chart_playcount.playcount, 0)`, "desc") .offset(skip) .limit(limit) .execute();