From 5fd5302a5f78eeb6d27b4c1ab2d96195585a2936 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 28 Sep 2021 22:49:42 +0100 Subject: [PATCH 1/7] Return some more information from /gpt. --- .../v1/games/_game/_playtype/router.test.ts | 4 +- .../api/v1/games/_game/_playtype/router.ts | 196 +++++++++++++++++- 2 files changed, 196 insertions(+), 4 deletions(-) diff --git a/server/src/server/router/api/v1/games/_game/_playtype/router.test.ts b/server/src/server/router/api/v1/games/_game/_playtype/router.test.ts index e23c1edbd..3aca598f7 100644 --- a/server/src/server/router/api/v1/games/_game/_playtype/router.test.ts +++ b/server/src/server/router/api/v1/games/_game/_playtype/router.test.ts @@ -11,7 +11,9 @@ t.test("GET /api/v1/games/:game/:playtype", (t) => { t.strictSame(res.body.body.config, GetGamePTConfig("iidx", "SP")); - // something else needs to go here? + t.equal(res.body.body.chartCount, 1); + t.equal(res.body.body.playerCount, 1); + t.equal(res.body.body.scoreCount, 1); t.end(); }); diff --git a/server/src/server/router/api/v1/games/_game/_playtype/router.ts b/server/src/server/router/api/v1/games/_game/_playtype/router.ts index 60618acfa..af7b376fb 100644 --- a/server/src/server/router/api/v1/games/_game/_playtype/router.ts +++ b/server/src/server/router/api/v1/games/_game/_playtype/router.ts @@ -1,7 +1,7 @@ import { Router } from "express"; import { SYMBOL_TachiData } from "lib/constants/tachi"; import { IsString } from "utils/misc"; -import { GetGamePTConfig, UserGameStats, FormatGame } from "tachi-common"; +import { GetGamePTConfig, UserGameStats, FormatGame, Game, Playtypes, integer } from "tachi-common"; import { FindOptions } from "monk"; import db from "external/mongo/db"; import { CheckStrProfileAlg, ParseStrPositiveNonZeroInt } from "utils/string-checks"; @@ -11,24 +11,66 @@ import songIDRouter from "./songs/_songID/router"; import { ValidatePlaytypeFromParam } from "./middleware"; import foldersRouter from "./folders/router"; import tablesRouter from "./tables/router"; +import NodeCache from "node-cache"; +import { ONE_HOUR } from "lib/constants/time"; +import prValidate from "server/middleware/prudence-validate"; +import p from "prudence"; +import { GetClassDistribution } from "utils/queries/stats"; +import scoresRouter from "./scores/router"; +import { GetRelevantSongsAndCharts } from "utils/db"; + const router: Router = Router({ mergeParams: true }); router.use(ValidatePlaytypeFromParam); +const gptStatCache = new NodeCache(); + +async function GetGameStats( + game: Game, + playtype: Playtypes[Game] +): Promise<{ scoreCount: integer; playerCount: integer; chartCount: integer }> { + const cacheRes = gptStatCache.get(`${game}:${playtype}`); + + if (!cacheRes) { + const [scoreCount, playerCount, chartCount] = await Promise.all([ + db.scores.count({ + game, + playtype, + }), + db["game-stats"].count({ + game, + playtype, + }), + db.charts[game].count({ playtype }), + ]); + + gptStatCache.set(`${game}:${playtype}`, { scoreCount, playerCount, chartCount }, ONE_HOUR); + + return { scoreCount, playerCount, chartCount }; + } + + return cacheRes as { scoreCount: integer; playerCount: integer; chartCount: integer }; +} + /** - * Returns the configuration for this game. + * Returns the configuration for this game along with some statistics. * * @name GET /api/v1/games/:game/:playtype */ -router.get("/", (req, res) => { +router.get("/", async (req, res) => { const game = req[SYMBOL_TachiData]!.game!; const playtype = req[SYMBOL_TachiData]!.playtype!; + const { scoreCount, playerCount, chartCount } = await GetGameStats(game, playtype); + return res.status(200).json({ success: true, description: `Retrieved information about ${FormatGame(game, playtype)}`, body: { config: GetGamePTConfig(game, playtype), + scoreCount, + playerCount, + chartCount, }, }); }); @@ -49,6 +91,13 @@ router.get("/leaderboard", async (req, res) => { const limit = ParseStrPositiveNonZeroInt(req.query.limit) ?? 50; + if (limit > 50) { + return res.status(400).json({ + success: false, + description: `Invalid limit. Limit is capped at 50.`, + }); + } + let alg = gptConfig.defaultProfileRatingAlg; if (IsString(req.query.alg)) { const temp = CheckStrProfileAlg(game, playtype, req.query.alg); @@ -92,11 +141,152 @@ router.get("/leaderboard", async (req, res) => { }); }); +/** + * Returns the best scores for this game. + * + * @param alg - An alternative algorithm to use instead of the gpts default. + * @param limit - How many scores to return. + * + * @name GET /api/v1/games/:game/:playtype/score-leaderboard + */ +router.get("/score-leaderboard", async (req, res) => { + const game = req[SYMBOL_TachiData]!.game!; + const playtype = req[SYMBOL_TachiData]!.playtype!; + const gptConfig = GetGamePTConfig(game, playtype); + + const limit = ParseStrPositiveNonZeroInt(req.query.limit) ?? 50; + + let alg = gptConfig.defaultProfileRatingAlg; + if (IsString(req.query.alg)) { + const temp = CheckStrProfileAlg(game, playtype, req.query.alg); + + if (temp === null) { + return res.status(400).json({ + success: false, + description: `Invalid value of ${ + req.query.alg + } for alg. Expected one of ${gptConfig.profileRatingAlgs.join(", ")}`, + }); + } + + alg = temp; + } + + const pbs = await db["personal-bests"].find( + { + game, + playtype, + }, + { + sort: { + [`calculatedData.${alg}`]: -1, + }, + limit, + } + ); + + const users = await GetUsersWithIDs(pbs.map((e) => e.userID)); + + const { songs, charts } = await GetRelevantSongsAndCharts(pbs, game); + + return res.status(200).send({ + success: true, + description: `Successfully returned ${pbs.length} pbs.`, + body: { + pbs, + songs, + charts, + users, + }, + }); +}); + +/** + * Return the distribution of players for the provided class. + * + * @param class - This should be one of the games supported classes. + * + * @name GET /api/v1/games/:game/:playtype/player-distribution + */ +router.get( + "/class-distribution", + prValidate({ + class: "string", + }), + async (req, res) => { + const game = req[SYMBOL_TachiData]!.game!; + const playtype = req[SYMBOL_TachiData]!.playtype!; + const gptConfig = GetGamePTConfig(game, playtype); + + const stat = req.query.class as string; + + const supportedClasses = Object.keys(gptConfig.classHumanisedFormat); + if (!supportedClasses.includes(stat)) { + return res.status(400).json({ + success: false, + description: `Invalid stat ${stat}. Expected any of ${supportedClasses.join( + ", " + )}.`, + }); + } + + // @hack This is asserted above. We're just going to cast as any because we know what + // we're doing. + const distribution = await GetClassDistribution(game, playtype, stat as any); + + return res.status(200).json({ + success: true, + description: `Successfully retrieved distribution.`, + body: distribution, + }); + } +); + +/** + * Returns recent class improvements for this GPT. + * + * @param limit - How many to return. Defaults to 10, caps at 50. + * + * @name GET /api/v1/games/:game/:playtype/recent-classes + */ +router.get( + "/recent-classes", + prValidate({ + limit: p.optional((self) => p.isBoundedInteger(1, 50)(Number(self))), + }), + async (req, res) => { + const game = req[SYMBOL_TachiData]!.game!; + const playtype = req[SYMBOL_TachiData]!.playtype!; + + const limit = req.query.limit ? Number(req.query.limit) : 10; + + const recentClasses = await db["class-achievements"].find( + { + game, + playtype, + }, + { + limit, + sort: { timeAchieved: -1 }, + } + ); + + const users = await GetUsersWithIDs(recentClasses.map((e) => e.userID)); + + return res.status(200).json({ + success: true, + description: `Returned ${recentClasses.length} recent classes.`, + body: { classes: recentClasses, users }, + }); + } +); + // @todo #196 Country Leaderboards? router.use("/charts", chartsRouter); router.use("/songs/:songID", songIDRouter); router.use("/folders", foldersRouter); router.use("/tables", tablesRouter); +router.use("/scores", scoresRouter); export default router; From f8125fa887a17b77b778d91fd189d99d7316931f Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 28 Sep 2021 22:49:51 +0100 Subject: [PATCH 2/7] Add a class achievements database. --- server/src/external/mongo/db.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index a021fdefd..bfc46aece 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -33,6 +33,7 @@ import { integer, FervidexSettingsDocument, OrphanChart, + ClassAchievementDocument, } from "tachi-common"; import monk, { TMiddleware } from "monk"; import CreateLogCtx from "lib/logger/logger"; @@ -177,6 +178,7 @@ const db = { "orphan-chart-queue": monkDB.get("orphan-chart-queue"), "password-reset-codes": monkDB.get<{ code: string; userID: integer; createdOn: number }>("password-reset-codes"), + "class-achievements": monkDB.get("class-achievements"), }; export type StaticDatabases = @@ -209,7 +211,8 @@ export type StaticDatabases = | "orphan-chart-queue" | "password-reset-codes" | "user-settings" - | "counters"; + | "counters" + | "class-achievements"; export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`; From 343f3acd4e8b4604314832e95c471645dde29014 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 28 Sep 2021 22:50:00 +0100 Subject: [PATCH 3/7] Add indexes for the class achivements Database --- server/src/external/mongo/indexes.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/external/mongo/indexes.ts b/server/src/external/mongo/indexes.ts index e60162d9b..9641a1058 100644 --- a/server/src/external/mongo/indexes.ts +++ b/server/src/external/mongo/indexes.ts @@ -81,6 +81,7 @@ const staticIndexes: Partial> = { "user-private-information": [index({ userID: 1 }, UNIQUE), index({ email: 1 }, UNIQUE)], "fer-settings": [index({ userID: 1 }, UNIQUE)], counters: [index({ counterName: 1 }, UNIQUE)], + "class-achievements": [index({ game: 1, playtype: 1, timeAchieved: 1 })], }; const indexes: Partial> = staticIndexes; From 275ec7330462515780a597c232820883c3813233 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 28 Sep 2021 22:50:21 +0100 Subject: [PATCH 4/7] install latest tachi-common --- server/package.json | 2 +- server/pnpm-lock.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/package.json b/server/package.json index da689d1ad..9f661f00d 100644 --- a/server/package.json +++ b/server/package.json @@ -74,7 +74,7 @@ "redis": "3.1.2", "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", - "tachi-common": "0.2.19", + "tachi-common": "0.2.25", "typescript": "4.3.4", "winston": "3.3.3", "winston-transport": "^4.4.0" diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index da963aaab..fb8145d3d 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -55,7 +55,7 @@ specifiers: rimraf: 3.0.2 safe-json-stringify: 1.2.0 supertest: 6.1.3 - tachi-common: 0.2.19 + tachi-common: 0.2.25 tap: 15.0.9 ts-node: 10.0.0 tsconfig-paths: 3.10.1 @@ -91,7 +91,7 @@ dependencies: redis: 3.1.2 rimraf: 3.0.2 safe-json-stringify: 1.2.0 - tachi-common: 0.2.19 + tachi-common: 0.2.25 typescript: 4.3.4 winston: 3.3.3 winston-transport: 4.4.0 @@ -3910,8 +3910,8 @@ packages: strip-ansi: 6.0.0 dev: true - /tachi-common/0.2.19: - resolution: {integrity: sha512-Gy4tr1XmwgpIfJqAKk6PUxTETg+4N00mLrSNBqBwKvRfINBBI6vQ5wp98PHTaCEE8h+dKkv1Uj0u0VnCRrLZig==} + /tachi-common/0.2.25: + resolution: {integrity: sha512-FSMwEWWcvP5Vw2heSD6vD7Iz1/lI+bO1HF0+0iURw5Ybj16yJZQ9wBiaIz2hotNweMhlQ73v57tj2TMUmFyG9w==} dependencies: monk: 7.3.4 transitivePeerDependencies: From a00858b0370e48d5178b2c7d43e5b5d2fef4f3d0 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 28 Sep 2021 22:51:34 +0100 Subject: [PATCH 5/7] Add a utility endpoint for a chart's parent folders. --- .../_game/_playtype/charts/_chartID/router.ts | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/server/src/server/router/api/v1/games/_game/_playtype/charts/_chartID/router.ts b/server/src/server/router/api/v1/games/_game/_playtype/charts/_chartID/router.ts index 274467e7a..4d27e95ff 100644 --- a/server/src/server/router/api/v1/games/_game/_playtype/charts/_chartID/router.ts +++ b/server/src/server/router/api/v1/games/_game/_playtype/charts/_chartID/router.ts @@ -1,5 +1,6 @@ import { Router } from "express"; import db from "external/mongo/db"; +import { FilterQuery } from "mongodb"; import { SYMBOL_TachiData } from "lib/constants/tachi"; import CreateLogCtx from "lib/logger/logger"; import { SearchUsersRegExp } from "lib/search/search"; @@ -7,7 +8,7 @@ import { IsString } from "utils/misc"; import { ParseStrPositiveNonZeroInt } from "utils/string-checks"; import { GetUsersWithIDs } from "utils/user"; import { ValidateAndGetChart } from "./middleware"; -import { FormatChart } from "tachi-common"; +import { FormatChart, FolderDocument } from "tachi-common"; const logger = CreateLogCtx(__filename); @@ -49,6 +50,44 @@ router.get("/", async (req, res) => { }); }); +/** + * Returns any folders that contain this chart. + * + * @param inactive - Also include inactive folders. + * + * @name GET /api/v1/games/:game/:playtype/charts/:chartID/folders + */ +router.get("/folders", async (req, res) => { + const chart = req[SYMBOL_TachiData]!.chartDoc!; + + const folderIDs = await db["folder-chart-lookup"].find( + { + chartID: chart.chartID, + }, + { + projection: { + folderID: 1, + }, + } + ); + + const query: FilterQuery = { + folderID: { $in: folderIDs.map((e) => e.folderID) }, + }; + + if (!req.query.inactive) { + query.inactive = false; + } + + const folders = await db.folders.find(query); + + return res.status(200).json({ + success: true, + description: `Found ${folders.length} folders that contain this chart.`, + body: folders, + }); +}); + /** * Returns the total amount of unique players that have played this chart. * From 7271f78b45c19156f540f8d98d70eeb188938ab8 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 28 Sep 2021 22:51:54 +0100 Subject: [PATCH 6/7] Refactor classes to emit a class-achieved document --- .../framework/user-game-stats/classes.test.ts | 31 ++++++++++------- .../framework/user-game-stats/classes.ts | 33 ++++++++++++++----- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/server/src/lib/score-import/framework/user-game-stats/classes.test.ts b/server/src/lib/score-import/framework/user-game-stats/classes.test.ts index d5d32a2a3..24943e0bc 100644 --- a/server/src/lib/score-import/framework/user-game-stats/classes.test.ts +++ b/server/src/lib/score-import/framework/user-game-stats/classes.test.ts @@ -2,7 +2,7 @@ import { UserGameStats } from "tachi-common"; import t from "tap"; import CreateLogCtx from "lib/logger/logger"; import ResetDBState from "test-utils/resets"; -import { CalculateClassDeltas, UpdateUGSClasses } from "./classes"; +import { ProcessClassDeltas, UpdateUGSClasses } from "./classes"; import { GitadoraColours } from "lib/constants/classes"; @@ -45,14 +45,15 @@ t.test("#UpdateUGSClasses", (t) => { t.end(); }); -t.test("#CalculateClassDeltas", (t) => { +t.test("#ProcessClassDeltas", (t) => { t.beforeEach(ResetDBState); - t.test("Should return improved classes from null", (t) => { - const res = CalculateClassDeltas("SP", { dan: 18 }, null, 1, logger); + t.test("Should return improved classes from null", async (t) => { + const res = await ProcessClassDeltas("iidx", "SP", { dan: 18 }, null, 1, logger); t.strictSame(res, [ { + game: "iidx", set: "dan", playtype: "SP", old: null, @@ -63,8 +64,9 @@ t.test("#CalculateClassDeltas", (t) => { t.end(); }); - t.test("Should return improved classes from null class", (t) => { - const res = CalculateClassDeltas( + t.test("Should return improved classes from null class", async (t) => { + const res = await ProcessClassDeltas( + "iidx", "SP", { dan: 18 }, { classes: {} } as UserGameStats, @@ -74,6 +76,7 @@ t.test("#CalculateClassDeltas", (t) => { t.strictSame(res, [ { + game: "iidx", set: "dan", playtype: "SP", old: null, @@ -84,8 +87,9 @@ t.test("#CalculateClassDeltas", (t) => { t.end(); }); - t.test("Should return improved classes", (t) => { - const res = CalculateClassDeltas( + t.test("Should return improved classes", async (t) => { + const res = await ProcessClassDeltas( + "iidx", "SP", { dan: 18 }, { classes: { dan: 17 } } as unknown as UserGameStats, @@ -95,6 +99,7 @@ t.test("#CalculateClassDeltas", (t) => { t.strictSame(res, [ { + game: "iidx", set: "dan", playtype: "SP", old: 17, @@ -105,8 +110,9 @@ t.test("#CalculateClassDeltas", (t) => { t.end(); }); - t.test("Should not return identical classes", (t) => { - const res = CalculateClassDeltas( + t.test("Should not return identical classes", async (t) => { + const res = await ProcessClassDeltas( + "iidx", "SP", { dan: 18 }, { classes: { dan: 18 } } as unknown as UserGameStats, @@ -119,8 +125,9 @@ t.test("#CalculateClassDeltas", (t) => { t.end(); }); - t.test("Should not return worse classes", (t) => { - const res = CalculateClassDeltas( + t.test("Should not return worse classes", async (t) => { + const res = await ProcessClassDeltas( + "iidx", "SP", { dan: 16 }, { classes: { dan: 18 } } as unknown as UserGameStats, diff --git a/server/src/lib/score-import/framework/user-game-stats/classes.ts b/server/src/lib/score-import/framework/user-game-stats/classes.ts index 8f1c41002..9ecdc3a2f 100644 --- a/server/src/lib/score-import/framework/user-game-stats/classes.ts +++ b/server/src/lib/score-import/framework/user-game-stats/classes.ts @@ -1,16 +1,16 @@ -import { Game, Playtypes, integer, UserGameStats, ClassDelta, IDStrings } from "tachi-common"; -import { GameClasses } from "tachi-common/js/game-classes"; import deepmerge from "deepmerge"; +import db from "external/mongo/db"; import { KtLogger } from "lib/logger/logger"; +import { EmitWebhookEvent } from "lib/webhooks/webhooks"; +import { ClassDelta, Game, IDStrings, integer, Playtypes, UserGameStats } from "tachi-common"; +import { GameClasses } from "tachi-common/js/game-classes"; +import { ReturnClassIfGreater } from "utils/class"; import { CalculateChunithmColour, CalculateGitadoraColour, CalculateSDVXClass, } from "./builtin-class-handlers"; -import { ReturnClassIfGreater } from "utils/class"; -import { RedisPub } from "external/redis/redis-IPC"; import { ClassHandler, ScoreClasses } from "./types"; -import { EmitWebhookEvent } from "lib/webhooks/webhooks"; type ClassHandlerMap = { [G in Game]: @@ -110,17 +110,20 @@ export async function UpdateUGSClasses( * This is for calculating scenarios where a users class has improved (i.e. they have gone from 9th dan to 10th dan). * * Knowing this information allows us to attach it onto the import, and also emit things on redis - * so that other services can listen for it. In the future we might allow webhooks, too. + * s**This function emits webhook events and inserts classachieved documents into the DB!** */ -export function CalculateClassDeltas( +export async function ProcessClassDeltas( + game: Game, playtype: Playtypes[Game], classes: ScoreClasses, userGameStats: UserGameStats | null, userID: integer, logger: KtLogger -): ClassDelta[] { +): Promise { const deltas: ClassDelta[] = []; + const updateOps = []; + for (const s in classes) { const classSet = s as keyof GameClasses; const classVal = classes[classSet]; @@ -139,6 +142,7 @@ export function CalculateClassDeltas( let delta: ClassDelta; if (isGreater === null) { delta = { + game, set: classSet, playtype, old: null, @@ -146,6 +150,7 @@ export function CalculateClassDeltas( }; } else { delta = { + game, set: classSet, playtype, old: userGameStats!.classes[classSet]!, @@ -155,6 +160,16 @@ export function CalculateClassDeltas( EmitWebhookEvent({ type: "class-update/v1", content: { userID, ...delta } }); + updateOps.push({ + userID, + classSet: delta.set, + classOldValue: delta.old, + classValue: delta.new, + game, + playtype, + timeAchieved: Date.now(), + }); + deltas.push(delta); } } catch (err) { @@ -162,5 +177,7 @@ export function CalculateClassDeltas( } } + await db["class-achievements"].insert(updateOps); + return deltas; } From b8068fe37215be95e2696db62a2ed3cc0dd92063 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 28 Sep 2021 22:52:24 +0100 Subject: [PATCH 7/7] Change how processclasses is used. --- .../lib/score-import/framework/user-game-stats/update-ugs.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/lib/score-import/framework/user-game-stats/update-ugs.ts b/server/src/lib/score-import/framework/user-game-stats/update-ugs.ts index 9ccba6ad4..5a77aa3ae 100644 --- a/server/src/lib/score-import/framework/user-game-stats/update-ugs.ts +++ b/server/src/lib/score-import/framework/user-game-stats/update-ugs.ts @@ -6,7 +6,7 @@ import { Game, Playtypes, integer, UserGameStats, ClassDelta } from "tachi-commo import db from "external/mongo/db"; import { CreateGameSettings } from "lib/game-settings/create-game-settings"; import { KtLogger } from "lib/logger/logger"; -import { CalculateClassDeltas, UpdateUGSClasses } from "./classes"; +import { ProcessClassDeltas, UpdateUGSClasses } from "./classes"; import { CalculateRatings } from "./rating"; import { ClassHandler } from "./types"; @@ -35,7 +35,7 @@ export async function UpdateUsersGamePlaytypeStats( logger.debug(`Calculating Class Deltas...`); - const deltas = CalculateClassDeltas(playtype, classes, userGameStats, userID, logger); + const deltas = await ProcessClassDeltas(game, playtype, classes, userGameStats, userID, logger); logger.debug(`Had ${deltas.length} deltas.`);