diff --git a/server/package.json b/server/package.json index 83cbd2f9f..934afb44b 100644 --- a/server/package.json +++ b/server/package.json @@ -25,6 +25,7 @@ "@types/range-parser": "1.2.3", "@types/rate-limit-redis": "1.7.1", "@types/rimraf": "3.0.0", + "@types/safe-json-stringify": "1.1.2", "@types/supertest": "2.0.11", "@types/tap": "15.0.4", "@types/uuid": "8.3.0", @@ -67,6 +68,7 @@ "rate-limit-redis": "2.1.0", "redis": "3.1.2", "rimraf": "3.0.2", + "safe-json-stringify": "1.2.0", "tachi-common": "github:zkldi/tachi-common", "typescript": "4.3.4", "winston": "3.3.3" diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index ddb1fe743..68dc58400 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -13,6 +13,7 @@ specifiers: '@types/range-parser': 1.2.3 '@types/rate-limit-redis': 1.7.1 '@types/rimraf': 3.0.0 + '@types/safe-json-stringify': 1.1.2 '@types/supertest': 2.0.11 '@types/tap': 15.0.4 '@types/uuid': 8.3.0 @@ -48,6 +49,7 @@ specifiers: rate-limit-redis: 2.1.0 redis: 3.1.2 rimraf: 3.0.2 + safe-json-stringify: 1.2.0 supertest: 6.1.3 tachi-common: github:zkldi/tachi-common tap: 15.0.9 @@ -81,7 +83,8 @@ dependencies: rate-limit-redis: 2.1.0 redis: 3.1.2 rimraf: 3.0.2 - tachi-common: github.com/zkldi/tachi-common/7a9f468590d81ea6309c34dc15a7619f9924ed9b_ts-node@10.0.0+typescript@4.3.4 + safe-json-stringify: 1.2.0 + tachi-common: github.com/zkldi/tachi-common/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9_ts-node@10.0.0+typescript@4.3.4 typescript: 4.3.4 winston: 3.3.3 @@ -98,6 +101,7 @@ devDependencies: '@types/range-parser': 1.2.3 '@types/rate-limit-redis': 1.7.1 '@types/rimraf': 3.0.0 + '@types/safe-json-stringify': 1.1.2 '@types/supertest': 2.0.11 '@types/tap': 15.0.4 '@types/uuid': 8.3.0 @@ -632,6 +636,10 @@ packages: '@types/node': 15.12.4 dev: true + /@types/safe-json-stringify/1.1.2: + resolution: {integrity: sha512-Hj/LZMBXFH3Qj9sNmNu6syBpkZqBaM00HgP7naf9CnZhB3kdxmrenWUD2cY6vvTWjuBM3NPOLknPKDUOQuTWXA==} + dev: true + /@types/serve-static/1.13.9: resolution: {integrity: sha512-ZFqF6qa48XsPdjXV5Gsz0Zqmux2PerNd3a/ktL45mHpa19cuMi/cL8tcxdAx497yRh+QtYPuofjT9oWw9P7nkA==} dependencies: @@ -3422,6 +3430,10 @@ packages: /safe-buffer/5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + /safe-json-stringify/1.2.0: + resolution: {integrity: sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==} + dev: false + /safer-buffer/2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -4214,9 +4226,9 @@ packages: engines: {node: '>=6'} dev: true - github.com/zkldi/tachi-common/7a9f468590d81ea6309c34dc15a7619f9924ed9b_ts-node@10.0.0+typescript@4.3.4: - resolution: {tarball: https://codeload.github.com/zkldi/tachi-common/tar.gz/7a9f468590d81ea6309c34dc15a7619f9924ed9b} - id: github.com/zkldi/tachi-common/7a9f468590d81ea6309c34dc15a7619f9924ed9b + github.com/zkldi/tachi-common/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9_ts-node@10.0.0+typescript@4.3.4: + resolution: {tarball: https://codeload.github.com/zkldi/tachi-common/tar.gz/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9} + id: github.com/zkldi/tachi-common/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9 name: tachi-common version: 0.1.0 dependencies: diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index 99e435bce..54fe69779 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -14,6 +14,7 @@ import { PrivateUserDocument, ScoreDocument, KaiAuthDocument, + UserGameStatsSnapshot, SessionDocument, AnySongDocument, AnyChartDocument, @@ -143,6 +144,7 @@ const db = { "import-locks": monkDB.get("import-locks"), tables: monkDB.get("tables"), "game-settings": monkDB.get("game-settings"), + "game-stats-snapshots": monkDB.get("game-stats-snapshots"), }; export default db; diff --git a/server/src/external/mongo/indexes.ts b/server/src/external/mongo/indexes.ts index f1baa9d15..9d5735437 100644 --- a/server/src/external/mongo/indexes.ts +++ b/server/src/external/mongo/indexes.ts @@ -79,6 +79,7 @@ const staticIndexes: Partial> = { "bms-course-lookup": [index({ md5sums: 1 }, UNIQUE)], "api-tokens": [index({ token: 1 }, UNIQUE), index({ userID: 1 })], tables: [index({ tableID: 1 }, UNIQUE), index({ game: 1, playtype: 1 })], + "game-stats-snapshots": [index({ timestamp: 1, userID: 1, game: 1, playtype: 1 }, UNIQUE)], }; const indexes: Partial> = staticIndexes; diff --git a/server/src/lib/jobs/jobconf.json5 b/server/src/lib/jobs/jobconf.json5 new file mode 100644 index 000000000..58d7374fb --- /dev/null +++ b/server/src/lib/jobs/jobconf.json5 @@ -0,0 +1,11 @@ +{ + // this format is just an example - @ereti will be designing the jobs system and will *definitely* want something + // better. This only exists as a rough example. + prefix: "tsnode -r tsconfig-paths/register", + jobs: [{ + path: "ugs-snapshot.ts", + name: "Snapshot User Game Stats", + // especially this part - i don't expect any machine to parse this text. + interval: "daily at midnight" + }] +} \ No newline at end of file diff --git a/server/src/lib/jobs/ugs-snapshot.ts b/server/src/lib/jobs/ugs-snapshot.ts new file mode 100644 index 000000000..6cdc74711 --- /dev/null +++ b/server/src/lib/jobs/ugs-snapshot.ts @@ -0,0 +1,87 @@ +import { UserGameStats, UserGameStatsSnapshot } from "tachi-common"; + +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import { GetUsersRanking } from "utils/user"; +import { GetMilisecondsSince } from "utils/misc"; + +const logger = CreateLogCtx(__filename); + +// get the time of this midnight. it's possible this script eclipses itself when weird timezone +// nonsense happens. we'll have to see. +const currentTime = new Date().setUTCHours(0, 0, 0, 0); + +let batchWrite: UserGameStatsSnapshot[] = []; + +(async () => { + const timeStart = process.hrtime.bigint(); + + const alreadyExists = await db["game-stats-snapshots"].findOne({ timestamp: currentTime }); + + if (alreadyExists) { + logger.severe( + `FATAL IN UGS-SNAPSHOT - There already exists snapshots at this time. Has this script been ran twice?` + ); + process.exit(1); + } + + logger.info(`Snapshotting UserGameStats.`); + + try { + await db["game-stats"] + .find({}) + // @ts-expect-error faulty TS types + .each(async (ugs: UserGameStats, { pause, resume }) => { + pause(); + + logger.debug(`Snapshotting ${ugs.userID} ${ugs.playtype} ${ugs.game}.`); + + const [playcount, ranking] = await Promise.all([ + db.scores.count({ userID: ugs.userID, playtype: ugs.playtype, game: ugs.game }), + GetUsersRanking(ugs), + ]); + + const ugsSnapshot: UserGameStatsSnapshot = { + ...ugs, + playcount, + ranking, + timestamp: currentTime, + }; + + batchWrite.push(ugsSnapshot); + + if (batchWrite.length >= 500) { + logger.verbose(`Flushed batch.`); + await db["game-stats-snapshots"].insert(batchWrite); + batchWrite = []; + } + + resume(); + }); + + if (batchWrite.length) { + await db["game-stats-snapshots"].insert(batchWrite); + } + + logger.info( + `Successfully snapshotted all data as of ${new Date( + currentTime + ).toString()}. Took ${GetMilisecondsSince(timeStart)} ms.` + ); + + process.exit(0); + } catch (err) { + // if we panic, we need to revert whatever we did. + logger.severe(`FATAL IN UGS-SNAPSHOT - Possibly failed midway through snapshotting.`, { + err, + }); + + logger.info(`Removing all snapshots at this timestamp (${currentTime}).`); + + await db["game-stats-snapshots"].remove({ timestamp: currentTime }); + + logger.info(`Removed.`); + + process.exit(1); + } +})(); diff --git a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts index b43437a8b..61909049c 100644 --- a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts +++ b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts @@ -2,7 +2,7 @@ import { Router } from "express"; import db from "../../../../../../../../../external/mongo/db"; import { SYMBOL_TachiData } from "../../../../../../../../../lib/constants/tachi"; import { GetRelevantSongsAndCharts } from "../../../../../../../../../utils/db"; -import { GetUsersRanking, GetUsersWithIDs } from "../../../../../../../../../utils/user"; +import { GetUsersRankingAndOutOf, GetUsersWithIDs } from "../../../../../../../../../utils/user"; import { CheckUserPlayedGamePlaytype } from "./middleware"; import { FilterQuery } from "mongodb"; import { @@ -69,7 +69,7 @@ router.get("/", async (req, res) => { }, } ), - GetUsersRanking(stats), + GetUsersRankingAndOutOf(stats), ]); return res.status(200).json({ @@ -386,7 +386,7 @@ router.get("/leaderboard-adjacent", async (req, res) => { ...below.map((e) => e.userID), ]); - const thisUsersRanking = await GetUsersRanking(thisUsersStats); + const thisUsersRanking = await GetUsersRankingAndOutOf(thisUsersStats); return res.status(200).json({ success: true, diff --git a/server/src/utils/user.ts b/server/src/utils/user.ts index 9cec7d04b..136786cc5 100644 --- a/server/src/utils/user.ts +++ b/server/src/utils/user.ts @@ -156,6 +156,40 @@ export function FormatUserDoc(userdoc: PublicUserDocument) { export async function GetUsersRanking(stats: UserGameStats) { const gptConfig = GetGamePTConfig(stats.game, stats.playtype); + const aggRes = await db["game-stats"].aggregate([ + { + $match: { + game: stats.game, + playtype: stats.playtype, + }, + }, + { + $group: { + _id: null, + ranking: { + $sum: { + $cond: { + if: { + $gt: [ + `$ratings.${gptConfig.defaultProfileRatingAlg}`, + stats.ratings[gptConfig.defaultProfileRatingAlg], + ], + }, + then: 1, + else: 0, + }, + }, + }, + }, + }, + ]); + + return (aggRes[0].ranking + 1) as integer; +} + +export async function GetUsersRankingAndOutOf(stats: UserGameStats) { + const gptConfig = GetGamePTConfig(stats.game, stats.playtype); + const aggRes = await db["game-stats"].aggregate([ { $match: {