Job framework for ugs-snapshots

server/package.json

@@ -25,6 +25,7 @@
         "@types/range-parser": "1.2.3",
         "@types/rate-limit-redis": "1.7.1",
         "@types/rimraf": "3.0.0",
+        "@types/safe-json-stringify": "1.1.2",
         "@types/supertest": "2.0.11",
         "@types/tap": "15.0.4",
         "@types/uuid": "8.3.0",
@@ -67,6 +68,7 @@
         "rate-limit-redis": "2.1.0",
         "redis": "3.1.2",
         "rimraf": "3.0.2",
+        "safe-json-stringify": "1.2.0",
         "tachi-common": "github:zkldi/tachi-common",
         "typescript": "4.3.4",
         "winston": "3.3.3"

server/pnpm-lock.yaml

@@ -13,6 +13,7 @@ specifiers:
   '@types/range-parser': 1.2.3
   '@types/rate-limit-redis': 1.7.1
   '@types/rimraf': 3.0.0
+  '@types/safe-json-stringify': 1.1.2
   '@types/supertest': 2.0.11
   '@types/tap': 15.0.4
   '@types/uuid': 8.3.0
@@ -48,6 +49,7 @@ specifiers:
   rate-limit-redis: 2.1.0
   redis: 3.1.2
   rimraf: 3.0.2
+  safe-json-stringify: 1.2.0
   supertest: 6.1.3
   tachi-common: github:zkldi/tachi-common
   tap: 15.0.9
@@ -81,7 +83,8 @@ dependencies:
   rate-limit-redis: 2.1.0
   redis: 3.1.2
   rimraf: 3.0.2
-  tachi-common: github.com/zkldi/tachi-common/7a9f468590d81ea6309c34dc15a7619f9924ed9b_ts-node@10.0.0+typescript@4.3.4
+  safe-json-stringify: 1.2.0
+  tachi-common: github.com/zkldi/tachi-common/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9_ts-node@10.0.0+typescript@4.3.4
   typescript: 4.3.4
   winston: 3.3.3
 
@@ -98,6 +101,7 @@ devDependencies:
   '@types/range-parser': 1.2.3
   '@types/rate-limit-redis': 1.7.1
   '@types/rimraf': 3.0.0
+  '@types/safe-json-stringify': 1.1.2
   '@types/supertest': 2.0.11
   '@types/tap': 15.0.4
   '@types/uuid': 8.3.0
@@ -632,6 +636,10 @@ packages:
       '@types/node': 15.12.4
     dev: true
 
+  /@types/safe-json-stringify/1.1.2:
+    resolution: {integrity: sha512-Hj/LZMBXFH3Qj9sNmNu6syBpkZqBaM00HgP7naf9CnZhB3kdxmrenWUD2cY6vvTWjuBM3NPOLknPKDUOQuTWXA==}
+    dev: true
+
   /@types/serve-static/1.13.9:
     resolution: {integrity: sha512-ZFqF6qa48XsPdjXV5Gsz0Zqmux2PerNd3a/ktL45mHpa19cuMi/cL8tcxdAx497yRh+QtYPuofjT9oWw9P7nkA==}
     dependencies:
@@ -3422,6 +3430,10 @@ packages:
   /safe-buffer/5.2.1:
     resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
 
+  /safe-json-stringify/1.2.0:
+    resolution: {integrity: sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==}
+    dev: false
+
   /safer-buffer/2.1.2:
     resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==}
 
@@ -4214,9 +4226,9 @@ packages:
     engines: {node: '>=6'}
     dev: true
 
-  github.com/zkldi/tachi-common/7a9f468590d81ea6309c34dc15a7619f9924ed9b_ts-node@10.0.0+typescript@4.3.4:
-    resolution: {tarball: https://codeload.github.com/zkldi/tachi-common/tar.gz/7a9f468590d81ea6309c34dc15a7619f9924ed9b}
-    id: github.com/zkldi/tachi-common/7a9f468590d81ea6309c34dc15a7619f9924ed9b
+  github.com/zkldi/tachi-common/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9_ts-node@10.0.0+typescript@4.3.4:
+    resolution: {tarball: https://codeload.github.com/zkldi/tachi-common/tar.gz/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9}
+    id: github.com/zkldi/tachi-common/dbfc3f67f73fb16e00adfaf5ec713f4e43cec2f9
     name: tachi-common
     version: 0.1.0
     dependencies:

server/src/external/mongo/db.ts

@@ -14,6 +14,7 @@ import {
 	PrivateUserDocument,
 	ScoreDocument,
 	KaiAuthDocument,
+	UserGameStatsSnapshot,
 	SessionDocument,
 	AnySongDocument,
 	AnyChartDocument,
@@ -143,6 +144,7 @@ const db = {
 	"import-locks": monkDB.get<ImportLockDocument>("import-locks"),
 	tables: monkDB.get<TableDocument>("tables"),
 	"game-settings": monkDB.get<UGPTSettings>("game-settings"),
+	"game-stats-snapshots": monkDB.get<UserGameStatsSnapshot>("game-stats-snapshots"),
 };
 
 export default db;

server/src/external/mongo/indexes.ts

@@ -79,6 +79,7 @@ const staticIndexes: Partial<Record<ValidDatabases, Index[]>> = {
 	"bms-course-lookup": [index({ md5sums: 1 }, UNIQUE)],
 	"api-tokens": [index({ token: 1 }, UNIQUE), index({ userID: 1 })],
 	tables: [index({ tableID: 1 }, UNIQUE), index({ game: 1, playtype: 1 })],
+	"game-stats-snapshots": [index({ timestamp: 1, userID: 1, game: 1, playtype: 1 }, UNIQUE)],
 };
 
 const indexes: Partial<Record<ValidDatabases, Index[]>> = staticIndexes;

server/src/lib/jobs/jobconf.json5

@@ -0,0 +1,11 @@
+{
+	// this format is just an example - @ereti will be designing the jobs system and will *definitely* want something
+	// better. This only exists as a rough example.
+	prefix: "tsnode -r tsconfig-paths/register",
+	jobs: [{
+		path: "ugs-snapshot.ts",
+		name: "Snapshot User Game Stats",
+		// especially this part - i don't expect any machine to parse this text.
+		interval: "daily at midnight"
+	}]
+}

server/src/lib/jobs/ugs-snapshot.ts

@@ -0,0 +1,87 @@
+import { UserGameStats, UserGameStatsSnapshot } from "tachi-common";
+
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { GetUsersRanking } from "utils/user";
+import { GetMilisecondsSince } from "utils/misc";
+
+const logger = CreateLogCtx(__filename);
+
+// get the time of this midnight. it's possible this script eclipses itself when weird timezone
+// nonsense happens. we'll have to see.
+const currentTime = new Date().setUTCHours(0, 0, 0, 0);
+
+let batchWrite: UserGameStatsSnapshot[] = [];
+
+(async () => {
+	const timeStart = process.hrtime.bigint();
+
+	const alreadyExists = await db["game-stats-snapshots"].findOne({ timestamp: currentTime });
+
+	if (alreadyExists) {
+		logger.severe(
+			`FATAL IN UGS-SNAPSHOT - There already exists snapshots at this time. Has this script been ran twice?`
+		);
+		process.exit(1);
+	}
+
+	logger.info(`Snapshotting UserGameStats.`);
+
+	try {
+		await db["game-stats"]
+			.find({})
+			// @ts-expect-error faulty TS types
+			.each(async (ugs: UserGameStats, { pause, resume }) => {
+				pause();
+
+				logger.debug(`Snapshotting ${ugs.userID} ${ugs.playtype} ${ugs.game}.`);
+
+				const [playcount, ranking] = await Promise.all([
+					db.scores.count({ userID: ugs.userID, playtype: ugs.playtype, game: ugs.game }),
+					GetUsersRanking(ugs),
+				]);
+
+				const ugsSnapshot: UserGameStatsSnapshot = {
+					...ugs,
+					playcount,
+					ranking,
+					timestamp: currentTime,
+				};
+
+				batchWrite.push(ugsSnapshot);
+
+				if (batchWrite.length >= 500) {
+					logger.verbose(`Flushed batch.`);
+					await db["game-stats-snapshots"].insert(batchWrite);
+					batchWrite = [];
+				}
+
+				resume();
+			});
+
+		if (batchWrite.length) {
+			await db["game-stats-snapshots"].insert(batchWrite);
+		}
+
+		logger.info(
+			`Successfully snapshotted all data as of ${new Date(
+				currentTime
+			).toString()}. Took ${GetMilisecondsSince(timeStart)} ms.`
+		);
+
+		process.exit(0);
+	} catch (err) {
+		// if we panic, we need to revert whatever we did.
+		logger.severe(`FATAL IN UGS-SNAPSHOT - Possibly failed midway through snapshotting.`, {
+			err,
+		});
+
+		logger.info(`Removing all snapshots at this timestamp (${currentTime}).`);
+
+		await db["game-stats-snapshots"].remove({ timestamp: currentTime });
+
+		logger.info(`Removed.`);
+
+		process.exit(1);
+	}
+})();

server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts

@@ -2,7 +2,7 @@ import { Router } from "express";
 import db from "../../../../../../../../../external/mongo/db";
 import { SYMBOL_TachiData } from "../../../../../../../../../lib/constants/tachi";
 import { GetRelevantSongsAndCharts } from "../../../../../../../../../utils/db";
-import { GetUsersRanking, GetUsersWithIDs } from "../../../../../../../../../utils/user";
+import { GetUsersRankingAndOutOf, GetUsersWithIDs } from "../../../../../../../../../utils/user";
 import { CheckUserPlayedGamePlaytype } from "./middleware";
 import { FilterQuery } from "mongodb";
 import {
@@ -69,7 +69,7 @@ router.get("/", async (req, res) => {
 				},
 			}
 		),
-		GetUsersRanking(stats),
+		GetUsersRankingAndOutOf(stats),
 	]);
 
 	return res.status(200).json({
@@ -386,7 +386,7 @@ router.get("/leaderboard-adjacent", async (req, res) => {
 		...below.map((e) => e.userID),
 	]);
 
-	const thisUsersRanking = await GetUsersRanking(thisUsersStats);
+	const thisUsersRanking = await GetUsersRankingAndOutOf(thisUsersStats);
 
 	return res.status(200).json({
 		success: true,

server/src/utils/user.ts

@@ -156,6 +156,40 @@ export function FormatUserDoc(userdoc: PublicUserDocument) {
 export async function GetUsersRanking(stats: UserGameStats) {
 	const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
 
+	const aggRes = await db["game-stats"].aggregate([
+		{
+			$match: {
+				game: stats.game,
+				playtype: stats.playtype,
+			},
+		},
+		{
+			$group: {
+				_id: null,
+				ranking: {
+					$sum: {
+						$cond: {
+							if: {
+								$gt: [
+									`$ratings.${gptConfig.defaultProfileRatingAlg}`,
+									stats.ratings[gptConfig.defaultProfileRatingAlg],
+								],
+							},
+							then: 1,
+							else: 0,
+						},
+					},
+				},
+			},
+		},
+	]);
+
+	return (aggRes[0].ranking + 1) as integer;
+}
+
+export async function GetUsersRankingAndOutOf(stats: UserGameStats) {
+	const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
+
 	const aggRes = await db["game-stats"].aggregate([
 		{
 			$match: {