Job framework for ugs-snapshots

server/src/external/mongo/db.ts

@@ -14,6 +14,7 @@ import {
 	PrivateUserDocument,
 	ScoreDocument,
 	KaiAuthDocument,
+	UserGameStatsSnapshot,
 	SessionDocument,
 	AnySongDocument,
 	AnyChartDocument,
@@ -143,6 +144,7 @@ const db = {
 	"import-locks": monkDB.get<ImportLockDocument>("import-locks"),
 	tables: monkDB.get<TableDocument>("tables"),
 	"game-settings": monkDB.get<UGPTSettings>("game-settings"),
+	"game-stats-snapshots": monkDB.get<UserGameStatsSnapshot>("game-stats-snapshots"),
 };
 
 export default db;

server/src/external/mongo/indexes.ts

@@ -79,6 +79,7 @@ const staticIndexes: Partial<Record<ValidDatabases, Index[]>> = {
 	"bms-course-lookup": [index({ md5sums: 1 }, UNIQUE)],
 	"api-tokens": [index({ token: 1 }, UNIQUE), index({ userID: 1 })],
 	tables: [index({ tableID: 1 }, UNIQUE), index({ game: 1, playtype: 1 })],
+	"game-stats-snapshots": [index({ timestamp: 1, userID: 1, game: 1, playtype: 1 }, UNIQUE)],
 };
 
 const indexes: Partial<Record<ValidDatabases, Index[]>> = staticIndexes;

server/src/lib/jobs/jobconf.json5

@@ -0,0 +1,11 @@
+{
+	// this format is just an example - @ereti will be designing the jobs system and will *definitely* want something
+	// better. This only exists as a rough example.
+	prefix: "tsnode -r tsconfig-paths/register",
+	jobs: [{
+		path: "ugs-snapshot.ts",
+		name: "Snapshot User Game Stats",
+		// especially this part - i don't expect any machine to parse this text.
+		interval: "daily at midnight"
+	}]
+}

server/src/lib/jobs/ugs-snapshot.ts

@@ -0,0 +1,87 @@
+import { UserGameStats, UserGameStatsSnapshot } from "tachi-common";
+
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { GetUsersRanking } from "utils/user";
+import { GetMilisecondsSince } from "utils/misc";
+
+const logger = CreateLogCtx(__filename);
+
+// get the time of this midnight. it's possible this script eclipses itself when weird timezone
+// nonsense happens. we'll have to see.
+const currentTime = new Date().setUTCHours(0, 0, 0, 0);
+
+let batchWrite: UserGameStatsSnapshot[] = [];
+
+(async () => {
+	const timeStart = process.hrtime.bigint();
+
+	const alreadyExists = await db["game-stats-snapshots"].findOne({ timestamp: currentTime });
+
+	if (alreadyExists) {
+		logger.severe(
+			`FATAL IN UGS-SNAPSHOT - There already exists snapshots at this time. Has this script been ran twice?`
+		);
+		process.exit(1);
+	}
+
+	logger.info(`Snapshotting UserGameStats.`);
+
+	try {
+		await db["game-stats"]
+			.find({})
+			// @ts-expect-error faulty TS types
+			.each(async (ugs: UserGameStats, { pause, resume }) => {
+				pause();
+
+				logger.debug(`Snapshotting ${ugs.userID} ${ugs.playtype} ${ugs.game}.`);
+
+				const [playcount, ranking] = await Promise.all([
+					db.scores.count({ userID: ugs.userID, playtype: ugs.playtype, game: ugs.game }),
+					GetUsersRanking(ugs),
+				]);
+
+				const ugsSnapshot: UserGameStatsSnapshot = {
+					...ugs,
+					playcount,
+					ranking,
+					timestamp: currentTime,
+				};
+
+				batchWrite.push(ugsSnapshot);
+
+				if (batchWrite.length >= 500) {
+					logger.verbose(`Flushed batch.`);
+					await db["game-stats-snapshots"].insert(batchWrite);
+					batchWrite = [];
+				}
+
+				resume();
+			});
+
+		if (batchWrite.length) {
+			await db["game-stats-snapshots"].insert(batchWrite);
+		}
+
+		logger.info(
+			`Successfully snapshotted all data as of ${new Date(
+				currentTime
+			).toString()}. Took ${GetMilisecondsSince(timeStart)} ms.`
+		);
+
+		process.exit(0);
+	} catch (err) {
+		// if we panic, we need to revert whatever we did.
+		logger.severe(`FATAL IN UGS-SNAPSHOT - Possibly failed midway through snapshotting.`, {
+			err,
+		});
+
+		logger.info(`Removing all snapshots at this timestamp (${currentTime}).`);
+
+		await db["game-stats-snapshots"].remove({ timestamp: currentTime });
+
+		logger.info(`Removed.`);
+
+		process.exit(1);
+	}
+})();

server/src/server/router/api/v1/users/_userID/games/_game/_playtype/router.ts

@@ -2,7 +2,7 @@ import { Router } from "express";
 import db from "../../../../../../../../../external/mongo/db";
 import { SYMBOL_TachiData } from "../../../../../../../../../lib/constants/tachi";
 import { GetRelevantSongsAndCharts } from "../../../../../../../../../utils/db";
-import { GetUsersRanking, GetUsersWithIDs } from "../../../../../../../../../utils/user";
+import { GetUsersRankingAndOutOf, GetUsersWithIDs } from "../../../../../../../../../utils/user";
 import { CheckUserPlayedGamePlaytype } from "./middleware";
 import { FilterQuery } from "mongodb";
 import {
@@ -69,7 +69,7 @@ router.get("/", async (req, res) => {
 				},
 			}
 		),
-		GetUsersRanking(stats),
+		GetUsersRankingAndOutOf(stats),
 	]);
 
 	return res.status(200).json({
@@ -386,7 +386,7 @@ router.get("/leaderboard-adjacent", async (req, res) => {
 		...below.map((e) => e.userID),
 	]);
 
-	const thisUsersRanking = await GetUsersRanking(thisUsersStats);
+	const thisUsersRanking = await GetUsersRankingAndOutOf(thisUsersStats);
 
 	return res.status(200).json({
 		success: true,

server/src/utils/user.ts

@@ -156,6 +156,40 @@ export function FormatUserDoc(userdoc: PublicUserDocument) {
 export async function GetUsersRanking(stats: UserGameStats) {
 	const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
 
+	const aggRes = await db["game-stats"].aggregate([
+		{
+			$match: {
+				game: stats.game,
+				playtype: stats.playtype,
+			},
+		},
+		{
+			$group: {
+				_id: null,
+				ranking: {
+					$sum: {
+						$cond: {
+							if: {
+								$gt: [
+									`$ratings.${gptConfig.defaultProfileRatingAlg}`,
+									stats.ratings[gptConfig.defaultProfileRatingAlg],
+								],
+							},
+							then: 1,
+							else: 0,
+						},
+					},
+				},
+			},
+		},
+	]);
+
+	return (aggRes[0].ranking + 1) as integer;
+}
+
+export async function GetUsersRankingAndOutOf(stats: UserGameStats) {
+	const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
+
 	const aggRes = await db["game-stats"].aggregate([
 		{
 			$match: {