feat: redo how kamai syncs updates to its seeds

server/src/lib/database-seeds/repo.ts

@@ -6,6 +6,7 @@ import fs from "fs/promises";
 import os from "os";
 import path from "path";
 import type { Game } from "tachi-common";
+import { ICollection } from "monk";
 
 const logger = CreateLogCtx(__filename);
 
@@ -269,3 +270,20 @@ export async function PullDatabaseSeeds(
 		throw err;
 	}
 }
+
+export async function BacksyncCollectionToBothBranches(collectionName: SeedsCollections, collection: ICollection, commitMessage: string) {
+	for (const branch of ["staging", "release/v2." + VERSION_INFO.minor]) {
+		const repo = await PullDatabaseSeeds(null, branch);
+
+		let charts = await collection.find({});
+
+		await repo.WriteCollection(collectionName, charts);
+
+		// @ts-expect-error Force node to free the memory.
+		charts = null;
+
+		await repo.CommitChangesBack(`${commitMessage} ${new Date().toISOString()}`);
+
+		await repo.Destroy();
+	}
+}

server/src/lib/jobs/backsync-kamai-data.ts

@@ -1,57 +0,0 @@
-/* eslint-disable no-await-in-loop */
-import db from "external/mongo/db";
-import { VERSION_INFO } from "lib/constants/version";
-import { PullDatabaseSeeds } from "lib/database-seeds/repo";
-import CreateLogCtx from "lib/logger/logger";
-
-const logger = CreateLogCtx(__filename);
-
-/**
- * The tachi-server may have its BMS or PMS database update. It needs to sync this
- * information back with the seeds.
- */
-export async function BacksyncKamaiSongsAndCharts() {
-	for (const branch of ["staging", `release/${VERSION_INFO.major}.${VERSION_INFO.minor}`]) {
-		const repo = await PullDatabaseSeeds(undefined, branch);
-
-		for (const game of ["iidx"] as const) {
-			logger.info(`Fetching ${game} songs from DB.`);
-
-			// did you know, this code is liable to blow up in my face and OOM one day?
-			let songs = await db.songs[game].find({});
-
-			logger.info(`Found ${songs.length} ${game} songs.`);
-
-			await repo.WriteCollection(`songs-${game}`, songs);
-
-			// @ts-expect-error This is obviously making something nullable when it shouldn't be.
-			// but if we don't *force* node to free this damn memory, it kills itself when it
-			// tries to read even more stuff.
-			songs = null;
-
-			logger.info(`Fetching ${game} charts from DB.`);
-			let charts = await db.charts[game].find({});
-
-			logger.info(`Found ${charts.length} bms charts.`);
-
-			await repo.WriteCollection(`charts-${game}`, charts);
-
-			// @ts-expect-error See previous expect-error.
-			charts = null;
-		}
-
-		await repo.CommitChangesBack(`Backsync Kamai Songs/Charts ${new Date().toISOString()}`);
-
-		await repo.Destroy();
-	}
-}
-
-if (require.main === module) {
-	 BacksyncKamaiSongsAndCharts()
-		.then(() => process.exit(0))
-		.catch((err: unknown) => {
-			logger.error(`Failed to backsync bms/pms songs and charts.`, { err }, () => {
-				process.exit(1);
-			});
-		});
-}

server/src/lib/jobs/update-bpi-data.ts

@@ -164,15 +164,7 @@ export async function UpdatePoyashiData() {
 	await repo.WriteCollection("charts-iidx", iidxCharts);
 
 	if (updatedChartIDs.length !== 0) {
-		logger.info(`Finished applying BPI changes. Writing back.`);
-
-		for (const branch of ["staging", `release/${VERSION_INFO.major}.${VERSION_INFO.minor}`]) {
-			const commitBackRepo = await PullDatabaseSeeds(branch);
-
-			await commitBackRepo.CommitChangesBack(`BPI Update ${new Date().toISOString()}`);
-
-			await commitBackRepo.Destroy();
-		}
+		logger.info(`Finished applying BPI changes. These changes will be backsynced later.`);
 
 		logger.info(`Recalcing scores.`);
 		await RecalcAllScores({

server/src/lib/jobs/update-dp-tiers.ts

@@ -1,7 +1,8 @@
 /* eslint-disable no-await-in-loop */
 import db from "external/mongo/db";
 import { decode } from "html-entities";
-import { PullDatabaseSeeds } from "lib/database-seeds/repo";
+import { VERSION_INFO } from "lib/constants/version";
+import { BacksyncCollectionToBothBranches, PullDatabaseSeeds } from "lib/database-seeds/repo";
 import CreateLogCtx from "lib/logger/logger";
 import { parse } from "node-html-parser";
 import { RecalcAllScores } from "utils/calculations/recalc-scores";
@@ -95,18 +96,7 @@ export async function UpdateDPTiers() {
 
 		logger.info(`Recalced those scores.`);
 
-		const repo = await PullDatabaseSeeds();
-
-		let charts = await db.charts.iidx.find({});
-
-		await repo.WriteCollection("charts-iidx", charts);
-
-		// @ts-expect-error Force node to free the memory.
-		charts = null;
-
-		await repo.CommitChangesBack(`Update DP Tierlist ${new Date().toISOString()}`);
-
-		await repo.Destroy();
+		await BacksyncCollectionToBothBranches("charts-iidx", db.charts.iidx, "Update DP Tierlist");
 	}
 
 	logger.info("Done.");

server/src/lib/jobs/update-sp12-data.ts

@@ -9,6 +9,8 @@ import { WrapScriptPromise } from "utils/misc";
 import { FindChartWithPTDF } from "utils/queries/charts";
 import { FindSongOnTitle } from "utils/queries/songs";
 import type { ChartDocument, Difficulties, integer } from "tachi-common";
+import { RecalcAllScores } from "utils/calculations/recalc-scores";
+import { BacksyncCollectionToBothBranches } from "lib/database-seeds/repo";
 
 const logger = CreateLogCtx(__filename);
 
@@ -56,6 +58,8 @@ async function FetchSP12Data() {
 		sheets: Array<SP12Data>;
 	};
 
+	const updatedChartIDs: Array<string> = [];
+
 	for (const sh of rj.sheets) {
 		let chart: ChartDocument | null;
 
@@ -138,7 +142,7 @@ async function FetchSP12Data() {
 
 			val = parseFloat(val.toFixed(2));
 
-			let ktKey;
+			let ktKey: keyof ChartDocument<"iidx:SP">["tierlistInfo"];
 
 			if (key === "exh") {
 				ktKey = "kt-EXHC";
@@ -155,6 +159,20 @@ async function FetchSP12Data() {
 					? val.toFixed(2)
 					: `12${stringVal.replace(/(個人差|地力)/u, "")}`;
 
+					const idvDiff =stringVal.includes("個人差");
+
+			const existingTlInfo = chart.tierlistInfo[ktKey];
+
+			if (existingTlInfo && 
+				
+				existingTlInfo.text === text &&
+				existingTlInfo.value === val &&
+				existingTlInfo.individualDifference === idvDiff) {
+					continue;
+				}
+
+				updatedChartIDs.push(chart.chartID);
+
 			await db.charts.iidx.update(
 				{
 					chartID: chart.chartID,
@@ -164,7 +182,7 @@ async function FetchSP12Data() {
 						[`tierlistInfo.${ktKey}`]: {
 							text,
 							value: val,
-							individualDifference: stringVal.includes("個人差"),
+							individualDifference: idvDiff,
 						},
 					},
 				}
@@ -174,7 +192,20 @@ async function FetchSP12Data() {
 		}
 	}
 
-	logger.info("Done!");
+	if (updatedChartIDs.length !== 0) {
+		logger.info(`Finished applying SP12 changes. Recalcing.`);
+		logger.info("These changes will be backsynced by a separate script.")
+
+		logger.info(`Recalcing scores.`);
+		await RecalcAllScores({
+			game: "iidx",
+			chartID: { $in: updatedChartIDs },
+		});
+
+		logger.info(`Finished recalcing scores.`);
+
+		await BacksyncCollectionToBothBranches("charts-iidx", db.charts.iidx, "Update SP12 Tierlist");
+	}
 
 	process.exit(0);
 }