Seeds Update Scripts

Fixes #658

server/src/scripts/sync-database.ts

@@ -1,21 +1,19 @@
 /* eslint-disable no-await-in-loop */
 // This script syncs this tachi instances database up with the tachi-database-seeds.
 
-import { execSync } from "child_process";
-import { monkDB } from "external/mongo/db";
+import db, { monkDB } from "external/mongo/db";
 import fjsh from "fast-json-stable-hash";
-import fs from "fs";
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
 import CreateLogCtx, { KtLogger } from "lib/logger/logger";
 import UpdateIsPrimaryStatus from "lib/score-mutation/update-isprimary";
-import { Environment, TachiConfig } from "lib/setup/config";
+import { TachiConfig } from "lib/setup/config";
 import { BulkWriteOperation } from "mongodb";
 import { ICollection } from "monk";
-import os from "os";
-import path from "path";
 import {
 	BMSCourseDocument,
 	ChartDocument,
 	FolderDocument,
+	Game,
 	SongDocument,
 	TableDocument,
 } from "tachi-common";
@@ -127,26 +125,49 @@ async function GenericUpsert<T>(
 
 const syncInstructions: SyncInstructions[] = [
 	{
-		pattern: /^charts-(usc|bms)$/u,
+		pattern: /^charts-bms$/u,
 		handler: async (
 			charts: ChartDocument[],
 			collection: ICollection<ChartDocument>,
 			logger
 		) => {
-			// Since the USC and BMS databases are managed Bokutachi-side, we
-			// shouldn't be honoring any sort of updates from tachi-database-seeds
-			// aside from an initial one.
-			//
-			// However, in practice, these syncs are the best way to actually update
-			// issues in the database.
-			// We're going to disable this anyway.
-			// const isInitial = (await collection.findOne()) === null;
-
 			const r = await GenericUpsert(charts, collection, "chartID", logger, false);
 
 			if (r.thingsChanged) {
 				await InitaliseFolderChartLookup();
 				await UpdateIsPrimaryStatus();
+
+				const largestBMSSongID = await db.songs.bms.findOne(
+					{},
+					{
+						sort: {
+							id: -1,
+						},
+					}
+				);
+
+				if (!largestBMSSongID) {
+					logger.severe(
+						`No BMS charts loaded, yet BMS sync was attempted? Lost state on bms-song-id counter. Panicking.`,
+						r
+					);
+					throw new Error(`No BMS charts loaded, yet BMS sync was attempted.`);
+				}
+
+				await db.counters.update(
+					{
+						counterName: "bms-song-id",
+					},
+					{
+						$set: {
+							value: largestBMSSongID.id + 1,
+						},
+					}
+				);
+
+				await RecalcAllScores({
+					chartID: { $in: r.changedFields },
+				});
 			}
 		},
 	},
@@ -172,7 +193,7 @@ const syncInstructions: SyncInstructions[] = [
 	{
 		pattern: /^songs-/u,
 		handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
-			const r = await GenericUpsert(songs, collection, "id", logger);
+			const r = await GenericUpsert(songs, collection, "id", logger, true);
 
 			if (r.thingsChanged) {
 				await RecalcAllScores({
@@ -213,34 +234,18 @@ const syncInstructions: SyncInstructions[] = [
 const logger = CreateLogCtx("Database Sync");
 
 async function SynchroniseDBWithSeeds() {
-	const seedsDir = fs.mkdtempSync(path.join(os.tmpdir(), "tachi-database-seeds-"));
-
-	logger.info(`Cloning data to ${seedsDir}.`);
-
-	fs.rmSync(seedsDir, { recursive: true, force: true });
-
 	// Wait for mongo to connect first.
 	await monkDB.then(() => void 0);
 
-	execSync(
-		`git clone https://github.com/TNG-dev/tachi-database-seeds -b "${
-			Environment.nodeEnv === "production" ? "master" : "develop"
-		}" --depth=1 "${seedsDir}"`,
-		{
-			stdio: "inherit",
-		}
-	);
+	const databaseSeedsRepo = await PullDatabaseSeeds();
 
-	const collections = fs.readdirSync(path.join(seedsDir, "collections"));
-
-	for (const jsonName of collections) {
-		const collectionName = path.parse(jsonName).name;
+	for await (const { collectionName, data } of databaseSeedsRepo.IterateCollections()) {
 		const spawnLogger = CreateLogCtx(`${collectionName} Sync`);
 
 		if (collectionName.startsWith("songs-") || collectionName.startsWith("charts-")) {
 			const game = collectionName.split("-")[1];
 
-			if (!TachiConfig.GAMES.includes(game as any)) {
+			if (!TachiConfig.GAMES.includes(game as Game)) {
 				spawnLogger.verbose(
 					`Skipping ${collectionName} (${game}) as it isn't for ${TachiConfig.NAME}.`
 				);
@@ -248,12 +253,6 @@ async function SynchroniseDBWithSeeds() {
 			}
 		}
 
-		spawnLogger.verbose(`Getting data.`);
-
-		let data = JSON.parse(
-			fs.readFileSync(path.join(seedsDir, "collections", jsonName), "utf-8")
-		);
-
 		spawnLogger.verbose(`Found ${data.length} documents.`);
 
 		let matchedSomething = false;
@@ -266,9 +265,6 @@ async function SynchroniseDBWithSeeds() {
 			}
 		}
 
-		// free memory forcefully
-		data = null;
-
 		if (!matchedSomething) {
 			spawnLogger.warn(
 				`Collection ${collectionName} didn't match any sync instructions. Skipping.`