Seeds Update Scripts

Fixes #658

server/.github/test.conf.json5

@@ -21,6 +21,7 @@
 	ARC_AUTH_TOKEN: "unused",
 	ENABLE_SERVER_HTTPS: false,
 	OUR_URL: "https://example.com",
+	SEEDS_URL: null,
 	INVITE_CODE_CONFIG: {
 		BATCH_SIZE: 2,
 		INVITE_CAP: 100,

server/src/external/mongo/sequence-docs.ts

@@ -15,14 +15,6 @@ export async function InitSequenceDocs() {
 		}
 	);
 
-	const largestUSCSongID = await db.songs.usc.findOne(
-		{},
-		{
-			sort: {
-				id: -1,
-			},
-		}
-	);
 	const largestBMSSongID = await db.songs.bms.findOne(
 		{},
 		{
@@ -37,10 +29,6 @@ export async function InitSequenceDocs() {
 			counterName: "users",
 			value: userWithLargestID ? userWithLargestID.id + 1 : 1,
 		},
-		{
-			counterName: "usc-song-id",
-			value: largestUSCSongID ? largestUSCSongID.id + 1 : 1,
-		},
 		{
 			counterName: "bms-song-id",
 			value: largestBMSSongID ? largestBMSSongID.id + 1 : 1,

server/src/lib/database-seeds/repo.ts

@@ -0,0 +1,166 @@
+import path from "path";
+import os from "os";
+import CreateLogCtx from "lib/logger/logger";
+import { Environment, ServerConfig } from "lib/setup/config";
+import { asyncExec } from "utils/misc";
+import { Game } from "tachi-common";
+import fs from "fs/promises";
+
+const logger = CreateLogCtx(__filename);
+
+export type SeedsCollections =
+	| `charts-${Game}`
+	| `songs-${Game}`
+	| "bms-course-lookup"
+	| "folders"
+	| "tables";
+
+/**
+ * Class that encapsulates the behaviour of a seeds repo.
+ */
+export class DatabaseSeedsRepo {
+	private baseDir: string;
+	private logger;
+
+	constructor(baseDir: string) {
+		this.baseDir = baseDir;
+		this.logger = CreateLogCtx(`DatabaseSeeds:${baseDir}`);
+	}
+
+	private CollectionNameToPath(collectionName: SeedsCollections) {
+		return path.join(this.baseDir, "collections", `${collectionName}.json`);
+	}
+
+	/**
+	 * Reads the data from a collection and returns the parsed JSON.
+	 *
+	 * @returns The data in the requested collection.
+	 */
+	async ReadCollection<D>(collectionName: SeedsCollections): Promise<D[]> {
+		const data = await fs.readFile(this.CollectionNameToPath(collectionName), {
+			encoding: "utf-8",
+		});
+
+		const parsedData = JSON.parse(data) as D[];
+
+		return parsedData;
+	}
+
+	/**
+	 * Writes a new array to the provided collectionName.
+	 *
+	 * @param collectionName - The collection to write to.
+	 * @param content - A new array of objects to write.
+	 */
+	async WriteCollection(collectionName: SeedsCollections, content: unknown[]) {
+		await fs.writeFile(this.CollectionNameToPath(collectionName), JSON.stringify(content));
+
+		// Deterministically sort whatever content we just wrote.
+		await asyncExec(
+			`cd "${this.baseDir}" || exit 1; node scripts/deterministic-collection-sort.js`
+		);
+	}
+
+	async *IterateCollections() {
+		const collectionNames = (await fs.readdir(path.join(this.baseDir, "collections"))).map(
+			(e) => path.parse(e).name
+		) as SeedsCollections[];
+
+		for (const collectionName of collectionNames) {
+			// eslint-disable-next-line no-await-in-loop
+			yield { collectionName, data: await this.ReadCollection(collectionName) };
+		}
+	}
+
+	/**
+	 * Mutate a collection with a given name.
+	 *
+	 * @param collectionName - The collection to mutate.
+	 * @param mutator - A function that takes the entire collection as an array, then returns a new array.
+	 */
+	async MutateCollection<D>(collectionName: SeedsCollections, mutator: (dataset: D[]) => D[]) {
+		const dataset = await this.ReadCollection<D>(collectionName);
+
+		const newData = mutator(dataset);
+
+		return this.WriteCollection(collectionName, newData);
+	}
+
+	/**
+	 * Checks for any diffs in the seeds repository we cloned. If there are any, commit them back
+	 * to the repository.
+	 *
+	 * @param commitMsg - The commit message.
+	 * @returns True when a commit has occured, false when it hasn't. Throws on failure.
+	 */
+	async CommitChangesBack(commitMsg: string) {
+		this.logger.verbose(`Received commit-back request.`);
+
+		try {
+			const { stdout: statusOut } = await asyncExec(
+				`cd "${this.baseDir}" || exit 1; git status --porcelain`
+			);
+
+			if (statusOut === "") {
+				this.logger.info(`No changes. Not committing any changes back.`);
+				return false;
+			}
+
+			this.logger.info(`Changes. Committing changes back.`);
+
+			const { stdout: commitOut, stderr: commitErr } = await asyncExec(
+				`cd "${this.baseDir}" || exit 2;
+				git add . || exit 3;
+				git commit -am "${commitMsg}" || exit 4;
+				git push`
+			);
+
+			this.logger.info(`Commit: ${commitOut}.`);
+
+			if (commitErr) {
+				this.logger.error(`Commit Err?: ${commitErr}`);
+			}
+
+			return true;
+		} catch (err) {
+			this.logger.error(`Failed to backport commits?`, { err });
+			throw err;
+		}
+	}
+}
+
+/**
+ * Pulls the database seeds from github, returns an object that can be used to manipulate them.
+ */
+export async function PullDatabaseSeeds() {
+	if (!ServerConfig.SEEDS_URL) {
+		throw new Error(`SEEDS_URL was null. You cannot pull a seeds repo.`);
+	}
+
+	const seedsDir = await fs.mkdtemp(path.join(os.tmpdir(), "tachi-database-seeds-"));
+
+	logger.info(`Cloning data to ${seedsDir}.`);
+
+	await fs.rm(seedsDir, { recursive: true, force: true });
+
+	try {
+		// stderr in git clone is normal output.
+		// stdout is for errors.
+		// there were expletives below this comment, but I have removed them.
+		const { stdout } = await asyncExec(
+			`git clone "${ServerConfig.SEEDS_URL}" -b "${
+				Environment.nodeEnv === "production" ? "master" : "develop"
+			}" --depth=1 '${seedsDir}'`
+		);
+
+		// isn't that confusing
+		if (stdout) {
+			logger.error(stdout);
+		}
+
+		return new DatabaseSeedsRepo(seedsDir);
+	} catch ({ err, stdout, stderr }) {
+		logger.error(`Error cloning database-seeds. ${stderr}.`);
+		throw err;
+	}
+}

server/src/lib/jobs/backsync-bms-data.ts

@@ -0,0 +1,42 @@
+import db from "external/mongo/db";
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * The tachi-server may have its BMS database update. It needs to sync this
+ * information back with the seeds.
+ */
+export async function BacksyncBMSSongsAndCharts() {
+	const repo = await PullDatabaseSeeds();
+
+	logger.info(`Fetching BMS songs from DB.`);
+	// did you know, this is liable to blow up in my face and OOM one day?
+	const bmsSongs = await db.songs.bms.find({});
+
+	logger.info(`Found ${bmsSongs.length} bms songs.`);
+
+	await repo.WriteCollection("songs-bms", bmsSongs);
+
+	// @ts-expect-error This is obviously making something nullable when it shouldn't be.
+	// but if we don't *force* node to free this damn memory, it kills itself when it
+	// tries to read even more stuff.
+	bmsSongs = null;
+
+	logger.info(`Fetching BMS charts from DB.`);
+	const bmsCharts = await db.charts.bms.find({});
+
+	logger.info(`Found ${bmsCharts.length} bms charts.`);
+
+	await repo.WriteCollection("charts-bms", bmsCharts);
+
+	// @ts-expect-error See previous expect-error.
+	bmsCharts = null;
+
+	await repo.CommitChangesBack(`Backsync BMS Songs/Charts ${new Date().toISOString()}`);
+}
+
+if (require.main === module) {
+	BacksyncBMSSongsAndCharts().then(() => process.exit(0));
+}

server/src/lib/jobs/deorphan-scores.ts

@@ -42,7 +42,5 @@ export async function DeoprhanScores() {
 }
 
 if (require.main === module) {
-	DeoprhanScores().then(() => {
-		process.exit(0);
-	});
+	DeoprhanScores().then(() => process.exit(0));
 }

server/src/lib/jobs/inline-job-runner/job-runner.ts

@@ -1,8 +1,11 @@
 import { Queue, Worker } from "bullmq";
 import CreateLogCtx from "lib/logger/logger";
+import { TachiConfig } from "lib/setup/config";
 import { DedupeArr } from "utils/misc";
+import { BacksyncBMSSongsAndCharts } from "../backsync-bms-data";
 import { DeoprhanScores } from "../deorphan-scores";
 import { UGSSnapshot } from "../ugs-snapshot";
+import { UpdatePoyashiData } from "../update-bpi-data";
 
 interface Job {
 	name: string;
@@ -25,6 +28,24 @@ const jobs: Job[] = [
 	},
 ];
 
+// if kamaitachi or omnitachi
+if (TachiConfig.TYPE !== "btchi") {
+	jobs.push({
+		name: "Update BPI",
+		cronFormat: "2 0 * * *",
+		run: UpdatePoyashiData,
+	});
+}
+
+// if bokutachi or omnimitachi
+if (TachiConfig.TYPE !== "ktchi") {
+	jobs.push({
+		name: "Backsync BMS",
+		cronFormat: "2 0 * * *",
+		run: BacksyncBMSSongsAndCharts,
+	});
+}
+
 const logger = CreateLogCtx("JOB_RUNNER");
 
 /**

server/src/lib/jobs/update-bpi-data.ts

@@ -0,0 +1,151 @@
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
+import CreateLogCtx from "lib/logger/logger";
+import {
+	ChartDocument,
+	Difficulties,
+	GPTSupportedVersions,
+	integer,
+	Playtypes,
+	SongDocument,
+} from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+const difficultyResolve = {
+	3: ["SP", "HYPER"],
+	4: ["SP", "ANOTHER"],
+	8: ["DP", "HYPER"],
+	9: ["DP", "ANOTHER"],
+	10: ["SP", "LEGGENDARIA"],
+	11: ["DP", "LEGGENDARIA"],
+} as Record<string, [Playtypes["iidx"], Difficulties["iidx:SP" | "iidx:DP"]]>;
+
+interface PoyashiProxyBPIInfo {
+	title: string;
+	difficulty: string;
+	wr: integer;
+	avg: integer;
+	notes: string;
+	bpm: string;
+	textage: string;
+	difficultyLevel: string;
+	dpLevel: string;
+	coef: number | null;
+	removed?: boolean;
+}
+
+interface PoyashiProxyData {
+	version: integer;
+	requireVersion: string;
+	body: PoyashiProxyBPIInfo[];
+}
+
+/**
+ * Fetches Poyashi BPI's latest information and syncs it back to the seeds repository.
+ *
+ * @note This function doesn't actually touch or backsync our database at all. Infact,
+ * the commit that hits the seeds repo will result in a database-sync, meaning this
+ * will all "just work"(tm)
+ */
+export async function UpdatePoyashiData() {
+	const repo = await PullDatabaseSeeds();
+
+	logger.info("Fetching data from proxy...");
+	const data = (await fetch("https://proxy.poyashi.me/?type=bpi").then((r) =>
+		r.json()
+	)) as PoyashiProxyData;
+
+	logger.info("Fetched data.");
+
+	const iidxSongs = (await repo.ReadCollection("songs-iidx")) as SongDocument<"iidx">[];
+	const iidxCharts = (await repo.ReadCollection("charts-iidx")) as ChartDocument<
+		"iidx:SP" | "iidx:DP"
+	>[];
+
+	// Utility functions for finding matching charts.
+	function FindSongOnTitle(title: string) {
+		for (const data of iidxSongs) {
+			if (data.title === title || data.altTitles.includes(title)) {
+				return data;
+			}
+		}
+
+		return null;
+	}
+
+	function FindChartWithPTDFVersion(
+		songID: integer,
+		playtype: Playtypes["iidx"],
+		diff: Difficulties["iidx:SP" | "iidx:DP"],
+		version: GPTSupportedVersions["iidx:SP" | "iidx:DP"]
+	) {
+		for (const chart of iidxCharts) {
+			if (
+				chart.songID === songID &&
+				chart.playtype === playtype &&
+				chart.difficulty === diff &&
+				chart.versions.includes(version)
+			) {
+				return chart;
+			}
+		}
+
+		return null;
+	}
+
+	// The actual mutation.
+	for (const d of data.body) {
+		const res = difficultyResolve[d.difficulty];
+
+		if (!res) {
+			throw new Error(`Unknown difficulty ${d.difficulty}`);
+		}
+
+		const [playtype, diff] = res;
+
+		const tachiSong = FindSongOnTitle(d.title);
+
+		if (!tachiSong) {
+			logger.warn(`Cannot find song ${d.title}?`);
+			continue;
+		}
+
+		// current poyashi version is 29
+		const tachiChart = FindChartWithPTDFVersion(tachiSong.id, playtype, diff, "29");
+
+		if (!tachiChart) {
+			logger.warn(
+				`Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?`
+			);
+			continue;
+		}
+
+		const kavg = Number(d.avg);
+
+		if (kavg < 0) {
+			logger.warn(
+				`${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.`
+			);
+			continue;
+		}
+
+		if (d.removed) {
+			logger.info(`Skipping removed chart ${tachiSong.title}.`);
+			continue;
+		}
+
+		tachiChart.data.bpiCoefficient = d.coef === -1 || d.coef === undefined ? null : d.coef;
+		tachiChart.data.kaidenAverage = Number(d.avg);
+		tachiChart.data.worldRecord = Number(d.wr);
+	}
+
+	await repo.WriteCollection("charts-iidx", iidxCharts);
+
+	logger.info(`Finished applying BPI changes. Writing back.`);
+
+	await repo.CommitChangesBack(`BPI Update ${new Date().toISOString()}`);
+}
+
+if (require.main === module) {
+	UpdatePoyashiData();
+}

server/src/lib/setup/config.ts

@@ -66,6 +66,7 @@ export interface TachiServerConfig {
 	OPTIONS_ALWAYS_SUCCEEDS?: boolean;
 	USE_EXTERNAL_SCORE_IMPORT_WORKER?: boolean;
 	EXTERNAL_SCORE_IMPORT_WORKER_CONCURRENCY?: integer;
+	SEEDS_URL: string | null;
 	EMAIL_CONFIG?: {
 		FROM: string;
 		DKIM?: SendMailOptions["dkim"];
@@ -194,6 +195,7 @@ const err = p(config, {
 			}
 		),
 	},
+	SEEDS_URL: "?string",
 });
 
 if (err) {

server/src/scripts/sync-database.ts

@@ -1,21 +1,19 @@
 /* eslint-disable no-await-in-loop */
 // This script syncs this tachi instances database up with the tachi-database-seeds.
 
-import { execSync } from "child_process";
-import { monkDB } from "external/mongo/db";
+import db, { monkDB } from "external/mongo/db";
 import fjsh from "fast-json-stable-hash";
-import fs from "fs";
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
 import CreateLogCtx, { KtLogger } from "lib/logger/logger";
 import UpdateIsPrimaryStatus from "lib/score-mutation/update-isprimary";
-import { Environment, TachiConfig } from "lib/setup/config";
+import { TachiConfig } from "lib/setup/config";
 import { BulkWriteOperation } from "mongodb";
 import { ICollection } from "monk";
-import os from "os";
-import path from "path";
 import {
 	BMSCourseDocument,
 	ChartDocument,
 	FolderDocument,
+	Game,
 	SongDocument,
 	TableDocument,
 } from "tachi-common";
@@ -127,26 +125,49 @@ async function GenericUpsert<T>(
 
 const syncInstructions: SyncInstructions[] = [
 	{
-		pattern: /^charts-(usc|bms)$/u,
+		pattern: /^charts-bms$/u,
 		handler: async (
 			charts: ChartDocument[],
 			collection: ICollection<ChartDocument>,
 			logger
 		) => {
-			// Since the USC and BMS databases are managed Bokutachi-side, we
-			// shouldn't be honoring any sort of updates from tachi-database-seeds
-			// aside from an initial one.
-			//
-			// However, in practice, these syncs are the best way to actually update
-			// issues in the database.
-			// We're going to disable this anyway.
-			// const isInitial = (await collection.findOne()) === null;
-
 			const r = await GenericUpsert(charts, collection, "chartID", logger, false);
 
 			if (r.thingsChanged) {
 				await InitaliseFolderChartLookup();
 				await UpdateIsPrimaryStatus();
+
+				const largestBMSSongID = await db.songs.bms.findOne(
+					{},
+					{
+						sort: {
+							id: -1,
+						},
+					}
+				);
+
+				if (!largestBMSSongID) {
+					logger.severe(
+						`No BMS charts loaded, yet BMS sync was attempted? Lost state on bms-song-id counter. Panicking.`,
+						r
+					);
+					throw new Error(`No BMS charts loaded, yet BMS sync was attempted.`);
+				}
+
+				await db.counters.update(
+					{
+						counterName: "bms-song-id",
+					},
+					{
+						$set: {
+							value: largestBMSSongID.id + 1,
+						},
+					}
+				);
+
+				await RecalcAllScores({
+					chartID: { $in: r.changedFields },
+				});
 			}
 		},
 	},
@@ -172,7 +193,7 @@ const syncInstructions: SyncInstructions[] = [
 	{
 		pattern: /^songs-/u,
 		handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
-			const r = await GenericUpsert(songs, collection, "id", logger);
+			const r = await GenericUpsert(songs, collection, "id", logger, true);
 
 			if (r.thingsChanged) {
 				await RecalcAllScores({
@@ -213,34 +234,18 @@ const syncInstructions: SyncInstructions[] = [
 const logger = CreateLogCtx("Database Sync");
 
 async function SynchroniseDBWithSeeds() {
-	const seedsDir = fs.mkdtempSync(path.join(os.tmpdir(), "tachi-database-seeds-"));
-
-	logger.info(`Cloning data to ${seedsDir}.`);
-
-	fs.rmSync(seedsDir, { recursive: true, force: true });
-
 	// Wait for mongo to connect first.
 	await monkDB.then(() => void 0);
 
-	execSync(
-		`git clone https://github.com/TNG-dev/tachi-database-seeds -b "${
-			Environment.nodeEnv === "production" ? "master" : "develop"
-		}" --depth=1 "${seedsDir}"`,
-		{
-			stdio: "inherit",
-		}
-	);
+	const databaseSeedsRepo = await PullDatabaseSeeds();
 
-	const collections = fs.readdirSync(path.join(seedsDir, "collections"));
-
-	for (const jsonName of collections) {
-		const collectionName = path.parse(jsonName).name;
+	for await (const { collectionName, data } of databaseSeedsRepo.IterateCollections()) {
 		const spawnLogger = CreateLogCtx(`${collectionName} Sync`);
 
 		if (collectionName.startsWith("songs-") || collectionName.startsWith("charts-")) {
 			const game = collectionName.split("-")[1];
 
-			if (!TachiConfig.GAMES.includes(game as any)) {
+			if (!TachiConfig.GAMES.includes(game as Game)) {
 				spawnLogger.verbose(
 					`Skipping ${collectionName} (${game}) as it isn't for ${TachiConfig.NAME}.`
 				);
@@ -248,12 +253,6 @@ async function SynchroniseDBWithSeeds() {
 			}
 		}
 
-		spawnLogger.verbose(`Getting data.`);
-
-		let data = JSON.parse(
-			fs.readFileSync(path.join(seedsDir, "collections", jsonName), "utf-8")
-		);
-
 		spawnLogger.verbose(`Found ${data.length} documents.`);
 
 		let matchedSomething = false;
@@ -266,9 +265,6 @@ async function SynchroniseDBWithSeeds() {
 			}
 		}
 
-		// free memory forcefully
-		data = null;
-
 		if (!matchedSomething) {
 			spawnLogger.warn(
 				`Collection ${collectionName} didn't match any sync instructions. Skipping.`

server/src/utils/misc.ts

@@ -1,3 +1,4 @@
+import { exec } from "child_process";
 import crypto from "crypto";
 import { ONE_HOUR } from "lib/constants/time";
 import { TachiConfig } from "lib/setup/config";
@@ -161,3 +162,24 @@ export function OmitUndefinedKeys<T>(obj: Partial<T>): Partial<T> {
 
 	return omittedObj;
 }
+
+/**
+ * Exec shellcode asynchronously.
+ *
+ * **DO NOT PASS USER INPUT INTO THIS FUNCTION!**
+ * **THIS EVALS A STRING AS BASH!**
+ * @param command A bash command to execute on the system.
+ * @returns stdout and stderr as strings.
+ */
+export function asyncExec(command: string) {
+	return new Promise<{ stdout: string; stderr: string }>((resolve, reject) => {
+		exec(command, (err, stdout, stderr) => {
+			if (err) {
+				// eslint-disable-next-line prefer-promise-reject-errors
+				return reject({ stdout, stderr, err });
+			}
+
+			return resolve({ stdout, stderr });
+		});
+	});
+}

server/tsconfig.json

@@ -31,11 +31,10 @@
 		]
 	},
 	"include": [
-		"src/**/*.ts"
+		"src/**/*.ts",
+		"src/lib/jobs/inline-job-runner"
 	],
 	"exclude": [
-		"node_modules",
-		// "src/test-utils",
-		// "src/**/*.test.ts"
+		"node_modules"
 	]
 }