Create Database Migration Scripts

server/scripts/__JSON-TO-KTDB/beatoraja/import.ts

@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
 import CreateLogCtx from "../../../src/common/logger";
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 const logger = CreateLogCtx(__filename);
 
 const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8"));

server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts

@@ -3,7 +3,7 @@
 
 import fs from "fs";
 import path from "path";
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 import { MDBJson } from "./types";
 import { Difficulties, Playtypes, ChartDocument } from "tachi-common";
 import CreateLogCtx from "../../../src/common/logger";

server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts

@@ -1,60 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { ChartDocument, SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
-import { gameOrders } from "tachi-common/js/config";
-import { oldKTDB } from "./old-db";
-
-const logger = CreateLogCtx(__filename);
-
-async function ConvertFn(c: any): Promise<ChartDocument<"gitadora:Gita" | "gitadora:Dora">> {
-	const song = (await db.songs.gitadora.findOne({
-		id: c.id,
-	})) as SongDocument<"gitadora">;
-
-	const oldSong = await oldKTDB.get("songs-gitadora").findOne({
-		id: c.id,
-	});
-
-	if (!song) {
-		logger.severe(`Cannot find song with ID ${c.id}?`);
-		throw new Error(`Cannot find song with ID ${c.id}?`);
-	}
-
-	const newChartDoc: ChartDocument<"gitadora:Gita" | "gitadora:Dora"> = {
-		rgcID: null,
-		chartID: c.chartID,
-		difficulty: c.difficulty,
-		songID: c.id,
-		playtype: c.playtype,
-		levelNum: c.levelNum,
-		level: c.level.toString(),
-		flags: {
-			"IN BASE GAME": true,
-			OMNIMIX: true,
-		},
-		data: {
-			inGameID: c.internals.inGameINTID,
-		},
-		isPrimary: true,
-		versions: [], // sentinel
-	};
-
-	const idx = gameOrders.gitadora.indexOf(song.firstVersion!);
-
-	if (idx === -1) {
-		logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
-		newChartDoc.versions = [song.firstVersion!];
-	} else {
-		newChartDoc.versions = gameOrders.gitadora.slice(idx);
-	}
-
-	return newChartDoc;
-}
-
-(async () => {
-	await MigrateRecords(db.charts.gitadora, "charts-gitadora", ConvertFn);
-
-	process.exit(0);
-})();

server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts

@@ -1,53 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
-import { gameOrders } from "tachi-common/js/config";
-
-const logger = CreateLogCtx(__filename);
-
-async function ConvertFn(c: any): Promise<ChartDocument<"museca:Single">> {
-	const song = await db.songs.museca.findOne({
-		id: c.id,
-	});
-
-	if (!song) {
-		logger.severe(`Cannot find song with ID ${c.id}?`);
-		throw new Error(`Cannot find song with ID ${c.id}?`);
-	}
-
-	const newChartDoc: ChartDocument<"museca:Single"> = {
-		rgcID: null,
-		chartID: c.chartID,
-		difficulty: c.difficulty,
-		songID: c.id,
-		playtype: c.playtype,
-		levelNum: c.levelNum,
-		level: c.level.toString(),
-		flags: {
-			"IN BASE GAME": true,
-			OMNIMIX: true, // false, needs to be overrode for 3 or so charts
-		},
-		data: {},
-		isPrimary: true,
-		versions: [], // sentinel
-	};
-
-	const idx = gameOrders.museca.indexOf(song.firstVersion!);
-
-	if (idx === -1) {
-		logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
-		newChartDoc.versions = [song.firstVersion!];
-	} else {
-		newChartDoc.versions = gameOrders.museca.slice(idx);
-	}
-
-	return newChartDoc;
-}
-
-(async () => {
-	await MigrateRecords(db.charts.museca, "charts-museca", ConvertFn);
-
-	process.exit(0);
-})();

server/scripts/__KT_DATABASE_MIGRATION/charts/charts-chunithm.ts

@@ -1,9 +1,9 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
+import MigrateRecords from "../migrate";
 import { gameOrders } from "tachi-common/js/config";
 
 const logger = CreateLogCtx(__filename);

server/scripts/__KT_DATABASE_MIGRATION/charts/charts-ddr.ts

@@ -1,11 +1,11 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { ChartDocument, SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import MigrateRecords from "../migrate";
 import { gameOrders } from "tachi-common/js/config";
-import { oldKTDB } from "./old-db";
+import { oldKTDB } from "../old-db";
 
 const logger = CreateLogCtx(__filename);
 

server/scripts/__KT_DATABASE_MIGRATION/charts/charts-iidx.ts

@@ -1,20 +1,32 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
-import { gameOrders } from "tachi-common/js/config";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import MigrateRecords from "../migrate";
 
 const logger = CreateLogCtx(__filename);
 
-async function ConvertFn(c: any): Promise<ChartDocument<"iidx:SP" | "iidx:DP">> {
+function Get2DXTraSet(flags: any) {
+	if (flags["All Scratch"]) {
+		return "All Scratch";
+	} else if (flags.Kiraku) {
+		return "Kiraku";
+	} else if (flags.Kichiku) {
+		return "Kichiku";
+	}
+
+	return null;
+}
+
+async function ConvertFn(c: any): Promise<ChartDocument<"iidx:SP" | "iidx:DP"> | null> {
 	const song = await db.songs.iidx.findOne({
 		id: c.id,
 	});
 
 	if (!song) {
-		logger.severe(`Cannot find song with ID ${c.id}?`);
+		logger.warn(`Cannot find song with ID ${c.id}?`);
+		return null;
 		throw new Error(`Cannot find song with ID ${c.id}?`);
 	}
 
@@ -26,29 +38,17 @@ async function ConvertFn(c: any): Promise<ChartDocument<"iidx:SP" | "iidx:DP">>
 		playtype: c.playtype,
 		levelNum: c.levelNum,
 		level: c.level.toString(),
-		flags: {
-			"IN BASE GAME": !!c.flags["IN BASE GAME"],
-			OMNIMIX: !!c.flags.OMNIMIX,
-			"N-1": !!c.flags["N-1"],
-		},
 		data: {
 			inGameID: c.internals.inGameINTID,
 			notecount: c.notedata.notecount,
 			arcChartID: null,
+			hashSHA256: c.internals.checksum2DXtra,
+			"2dxtraSet": Get2DXTraSet(c.flags),
 		},
 		isPrimary: true,
-		versions: [], // sentinel
+		versions: [], // handled by scripts/single-use/fix-iidx-versions atm
 	};
 
-	const idx = gameOrders.iidx.indexOf(song.firstVersion!);
-
-	if (idx === -1) {
-		logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
-		newChartDoc.versions = [song.firstVersion!];
-	} else {
-		newChartDoc.versions = gameOrders.iidx.slice(idx);
-	}
-
 	return newChartDoc;
 }
 

server/scripts/__KT_DATABASE_MIGRATION/charts/charts-museca.ts

@@ -0,0 +1,41 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { ChartDocument } from "tachi-common";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import MigrateRecords from "../migrate";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ChartDocument<"museca:Single"> | null> {
+	const song = await db.songs.museca.findOne({
+		id: c.id,
+	});
+
+	// this is a warn instead of severe because some songs have charts but no songs
+	// presumably removed stuff like brain power?
+	if (!song) {
+		logger.warn(`Cannot find song with ID ${c.id}?`);
+		return null;
+	}
+
+	const newChartDoc: ChartDocument<"museca:Single"> = {
+		rgcID: null,
+		chartID: c.chartID,
+		difficulty: c.difficulty,
+		songID: c.id,
+		playtype: c.playtype,
+		levelNum: c.levelNum,
+		level: c.level.toString(),
+		data: {},
+		isPrimary: true,
+		versions: ["1+1/2"],
+	};
+
+	return newChartDoc;
+}
+
+(async () => {
+	await MigrateRecords(db.charts.museca, "charts-museca", ConvertFn);
+
+	process.exit(0);
+})();

server/scripts/__KT_DATABASE_MIGRATION/charts/charts-sdvx.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 import { gameOrders } from "tachi-common/js/config";

server/scripts/__KT_DATABASE_MIGRATION/folders.ts

@@ -2,7 +2,7 @@
 
 import { FolderDocument, Game } from "tachi-common";
 import { validPlaytypes, gameHuman } from "tachi-common/js/config";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 import crypto from "crypto";

server/scripts/__KT_DATABASE_MIGRATION/goals.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { GoalDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 

server/scripts/__KT_DATABASE_MIGRATION/migrate.ts

@@ -4,45 +4,60 @@ import { oldKTDB } from "./old-db";
 
 const logger = CreateLogCtx(__filename);
 
-const newDocuments: unknown[] = [];
-
 export default async function MigrateRecords(
 	existingCollection: ICollection,
 	collectionName: string,
-	HandlerFN: (c: unknown) => unknown
+	HandlerFN: (c: unknown) => unknown,
+	filter: Record<string, any> = {},
+	force = false
 ) {
 	const existingRecords = await existingCollection.count({});
 	logger.info(`Starting migration for ${collectionName}...`);
 
 	if (existingRecords > 0) {
-		logger.error(`${existingRecords} documents already exist in db, terminating.`);
-		process.exit(1);
+		if (force) {
+			logger.warn(
+				`Forcing migration of documents - appending to ${existingRecords} existing records.`
+			);
+		} else {
+			logger.error(`${existingRecords} documents already exist in db, terminating.`);
+			process.exit(1);
+		}
 	}
 
 	let i = 0;
+	const bucketSize = 10_000;
 
-	await oldKTDB
-		.get(collectionName)
-		.find({})
-		// @ts-expect-error it exists.
-		.each(async (c, { pause, resume }) => {
-			i++;
-			pause();
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		// eslint-disable-next-line no-await-in-loop
+		const docs = await oldKTDB.get(collectionName).find(filter, { limit: bucketSize, skip: i });
 
-			if (i % 10000 === 0) {
-				logger.info(`Processed ${i} documents.`);
-			}
-			const newDoc = await HandlerFN(c);
+		logger.info(`Migrating ${i} - ${i + docs.length} documents.`);
 
-			if (newDoc !== null) {
-				newDocuments.push(newDoc);
-			}
+		if (docs.length === 0) {
+			break;
+		}
 
-			resume();
-		});
+		// eslint-disable-next-line no-await-in-loop
+		const newDocs = await Promise.all(docs.map(HandlerFN));
+
+		i += bucketSize;
+
+		const notNulls = newDocs.filter((e) => e !== null);
+
+		logger.info(`Success: ${notNulls.length}, Failures: ${newDocs.length - notNulls.length}.`);
+
+		// eslint-disable-next-line no-await-in-loop
+		await existingCollection.insert(notNulls);
+
+		logger.info(`Migrated ${docs.length} documents.`);
+
+		if (docs.length < bucketSize) {
+			break;
+		}
+	}
 
-	logger.info(`Inserting ${newDocuments.length} documents.`);
-	await existingCollection.insert(newDocuments.flat());
 	logger.info(`Done!`);
 
 	process.exit(0);

server/scripts/__KT_DATABASE_MIGRATION/old-db.ts

@@ -1,3 +1,18 @@
 import monk from "monk";
+import { ServerConfig } from "lib/setup/config";
+import CreateLogCtx from "lib/logger/logger";
 
-export const oldKTDB = monk(`${process.env.MONGO_BASE_URL}/kamaitachidb`);
+const logger = CreateLogCtx(__filename);
+
+logger.info(`Connecting to Old KTDB at ${ServerConfig.MONGO_CONNECTION_URL}/kamaitachidb.`);
+
+export const oldKTDB = monk(`${ServerConfig.MONGO_CONNECTION_URL}/kamaitachidb`);
+
+oldKTDB
+	.then(() => {
+		logger.info(`Done.`);
+	})
+	.catch((err) => {
+		logger.crit(`Fatal in connecting to old KT DB.`, { err });
+		process.exit(1);
+	});

server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts

@@ -1,4 +1,4 @@
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 import { TierlistDataDocument } from "tachi-common";
 import { CalculateTierlistDataID } from "../../../src/common/tierlist";
 import { oldKTDB } from "../old-db";

server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts

@@ -0,0 +1,138 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
+	if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
+		base[baseProp] = other[otherProp];
+	}
+}
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+	const playtype = c.scoreData.playtype;
+
+	const oldChartDoc = await oldKTDB.get("charts-bms").findOne({
+		chartID: c.chartID,
+	});
+
+	if (!oldChartDoc) {
+		logger.warn(`Cannot find oldChartDoc for ${c.chartID}`);
+	}
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		"data.hashMD5": oldChartDoc.internals.hash,
+	});
+
+	if (!chartDoc) {
+		const oldSong = await oldKTDB.get("songs-bms").findOne({ id: c.songID });
+		logger.warn(`Cannot find ChartDoc for ${oldSong.title} ${oldChartDoc.level} ${playtype}`);
+		return null;
+		throw new Error(`Cannot find ChartDoc for ${c.scoreID} (${c.chartID})`);
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {
+			random: c.scoreMeta?.optionsRandom ?? null,
+			inputDevice: c.scoreMeta?.inputDevice ?? null,
+		},
+	};
+
+	// @ts-expect-error shut
+	if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
+		// @ts-expect-error shut
+		base.scoreData.hitMeta.bp = null;
+	}
+
+	// @ts-expect-error shut
+	if (base.scoreData.hitMeta.gauge === -1) {
+		// @ts-expect-error shut
+		base.scoreData.hitMeta.gauge = null;
+	}
+
+	// @ts-expect-error shut
+	delete base.scoreData.hitMeta.inputDevice;
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	await oldKTDB.get("score-id-lookup").insert({
+		old: c.scoreID,
+		new: scoreID,
+	});
+
+	return score;
+}
+
+export async function MigrateBMSScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "bms" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateBMSScores();
+		process.exit(0);
+	})();
+}

server/scripts/__KT_DATABASE_MIGRATION/scores/ddr.ts

@@ -0,0 +1,101 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { ScoreDocument, GetGamePTConfig, StaticConfig } from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		chartID: c.chartID,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
+		return null;
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument<"ddr:SP" | "ddr:DP">, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {},
+	};
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	try {
+		await oldKTDB.get("score-id-lookup").insert({
+			old: c.scoreID,
+			new: scoreID,
+		});
+	} catch (err) {
+		// whoops
+	}
+
+	return score;
+}
+
+export async function MigrateDDRScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "ddr" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateDDRScores();
+		process.exit(0);
+	})();
+}

server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts

@@ -0,0 +1,159 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
+	if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
+		base[baseProp] = other[otherProp];
+	}
+}
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	const game = c.game;
+	const playtype = c.scoreData.playtype;
+
+	const gameConfig = GetGameConfig(game);
+
+	if (!gameConfig.validPlaytypes.includes(playtype)) {
+		logger.warn(`Ignored game pt ${game}, ${playtype}`);
+		throw new Error(`Ignored game pt ${game}, ${playtype}`);
+	}
+
+	const gptConfig = GetGamePTConfig(game, playtype);
+
+	if (!gptConfig) {
+		logger.error(`No gptconfig for ${game} ${playtype}?`);
+		throw new Error(`No gptconfig for ${game} ${playtype}?`);
+	}
+
+	if (c.songID === 1819) {
+		c.songID = 1121; // praludium has two songs in kt1 for some reason (tricoro 6)
+		// this overrides that
+	}
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		songID: c.songID,
+		difficulty: c.scoreData.difficulty,
+		playtype,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.scoreData.difficulty} ${playtype} `);
+		return null;
+		throw new Error(`Cannot find ChartDoc for ${c.scoreID} (${c.chartID})`);
+	}
+
+	const base: Omit<ScoreDocument, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {
+			random: c.scoreMeta?.optionsRandom ?? null,
+			gauge: c.scoreMeta?.optionsGauge ?? null,
+			assist: c.scoreMeta?.optionsAssist ?? null,
+			range: c.scoreMeta?.optionsRange ?? null,
+		},
+	};
+
+	if (c.scoreMeta) {
+		// @ts-expect-error this game is iidx
+		if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
+			// @ts-expect-error this game is iidx
+			base.scoreMeta.random = null;
+		}
+
+		// @ts-expect-error asdf
+		if (base.scoreMeta.range === "") {
+			// @ts-expect-error asdf
+			base.scoreMeta.range = "NONE";
+		}
+	}
+
+	// @ts-expect-error shut
+	if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
+		// @ts-expect-error shut
+		base.scoreData.hitMeta.bp = null;
+	}
+
+	// @ts-expect-error nope
+	if (base.scoreData.hitMeta.gauge > 200) {
+		// @ts-expect-error nope
+		base.scoreData.hitMeta.gauge = null;
+	}
+
+	// @ts-expect-error nope
+	if (base.scoreData.hitMeta.gaugeHistory) {
+		// @ts-expect-error nope
+		base.scoreData.hitMeta.gaugeHistory = base.scoreData.hitMeta.gaugeHistory.map((e) =>
+			e > 200 ? null : e
+		);
+	}
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	await oldKTDB.get("score-id-lookup").insert({
+		old: c.scoreID,
+		new: scoreID,
+	});
+
+	return score;
+}
+
+export async function MigrateIIDXScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "iidx" });
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateIIDXScores();
+		process.exit(0);
+	})();
+}

server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts

@@ -0,0 +1,15 @@
+import CreateLogCtx from "lib/logger/logger";
+import { MigrateBMSScores } from "./bms";
+import { MigrateIIDXScores } from "./iidx";
+
+const logger = CreateLogCtx(__filename);
+
+(async () => {
+	logger.info(`Migrating IIDX Scores.`);
+
+	await MigrateIIDXScores();
+
+	logger.info(`Migrating BMS Scores.`);
+
+	await MigrateBMSScores();
+})();

server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts

@@ -0,0 +1,107 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		chartID: c.chartID,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
+		return null;
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument<"museca:Single">, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {},
+	};
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	try {
+		await oldKTDB.get("score-id-lookup").insert({
+			old: c.scoreID,
+			new: scoreID,
+		});
+	} catch (err) {
+		// whoops
+	}
+
+	return score;
+}
+
+export async function MigrateMusecaScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "museca" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateMusecaScores();
+		process.exit(0);
+	})();
+}

server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts

@@ -1,41 +1,74 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-import { PrivateUserDocument, ScoreDocument } from "tachi-common";
-import { grades, lamps, validPlaytypes } from "tachi-common/js/config";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
-import CreateLogCtx from "../../src/common/logger";
-import { CreateScoreID } from "../../src/score-import/framework/common/score-utils";
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
 
 const logger = CreateLogCtx(__filename);
 
-// HERE. WE. GO!
-
 function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
 	if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
 		base[baseProp] = other[otherProp];
 	}
 }
 
-function ConvertFn(c: any): ScoreDocument | null {
-	if (!supportedGames.includes(c.game)) {
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
 		logger.warn(`Ignored game ${c.game}`);
 		return null;
 	}
+	const game = c.game;
+	const playtype = c.scoreData.playtype;
 
 	if (["popn", "jubeat", "gitadora", "usc"].includes(c.game)) {
 		return null;
 	}
 
-	if (validPlaytypes[c.game].includes(c.playtype)) {
-		logger.warn(`Ignored game pt ${c.game}, ${c.playtype}`);
+	const gameConfig = GetGameConfig(game);
+
+	if (!gameConfig.validPlaytypes.includes(playtype)) {
+		logger.warn(`Ignored game pt ${game}, ${playtype}`);
+		throw new Error(`Ignored game pt ${game}, ${playtype}`);
+	}
+
+	const gptConfig = GetGamePTConfig(game, playtype);
+
+	if (!gptConfig) {
+		logger.error(`No gptconfig for ${game} ${playtype}?`);
+		throw new Error(`No gptconfig for ${game} ${playtype}?`);
+	}
+
+	if (game === "iidx" && c.songID === 1819) {
+		c.songID = 1121; // praludium has two songs in kt1 for some reason (tricoro 6)
+		// this overrides that
+	}
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		songID: c.songID,
+		difficulty: c.scoreData.difficulty,
+		playtype,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.scoreData.difficulty} ${playtype} `);
+		return null;
+		throw new Error(`Cannot find ChartDoc for ${c.scoreID} (${c.chartID})`);
 	}
 
 	const base: Omit<ScoreDocument, "scoreID"> = {
 		userID: c.userID,
-		songID: c.songID,
-		playtype: c.scoreData.playtype,
-		chartID: c.chartID,
+		songID: chartDoc.songID,
+		playtype,
+		chartID: chartDoc.chartID,
 		game: c.game,
 		timeAdded: c.timeAdded,
 		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
@@ -43,16 +76,18 @@ function ConvertFn(c: any): ScoreDocument | null {
 		highlight: c.highlight ?? false,
 		service: c.service,
 		importType: null,
-		calculatedData: c.calculatedData,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
 		scoreData: {
 			esd: c.scoreData.esd ?? null,
 			grade: c.scoreData.grade,
-			gradeIndex: grades[c.game].indexOf(c.scoreData.grade),
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
 			lamp: c.scoreData.lamp,
-			lampIndex: lamps[c.game].indexOf(c.scoreData.lamp),
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
 			percent: c.scoreData.percent,
 			score: c.scoreData.score,
-			judgements: c.scoreData.judgements,
+			judgements: c.scoreData.hitData,
 			hitMeta: c.scoreData.hitMeta,
 		},
 		scoreMeta: {},
@@ -68,7 +103,9 @@ function ConvertFn(c: any): ScoreDocument | null {
 			ConditionalAssign(base.scoreMeta, "pacemakerName", c.scoreMeta, "pacemakerName");
 			ConditionalAssign(base.scoreMeta, "pacemakerTarget", c.scoreMeta, "pacemakerTarget");
 
+			// @ts-expect-error this game is iidx
 			if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
+				// @ts-expect-error this game is iidx
 				base.scoreMeta.random = null;
 			}
 
@@ -119,7 +156,7 @@ function ConvertFn(c: any): ScoreDocument | null {
 		delete base.scoreData.judgements.error;
 	} else if (base.game === "chunithm") {
 		if (base.scoreData.percent === 101) {
-			base.scoreData.grade === "SSS";
+			base.scoreData.grade = "SSS";
 		}
 	}
 

server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts

@@ -0,0 +1,105 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		chartID: c.chartID,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
+		return null;
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument<"sdvx:Single">, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {
+			inSkillAnalyser: null,
+		},
+	};
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	await oldKTDB.get("score-id-lookup").insert({
+		old: c.scoreID,
+		new: scoreID,
+	});
+
+	return score;
+}
+
+export async function MigrateSDVXScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "sdvx" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateSDVXScores();
+		process.exit(0);
+	})();
+}

server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts

@@ -1,27 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
-
-function ConvertFn(c: any): SongDocument<"gitadora"> {
-	const newSongDoc: SongDocument<"gitadora"> = {
-		title: c.title,
-		artist: c.artist,
-		id: c.id,
-		firstVersion: c.firstAppearance,
-		"alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [],
-		"search-titles": c["search-titles"]
-			.filter((e: string) => !!e)
-			.map((e: string) => e.toString())
-			.filter((e: string) => e !== c.title),
-		data: {},
-	};
-
-	return newSongDoc;
-}
-
-(async () => {
-	await MigrateRecords(db.songs.gitadora, "songs-gitadora", ConvertFn);
-
-	process.exit(0);
-})();

server/scripts/__KT_DATABASE_MIGRATION/songs/songs-chunithm.ts

@@ -1,8 +1,8 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
 
 function ConvertFn(c: any): SongDocument<"chunithm"> {
 	const newSongDoc: SongDocument<"chunithm"> = {

server/scripts/__KT_DATABASE_MIGRATION/songs/songs-ddr.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
 import { gameOrders } from "tachi-common/js/config";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import MigrateRecords from "./migrate";
 
 function ConvertFn(c: any): SongDocument<"ddr"> {

server/scripts/__KT_DATABASE_MIGRATION/songs/songs-iidx.ts

@@ -1,8 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
-import { gameOrders } from "tachi-common/js/config";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
 
 function ConvertFn(c: any): SongDocument<"iidx"> {
 	const newSongDoc: SongDocument<"iidx"> = {

server/scripts/__KT_DATABASE_MIGRATION/songs/songs-museca.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
 
 function ConvertFn(c: any): SongDocument<"museca"> {
 	const newSongDoc: SongDocument<"museca"> = {

server/scripts/__KT_DATABASE_MIGRATION/songs/songs-sdvx.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import MigrateRecords from "./migrate";
 
 function ConvertFn(c: any): SongDocument<"sdvx"> {

server/scripts/__KT_DATABASE_MIGRATION/state-sync/create-game-profiles.ts

@@ -0,0 +1,43 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { UpdateUsersGamePlaytypeStats } from "lib/score-import/framework/user-game-stats/update-ugs";
+import { FormatUserDoc } from "utils/user";
+
+const logger = CreateLogCtx(__filename);
+
+export async function CreateGameProfiles() {
+	const users = await db.users.find({});
+
+	for (const user of users) {
+		const gpts = await db.scores.aggregate([
+			{
+				$match: {
+					userID: user.id,
+				},
+			},
+			{
+				$group: {
+					_id: {
+						game: "$game",
+						playtype: "$playtype",
+					},
+				},
+			},
+		]);
+
+		logger.info(`Found ${gpts.length} GPTs for ${FormatUserDoc(user)}`);
+
+		for (const gpt of gpts) {
+			const { game, playtype } = gpt._id;
+			logger.info(`Updating ${FormatUserDoc(user)}'s ${game} ${playtype} stats.`);
+			await UpdateUsersGamePlaytypeStats(game, playtype, user.id, null, logger);
+		}
+	}
+
+	logger.info(`Done.`);
+}
+
+if (require.main === module) {
+	CreateGameProfiles().then(() => process.exit(0));
+}

server/scripts/__KT_DATABASE_MIGRATION/state-sync/dedupe-score-ids.ts

@@ -0,0 +1,36 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+async function DedupeScoreIDs() {
+	const dups = await db.scores.aggregate(
+		[
+			{
+				$group: {
+					_id: "$scoreID",
+					dups: { $addToSet: "$_id" },
+					count: { $sum: 1 },
+				},
+			},
+			{
+				$match: {
+					count: { $gt: 1 },
+				},
+			},
+		],
+		{ allowDiskUse: true }
+	);
+
+	logger.info(`Found ${dups.length} dups.`);
+
+	for (const dup of dups) {
+		dup.dups.shift();
+		await db.scores.remove({ _id: { $in: dup.dups } });
+	}
+
+	logger.info(`Done.`);
+	process.exit(0);
+}
+
+DedupeScoreIDs();

server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts

@@ -0,0 +1,47 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { CreateCalculatedData } from "lib/score-import/framework/calculated-data/calculated-data";
+import { EfficientDBIterate } from "../../util/efficient-db-iterate";
+
+const logger = CreateLogCtx(__filename);
+
+export async function RecalcAllScores(filter = {}) {
+	logger.info(`Recalcing All Scores.`);
+
+	await EfficientDBIterate(
+		db.scores,
+		async (c) => {
+			const chart = await db.charts[c.game].findOne({ chartID: c.chartID });
+
+			if (!chart) {
+				logger.error(`Can't find chartID ${c.chartID} ${c.scoreID} (${c.game})`, {
+					score: c,
+				});
+
+				throw new Error(`screwed`);
+			}
+			const calculatedData = await CreateCalculatedData(c, chart, c.scoreData.esd, logger);
+
+			return { scoreID: c.scoreID, calculatedData };
+		},
+		async (updates) => {
+			await db.scores.bulkWrite(
+				updates.map((e) => ({
+					updateOne: {
+						filter: {
+							scoreID: e.scoreID,
+						},
+						update: {
+							$set: {
+								calculatedData: e.calculatedData,
+							},
+						},
+					},
+				}))
+			);
+		},
+		filter
+	);
+
+	logger.info(`Done!`);
+}

server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts

@@ -0,0 +1,15 @@
+import { CreateGameProfiles } from "./create-game-profiles";
+import { UpdateAllPBs } from "./update-all-pbs";
+import { RecalcAllScores } from "./recalc-all-scores";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+(async () => {
+	await RecalcAllScores({ game: "ddr" });
+	await UpdateAllPBs({ game: "ddr" });
+	await CreateGameProfiles();
+
+	logger.info(`Completely done!`);
+	process.exit(0);
+})();

server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts

@@ -0,0 +1,25 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { ProcessPBs } from "lib/score-import/framework/pb/process-pbs";
+import { FormatUserDoc } from "utils/user";
+import deepmerge from "deepmerge";
+
+const logger = CreateLogCtx(__filename);
+
+export async function UpdateAllPBs(filter = {}) {
+	const allUsers = await db.users.find({});
+
+	for (const user of allUsers) {
+		logger.info(`Finding ${FormatUserDoc(user)}'s scores.`);
+
+		const scores = await db.scores.find(deepmerge({ userID: user.id }, filter), {
+			projection: { chartID: 1 },
+		});
+
+		logger.info(`PBing ${FormatUserDoc(user)}'s scores.`);
+		await ProcessPBs(user.id, new Set(scores.map((e) => e.chartID)), logger);
+	}
+
+	logger.info(`Done!`);
+}

server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { UserGoalDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 

server/scripts/__KT_DATABASE_MIGRATION/users.ts

@@ -1,5 +1,5 @@
 import { PrivateUserDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import MigrateRecords from "./migrate";
 
 function ConvertFn(c: any): PrivateUserDocument {
@@ -11,15 +11,9 @@ function ConvertFn(c: any): PrivateUserDocument {
 		customBanner: c.custombanner,
 		customPfp: c.custompfp,
 		email: c.email,
-		friends: c.friends,
 		id: c.id,
 		lastSeen: c.lastSeen,
 		password: c.password,
-		permissions: {},
-		settings: {
-			invisible: !!c.settings.invisible,
-			nsfwSplashes: !!c.settings.nsfwsplashes,
-		},
 		socialMedia: {
 			discord: c.socialmedia.discord || null,
 			twitter: c.socialmedia.twitter || null,
@@ -28,6 +22,9 @@ function ConvertFn(c: any): PrivateUserDocument {
 			github: c.socialmedia.github || null,
 			steam: c.socialmedia.steam || null,
 		},
+		joinDate: 0,
+		authLevel: "user",
+		badges: ["beta"],
 	};
 
 	return newUserDoc;

server/scripts/arc-db-sync/arc-db-merge.ts

@@ -1,7 +1,7 @@
 import fs from "fs";
 import { ICollection } from "monk";
 import path from "path";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import { ChartDocument } from "tachi-common";
 import { FindSongOnTitle } from "../../src/common/database-lookup/song";

server/scripts/bms-tables/table-sync.ts

@@ -0,0 +1,60 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { ChartDocument } from "tachi-common";
+import fetch from "node-fetch";
+
+// this seems to be all we care about
+interface TableJSONDoc {
+	md5: string;
+	title?: string;
+	level: string;
+}
+
+const logger = CreateLogCtx(__filename);
+
+async function ImportTableLevels(tableJSON: TableJSONDoc[], prefix: string) {
+	let failures = 0;
+	let success = 0;
+	const total = tableJSON.length;
+
+	for (const td of tableJSON) {
+		const chart = (await db.charts.bms.findOne({ "data.hashMD5": td.md5 })) as ChartDocument<
+			"bms:7K" | "bms:14K"
+		>;
+
+		if (!chart) {
+			logger.warn(
+				`No chart exists in table fro ${td.md5} Possible title: ${td.title} ${prefix}${td.level}`
+			);
+			failures++;
+			continue;
+		}
+
+		const tableFolders = chart.data.tableFolders.filter((e) => e.table !== prefix);
+
+		tableFolders.push({ table: prefix, level: td.level });
+
+		await db.charts.bms.update(
+			{
+				chartID: chart.chartID,
+			},
+			{
+				$set: {
+					"data.tableFolders": tableFolders,
+				},
+			}
+		);
+
+		success++;
+	}
+
+	logger.info(`Finished updating table ${prefix}.`);
+	logger.info(`${success} Success | ${failures} Failures | ${total} Total.`);
+}
+
+export async function UpdateTable(prefix: string, url: string) {
+	const tableJSON = await fetch(url).then((r) => r.json());
+
+	return ImportTableLevels(tableJSON, prefix);
+}

server/scripts/bms-tables/update.ts

@@ -0,0 +1,48 @@
+import { Command } from "commander";
+import { UpdateTable } from "./table-sync";
+import { BMS_TABLES } from "lib/constants/bms-tables";
+
+const program = new Command();
+program.option("-t, --table <table>");
+program.option("-a, --all");
+
+program.parse(process.argv);
+const options = program.opts();
+
+// I have no confidence in half of these links surviving.
+// Eventually, some of these are going to disappear, and we'll have to find
+// something else. probably.
+const registered: any = {
+	insane: ["http://www.ribbit.xyz/bms/tables/insane_body.json", BMS_TABLES.insane],
+	normal: ["http://www.ribbit.xyz/bms/tables/normal_body.json", BMS_TABLES.normal],
+	stella: ["https://stellabms.xyz/st/score.json", BMS_TABLES.stella],
+	satellite: ["https://stellabms.xyz/sl/score.json", BMS_TABLES.satellite],
+	dpSatellite: ["https://stellabms.xyz/dp/score.json", BMS_TABLES.satellite],
+	insane2: ["http://rattoto10.jounin.jp/js/insane_data.json", BMS_TABLES.insane2],
+	normal2: ["http://rattoto10.jounin.jp/js/score.json", BMS_TABLES.normal2],
+	overjoy: ["http://lr2.sakura.ne.jp/data/score.json", BMS_TABLES.overjoy],
+	dpInsane: ["http://dpbmsdelta.web.fc2.com/table/data/insane_data.json", BMS_TABLES.dpInsane],
+	dpNormal: ["http://dpbmsdelta.web.fc2.com/table/data/dpdelta_data.json", BMS_TABLES.dpNormal],
+};
+
+(async () => {
+	if (options.all) {
+		for (const table of Object.values(registered)) {
+			// eslint-disable-next-line no-await-in-loop
+			await UpdateTable((table as any)[1], (table as any)[0]);
+		}
+
+		process.exit(0);
+	} else {
+		const tables = Object.keys(registered);
+		if (!tables.includes(options.table)) {
+			throw new Error(
+				`Invalid table ${options.table} - expected any of ${tables.join(", ")}.`
+			);
+		}
+
+		await UpdateTable(registered[options.table][1], registered[options.table][0]);
+
+		process.exit(0);
+	}
+})();

server/scripts/iidx-tierlist/get-sp12-data.ts

@@ -1,6 +1,6 @@
 /* eslint-disable no-case-declarations */
 /* eslint-disable no-param-reassign */
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import fetch from "node-fetch";
 import { FindSongOnTitle } from "../../src/common/database-lookup/song";
 import { FindChartWithPTDF } from "../../src/common/database-lookup/chart";

server/scripts/iidx-tierlist/parse-iidxkr-data.ts

@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import fs from "fs";
 import path from "path";
 import { CalculateTierlistDataID } from "../../src/common/tierlist";

server/scripts/single-use/add-2dxtra-flag.ts

@@ -1,6 +1,6 @@
 // @ts-nocheck
 
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 
 db.charts.iidx
 	.find({})

server/scripts/single-use/add-folders-array.ts

@@ -0,0 +1,16 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+(async () => {
+	logger.info(`Setting tableFolders.`);
+	const r = await db.charts.bms.update(
+		{},
+		{ $set: { "data.tableFolders": [] } },
+		{ multi: true }
+	);
+	logger.info(`Done!`, { r });
+
+	process.exit(0);
+})();

server/scripts/single-use/add-hcnver.ts

@@ -2,7 +2,7 @@ import fs from "fs";
 import path from "path";
 import { PRUDENCE_CHART_SCHEMAS } from "../../src/db/schemas";
 import p from "prudence";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 
 const logger = CreateLogCtx(__filename);

server/scripts/single-use/fix-chuni-port.ts

@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 
 (async () => {
 	const charts = await db.charts.chunithm.find({});

server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts

@@ -1,28 +0,0 @@
-import db from "../../../src/db/db";
-import { ChartDocument } from "tachi-common";
-
-db.charts.iidx
-	.find({})
-	// @ts-expect-error it exists
-	.each(async (c: ChartDocument<"iidx:SP" | "iidx:DP">, { pause, resume }: any) => {
-		pause();
-
-		const song = (await db.songs.iidx.findOne({ id: c.songID }))!;
-
-		let versions = [];
-
-		if (song.firstVersion === "27") {
-			versions = ["27", "28", "27-omni", "28-omni"];
-		} else if (song.firstVersion === "28") {
-			versions = ["28", "28-omni"];
-		} else if (song.firstVersion === "inf") {
-			versions = ["inf"];
-		} else {
-			versions = ["26", "27", "28", "26-omni", "27-omni", "28-omni"];
-		}
-
-		await db.charts.iidx.update({ _id: c._id }, { $set: { versions } });
-
-		resume();
-	})
-	.then(() => console.log("done"));

server/scripts/single-use/fix-iidx-versions/stat.ts

@@ -1,20 +0,0 @@
-import CreateLogCtx from "../../../src/common/logger";
-import db from "../../../src/db/db";
-
-const logger = CreateLogCtx(__filename);
-
-(async () => {
-	const charts = await db.charts.iidx.find({ versions: { $size: 0 } });
-
-	for (const chart of charts) {
-		// eslint-disable-next-line no-await-in-loop
-		const song = await db.songs.iidx.findOne({ id: chart.songID });
-
-		if (!song) {
-			logger.error(`cannot find parent song ${chart.songID}`);
-			continue;
-		}
-
-		logger.info(`${song.title} ${chart.playtype} ${chart.difficulty}`);
-	}
-})();

server/scripts/single-use/fix-iidx-versions/update-versions.ts

@@ -1,9 +1,10 @@
 /* eslint-disable no-await-in-loop */
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 import fs from "fs";
 import path from "path";
-import CreateLogCtx from "../../../src/common/logger";
-import { FindSongOnTitleInsensitive } from "../../../src/common/database-lookup/song";
+import CreateLogCtx from "lib/logger/logger";
+import { FindSongOnTitleInsensitive } from "utils/queries/songs";
+
 const logger = CreateLogCtx(__filename);
 
 const comicallyLargeJSONData = JSON.parse(
@@ -78,7 +79,7 @@ function ConvertVersion(c: string) {
 			continue;
 		}
 
-		// logger.info(relevantVersions);q
+		// logger.info(relevantVersions);
 
 		const chart = await db.charts.iidx.findOne({ songID: song.id, difficulty, playtype });
 

server/scripts/single-use/fix-legacy-folder-format.ts

@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import { FolderDocument } from "tachi-common";
 
 (async () => {

server/scripts/single-use/fix-sdvx-judgements.ts

@@ -0,0 +1,30 @@
+import db from "external/mongo/db";
+
+(async () => {
+	await db.scores.update(
+		{ game: "sdvx", "scoreData.lamp": "ULTIMATE CHAIN" },
+		{
+			$set: {
+				"scoreData.judgements.miss": 0,
+			},
+		},
+		{
+			multi: true,
+		}
+	);
+
+	await db.scores.update(
+		{ game: "sdvx", "scoreData.lamp": "PERFECT ULTIMATE CHAIN" },
+		{
+			$set: {
+				"scoreData.judgements.miss": 0,
+				"scoreData.judgements.near": 0,
+			},
+		},
+		{
+			multi: true,
+		}
+	);
+
+	process.exit(0);
+})();

server/scripts/single-use/get-needs-humanise-sdvx.ts

@@ -0,0 +1,25 @@
+import db from "external/mongo/db";
+import fs from "fs";
+
+(async () => {
+	const all = await db.songs.sdvx.find({});
+
+	const humanise = [];
+	for (const h of all) {
+		if (!h.title.match(/^[a-zA-Z0-9 ().!?-_]*$/)) {
+			humanise.push(h);
+		}
+	}
+
+	fs.writeFileSync(
+		"needs-humanise.csv",
+		humanise
+			.map(
+				(e) =>
+					`${e.id}, "${e.title.replace('"', '\\"')}", "${e.artist.replace('"', '\\"')}"`
+			)
+			.join("\n")
+	);
+
+	process.exit(0);
+})();

server/scripts/single-use/numify-tableindex.ts

@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 
 (async () => {
 	db.folders

server/scripts/update-bpi-poyashi.ts

@@ -9,7 +9,7 @@ import {
 	FindChartWithPTDF,
 	FindChartWithPTDFVersion,
 } from "../src/score-import/database-lookup/chart-ptdf";
-import db from "../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../src/common/logger";
 const program = new Command();
 

server/scripts/util/efficient-db-iterate.ts

@@ -0,0 +1,35 @@
+/* eslint-disable no-await-in-loop */
+import CreateLogCtx from "lib/logger/logger";
+import { ICollection } from "monk";
+import { oldKTDB } from "../__KT_DATABASE_MIGRATION/old-db";
+
+const logger = CreateLogCtx(__filename);
+
+export async function EfficientDBIterate<T, R>(
+	collection: ICollection<T>,
+	callbackFn: (c: T) => Promise<R>,
+	saveOp: (c: R[]) => Promise<void>,
+	filter: any = {},
+	bucketSize = 10_000
+) {
+	let i = 0;
+
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		logger.info(`Migrating ${i} - ${i + bucketSize} documents.`);
+		// eslint-disable-next-line no-await-in-loop
+		const docs = await collection.find(filter, { limit: bucketSize, skip: i });
+
+		if (docs.length === 0) {
+			break;
+		}
+
+		const rDocs = await Promise.all(docs.map(callbackFn));
+
+		i += bucketSize;
+
+		await saveOp(rDocs);
+
+		logger.info(`Migrated ${i} documents.`);
+	}
+}

server/scripts/validate-database.ts

@@ -1,12 +1,11 @@
 import Prudence, { PrudenceSchema } from "prudence";
-import db from "../src/db/db";
-import { PRUDENCE_SCORE_SCHEMAS, STATIC_SCHEMAS } from "../src/db/schemas";
-import CreateLogCtx from "../src/common/logger";
+import db from "external/mongo/db";
+import { PRUDENCE_SCORE_SCHEMAS, STATIC_SCHEMAS } from "external/mongo/schemas";
+import CreateLogCtx from "lib/logger/logger";
 import fs from "fs";
 import path from "path";
 import { PrudenceError } from "prudence/js/error";
-import { validPlaytypes } from "tachi-common/js/config";
-import { CONF_INFO } from "../src/lib/setup/config";
+import { StaticConfig, GetGameConfig } from "tachi-common";
 
 const BASE_DIR = path.join(__dirname, "./validate-database-errs");
 
@@ -63,8 +62,9 @@ async function ValidateScores(): Promise<void> {
 	let globalTotal = 0;
 	let globalSuccess = 0;
 	let globalFail = 0;
-	for (const game of CONF_INFO.supportedGames) {
-		for (const playtype of validPlaytypes[game]) {
+	for (const game of StaticConfig.allSupportedGames) {
+		const gameConfig = GetGameConfig(game);
+		for (const playtype of gameConfig.validPlaytypes) {
 			// @ts-expect-error shut up
 			const schema = PRUDENCE_SCORE_SCHEMAS[game][playtype];
 
@@ -135,7 +135,7 @@ async function ValidateChartsOrSongs(type: "songs" | "charts"): Promise<void> {
 	let globalTotal = 0;
 	let globalSuccess = 0;
 	let globalFail = 0;
-	for (const game of CONF_INFO.supportedGames) {
+	for (const game of StaticConfig.allSupportedGames) {
 		const schema = STATIC_SCHEMAS[type]![game];
 
 		let successCount = 0;

server/src/lib/constants/bms-tables.ts

@@ -0,0 +1,11 @@
+export const BMS_TABLES = {
+	insane: "🟊",
+	overjoy: "🟊🟊",
+	normal: "☆",
+	normal2: "▽",
+	insane2: "▼",
+	stella: "st",
+	satellite: "sl",
+	dpNormal: "δ",
+	dpInsane: "★",
+};