From 2e4bd5515fc650d1b6637b5f04ef3cccba810995 Mon Sep 17 00:00:00 2001
From: zkldi <ktchidev@gmail.com>
Date: Sat, 21 Aug 2021 01:52:21 +0100
Subject: [PATCH] Create Database Migration Scripts

---
 .../__JSON-TO-KTDB/beatoraja/import.ts        |   2 +-
 .../scripts/__JSON-TO-KTDB/iidx/diff-sync.ts  |   2 +-
 .../charts-gitadora.ts                        |  60 -------
 .../__KT_DATABASE_MIGRATION/charts-museca.ts  |  53 ------
 .../{ => charts}/charts-chunithm.ts           |   4 +-
 .../{ => charts}/charts-ddr.ts                |   8 +-
 .../{ => charts}/charts-iidx.ts               |  42 ++---
 .../charts/charts-museca.ts                   |  41 +++++
 .../{ => charts}/charts-sdvx.ts               |   2 +-
 .../__KT_DATABASE_MIGRATION/folders.ts        |   2 +-
 .../scripts/__KT_DATABASE_MIGRATION/goals.ts  |   2 +-
 .../__KT_DATABASE_MIGRATION/migrate.ts        |  61 ++++---
 .../scripts/__KT_DATABASE_MIGRATION/old-db.ts |  17 +-
 .../other/museca-tierlist.ts                  |   2 +-
 .../__KT_DATABASE_MIGRATION/scores/bms.ts     | 138 +++++++++++++++
 .../__KT_DATABASE_MIGRATION/scores/ddr.ts     | 101 +++++++++++
 .../__KT_DATABASE_MIGRATION/scores/iidx.ts    | 159 ++++++++++++++++++
 .../scores/migrate-all.ts                     |  15 ++
 .../__KT_DATABASE_MIGRATION/scores/museca.ts  | 107 ++++++++++++
 .../{scores.ts => scores/reference.ts}        |  77 ++++++---
 .../__KT_DATABASE_MIGRATION/scores/sdvx.ts    | 105 ++++++++++++
 .../__KT_DATABASE_MIGRATION/songs-gitadora.ts |  27 ---
 .../{ => songs}/songs-chunithm.ts             |   4 +-
 .../{ => songs}/songs-ddr.ts                  |   2 +-
 .../{ => songs}/songs-iidx.ts                 |   5 +-
 .../{ => songs}/songs-museca.ts               |   4 +-
 .../{ => songs}/songs-sdvx.ts                 |   2 +-
 .../state-sync/create-game-profiles.ts        |  43 +++++
 .../state-sync/dedupe-score-ids.ts            |  36 ++++
 .../state-sync/recalc-all-scores.ts           |  47 ++++++
 .../state-sync/sync-state.ts                  |  15 ++
 .../state-sync/update-all-pbs.ts              |  25 +++
 .../__KT_DATABASE_MIGRATION/user-goals.ts     |   2 +-
 .../scripts/__KT_DATABASE_MIGRATION/users.ts  |  11 +-
 server/scripts/arc-db-sync/arc-db-merge.ts    |   2 +-
 server/scripts/bms-tables/table-sync.ts       |  60 +++++++
 server/scripts/bms-tables/update.ts           |  48 ++++++
 server/scripts/iidx-tierlist/get-sp12-data.ts |   2 +-
 .../iidx-tierlist/parse-iidxkr-data.ts        |   2 +-
 server/scripts/single-use/add-2dxtra-flag.ts  |   2 +-
 .../scripts/single-use/add-folders-array.ts   |  16 ++
 server/scripts/single-use/add-hcnver.ts       |   2 +-
 server/scripts/single-use/fix-chuni-port.ts   |   2 +-
 .../fix-iidx-versions/fix-iidx-versions.ts    |  28 ---
 .../single-use/fix-iidx-versions/stat.ts      |  20 ---
 .../fix-iidx-versions/update-versions.ts      |   9 +-
 .../single-use/fix-legacy-folder-format.ts    |   2 +-
 .../scripts/single-use/fix-sdvx-judgements.ts |  30 ++++
 .../single-use/get-needs-humanise-sdvx.ts     |  25 +++
 .../scripts/single-use/numify-tableindex.ts   |   2 +-
 server/scripts/update-bpi-poyashi.ts          |   2 +-
 server/scripts/util/efficient-db-iterate.ts   |  35 ++++
 server/scripts/validate-database.ts           |  16 +-
 server/src/lib/constants/bms-tables.ts        |  11 ++
 54 files changed, 1236 insertions(+), 303 deletions(-)
 delete mode 100644 server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts
 delete mode 100644 server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => charts}/charts-chunithm.ts (94%)
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => charts}/charts-ddr.ts (89%)
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => charts}/charts-iidx.ts (55%)
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/charts/charts-museca.ts
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => charts}/charts-sdvx.ts (97%)
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/scores/ddr.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts
 rename server/scripts/__KT_DATABASE_MIGRATION/{scores.ts => scores/reference.ts} (64%)
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts
 delete mode 100644 server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => songs}/songs-chunithm.ts (89%)
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => songs}/songs-ddr.ts (95%)
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => songs}/songs-iidx.ts (83%)
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => songs}/songs-museca.ts (87%)
 rename server/scripts/__KT_DATABASE_MIGRATION/{ => songs}/songs-sdvx.ts (94%)
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/state-sync/create-game-profiles.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/state-sync/dedupe-score-ids.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts
 create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts
 create mode 100644 server/scripts/bms-tables/table-sync.ts
 create mode 100644 server/scripts/bms-tables/update.ts
 create mode 100644 server/scripts/single-use/add-folders-array.ts
 delete mode 100644 server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts
 delete mode 100644 server/scripts/single-use/fix-iidx-versions/stat.ts
 create mode 100644 server/scripts/single-use/fix-sdvx-judgements.ts
 create mode 100644 server/scripts/single-use/get-needs-humanise-sdvx.ts
 create mode 100644 server/scripts/util/efficient-db-iterate.ts
 create mode 100644 server/src/lib/constants/bms-tables.ts

diff --git a/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts b/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts
index 319e0cbdd..f7bf91642 100644
--- a/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts
+++ b/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts
@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
 import CreateLogCtx from "../../../src/common/logger";
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 const logger = CreateLogCtx(__filename);
 
 const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8"));
diff --git a/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts b/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts
index ea7ce62f7..c70816964 100644
--- a/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts
+++ b/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts
@@ -3,7 +3,7 @@
 
 import fs from "fs";
 import path from "path";
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 import { MDBJson } from "./types";
 import { Difficulties, Playtypes, ChartDocument } from "tachi-common";
 import CreateLogCtx from "../../../src/common/logger";
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts
deleted file mode 100644
index 7ccbae4ea..000000000
--- a/server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts
+++ /dev/null
@@ -1,60 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { ChartDocument, SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
-import { gameOrders } from "tachi-common/js/config";
-import { oldKTDB } from "./old-db";
-
-const logger = CreateLogCtx(__filename);
-
-async function ConvertFn(c: any): Promise<ChartDocument<"gitadora:Gita" | "gitadora:Dora">> {
-	const song = (await db.songs.gitadora.findOne({
-		id: c.id,
-	})) as SongDocument<"gitadora">;
-
-	const oldSong = await oldKTDB.get("songs-gitadora").findOne({
-		id: c.id,
-	});
-
-	if (!song) {
-		logger.severe(`Cannot find song with ID ${c.id}?`);
-		throw new Error(`Cannot find song with ID ${c.id}?`);
-	}
-
-	const newChartDoc: ChartDocument<"gitadora:Gita" | "gitadora:Dora"> = {
-		rgcID: null,
-		chartID: c.chartID,
-		difficulty: c.difficulty,
-		songID: c.id,
-		playtype: c.playtype,
-		levelNum: c.levelNum,
-		level: c.level.toString(),
-		flags: {
-			"IN BASE GAME": true,
-			OMNIMIX: true,
-		},
-		data: {
-			inGameID: c.internals.inGameINTID,
-		},
-		isPrimary: true,
-		versions: [], // sentinel
-	};
-
-	const idx = gameOrders.gitadora.indexOf(song.firstVersion!);
-
-	if (idx === -1) {
-		logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
-		newChartDoc.versions = [song.firstVersion!];
-	} else {
-		newChartDoc.versions = gameOrders.gitadora.slice(idx);
-	}
-
-	return newChartDoc;
-}
-
-(async () => {
-	await MigrateRecords(db.charts.gitadora, "charts-gitadora", ConvertFn);
-
-	process.exit(0);
-})();
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts
deleted file mode 100644
index eeab079a6..000000000
--- a/server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
-import { gameOrders } from "tachi-common/js/config";
-
-const logger = CreateLogCtx(__filename);
-
-async function ConvertFn(c: any): Promise<ChartDocument<"museca:Single">> {
-	const song = await db.songs.museca.findOne({
-		id: c.id,
-	});
-
-	if (!song) {
-		logger.severe(`Cannot find song with ID ${c.id}?`);
-		throw new Error(`Cannot find song with ID ${c.id}?`);
-	}
-
-	const newChartDoc: ChartDocument<"museca:Single"> = {
-		rgcID: null,
-		chartID: c.chartID,
-		difficulty: c.difficulty,
-		songID: c.id,
-		playtype: c.playtype,
-		levelNum: c.levelNum,
-		level: c.level.toString(),
-		flags: {
-			"IN BASE GAME": true,
-			OMNIMIX: true, // false, needs to be overrode for 3 or so charts
-		},
-		data: {},
-		isPrimary: true,
-		versions: [], // sentinel
-	};
-
-	const idx = gameOrders.museca.indexOf(song.firstVersion!);
-
-	if (idx === -1) {
-		logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
-		newChartDoc.versions = [song.firstVersion!];
-	} else {
-		newChartDoc.versions = gameOrders.museca.slice(idx);
-	}
-
-	return newChartDoc;
-}
-
-(async () => {
-	await MigrateRecords(db.charts.museca, "charts-museca", ConvertFn);
-
-	process.exit(0);
-})();
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-chunithm.ts
similarity index 94%
rename from server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/charts/charts-chunithm.ts
index 21e880a18..221ba31c8 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-chunithm.ts
@@ -1,9 +1,9 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
+import MigrateRecords from "../migrate";
 import { gameOrders } from "tachi-common/js/config";
 
 const logger = CreateLogCtx(__filename);
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-ddr.ts
similarity index 89%
rename from server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/charts/charts-ddr.ts
index c3351ed5c..d20995b2d 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-ddr.ts
@@ -1,11 +1,11 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { ChartDocument, SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import MigrateRecords from "../migrate";
 import { gameOrders } from "tachi-common/js/config";
-import { oldKTDB } from "./old-db";
+import { oldKTDB } from "../old-db";
 
 const logger = CreateLogCtx(__filename);
 
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-iidx.ts
similarity index 55%
rename from server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/charts/charts-iidx.ts
index e14884cf0..a10725537 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-iidx.ts
@@ -1,20 +1,32 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
-import CreateLogCtx from "../../src/common/logger";
-import MigrateRecords from "./migrate";
-import { gameOrders } from "tachi-common/js/config";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import MigrateRecords from "../migrate";
 
 const logger = CreateLogCtx(__filename);
 
-async function ConvertFn(c: any): Promise<ChartDocument<"iidx:SP" | "iidx:DP">> {
+function Get2DXTraSet(flags: any) {
+	if (flags["All Scratch"]) {
+		return "All Scratch";
+	} else if (flags.Kiraku) {
+		return "Kiraku";
+	} else if (flags.Kichiku) {
+		return "Kichiku";
+	}
+
+	return null;
+}
+
+async function ConvertFn(c: any): Promise<ChartDocument<"iidx:SP" | "iidx:DP"> | null> {
 	const song = await db.songs.iidx.findOne({
 		id: c.id,
 	});
 
 	if (!song) {
-		logger.severe(`Cannot find song with ID ${c.id}?`);
+		logger.warn(`Cannot find song with ID ${c.id}?`);
+		return null;
 		throw new Error(`Cannot find song with ID ${c.id}?`);
 	}
 
@@ -26,29 +38,17 @@ async function ConvertFn(c: any): Promise<ChartDocument<"iidx:SP" | "iidx:DP">>
 		playtype: c.playtype,
 		levelNum: c.levelNum,
 		level: c.level.toString(),
-		flags: {
-			"IN BASE GAME": !!c.flags["IN BASE GAME"],
-			OMNIMIX: !!c.flags.OMNIMIX,
-			"N-1": !!c.flags["N-1"],
-		},
 		data: {
 			inGameID: c.internals.inGameINTID,
 			notecount: c.notedata.notecount,
 			arcChartID: null,
+			hashSHA256: c.internals.checksum2DXtra,
+			"2dxtraSet": Get2DXTraSet(c.flags),
 		},
 		isPrimary: true,
-		versions: [], // sentinel
+		versions: [], // handled by scripts/single-use/fix-iidx-versions atm
 	};
 
-	const idx = gameOrders.iidx.indexOf(song.firstVersion!);
-
-	if (idx === -1) {
-		logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
-		newChartDoc.versions = [song.firstVersion!];
-	} else {
-		newChartDoc.versions = gameOrders.iidx.slice(idx);
-	}
-
 	return newChartDoc;
 }
 
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-museca.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-museca.ts
new file mode 100644
index 000000000..98876c91f
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-museca.ts
@@ -0,0 +1,41 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { ChartDocument } from "tachi-common";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import MigrateRecords from "../migrate";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ChartDocument<"museca:Single"> | null> {
+	const song = await db.songs.museca.findOne({
+		id: c.id,
+	});
+
+	// this is a warn instead of severe because some songs have charts but no songs
+	// presumably removed stuff like brain power?
+	if (!song) {
+		logger.warn(`Cannot find song with ID ${c.id}?`);
+		return null;
+	}
+
+	const newChartDoc: ChartDocument<"museca:Single"> = {
+		rgcID: null,
+		chartID: c.chartID,
+		difficulty: c.difficulty,
+		songID: c.id,
+		playtype: c.playtype,
+		levelNum: c.levelNum,
+		level: c.level.toString(),
+		data: {},
+		isPrimary: true,
+		versions: ["1+1/2"],
+	};
+
+	return newChartDoc;
+}
+
+(async () => {
+	await MigrateRecords(db.charts.museca, "charts-museca", ConvertFn);
+
+	process.exit(0);
+})();
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-sdvx.ts
similarity index 97%
rename from server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/charts/charts-sdvx.ts
index f9c061ab7..04e31e033 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/charts/charts-sdvx.ts
@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { ChartDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 import { gameOrders } from "tachi-common/js/config";
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/folders.ts b/server/scripts/__KT_DATABASE_MIGRATION/folders.ts
index 5ee11c210..7a35ee85b 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/folders.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/folders.ts
@@ -2,7 +2,7 @@
 
 import { FolderDocument, Game } from "tachi-common";
 import { validPlaytypes, gameHuman } from "tachi-common/js/config";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 import crypto from "crypto";
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/goals.ts b/server/scripts/__KT_DATABASE_MIGRATION/goals.ts
index f6eb1fc9a..028fd1dd8 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/goals.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/goals.ts
@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { GoalDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts
index f15bd5640..88a5bdf01 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts
@@ -4,45 +4,60 @@ import { oldKTDB } from "./old-db";
 
 const logger = CreateLogCtx(__filename);
 
-const newDocuments: unknown[] = [];
-
 export default async function MigrateRecords(
 	existingCollection: ICollection,
 	collectionName: string,
-	HandlerFN: (c: unknown) => unknown
+	HandlerFN: (c: unknown) => unknown,
+	filter: Record<string, any> = {},
+	force = false
 ) {
 	const existingRecords = await existingCollection.count({});
 	logger.info(`Starting migration for ${collectionName}...`);
 
 	if (existingRecords > 0) {
-		logger.error(`${existingRecords} documents already exist in db, terminating.`);
-		process.exit(1);
+		if (force) {
+			logger.warn(
+				`Forcing migration of documents - appending to ${existingRecords} existing records.`
+			);
+		} else {
+			logger.error(`${existingRecords} documents already exist in db, terminating.`);
+			process.exit(1);
+		}
 	}
 
 	let i = 0;
+	const bucketSize = 10_000;
 
-	await oldKTDB
-		.get(collectionName)
-		.find({})
-		// @ts-expect-error it exists.
-		.each(async (c, { pause, resume }) => {
-			i++;
-			pause();
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		// eslint-disable-next-line no-await-in-loop
+		const docs = await oldKTDB.get(collectionName).find(filter, { limit: bucketSize, skip: i });
 
-			if (i % 10000 === 0) {
-				logger.info(`Processed ${i} documents.`);
-			}
-			const newDoc = await HandlerFN(c);
+		logger.info(`Migrating ${i} - ${i + docs.length} documents.`);
 
-			if (newDoc !== null) {
-				newDocuments.push(newDoc);
-			}
+		if (docs.length === 0) {
+			break;
+		}
 
-			resume();
-		});
+		// eslint-disable-next-line no-await-in-loop
+		const newDocs = await Promise.all(docs.map(HandlerFN));
+
+		i += bucketSize;
+
+		const notNulls = newDocs.filter((e) => e !== null);
+
+		logger.info(`Success: ${notNulls.length}, Failures: ${newDocs.length - notNulls.length}.`);
+
+		// eslint-disable-next-line no-await-in-loop
+		await existingCollection.insert(notNulls);
+
+		logger.info(`Migrated ${docs.length} documents.`);
+
+		if (docs.length < bucketSize) {
+			break;
+		}
+	}
 
-	logger.info(`Inserting ${newDocuments.length} documents.`);
-	await existingCollection.insert(newDocuments.flat());
 	logger.info(`Done!`);
 
 	process.exit(0);
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/old-db.ts b/server/scripts/__KT_DATABASE_MIGRATION/old-db.ts
index 751231db5..3ed4ec313 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/old-db.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/old-db.ts
@@ -1,3 +1,18 @@
 import monk from "monk";
+import { ServerConfig } from "lib/setup/config";
+import CreateLogCtx from "lib/logger/logger";
 
-export const oldKTDB = monk(`${process.env.MONGO_BASE_URL}/kamaitachidb`);
+const logger = CreateLogCtx(__filename);
+
+logger.info(`Connecting to Old KTDB at ${ServerConfig.MONGO_CONNECTION_URL}/kamaitachidb.`);
+
+export const oldKTDB = monk(`${ServerConfig.MONGO_CONNECTION_URL}/kamaitachidb`);
+
+oldKTDB
+	.then(() => {
+		logger.info(`Done.`);
+	})
+	.catch((err) => {
+		logger.crit(`Fatal in connecting to old KT DB.`, { err });
+		process.exit(1);
+	});
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts b/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts
index 1810e213e..28e9496fb 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts
@@ -1,4 +1,4 @@
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 import { TierlistDataDocument } from "tachi-common";
 import { CalculateTierlistDataID } from "../../../src/common/tierlist";
 import { oldKTDB } from "../old-db";
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts
new file mode 100644
index 000000000..502940089
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts
@@ -0,0 +1,138 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
+	if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
+		base[baseProp] = other[otherProp];
+	}
+}
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+	const playtype = c.scoreData.playtype;
+
+	const oldChartDoc = await oldKTDB.get("charts-bms").findOne({
+		chartID: c.chartID,
+	});
+
+	if (!oldChartDoc) {
+		logger.warn(`Cannot find oldChartDoc for ${c.chartID}`);
+	}
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		"data.hashMD5": oldChartDoc.internals.hash,
+	});
+
+	if (!chartDoc) {
+		const oldSong = await oldKTDB.get("songs-bms").findOne({ id: c.songID });
+		logger.warn(`Cannot find ChartDoc for ${oldSong.title} ${oldChartDoc.level} ${playtype}`);
+		return null;
+		throw new Error(`Cannot find ChartDoc for ${c.scoreID} (${c.chartID})`);
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {
+			random: c.scoreMeta?.optionsRandom ?? null,
+			inputDevice: c.scoreMeta?.inputDevice ?? null,
+		},
+	};
+
+	// @ts-expect-error shut
+	if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
+		// @ts-expect-error shut
+		base.scoreData.hitMeta.bp = null;
+	}
+
+	// @ts-expect-error shut
+	if (base.scoreData.hitMeta.gauge === -1) {
+		// @ts-expect-error shut
+		base.scoreData.hitMeta.gauge = null;
+	}
+
+	// @ts-expect-error shut
+	delete base.scoreData.hitMeta.inputDevice;
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	await oldKTDB.get("score-id-lookup").insert({
+		old: c.scoreID,
+		new: scoreID,
+	});
+
+	return score;
+}
+
+export async function MigrateBMSScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "bms" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateBMSScores();
+		process.exit(0);
+	})();
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/ddr.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/ddr.ts
new file mode 100644
index 000000000..8e92e51ad
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/ddr.ts
@@ -0,0 +1,101 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { ScoreDocument, GetGamePTConfig, StaticConfig } from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		chartID: c.chartID,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
+		return null;
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument<"ddr:SP" | "ddr:DP">, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {},
+	};
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	try {
+		await oldKTDB.get("score-id-lookup").insert({
+			old: c.scoreID,
+			new: scoreID,
+		});
+	} catch (err) {
+		// whoops
+	}
+
+	return score;
+}
+
+export async function MigrateDDRScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "ddr" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateDDRScores();
+		process.exit(0);
+	})();
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts
new file mode 100644
index 000000000..6da9400b9
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts
@@ -0,0 +1,159 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
+	if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
+		base[baseProp] = other[otherProp];
+	}
+}
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	const game = c.game;
+	const playtype = c.scoreData.playtype;
+
+	const gameConfig = GetGameConfig(game);
+
+	if (!gameConfig.validPlaytypes.includes(playtype)) {
+		logger.warn(`Ignored game pt ${game}, ${playtype}`);
+		throw new Error(`Ignored game pt ${game}, ${playtype}`);
+	}
+
+	const gptConfig = GetGamePTConfig(game, playtype);
+
+	if (!gptConfig) {
+		logger.error(`No gptconfig for ${game} ${playtype}?`);
+		throw new Error(`No gptconfig for ${game} ${playtype}?`);
+	}
+
+	if (c.songID === 1819) {
+		c.songID = 1121; // praludium has two songs in kt1 for some reason (tricoro 6)
+		// this overrides that
+	}
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		songID: c.songID,
+		difficulty: c.scoreData.difficulty,
+		playtype,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.scoreData.difficulty} ${playtype} `);
+		return null;
+		throw new Error(`Cannot find ChartDoc for ${c.scoreID} (${c.chartID})`);
+	}
+
+	const base: Omit<ScoreDocument, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {
+			random: c.scoreMeta?.optionsRandom ?? null,
+			gauge: c.scoreMeta?.optionsGauge ?? null,
+			assist: c.scoreMeta?.optionsAssist ?? null,
+			range: c.scoreMeta?.optionsRange ?? null,
+		},
+	};
+
+	if (c.scoreMeta) {
+		// @ts-expect-error this game is iidx
+		if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
+			// @ts-expect-error this game is iidx
+			base.scoreMeta.random = null;
+		}
+
+		// @ts-expect-error asdf
+		if (base.scoreMeta.range === "") {
+			// @ts-expect-error asdf
+			base.scoreMeta.range = "NONE";
+		}
+	}
+
+	// @ts-expect-error shut
+	if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
+		// @ts-expect-error shut
+		base.scoreData.hitMeta.bp = null;
+	}
+
+	// @ts-expect-error nope
+	if (base.scoreData.hitMeta.gauge > 200) {
+		// @ts-expect-error nope
+		base.scoreData.hitMeta.gauge = null;
+	}
+
+	// @ts-expect-error nope
+	if (base.scoreData.hitMeta.gaugeHistory) {
+		// @ts-expect-error nope
+		base.scoreData.hitMeta.gaugeHistory = base.scoreData.hitMeta.gaugeHistory.map((e) =>
+			e > 200 ? null : e
+		);
+	}
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	await oldKTDB.get("score-id-lookup").insert({
+		old: c.scoreID,
+		new: scoreID,
+	});
+
+	return score;
+}
+
+export async function MigrateIIDXScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "iidx" });
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateIIDXScores();
+		process.exit(0);
+	})();
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts
new file mode 100644
index 000000000..1435cffee
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts
@@ -0,0 +1,15 @@
+import CreateLogCtx from "lib/logger/logger";
+import { MigrateBMSScores } from "./bms";
+import { MigrateIIDXScores } from "./iidx";
+
+const logger = CreateLogCtx(__filename);
+
+(async () => {
+	logger.info(`Migrating IIDX Scores.`);
+
+	await MigrateIIDXScores();
+
+	logger.info(`Migrating BMS Scores.`);
+
+	await MigrateBMSScores();
+})();
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts
new file mode 100644
index 000000000..e9ab0e0c3
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts
@@ -0,0 +1,107 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		chartID: c.chartID,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
+		return null;
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument<"museca:Single">, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {},
+	};
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	try {
+		await oldKTDB.get("score-id-lookup").insert({
+			old: c.scoreID,
+			new: scoreID,
+		});
+	} catch (err) {
+		// whoops
+	}
+
+	return score;
+}
+
+export async function MigrateMusecaScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "museca" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateMusecaScores();
+		process.exit(0);
+	})();
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts
similarity index 64%
rename from server/scripts/__KT_DATABASE_MIGRATION/scores.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts
index b058ccfd4..97ce9904b 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/scores.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts
@@ -1,41 +1,74 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-import { PrivateUserDocument, ScoreDocument } from "tachi-common";
-import { grades, lamps, validPlaytypes } from "tachi-common/js/config";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
-import CreateLogCtx from "../../src/common/logger";
-import { CreateScoreID } from "../../src/score-import/framework/common/score-utils";
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
 
 const logger = CreateLogCtx(__filename);
 
-// HERE. WE. GO!
-
 function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
 	if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
 		base[baseProp] = other[otherProp];
 	}
 }
 
-function ConvertFn(c: any): ScoreDocument | null {
-	if (!supportedGames.includes(c.game)) {
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
 		logger.warn(`Ignored game ${c.game}`);
 		return null;
 	}
+	const game = c.game;
+	const playtype = c.scoreData.playtype;
 
 	if (["popn", "jubeat", "gitadora", "usc"].includes(c.game)) {
 		return null;
 	}
 
-	if (validPlaytypes[c.game].includes(c.playtype)) {
-		logger.warn(`Ignored game pt ${c.game}, ${c.playtype}`);
+	const gameConfig = GetGameConfig(game);
+
+	if (!gameConfig.validPlaytypes.includes(playtype)) {
+		logger.warn(`Ignored game pt ${game}, ${playtype}`);
+		throw new Error(`Ignored game pt ${game}, ${playtype}`);
+	}
+
+	const gptConfig = GetGamePTConfig(game, playtype);
+
+	if (!gptConfig) {
+		logger.error(`No gptconfig for ${game} ${playtype}?`);
+		throw new Error(`No gptconfig for ${game} ${playtype}?`);
+	}
+
+	if (game === "iidx" && c.songID === 1819) {
+		c.songID = 1121; // praludium has two songs in kt1 for some reason (tricoro 6)
+		// this overrides that
+	}
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		songID: c.songID,
+		difficulty: c.scoreData.difficulty,
+		playtype,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.scoreData.difficulty} ${playtype} `);
+		return null;
+		throw new Error(`Cannot find ChartDoc for ${c.scoreID} (${c.chartID})`);
 	}
 
 	const base: Omit<ScoreDocument, "scoreID"> = {
 		userID: c.userID,
-		songID: c.songID,
-		playtype: c.scoreData.playtype,
-		chartID: c.chartID,
+		songID: chartDoc.songID,
+		playtype,
+		chartID: chartDoc.chartID,
 		game: c.game,
 		timeAdded: c.timeAdded,
 		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
@@ -43,16 +76,18 @@ function ConvertFn(c: any): ScoreDocument | null {
 		highlight: c.highlight ?? false,
 		service: c.service,
 		importType: null,
-		calculatedData: c.calculatedData,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
 		scoreData: {
 			esd: c.scoreData.esd ?? null,
 			grade: c.scoreData.grade,
-			gradeIndex: grades[c.game].indexOf(c.scoreData.grade),
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
 			lamp: c.scoreData.lamp,
-			lampIndex: lamps[c.game].indexOf(c.scoreData.lamp),
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
 			percent: c.scoreData.percent,
 			score: c.scoreData.score,
-			judgements: c.scoreData.judgements,
+			judgements: c.scoreData.hitData,
 			hitMeta: c.scoreData.hitMeta,
 		},
 		scoreMeta: {},
@@ -68,7 +103,9 @@ function ConvertFn(c: any): ScoreDocument | null {
 			ConditionalAssign(base.scoreMeta, "pacemakerName", c.scoreMeta, "pacemakerName");
 			ConditionalAssign(base.scoreMeta, "pacemakerTarget", c.scoreMeta, "pacemakerTarget");
 
+			// @ts-expect-error this game is iidx
 			if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
+				// @ts-expect-error this game is iidx
 				base.scoreMeta.random = null;
 			}
 
@@ -119,7 +156,7 @@ function ConvertFn(c: any): ScoreDocument | null {
 		delete base.scoreData.judgements.error;
 	} else if (base.game === "chunithm") {
 		if (base.scoreData.percent === 101) {
-			base.scoreData.grade === "SSS";
+			base.scoreData.grade = "SSS";
 		}
 	}
 
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts
new file mode 100644
index 000000000..effc8ac4d
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts
@@ -0,0 +1,105 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import {
+	PrivateUserDocument,
+	ScoreDocument,
+	GetGameConfig,
+	GetGamePTConfig,
+	StaticConfig,
+} from "tachi-common";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
+import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
+import CreateLogCtx from "lib/logger/logger";
+import { oldKTDB } from "../old-db";
+
+const logger = CreateLogCtx(__filename);
+
+async function ConvertFn(c: any): Promise<ScoreDocument | null> {
+	if (!StaticConfig.allSupportedGames.includes(c.game)) {
+		logger.warn(`Ignored game ${c.game}`);
+		return null;
+	}
+	const game = c.game;
+
+	// @ts-expect-error implicit any is expected
+	const chartDoc = await db.charts[game].findOne({
+		chartID: c.chartID,
+	});
+
+	if (!chartDoc) {
+		logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
+		return null;
+	}
+
+	const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
+
+	if (!gptConfig) {
+		logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
+
+		throw new Error(`Screwed`);
+	}
+
+	const base: Omit<ScoreDocument<"sdvx:Single">, "scoreID"> = {
+		userID: c.userID,
+		songID: chartDoc.songID,
+		playtype: chartDoc.playtype,
+		chartID: chartDoc.chartID,
+		game: c.game,
+		timeAdded: c.timeAdded,
+		timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
+		comment: c.comment ?? null,
+		highlight: c.highlight ?? false,
+		service: c.service,
+		importType: null,
+		// we'll just recalc calculated data instead of trying to update
+		calculatedData: {},
+		isPrimary: chartDoc.isPrimary,
+		scoreData: {
+			esd: c.scoreData.esd ?? null,
+			grade: c.scoreData.grade,
+			gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
+			lamp: c.scoreData.lamp,
+			lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
+			percent: c.scoreData.percent,
+			score: c.scoreData.score,
+			judgements: c.scoreData.hitData,
+			hitMeta: c.scoreData.hitMeta,
+		},
+		scoreMeta: {
+			inSkillAnalyser: null,
+		},
+	};
+
+	const scoreID = CreateScoreID(base.userID, base, base.chartID);
+
+	const exists = await db.scores.findOne({ scoreID });
+
+	if (exists) {
+		logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
+		return null;
+	}
+
+	const score: ScoreDocument = {
+		...base,
+		scoreID,
+	};
+
+	await oldKTDB.get("score-id-lookup").insert({
+		old: c.scoreID,
+		new: scoreID,
+	});
+
+	return score;
+}
+
+export async function MigrateSDVXScores() {
+	await MigrateRecords(db.scores, "scores", ConvertFn, { game: "sdvx" }, true);
+}
+
+if (require.main === module) {
+	(async () => {
+		await MigrateSDVXScores();
+		process.exit(0);
+	})();
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts
deleted file mode 100644
index c24fb8d44..000000000
--- a/server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
-
-function ConvertFn(c: any): SongDocument<"gitadora"> {
-	const newSongDoc: SongDocument<"gitadora"> = {
-		title: c.title,
-		artist: c.artist,
-		id: c.id,
-		firstVersion: c.firstAppearance,
-		"alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [],
-		"search-titles": c["search-titles"]
-			.filter((e: string) => !!e)
-			.map((e: string) => e.toString())
-			.filter((e: string) => e !== c.title),
-		data: {},
-	};
-
-	return newSongDoc;
-}
-
-(async () => {
-	await MigrateRecords(db.songs.gitadora, "songs-gitadora", ConvertFn);
-
-	process.exit(0);
-})();
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-chunithm.ts
similarity index 89%
rename from server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/songs/songs-chunithm.ts
index 771928e04..3e9839e44 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-chunithm.ts
@@ -1,8 +1,8 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
 
 function ConvertFn(c: any): SongDocument<"chunithm"> {
 	const newSongDoc: SongDocument<"chunithm"> = {
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-ddr.ts
similarity index 95%
rename from server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/songs/songs-ddr.ts
index f4b112aaa..ca03565f8 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-ddr.ts
@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
 import { gameOrders } from "tachi-common/js/config";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import MigrateRecords from "./migrate";
 
 function ConvertFn(c: any): SongDocument<"ddr"> {
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-iidx.ts
similarity index 83%
rename from server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/songs/songs-iidx.ts
index ca09f47b9..e6bad7f66 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-iidx.ts
@@ -1,8 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
-import { gameOrders } from "tachi-common/js/config";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
 
 function ConvertFn(c: any): SongDocument<"iidx"> {
 	const newSongDoc: SongDocument<"iidx"> = {
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-museca.ts
similarity index 87%
rename from server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/songs/songs-museca.ts
index 2b69bddb2..d455d6be5 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-museca.ts
@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
-import MigrateRecords from "./migrate";
+import db from "external/mongo/db";
+import MigrateRecords from "../migrate";
 
 function ConvertFn(c: any): SongDocument<"museca"> {
 	const newSongDoc: SongDocument<"museca"> = {
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-sdvx.ts
similarity index 94%
rename from server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts
rename to server/scripts/__KT_DATABASE_MIGRATION/songs/songs-sdvx.ts
index aac8faf03..7b2ba3295 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/songs/songs-sdvx.ts
@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { SongDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import MigrateRecords from "./migrate";
 
 function ConvertFn(c: any): SongDocument<"sdvx"> {
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/create-game-profiles.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/create-game-profiles.ts
new file mode 100644
index 000000000..87bc20540
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/create-game-profiles.ts
@@ -0,0 +1,43 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { UpdateUsersGamePlaytypeStats } from "lib/score-import/framework/user-game-stats/update-ugs";
+import { FormatUserDoc } from "utils/user";
+
+const logger = CreateLogCtx(__filename);
+
+export async function CreateGameProfiles() {
+	const users = await db.users.find({});
+
+	for (const user of users) {
+		const gpts = await db.scores.aggregate([
+			{
+				$match: {
+					userID: user.id,
+				},
+			},
+			{
+				$group: {
+					_id: {
+						game: "$game",
+						playtype: "$playtype",
+					},
+				},
+			},
+		]);
+
+		logger.info(`Found ${gpts.length} GPTs for ${FormatUserDoc(user)}`);
+
+		for (const gpt of gpts) {
+			const { game, playtype } = gpt._id;
+			logger.info(`Updating ${FormatUserDoc(user)}'s ${game} ${playtype} stats.`);
+			await UpdateUsersGamePlaytypeStats(game, playtype, user.id, null, logger);
+		}
+	}
+
+	logger.info(`Done.`);
+}
+
+if (require.main === module) {
+	CreateGameProfiles().then(() => process.exit(0));
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/dedupe-score-ids.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/dedupe-score-ids.ts
new file mode 100644
index 000000000..e5ef0eb74
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/dedupe-score-ids.ts
@@ -0,0 +1,36 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+async function DedupeScoreIDs() {
+	const dups = await db.scores.aggregate(
+		[
+			{
+				$group: {
+					_id: "$scoreID",
+					dups: { $addToSet: "$_id" },
+					count: { $sum: 1 },
+				},
+			},
+			{
+				$match: {
+					count: { $gt: 1 },
+				},
+			},
+		],
+		{ allowDiskUse: true }
+	);
+
+	logger.info(`Found ${dups.length} dups.`);
+
+	for (const dup of dups) {
+		dup.dups.shift();
+		await db.scores.remove({ _id: { $in: dup.dups } });
+	}
+
+	logger.info(`Done.`);
+	process.exit(0);
+}
+
+DedupeScoreIDs();
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts
new file mode 100644
index 000000000..a727bde06
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts
@@ -0,0 +1,47 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { CreateCalculatedData } from "lib/score-import/framework/calculated-data/calculated-data";
+import { EfficientDBIterate } from "../../util/efficient-db-iterate";
+
+const logger = CreateLogCtx(__filename);
+
+export async function RecalcAllScores(filter = {}) {
+	logger.info(`Recalcing All Scores.`);
+
+	await EfficientDBIterate(
+		db.scores,
+		async (c) => {
+			const chart = await db.charts[c.game].findOne({ chartID: c.chartID });
+
+			if (!chart) {
+				logger.error(`Can't find chartID ${c.chartID} ${c.scoreID} (${c.game})`, {
+					score: c,
+				});
+
+				throw new Error(`screwed`);
+			}
+			const calculatedData = await CreateCalculatedData(c, chart, c.scoreData.esd, logger);
+
+			return { scoreID: c.scoreID, calculatedData };
+		},
+		async (updates) => {
+			await db.scores.bulkWrite(
+				updates.map((e) => ({
+					updateOne: {
+						filter: {
+							scoreID: e.scoreID,
+						},
+						update: {
+							$set: {
+								calculatedData: e.calculatedData,
+							},
+						},
+					},
+				}))
+			);
+		},
+		filter
+	);
+
+	logger.info(`Done!`);
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts
new file mode 100644
index 000000000..3eb27c716
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts
@@ -0,0 +1,15 @@
+import { CreateGameProfiles } from "./create-game-profiles";
+import { UpdateAllPBs } from "./update-all-pbs";
+import { RecalcAllScores } from "./recalc-all-scores";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+(async () => {
+	await RecalcAllScores({ game: "ddr" });
+	await UpdateAllPBs({ game: "ddr" });
+	await CreateGameProfiles();
+
+	logger.info(`Completely done!`);
+	process.exit(0);
+})();
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts
new file mode 100644
index 000000000..59f2b94a7
--- /dev/null
+++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts
@@ -0,0 +1,25 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { ProcessPBs } from "lib/score-import/framework/pb/process-pbs";
+import { FormatUserDoc } from "utils/user";
+import deepmerge from "deepmerge";
+
+const logger = CreateLogCtx(__filename);
+
+export async function UpdateAllPBs(filter = {}) {
+	const allUsers = await db.users.find({});
+
+	for (const user of allUsers) {
+		logger.info(`Finding ${FormatUserDoc(user)}'s scores.`);
+
+		const scores = await db.scores.find(deepmerge({ userID: user.id }, filter), {
+			projection: { chartID: 1 },
+		});
+
+		logger.info(`PBing ${FormatUserDoc(user)}'s scores.`);
+		await ProcessPBs(user.id, new Set(scores.map((e) => e.chartID)), logger);
+	}
+
+	logger.info(`Done!`);
+}
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts b/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts
index 2e0b41953..2a901252c 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts
@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { UserGoalDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import MigrateRecords from "./migrate";
 
diff --git a/server/scripts/__KT_DATABASE_MIGRATION/users.ts b/server/scripts/__KT_DATABASE_MIGRATION/users.ts
index 48fa8d8d2..a94c55cd4 100644
--- a/server/scripts/__KT_DATABASE_MIGRATION/users.ts
+++ b/server/scripts/__KT_DATABASE_MIGRATION/users.ts
@@ -1,5 +1,5 @@
 import { PrivateUserDocument } from "tachi-common";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import MigrateRecords from "./migrate";
 
 function ConvertFn(c: any): PrivateUserDocument {
@@ -11,15 +11,9 @@ function ConvertFn(c: any): PrivateUserDocument {
 		customBanner: c.custombanner,
 		customPfp: c.custompfp,
 		email: c.email,
-		friends: c.friends,
 		id: c.id,
 		lastSeen: c.lastSeen,
 		password: c.password,
-		permissions: {},
-		settings: {
-			invisible: !!c.settings.invisible,
-			nsfwSplashes: !!c.settings.nsfwsplashes,
-		},
 		socialMedia: {
 			discord: c.socialmedia.discord || null,
 			twitter: c.socialmedia.twitter || null,
@@ -28,6 +22,9 @@ function ConvertFn(c: any): PrivateUserDocument {
 			github: c.socialmedia.github || null,
 			steam: c.socialmedia.steam || null,
 		},
+		joinDate: 0,
+		authLevel: "user",
+		badges: ["beta"],
 	};
 
 	return newUserDoc;
diff --git a/server/scripts/arc-db-sync/arc-db-merge.ts b/server/scripts/arc-db-sync/arc-db-merge.ts
index 01641440a..e2a3ae3b2 100644
--- a/server/scripts/arc-db-sync/arc-db-merge.ts
+++ b/server/scripts/arc-db-sync/arc-db-merge.ts
@@ -1,7 +1,7 @@
 import fs from "fs";
 import { ICollection } from "monk";
 import path from "path";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 import { ChartDocument } from "tachi-common";
 import { FindSongOnTitle } from "../../src/common/database-lookup/song";
diff --git a/server/scripts/bms-tables/table-sync.ts b/server/scripts/bms-tables/table-sync.ts
new file mode 100644
index 000000000..76d06bea7
--- /dev/null
+++ b/server/scripts/bms-tables/table-sync.ts
@@ -0,0 +1,60 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { ChartDocument } from "tachi-common";
+import fetch from "node-fetch";
+
+// this seems to be all we care about
+interface TableJSONDoc {
+	md5: string;
+	title?: string;
+	level: string;
+}
+
+const logger = CreateLogCtx(__filename);
+
+async function ImportTableLevels(tableJSON: TableJSONDoc[], prefix: string) {
+	let failures = 0;
+	let success = 0;
+	const total = tableJSON.length;
+
+	for (const td of tableJSON) {
+		const chart = (await db.charts.bms.findOne({ "data.hashMD5": td.md5 })) as ChartDocument<
+			"bms:7K" | "bms:14K"
+		>;
+
+		if (!chart) {
+			logger.warn(
+				`No chart exists in table fro ${td.md5} Possible title: ${td.title} ${prefix}${td.level}`
+			);
+			failures++;
+			continue;
+		}
+
+		const tableFolders = chart.data.tableFolders.filter((e) => e.table !== prefix);
+
+		tableFolders.push({ table: prefix, level: td.level });
+
+		await db.charts.bms.update(
+			{
+				chartID: chart.chartID,
+			},
+			{
+				$set: {
+					"data.tableFolders": tableFolders,
+				},
+			}
+		);
+
+		success++;
+	}
+
+	logger.info(`Finished updating table ${prefix}.`);
+	logger.info(`${success} Success | ${failures} Failures | ${total} Total.`);
+}
+
+export async function UpdateTable(prefix: string, url: string) {
+	const tableJSON = await fetch(url).then((r) => r.json());
+
+	return ImportTableLevels(tableJSON, prefix);
+}
diff --git a/server/scripts/bms-tables/update.ts b/server/scripts/bms-tables/update.ts
new file mode 100644
index 000000000..f0c4cc64b
--- /dev/null
+++ b/server/scripts/bms-tables/update.ts
@@ -0,0 +1,48 @@
+import { Command } from "commander";
+import { UpdateTable } from "./table-sync";
+import { BMS_TABLES } from "lib/constants/bms-tables";
+
+const program = new Command();
+program.option("-t, --table <table>");
+program.option("-a, --all");
+
+program.parse(process.argv);
+const options = program.opts();
+
+// I have no confidence in half of these links surviving.
+// Eventually, some of these are going to disappear, and we'll have to find
+// something else. probably.
+const registered: any = {
+	insane: ["http://www.ribbit.xyz/bms/tables/insane_body.json", BMS_TABLES.insane],
+	normal: ["http://www.ribbit.xyz/bms/tables/normal_body.json", BMS_TABLES.normal],
+	stella: ["https://stellabms.xyz/st/score.json", BMS_TABLES.stella],
+	satellite: ["https://stellabms.xyz/sl/score.json", BMS_TABLES.satellite],
+	dpSatellite: ["https://stellabms.xyz/dp/score.json", BMS_TABLES.satellite],
+	insane2: ["http://rattoto10.jounin.jp/js/insane_data.json", BMS_TABLES.insane2],
+	normal2: ["http://rattoto10.jounin.jp/js/score.json", BMS_TABLES.normal2],
+	overjoy: ["http://lr2.sakura.ne.jp/data/score.json", BMS_TABLES.overjoy],
+	dpInsane: ["http://dpbmsdelta.web.fc2.com/table/data/insane_data.json", BMS_TABLES.dpInsane],
+	dpNormal: ["http://dpbmsdelta.web.fc2.com/table/data/dpdelta_data.json", BMS_TABLES.dpNormal],
+};
+
+(async () => {
+	if (options.all) {
+		for (const table of Object.values(registered)) {
+			// eslint-disable-next-line no-await-in-loop
+			await UpdateTable((table as any)[1], (table as any)[0]);
+		}
+
+		process.exit(0);
+	} else {
+		const tables = Object.keys(registered);
+		if (!tables.includes(options.table)) {
+			throw new Error(
+				`Invalid table ${options.table} - expected any of ${tables.join(", ")}.`
+			);
+		}
+
+		await UpdateTable(registered[options.table][1], registered[options.table][0]);
+
+		process.exit(0);
+	}
+})();
diff --git a/server/scripts/iidx-tierlist/get-sp12-data.ts b/server/scripts/iidx-tierlist/get-sp12-data.ts
index 1bb74f5f1..a6861f634 100644
--- a/server/scripts/iidx-tierlist/get-sp12-data.ts
+++ b/server/scripts/iidx-tierlist/get-sp12-data.ts
@@ -1,6 +1,6 @@
 /* eslint-disable no-case-declarations */
 /* eslint-disable no-param-reassign */
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import fetch from "node-fetch";
 import { FindSongOnTitle } from "../../src/common/database-lookup/song";
 import { FindChartWithPTDF } from "../../src/common/database-lookup/chart";
diff --git a/server/scripts/iidx-tierlist/parse-iidxkr-data.ts b/server/scripts/iidx-tierlist/parse-iidxkr-data.ts
index b16d01d7e..cd3bdc2d0 100644
--- a/server/scripts/iidx-tierlist/parse-iidxkr-data.ts
+++ b/server/scripts/iidx-tierlist/parse-iidxkr-data.ts
@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import fs from "fs";
 import path from "path";
 import { CalculateTierlistDataID } from "../../src/common/tierlist";
diff --git a/server/scripts/single-use/add-2dxtra-flag.ts b/server/scripts/single-use/add-2dxtra-flag.ts
index 0f7421787..cf040d8d4 100644
--- a/server/scripts/single-use/add-2dxtra-flag.ts
+++ b/server/scripts/single-use/add-2dxtra-flag.ts
@@ -1,6 +1,6 @@
 // @ts-nocheck
 
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 
 db.charts.iidx
 	.find({})
diff --git a/server/scripts/single-use/add-folders-array.ts b/server/scripts/single-use/add-folders-array.ts
new file mode 100644
index 000000000..4d188d4ec
--- /dev/null
+++ b/server/scripts/single-use/add-folders-array.ts
@@ -0,0 +1,16 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+(async () => {
+	logger.info(`Setting tableFolders.`);
+	const r = await db.charts.bms.update(
+		{},
+		{ $set: { "data.tableFolders": [] } },
+		{ multi: true }
+	);
+	logger.info(`Done!`, { r });
+
+	process.exit(0);
+})();
diff --git a/server/scripts/single-use/add-hcnver.ts b/server/scripts/single-use/add-hcnver.ts
index c945a32de..19ea4c53b 100644
--- a/server/scripts/single-use/add-hcnver.ts
+++ b/server/scripts/single-use/add-hcnver.ts
@@ -2,7 +2,7 @@ import fs from "fs";
 import path from "path";
 import { PRUDENCE_CHART_SCHEMAS } from "../../src/db/schemas";
 import p from "prudence";
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../../src/common/logger";
 
 const logger = CreateLogCtx(__filename);
diff --git a/server/scripts/single-use/fix-chuni-port.ts b/server/scripts/single-use/fix-chuni-port.ts
index f180f9152..e929fd00a 100644
--- a/server/scripts/single-use/fix-chuni-port.ts
+++ b/server/scripts/single-use/fix-chuni-port.ts
@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 
 (async () => {
 	const charts = await db.charts.chunithm.find({});
diff --git a/server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts b/server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts
deleted file mode 100644
index cdf2498f0..000000000
--- a/server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import db from "../../../src/db/db";
-import { ChartDocument } from "tachi-common";
-
-db.charts.iidx
-	.find({})
-	// @ts-expect-error it exists
-	.each(async (c: ChartDocument<"iidx:SP" | "iidx:DP">, { pause, resume }: any) => {
-		pause();
-
-		const song = (await db.songs.iidx.findOne({ id: c.songID }))!;
-
-		let versions = [];
-
-		if (song.firstVersion === "27") {
-			versions = ["27", "28", "27-omni", "28-omni"];
-		} else if (song.firstVersion === "28") {
-			versions = ["28", "28-omni"];
-		} else if (song.firstVersion === "inf") {
-			versions = ["inf"];
-		} else {
-			versions = ["26", "27", "28", "26-omni", "27-omni", "28-omni"];
-		}
-
-		await db.charts.iidx.update({ _id: c._id }, { $set: { versions } });
-
-		resume();
-	})
-	.then(() => console.log("done"));
diff --git a/server/scripts/single-use/fix-iidx-versions/stat.ts b/server/scripts/single-use/fix-iidx-versions/stat.ts
deleted file mode 100644
index 5644d31b5..000000000
--- a/server/scripts/single-use/fix-iidx-versions/stat.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-import CreateLogCtx from "../../../src/common/logger";
-import db from "../../../src/db/db";
-
-const logger = CreateLogCtx(__filename);
-
-(async () => {
-	const charts = await db.charts.iidx.find({ versions: { $size: 0 } });
-
-	for (const chart of charts) {
-		// eslint-disable-next-line no-await-in-loop
-		const song = await db.songs.iidx.findOne({ id: chart.songID });
-
-		if (!song) {
-			logger.error(`cannot find parent song ${chart.songID}`);
-			continue;
-		}
-
-		logger.info(`${song.title} ${chart.playtype} ${chart.difficulty}`);
-	}
-})();
diff --git a/server/scripts/single-use/fix-iidx-versions/update-versions.ts b/server/scripts/single-use/fix-iidx-versions/update-versions.ts
index 0872622a3..38c900a2d 100644
--- a/server/scripts/single-use/fix-iidx-versions/update-versions.ts
+++ b/server/scripts/single-use/fix-iidx-versions/update-versions.ts
@@ -1,9 +1,10 @@
 /* eslint-disable no-await-in-loop */
-import db from "../../../src/db/db";
+import db from "external/mongo/db";
 import fs from "fs";
 import path from "path";
-import CreateLogCtx from "../../../src/common/logger";
-import { FindSongOnTitleInsensitive } from "../../../src/common/database-lookup/song";
+import CreateLogCtx from "lib/logger/logger";
+import { FindSongOnTitleInsensitive } from "utils/queries/songs";
+
 const logger = CreateLogCtx(__filename);
 
 const comicallyLargeJSONData = JSON.parse(
@@ -78,7 +79,7 @@ function ConvertVersion(c: string) {
 			continue;
 		}
 
-		// logger.info(relevantVersions);q
+		// logger.info(relevantVersions);
 
 		const chart = await db.charts.iidx.findOne({ songID: song.id, difficulty, playtype });
 
diff --git a/server/scripts/single-use/fix-legacy-folder-format.ts b/server/scripts/single-use/fix-legacy-folder-format.ts
index c40c5f4f8..ae43145f2 100644
--- a/server/scripts/single-use/fix-legacy-folder-format.ts
+++ b/server/scripts/single-use/fix-legacy-folder-format.ts
@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 import { FolderDocument } from "tachi-common";
 
 (async () => {
diff --git a/server/scripts/single-use/fix-sdvx-judgements.ts b/server/scripts/single-use/fix-sdvx-judgements.ts
new file mode 100644
index 000000000..6cebcd486
--- /dev/null
+++ b/server/scripts/single-use/fix-sdvx-judgements.ts
@@ -0,0 +1,30 @@
+import db from "external/mongo/db";
+
+(async () => {
+	await db.scores.update(
+		{ game: "sdvx", "scoreData.lamp": "ULTIMATE CHAIN" },
+		{
+			$set: {
+				"scoreData.judgements.miss": 0,
+			},
+		},
+		{
+			multi: true,
+		}
+	);
+
+	await db.scores.update(
+		{ game: "sdvx", "scoreData.lamp": "PERFECT ULTIMATE CHAIN" },
+		{
+			$set: {
+				"scoreData.judgements.miss": 0,
+				"scoreData.judgements.near": 0,
+			},
+		},
+		{
+			multi: true,
+		}
+	);
+
+	process.exit(0);
+})();
diff --git a/server/scripts/single-use/get-needs-humanise-sdvx.ts b/server/scripts/single-use/get-needs-humanise-sdvx.ts
new file mode 100644
index 000000000..83b4492de
--- /dev/null
+++ b/server/scripts/single-use/get-needs-humanise-sdvx.ts
@@ -0,0 +1,25 @@
+import db from "external/mongo/db";
+import fs from "fs";
+
+(async () => {
+	const all = await db.songs.sdvx.find({});
+
+	const humanise = [];
+	for (const h of all) {
+		if (!h.title.match(/^[a-zA-Z0-9 ().!?-_]*$/)) {
+			humanise.push(h);
+		}
+	}
+
+	fs.writeFileSync(
+		"needs-humanise.csv",
+		humanise
+			.map(
+				(e) =>
+					`${e.id}, "${e.title.replace('"', '\\"')}", "${e.artist.replace('"', '\\"')}"`
+			)
+			.join("\n")
+	);
+
+	process.exit(0);
+})();
diff --git a/server/scripts/single-use/numify-tableindex.ts b/server/scripts/single-use/numify-tableindex.ts
index 8782be945..8aeb7011d 100644
--- a/server/scripts/single-use/numify-tableindex.ts
+++ b/server/scripts/single-use/numify-tableindex.ts
@@ -1,4 +1,4 @@
-import db from "../../src/db/db";
+import db from "external/mongo/db";
 
 (async () => {
 	db.folders
diff --git a/server/scripts/update-bpi-poyashi.ts b/server/scripts/update-bpi-poyashi.ts
index 24af9a128..c07a3b4f4 100644
--- a/server/scripts/update-bpi-poyashi.ts
+++ b/server/scripts/update-bpi-poyashi.ts
@@ -9,7 +9,7 @@ import {
 	FindChartWithPTDF,
 	FindChartWithPTDFVersion,
 } from "../src/score-import/database-lookup/chart-ptdf";
-import db from "../src/db/db";
+import db from "external/mongo/db";
 import CreateLogCtx from "../src/common/logger";
 const program = new Command();
 
diff --git a/server/scripts/util/efficient-db-iterate.ts b/server/scripts/util/efficient-db-iterate.ts
new file mode 100644
index 000000000..81c1338d1
--- /dev/null
+++ b/server/scripts/util/efficient-db-iterate.ts
@@ -0,0 +1,35 @@
+/* eslint-disable no-await-in-loop */
+import CreateLogCtx from "lib/logger/logger";
+import { ICollection } from "monk";
+import { oldKTDB } from "../__KT_DATABASE_MIGRATION/old-db";
+
+const logger = CreateLogCtx(__filename);
+
+export async function EfficientDBIterate<T, R>(
+	collection: ICollection<T>,
+	callbackFn: (c: T) => Promise<R>,
+	saveOp: (c: R[]) => Promise<void>,
+	filter: any = {},
+	bucketSize = 10_000
+) {
+	let i = 0;
+
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		logger.info(`Migrating ${i} - ${i + bucketSize} documents.`);
+		// eslint-disable-next-line no-await-in-loop
+		const docs = await collection.find(filter, { limit: bucketSize, skip: i });
+
+		if (docs.length === 0) {
+			break;
+		}
+
+		const rDocs = await Promise.all(docs.map(callbackFn));
+
+		i += bucketSize;
+
+		await saveOp(rDocs);
+
+		logger.info(`Migrated ${i} documents.`);
+	}
+}
diff --git a/server/scripts/validate-database.ts b/server/scripts/validate-database.ts
index be15c2ec3..a0e703421 100644
--- a/server/scripts/validate-database.ts
+++ b/server/scripts/validate-database.ts
@@ -1,12 +1,11 @@
 import Prudence, { PrudenceSchema } from "prudence";
-import db from "../src/db/db";
-import { PRUDENCE_SCORE_SCHEMAS, STATIC_SCHEMAS } from "../src/db/schemas";
-import CreateLogCtx from "../src/common/logger";
+import db from "external/mongo/db";
+import { PRUDENCE_SCORE_SCHEMAS, STATIC_SCHEMAS } from "external/mongo/schemas";
+import CreateLogCtx from "lib/logger/logger";
 import fs from "fs";
 import path from "path";
 import { PrudenceError } from "prudence/js/error";
-import { validPlaytypes } from "tachi-common/js/config";
-import { CONF_INFO } from "../src/lib/setup/config";
+import { StaticConfig, GetGameConfig } from "tachi-common";
 
 const BASE_DIR = path.join(__dirname, "./validate-database-errs");
 
@@ -63,8 +62,9 @@ async function ValidateScores(): Promise<void> {
 	let globalTotal = 0;
 	let globalSuccess = 0;
 	let globalFail = 0;
-	for (const game of CONF_INFO.supportedGames) {
-		for (const playtype of validPlaytypes[game]) {
+	for (const game of StaticConfig.allSupportedGames) {
+		const gameConfig = GetGameConfig(game);
+		for (const playtype of gameConfig.validPlaytypes) {
 			// @ts-expect-error shut up
 			const schema = PRUDENCE_SCORE_SCHEMAS[game][playtype];
 
@@ -135,7 +135,7 @@ async function ValidateChartsOrSongs(type: "songs" | "charts"): Promise<void> {
 	let globalTotal = 0;
 	let globalSuccess = 0;
 	let globalFail = 0;
-	for (const game of CONF_INFO.supportedGames) {
+	for (const game of StaticConfig.allSupportedGames) {
 		const schema = STATIC_SCHEMAS[type]![game];
 
 		let successCount = 0;
diff --git a/server/src/lib/constants/bms-tables.ts b/server/src/lib/constants/bms-tables.ts
new file mode 100644
index 000000000..d8b1b43fa
--- /dev/null
+++ b/server/src/lib/constants/bms-tables.ts
@@ -0,0 +1,11 @@
+export const BMS_TABLES = {
+	insane: "🟊",
+	overjoy: "🟊🟊",
+	normal: "☆",
+	normal2: "▽",
+	insane2: "▼",
+	stella: "st",
+	satellite: "sl",
+	dpNormal: "δ",
+	dpInsane: "★",
+};