From 4f17546c4dfc7ba66b1748b7e0dec13c78956681 Mon Sep 17 00:00:00 2001
From: zkldi <20380519+zkldi@users.noreply.github.com>
Date: Thu, 6 Oct 2022 21:57:57 +0100
Subject: [PATCH 1/6] feat: allow invoking parser.ts to debug.

---
 .../rerunners/iidx-mdb-parse/.gitignore       |  5 +-
 .../iidx-mdb-parse/dot-one-parser/parser.ts   | 50 ++++++++++++++-----
 2 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore b/database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore
index eb80f3b1e..de1745214 100644
--- a/database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore
+++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore
@@ -1,2 +1,5 @@
 # this gets flooded with extracted .ifses
-ifs-output/*
\ No newline at end of file
+ifs-output/*
+
+# parser.ts spits here if invoked by hand
+debug.json
\ No newline at end of file
diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts b/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts
index 4b792baa8..a7a825e60 100644
--- a/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts
+++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts
@@ -18,6 +18,7 @@ import {
 } from "./types";
 import fs from "fs/promises";
 import logger from "../../../logger";
+import path from "path";
 
 interface EventParserContext {
 	isDP?: boolean;
@@ -228,7 +229,11 @@ function ParseEvent(
 				obj.col = 1;
 			} else {
 				logger.warn(
-					`Unknown column of scratchObj sent to EventParser: ${data.param}, expected 7 or 14. Skipping.`
+					`(${
+						parseContext.songTitle ?? "UNKNOWN"
+					}) Unknown column of scratchObj sent to EventParser: ${
+						data.param
+					}, expected 7 or 14. Skipping.`
 				);
 				return null;
 			}
@@ -251,7 +256,13 @@ function ParseEvent(
 		};
 	} else if (type === "BPM") {
 		if (data.param === 0) {
-			logger.warn("Invalid value of 0 for denominator in BPM event. Skipping.");
+			logger.warn(
+				`(${
+					parseContext.songTitle ?? "UNKNOWN"
+				}) Invalid value of 0 for denominator in BPM event (numerator was ${
+					data.val
+				}). Skipping.`
+			);
 			return null;
 		}
 
@@ -291,7 +302,9 @@ function ParseEvent(
 
 		if (!window) {
 			logger.warn(
-				`Unknown type of timingWindow: ${window}, from .1 param ${data.param}. Skipping.`
+				`(${
+					parseContext.songTitle ?? "UNKNOWN"
+				}) Unknown type of timingWindow: ${window}, from .1 param ${data.param}. Skipping.`
 			);
 			return null;
 		}
@@ -328,7 +341,9 @@ function ParseEvent(
 		if (parseContext.isDP) {
 			if (data.param !== 0 && data.param !== 1) {
 				logger.warn(
-					`Invalid parameter for measurebar of ${data.param}. Defaulting to 0 (left hand side).`
+					`(${parseContext.songTitle ?? "UNKNOWN"}) Invalid parameter for measurebar of ${
+						data.param
+					}. Defaulting to 0 (left hand side).`
 				);
 				obj.side = 0;
 			} else {
@@ -353,15 +368,15 @@ function ParseEvent(
 		// this chart is using HCNs.
 		options.isHellCharge = true;
 		return null;
+	} else if (data.type === 11) {
+		// it's some sort of unknown BGA-related thing. Only seems to happen in older
+		// charts. Maybe it was some sort of diverging bga instruction at one point?
+		return null;
 	} else {
 		logger.warn(
 			`(${parseContext.songTitle ?? "UNKNOWN"}) Unknown event type in .1: ${
 				data.type
-			} at ${ms.toFixed(2)}ms. Ignoring.${
-				data.type === 11
-					? " (It's BGA Related. Not sure what it does, but don't worry.)"
-					: ""
-			}`
+			} at ${ms.toFixed(2)}ms. Ignoring.`
 		);
 		return null;
 	}
@@ -457,7 +472,9 @@ export function ParseDotOne(
 
 		if (!difficulty) {
 			logger.warn(
-				`Data present inside directory ${i}, but this corresponds to an unknown difficulty. Ignoring.`
+				`(${
+					context.songTitle ?? "UNKNOWN"
+				}) Data present inside directory ${i}, but this corresponds to an unknown difficulty. Ignoring.`
 			);
 			continue;
 		}
@@ -506,11 +523,20 @@ function getNotecount(events: IIDXChartEvents) {
 }
 
 export async function ParseDotOneFile(
-	fileath: string,
+	filepath: string,
 	context: IIDXParserContext = {},
 	options: IIDXParserOptions = {}
 ) {
-	const data = await fs.readFile(fileath);
+	const data = await fs.readFile(filepath);
 
 	return ParseDotOne(data, context, options);
 }
+
+// if invoked as a script, parse the dotone and spit it out to debug.json.
+if (require.main === module) {
+	(async () => {
+		const res = await ParseDotOneFile(process.argv.slice(2).join(" "));
+
+		await fs.writeFile(path.join(__dirname, "debug.json"), JSON.stringify(res, null, "\t"));
+	})();
+}

From d3cd2768d4cda0db937362acf08c82ea21c2e114 Mon Sep 17 00:00:00 2001
From: zkldi <20380519+zkldi@users.noreply.github.com>
Date: Thu, 6 Oct 2022 22:41:31 +0100
Subject: [PATCH 2/6] fix: IMPORTANT IIDX OMNI PATCHES
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch
- Updates broken notecounts on a series of charts

these broken notecounts are *extremely* sinister, only off by one
or two in some cases. Furthermore, this has happened on relatively underplayed charts.
As such, this has gone unnoticed for a while!

- theme of denjin J is now rated 7 instead of 0
no idea why it was 0, but it was

- jelly kiss's another charts now appear
no idea why they didn't

- back into the light (recut) and the hope of tomorrow (recut) now appear in 28-omni
they didn't before, no idea why.

- サナ・モレッテ・ネ・エンテ(B.L.T.STYLE) now appears in 28-omni
same as above, no idea why.
---
 database-seeds/collections/charts-iidx.json   | 196 +++++++++++++-----
 .../rerunners/iidx-mdb-parse/blacklist.txt    |   6 +
 2 files changed, 146 insertions(+), 56 deletions(-)

diff --git a/database-seeds/collections/charts-iidx.json b/database-seeds/collections/charts-iidx.json
index 5782bf634..86972029d 100644
--- a/database-seeds/collections/charts-iidx.json
+++ b/database-seeds/collections/charts-iidx.json
@@ -14801,7 +14801,7 @@
 			"hashSHA256": null,
 			"inGameID": 1208,
 			"kaidenAverage": null,
-			"notecount": 209,
+			"notecount": 184,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -80124,7 +80124,9 @@
 		"rgcID": null,
 		"songID": 143,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "96879b78bc80745249933dbb9a8ebe6c8f3ec8f7",
@@ -80196,7 +80198,9 @@
 		"rgcID": null,
 		"songID": 143,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "9292d34c2132c1da50b1d748e9c25fead37a2c98",
@@ -80268,7 +80272,9 @@
 		"rgcID": null,
 		"songID": 143,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "e5db54b7385da1557af89512b658cb18a22ee5f3",
@@ -80354,7 +80360,7 @@
 			"hashSHA256": null,
 			"inGameID": 4216,
 			"kaidenAverage": null,
-			"notecount": 442,
+			"notecount": 443,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -80365,7 +80371,9 @@
 		"rgcID": null,
 		"songID": 143,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "84dc89843e9859fef23c5a2c3b1913c43f81116f",
@@ -124603,8 +124611,8 @@
 		},
 		"difficulty": "ANOTHER",
 		"isPrimary": true,
-		"level": "0",
-		"levelNum": 0,
+		"level": "7",
+		"levelNum": 7,
 		"playtype": "SP",
 		"rgcID": null,
 		"songID": 211,
@@ -175700,7 +175708,7 @@
 		]
 	},
 	{
-		"chartID": "cd89731d004cdcc5292aa10d11c181d8d0545aa5",
+		"chartID": "e17adc8e3575d2c0cb873f913e3411e6527e3d23",
 		"data": {
 			"2dxtraSet": null,
 			"arcChartID": null,
@@ -175711,6 +175719,30 @@
 			"notecount": 682,
 			"worldRecord": null
 		},
+		"difficulty": "ANOTHER",
+		"isPrimary": true,
+		"level": "8",
+		"levelNum": 8,
+		"playtype": "DP",
+		"rgcID": null,
+		"songID": 290,
+		"tierlistInfo": {},
+		"versions": [
+			"28-omni"
+		]
+	},
+	{
+		"chartID": "cd89731d004cdcc5292aa10d11c181d8d0545aa5",
+		"data": {
+			"2dxtraSet": null,
+			"arcChartID": null,
+			"bpiCoefficient": null,
+			"hashSHA256": null,
+			"inGameID": 8022,
+			"kaidenAverage": null,
+			"notecount": 503,
+			"worldRecord": null
+		},
 		"difficulty": "HYPER",
 		"isPrimary": true,
 		"level": "9",
@@ -175835,7 +175867,7 @@
 			"hashSHA256": null,
 			"inGameID": 8022,
 			"kaidenAverage": null,
-			"notecount": 503,
+			"notecount": 267,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -176001,6 +176033,30 @@
 			"28-2dxtra"
 		]
 	},
+	{
+		"chartID": "0b4ae8fb5ae20b5277d52953c7cd26ff84d67f18",
+		"data": {
+			"2dxtraSet": null,
+			"arcChartID": null,
+			"bpiCoefficient": null,
+			"hashSHA256": null,
+			"inGameID": 8022,
+			"kaidenAverage": null,
+			"notecount": 656,
+			"worldRecord": null
+		},
+		"difficulty": "ANOTHER",
+		"isPrimary": true,
+		"level": "8",
+		"levelNum": 8,
+		"playtype": "SP",
+		"rgcID": null,
+		"songID": 290,
+		"tierlistInfo": {},
+		"versions": [
+			"28-omni"
+		]
+	},
 	{
 		"chartID": "06a866286936157f613ec50e26b4d774bfc70b2d",
 		"data": {
@@ -181969,7 +182025,8 @@
 		"songID": 298,
 		"tierlistInfo": {},
 		"versions": [
-			"inf"
+			"inf",
+			"28-omni"
 		]
 	},
 	{
@@ -182141,7 +182198,8 @@
 		"songID": 298,
 		"tierlistInfo": {},
 		"versions": [
-			"inf"
+			"inf",
+			"28-omni"
 		]
 	},
 	{
@@ -182239,7 +182297,8 @@
 		"songID": 298,
 		"tierlistInfo": {},
 		"versions": [
-			"inf"
+			"inf",
+			"28-omni"
 		]
 	},
 	{
@@ -182411,7 +182470,8 @@
 		"songID": 298,
 		"tierlistInfo": {},
 		"versions": [
-			"inf"
+			"inf",
+			"28-omni"
 		]
 	},
 	{
@@ -219920,8 +219980,8 @@
 		},
 		"difficulty": "NORMAL",
 		"isPrimary": true,
-		"level": "4",
-		"levelNum": 4,
+		"level": "3",
+		"levelNum": 3,
 		"playtype": "SP",
 		"rgcID": null,
 		"songID": 356,
@@ -402332,7 +402392,9 @@
 		"rgcID": null,
 		"songID": 629,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "afb62aab0829002d5cc7280749c6580aa1f378cd",
@@ -402354,7 +402416,9 @@
 		"rgcID": null,
 		"songID": 629,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "9ec852789f818e7497275cd533458c9d5e068da5",
@@ -402664,7 +402728,9 @@
 		"rgcID": null,
 		"songID": 629,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "c5650359963ba876e6ec18f2eb0332ebaeddc9e4",
@@ -402830,7 +402896,9 @@
 		"rgcID": null,
 		"songID": 629,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "c49713794172b674d059270ff96e9e9d5f1e1c45",
@@ -402852,7 +402920,9 @@
 		"rgcID": null,
 		"songID": 629,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "9e597d306bc3b54117947e914e6189d9a849ff06",
@@ -403162,7 +403232,9 @@
 		"rgcID": null,
 		"songID": 629,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "6f7e9548837a038af0fb1aebdcdd41368a46a4c1",
@@ -407027,7 +407099,7 @@
 			"hashSHA256": null,
 			"inGameID": 13209,
 			"kaidenAverage": null,
-			"notecount": 529,
+			"notecount": 531,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -408634,7 +408706,9 @@
 		"rgcID": null,
 		"songID": 636,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "3cb55a166faa0a3f537bd2fb68e6718dfc92be34",
@@ -408656,7 +408730,9 @@
 		"rgcID": null,
 		"songID": 636,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "1e0735f550dc50c6ff6e66f7e6d402b41561be05",
@@ -408966,7 +409042,9 @@
 		"rgcID": null,
 		"songID": 636,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "33c475f343e10212508dd71c3f7aa7b4bafb9493",
@@ -409132,7 +409210,9 @@
 		"rgcID": null,
 		"songID": 636,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "7a612f4f90a1349a2942412fa81bf7082817ec22",
@@ -409154,7 +409234,9 @@
 		"rgcID": null,
 		"songID": 636,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "0c7fb1fff2e323e3ccb0dce1da1ad848952efb96",
@@ -409464,7 +409546,9 @@
 		"rgcID": null,
 		"songID": 636,
 		"tierlistInfo": {},
-		"versions": []
+		"versions": [
+			"28-omni"
+		]
 	},
 	{
 		"chartID": "0b5216ff09349bf48c3e9013ccf35a3238039333",
@@ -588714,7 +588798,7 @@
 			"hashSHA256": null,
 			"inGameID": 17030,
 			"kaidenAverage": null,
-			"notecount": 897,
+			"notecount": 901,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -588748,7 +588832,7 @@
 			"hashSHA256": null,
 			"inGameID": 17030,
 			"kaidenAverage": null,
-			"notecount": 703,
+			"notecount": 707,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -588932,7 +589016,7 @@
 			"hashSHA256": null,
 			"inGameID": 17030,
 			"kaidenAverage": null,
-			"notecount": 409,
+			"notecount": 414,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -589041,7 +589125,7 @@
 			"hashSHA256": null,
 			"inGameID": 17030,
 			"kaidenAverage": null,
-			"notecount": 892,
+			"notecount": 894,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -589075,7 +589159,7 @@
 			"hashSHA256": null,
 			"inGameID": 17030,
 			"kaidenAverage": null,
-			"notecount": 673,
+			"notecount": 676,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -589259,7 +589343,7 @@
 			"hashSHA256": null,
 			"inGameID": 17030,
 			"kaidenAverage": null,
-			"notecount": 409,
+			"notecount": 412,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -606168,8 +606252,8 @@
 		},
 		"difficulty": "LEGGENDARIA",
 		"isPrimary": true,
-		"level": "0",
-		"levelNum": 0,
+		"level": "12",
+		"levelNum": 12,
 		"playtype": "DP",
 		"rgcID": null,
 		"songID": 908,
@@ -619861,7 +619945,7 @@
 			"hashSHA256": null,
 			"inGameID": 18007,
 			"kaidenAverage": null,
-			"notecount": 582,
+			"notecount": 583,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -666522,7 +666606,7 @@
 			"hashSHA256": null,
 			"inGameID": 18102,
 			"kaidenAverage": null,
-			"notecount": 427,
+			"notecount": 429,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -679696,7 +679780,7 @@
 			"hashSHA256": null,
 			"inGameID": 19017,
 			"kaidenAverage": null,
-			"notecount": 1067,
+			"notecount": 1068,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -679910,7 +679994,7 @@
 			"hashSHA256": null,
 			"inGameID": 19017,
 			"kaidenAverage": null,
-			"notecount": 315,
+			"notecount": 316,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -680231,7 +680315,7 @@
 			"hashSHA256": null,
 			"inGameID": 19017,
 			"kaidenAverage": null,
-			"notecount": 295,
+			"notecount": 297,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -933000,7 +933084,7 @@
 			"hashSHA256": null,
 			"inGameID": 23023,
 			"kaidenAverage": null,
-			"notecount": 1182,
+			"notecount": 1183,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -933032,7 +933116,7 @@
 			"hashSHA256": null,
 			"inGameID": 23023,
 			"kaidenAverage": null,
-			"notecount": 541,
+			"notecount": 543,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -933321,7 +933405,7 @@
 			"hashSHA256": null,
 			"inGameID": 23023,
 			"kaidenAverage": null,
-			"notecount": 1185,
+			"notecount": 1186,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -933353,7 +933437,7 @@
 			"hashSHA256": null,
 			"inGameID": 23023,
 			"kaidenAverage": null,
-			"notecount": 561,
+			"notecount": 562,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -944901,7 +944985,7 @@
 			"hashSHA256": null,
 			"inGameID": 23040,
 			"kaidenAverage": null,
-			"notecount": 939,
+			"notecount": 940,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -945083,7 +945167,7 @@
 			"hashSHA256": null,
 			"inGameID": 23040,
 			"kaidenAverage": null,
-			"notecount": 491,
+			"notecount": 492,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -1025917,7 +1026001,7 @@
 			"hashSHA256": null,
 			"inGameID": 24064,
 			"kaidenAverage": null,
-			"notecount": 1223,
+			"notecount": 1225,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -1025950,7 +1026034,7 @@
 			"hashSHA256": null,
 			"inGameID": 24064,
 			"kaidenAverage": null,
-			"notecount": 936,
+			"notecount": 940,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -1026133,7 +1026217,7 @@
 			"hashSHA256": null,
 			"inGameID": 24064,
 			"kaidenAverage": null,
-			"notecount": 388,
+			"notecount": 393,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -1026241,7 +1026325,7 @@
 			"hashSHA256": null,
 			"inGameID": 24064,
 			"kaidenAverage": null,
-			"notecount": 1098,
+			"notecount": 1100,
 			"worldRecord": null
 		},
 		"difficulty": "ANOTHER",
@@ -1026274,7 +1026358,7 @@
 			"hashSHA256": null,
 			"inGameID": 24064,
 			"kaidenAverage": null,
-			"notecount": 694,
+			"notecount": 696,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -1026457,7 +1026541,7 @@
 			"hashSHA256": null,
 			"inGameID": 24064,
 			"kaidenAverage": null,
-			"notecount": 387,
+			"notecount": 389,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
@@ -1191109,7 +1191193,7 @@
 			"hashSHA256": null,
 			"inGameID": 20042,
 			"kaidenAverage": null,
-			"notecount": 857,
+			"notecount": 859,
 			"worldRecord": null
 		},
 		"difficulty": "HYPER",
@@ -1191293,7 +1191377,7 @@
 			"hashSHA256": null,
 			"inGameID": 20042,
 			"kaidenAverage": null,
-			"notecount": 447,
+			"notecount": 448,
 			"worldRecord": null
 		},
 		"difficulty": "NORMAL",
diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt b/database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt
index 087523a93..458dcf0dc 100644
--- a/database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt
+++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt
@@ -14,3 +14,9 @@ S16072$
 S16080$
 S16081$
 S16082$
+
+# there are two songs called Changes in omnimix. We support neither.
+# this is because it means we can't use songTitle as a matching thing
+# even though we really want to.
+S10029$
+S6211$
\ No newline at end of file

From 18b0eac8213b293c17a6506586f613b862e329cc Mon Sep 17 00:00:00 2001
From: zkldi <20380519+zkldi@users.noreply.github.com>
Date: Thu, 6 Oct 2022 23:07:08 +0100
Subject: [PATCH 3/6] fix: make sync-database-local work

also, use sync-database-local instead of sync-database when running
bootstrap.sh
---
 _scripts/bootstrap.sh               | 3 +--
 server/package.json                 | 2 +-
 server/src/scripts/sync-database.ts | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/_scripts/bootstrap.sh b/_scripts/bootstrap.sh
index c36f570f1..88ff7d2d7 100755
--- a/_scripts/bootstrap.sh
+++ b/_scripts/bootstrap.sh
@@ -59,7 +59,7 @@ function syncDatabaseWithSeeds {
 
 	cd server
 
-	pnpm run sync-database
+	pnpm run sync-database-local
 
 	echo "Synced."
 }
@@ -69,7 +69,6 @@ selfSignHTTPS
 pnpmInstall
 syncDatabaseWithSeeds
 
-
 tput setaf 1
 
 cat << EOF
diff --git a/server/package.json b/server/package.json
index 7f1614a73..d76de5731 100644
--- a/server/package.json
+++ b/server/package.json
@@ -15,7 +15,7 @@
 		"start-no-build": "node -r ts-node/register -r tsconfig-paths/register js/server/src/main.js",
 		"runscoreworker": "node -r ts-node/register -r tsconfig-paths/register js/server/src/lib/score-import/worker/worker.js",
 		"sync-database": "ts-node src/scripts/sync-database",
-		"sync-database-local": "ts-node src/scripts/sync-database --localPath '../database-seeds'",
+		"sync-database-local": "ts-node src/scripts/sync-database --localPath '../database-seeds/collections'",
 		"recalc-everything": "ts-node src/scripts/state-sync/sync-state.ts"
 	},
 	"author": "zkldi",
diff --git a/server/src/scripts/sync-database.ts b/server/src/scripts/sync-database.ts
index d9840d5fd..7b626eb23 100644
--- a/server/src/scripts/sync-database.ts
+++ b/server/src/scripts/sync-database.ts
@@ -340,7 +340,7 @@ const logger = CreateLogCtx("Database Sync");
 
 const program = new Command();
 
-program.option("-l, --localPath <path/to/database-seeds>");
+program.option("-l, --localPath <path/to/database-seeds/collections>");
 
 program.parse(process.argv);
 const options: {

From ac0debe2e9e9d7490f26aaff677e93e78459b77d Mon Sep 17 00:00:00 2001
From: zkldi <20380519+zkldi@users.noreply.github.com>
Date: Fri, 7 Oct 2022 00:20:59 +0100
Subject: [PATCH 4/6] feat: add migration script for iidx notecounts

---
 .../recalc-broken-iidx-notecounts.ts          | 92 +++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts

diff --git a/server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts b/server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts
new file mode 100644
index 000000000..be6eb8ebf
--- /dev/null
+++ b/server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts
@@ -0,0 +1,92 @@
+// Charts that had their notecounts changed as the result of a recent discovered
+
+import db from "external/mongo/db";
+import { GenericGetGradeAndPercent } from "lib/score-import/framework/common/score-utils";
+import UpdateScore from "lib/score-mutation/update-score";
+import { GetGamePTConfig } from "tachi-common";
+import type { ScoreDocument, ChartDocument } from "tachi-common";
+import type { Migration } from "utils/types";
+
+// bug in the seeds.
+const recalcChartIDs = [
+	"38ebf85a6b9079bed6acb418cc3866f3a6b6adb7",
+	"88a5f281abf7ce08fde62769edb50b2ed8a1eeb9",
+	"cd89731d004cdcc5292aa10d11c181d8d0545aa5",
+	"8520f95da6f51548adde23ce84bf800e4a19334b",
+	"fcb31b1536d97bfb213cc574462d68d40f9fb15e",
+	"bd67cde14867804902f526c61df59982cb4ffa3e",
+	"241d190af5648d98a91f3b37585d648020d272ff",
+	"7275b7c88df34b6401fb99da7dece2af785cfc4f",
+	"ce71f9c794c429b71ca37ca0615b7b983b3d16ef",
+	"13d6019c21f8fd53b141000c742d4d2a37616aea",
+	"e00c3264172f717675e9c4c2d4ee82f2540a6772",
+	"76b502d2d39861095dc10f691d214bd9f482a815",
+	"4ad68e155dae973d737d7d8ac57453abcc4ac035",
+	"8e60d1ad926add5e84fb6de5a0924c89c365f80f",
+	"a73460d18cf2c6d13a09a46e83073fbd989d2222",
+	"21a475985581c40b616fdfb8ee60d16f64c9bbfa",
+	"a0687bc6951293e803bc121526948dc49aa1394f",
+	"2a80757be4c72f3674c96ff4bdff75deea120892",
+	"37817b02053862c7887f8e30cda23cd5f2c3be4d",
+	"df5543a6822c307dfdbdf162fecf6da013043fff",
+	"828dc62f7cabaf33db72158fdfc5c9ffb89adb4e",
+	"7e7e6866fc5c587a8029aa1bf0bec56441c8c519",
+	"8a51375d59bbe382f9e951d69a8f814b359b570b",
+	"8ce008ee846349caba96edd4538e7a9c148f8af7",
+	"c8805a4e481bb124b6687c44d3d359bdca4dd50e",
+	"adc8458064d179fef6302c12d691f9ff58d92163",
+	"f654276295ca8c228da91c230cc8ac86b23d16e9",
+	"1a856ae72278eedc5ac19050bc1cb441885ac46d",
+	"d87e7aba6edf70bd4b0087165afaa82fad801db3",
+	"85a4e8cebef9a12dddb407ff581f499a078f6010",
+];
+
+const migration: Migration = {
+	id: "recalc-broken-iidx-notecounts",
+	up: async () => {
+		await Promise.all(recalcChartIDs.map((e) => HandleChangedIIDXNotecount(e)));
+	},
+	down: () => {
+		throw new Error(`Reverting this change is not possible.`);
+	},
+};
+
+async function HandleChangedIIDXNotecount(chartID: string) {
+	const scores = await db.scores.find({ chartID });
+	const chart = await db.charts.iidx.findOne({ chartID });
+
+	if (!chart) {
+		throw new Error(`No such chart with the ID ${chartID} exists.`);
+	}
+
+	return Promise.all(scores.map((e) => RecalcPercentForIIDXScore(e, chart)));
+}
+
+function RecalcPercentForIIDXScore(score: ScoreDocument, chart: ChartDocument) {
+	const { percent, grade } = GenericGetGradeAndPercent("iidx", score.scoreData.score, chart);
+
+	const gptConfig = GetGamePTConfig("iidx", score.playtype);
+
+	const gradeIndex = gptConfig.grades.indexOf(grade);
+
+	// shouldn't happen, but lets check anyway.
+	if (gradeIndex === -1) {
+		throw new Error(
+			`Failed to calculate new grade? ${percent} ${grade} on chartID:${chart.chartID}.`
+		);
+	}
+
+	const newScore: ScoreDocument = {
+		...score,
+		scoreData: {
+			...score.scoreData,
+			percent,
+			grade,
+			gradeIndex,
+		},
+	};
+
+	return UpdateScore(score, newScore);
+}
+
+export default migration;

From dc5ca0f206bf4c307710e37f4d3f1e36060486ad Mon Sep 17 00:00:00 2001
From: zkldi <20380519+zkldi@users.noreply.github.com>
Date: Fri, 7 Oct 2022 01:28:09 +0100
Subject: [PATCH 5/6] fix: remove invalid versions from jelly kiss DPN and DPH

between 8th CS and 16th CS, jelly kiss was changed such that its
DPN became a DPH, and its DPH became a DPA.

this is a problem, as we haven't accounted for this change properly.

we're just going to remove those versions from the db, and change it to
16-cs.
---
 database-seeds/collections/charts-iidx.json | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/database-seeds/collections/charts-iidx.json b/database-seeds/collections/charts-iidx.json
index 86972029d..34027c252 100644
--- a/database-seeds/collections/charts-iidx.json
+++ b/database-seeds/collections/charts-iidx.json
@@ -175754,12 +175754,10 @@
 		"versions": [
 			"27-omni",
 			"26-omni",
-			"8-cs",
-			"bmus",
-			"6-cs",
 			"27-2dxtra",
 			"28-omni",
-			"28-2dxtra"
+			"28-2dxtra",
+			"16-cs"
 		]
 	},
 	{
@@ -175881,12 +175879,10 @@
 		"versions": [
 			"27-omni",
 			"26-omni",
-			"8-cs",
-			"bmus",
-			"6-cs",
 			"27-2dxtra",
 			"28-omni",
-			"28-2dxtra"
+			"28-2dxtra",
+			"16-cs"
 		]
 	},
 	{

From 695fa98fecaa4969d0f13e196980ff18032b138b Mon Sep 17 00:00:00 2001
From: zkldi <20380519+zkldi@users.noreply.github.com>
Date: Fri, 7 Oct 2022 01:48:21 +0100
Subject: [PATCH 6/6] feat: mount migration

confirmed to work on local hardware, should be good.
---
 server/src/lib/migration/migrations.ts        |  2 ++
 .../recalc-broken-iidx-notecounts.ts          | 19 ++++++++++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/server/src/lib/migration/migrations.ts b/server/src/lib/migration/migrations.ts
index 246f11dc6..db4e4ba19 100644
--- a/server/src/lib/migration/migrations.ts
+++ b/server/src/lib/migration/migrations.ts
@@ -1,4 +1,5 @@
 import UGPTRivalsMigration from "./migrations/add-rivals-to-ugpt";
+import RecalcBrokenIIDXNotecounts from "./migrations/recalc-broken-iidx-notecounts";
 import RemoveIIDXBeginners from "./migrations/remove-iidx-beginners";
 import RemoveMultifolderStats from "./migrations/remove-multifolder-stats";
 import UpdateJubeatPreferredTables from "./migrations/update-jubeat-preferred-tables";
@@ -31,6 +32,7 @@ const REGISTERED_MIGRATIONS: Array<Migration> =
 				RemoveMultifolderStats,
 				RemoveIIDXBeginners,
 				UpdateJubeatPreferredTables,
+				RecalcBrokenIIDXNotecounts,
 		  ];
 
 function CreateMigrationLookupMap(migrations: Array<Migration>) {
diff --git a/server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts b/server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts
index be6eb8ebf..c076bc50b 100644
--- a/server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts
+++ b/server/src/lib/migration/migrations/recalc-broken-iidx-notecounts.ts
@@ -1,7 +1,7 @@
 // Charts that had their notecounts changed as the result of a recent discovered
-
 import db from "external/mongo/db";
 import { GenericGetGradeAndPercent } from "lib/score-import/framework/common/score-utils";
+import { DeleteMultipleScores } from "lib/score-mutation/delete-scores";
 import UpdateScore from "lib/score-mutation/update-score";
 import { GetGamePTConfig } from "tachi-common";
 import type { ScoreDocument, ChartDocument } from "tachi-common";
@@ -11,8 +11,6 @@ import type { Migration } from "utils/types";
 const recalcChartIDs = [
 	"38ebf85a6b9079bed6acb418cc3866f3a6b6adb7",
 	"88a5f281abf7ce08fde62769edb50b2ed8a1eeb9",
-	"cd89731d004cdcc5292aa10d11c181d8d0545aa5",
-	"8520f95da6f51548adde23ce84bf800e4a19334b",
 	"fcb31b1536d97bfb213cc574462d68d40f9fb15e",
 	"bd67cde14867804902f526c61df59982cb4ffa3e",
 	"241d190af5648d98a91f3b37585d648020d272ff",
@@ -41,9 +39,24 @@ const recalcChartIDs = [
 	"85a4e8cebef9a12dddb407ff581f499a078f6010",
 ];
 
+// jelly kiss DPN and DPH are now unreliable
+// their charts were changed at some point, but the versions are wrong,
+// we have to wipe these charts, sadly.
+const toWipe = [
+	"cd89731d004cdcc5292aa10d11c181d8d0545aa5",
+	"8520f95da6f51548adde23ce84bf800e4a19334b",
+];
+
 const migration: Migration = {
 	id: "recalc-broken-iidx-notecounts",
 	up: async () => {
+		const toDelete = await db.scores.find({
+			game: "iidx",
+			chartID: { $in: toWipe },
+		});
+
+		await DeleteMultipleScores(toDelete);
+
 		await Promise.all(recalcChartIDs.map((e) => HandleChangedIIDXNotecount(e)));
 	},
 	down: () => {