feat: iidx-mdb-parser + merger

database-seeds/scripts/package.json

@@ -21,12 +21,12 @@
 		"fast-xml-parser": "^4.0.2",
 		"glob": "^7.2.0",
 		"html-entities": "^2.3.2",
+		"iconv-lite": "^0.6.3",
 		"lodash.get": "^4.4.2",
 		"mei-logger": "^1.0.1",
 		"monk": "^7.3.4",
 		"node-fetch": "2.6.7",
 		"prudence": "^0.9.7",
 		"xml2js": "^0.4.23"
-	},
-	"devDependencies": {}
+	}
 }

database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore

@@ -0,0 +1,2 @@
+# this gets flooded with extracted .ifses
+ifs-output/*

database-seeds/scripts/rerunners/iidx-mdb-parse/HOWTO.md

@@ -0,0 +1,11 @@
+# How do I use this?
+
+[Make sure you have the repository set up.](https://docs.bokutachi.xyz/contributing/setup/)
+
+With that done, run `pnpx ts-node merge-mdb.ts --help`. Provide the right data for `--basedir, --index and --version`.
+
+Make sure you have `ifstools` installed in your PATH. You'll need python3 installed.
+
+Once you have `python3` installed, run `pip install ifstools`.
+
+The script should now "just work"(TM).

database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt

@@ -0,0 +1,16 @@
+# songID + playtype + difficulty combos to ignore
+# these are regexes, and parse things in the format of
+# C$songid-$playtype-$difficulty
+# they also receive strings in the format of
+# S$songid
+# this is to support features like ignoring entire songs, with something like
+# S1234$
+# lines starting with '#' are ignored
+
+# we don't care about beginners
+BEGINNER
+
+S16072$
+S16080$
+S16081$
+S16082$

database-seeds/scripts/rerunners/iidx-mdb-parse/convert.ts

@@ -0,0 +1,219 @@
+import fs, { mkdirSync } from "fs";
+import path from "path";
+import iconv from "iconv-lite";
+import { execSync } from "child_process";
+import { ParseDotOneFile } from "./dot-one-parser/parser";
+import logger from "../../logger";
+import { IIDXConvertOutput } from "./dot-one-parser/types";
+import { integer } from "tachi-common";
+
+function sjisHack(b: Buffer, i: number) {
+	return iconv.decode(b.slice(i, i + 64).filter((e) => e !== 0) as Buffer, "shift_jis");
+}
+
+// where to store extracted IFS output
+const ifsOUT = path.join(__dirname, "./ifs-output");
+mkdirSync(ifsOUT, { recursive: true });
+
+/**
+ * Extracts an IFS file into the ./ifs-output cache directory.
+ */
+function ifsExtract(ifsPath: string) {
+	logger.info(`Extracting ${ifsPath}...`);
+	execSync(`ifstools "${ifsPath}" -o "${ifsOUT}" -y`, { stdio: "ignore" });
+}
+
+// @optimisable
+// this can be processed in parallel
+// but like. this basically only has to be done once a year.
+// https://xkcd.com/1205/
+export async function ParseIIDXData(
+	basedir: string,
+	index: "0" | "1",
+	omni: boolean,
+	alwaysExtract: boolean
+) {
+	logger.info(`Parsing from ${basedir}.`);
+
+	const mdbPath = path.join(
+		basedir,
+		"data/info",
+		index,
+		omni ? "music_omni.bin" : "music_data.bin"
+	);
+
+	logger.info(`Parsing MDB ${mdbPath}.`);
+
+	const buffer = fs.readFileSync(mdbPath);
+
+	if (buffer.slice(0, 4).toString() !== "IIDX") {
+		throw new Error("Invalid MDB File.");
+	}
+
+	const start = buffer.readInt16LE(0xa) * 2 + 0x10;
+
+	let structSize: integer;
+
+	switch (buffer[4]) {
+		case 27:
+		case 28:
+		case 29:
+			structSize = 0x52c;
+			break;
+		default:
+			throw new Error("Unknown version of MDB.");
+	}
+
+	let moreData = true;
+
+	let curLoc = start;
+
+	const parsedData: Array<IIDXConvertOutput> = [];
+
+	while (moreData) {
+		const struct = buffer.slice(curLoc, curLoc + structSize);
+
+		// laziest hack
+		const levels = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].map((e) => struct.readUInt8(288 + e));
+
+		// remember that i get paid real money to program in this language
+		const [spb, spn, sph, spa, spl, _dpb, dpn, dph, dpa, dpl] = levels as [
+			number,
+			number,
+			number,
+			number,
+			number,
+			number, // dpb has nothing defined, or maybe it does, nobody knows, or cares
+			number,
+			number,
+			number,
+			number
+		];
+
+		const songID = struct.readUInt16LE(944);
+
+		const title = sjisHack(struct, 0);
+		const marquee = sjisHack(struct, 64);
+		const genre = sjisHack(struct, 128);
+		const artist = sjisHack(struct, 192);
+		const folder = struct.readUInt8(280);
+
+		const notecounts: Record<string, number> = {};
+
+		// internal SongID - songIDs are not really an int, and are sometimes prefixed with a 0
+		// to fit as a 5 digit string.
+		const iSongID = songID < 10_000 ? `0${songID}` : songID.toString();
+
+		// where the IFS for this song might be located
+		const ifsPath = path.join(basedir, `data/sound/${iSongID}.ifs`);
+		const ifsLeggPath = path.join(basedir, `data/sound/${iSongID}-p0.ifs`);
+
+		// where the dotOne for this file might be located.
+		let dotOnePath = path.join(basedir, `data/sound/${iSongID}/${iSongID}.1`);
+
+		// where the dotone file will be after extraction (if necessary)
+		const extractedIFSLoc = path.join(ifsOUT, `${iSongID}_ifs/${iSongID}/${iSongID}.1`);
+
+		const extractedIFSLeggLoc = path.join(ifsOUT, `${iSongID}-p0_ifs/${iSongID}/${iSongID}.1`);
+
+		if (fs.existsSync(ifsLeggPath)) {
+			if (!fs.existsSync(extractedIFSLeggLoc)) {
+				ifsExtract(ifsLeggPath);
+			}
+
+			const charts = await ParseDotOneFile(extractedIFSLeggLoc, {
+				genre,
+				marquee,
+				songArtist: artist,
+				songTitle: title,
+			});
+
+			for (const chart of charts) {
+				notecounts[`${chart.playtype}-${chart.difficulty}`] = chart.notecount;
+			}
+		}
+
+		// if we've already extracted this, and --always-extract isn't set
+		if (fs.existsSync(extractedIFSLoc) && !alwaysExtract) {
+			dotOnePath = extractedIFSLoc;
+		} else if (fs.existsSync(ifsPath)) {
+			ifsExtract(ifsPath);
+
+			dotOnePath = extractedIFSLoc;
+		}
+
+		if (!fs.existsSync(dotOnePath)) {
+			curLoc += structSize;
+			logger.error(`${dotOnePath} cannot find file. neither exist.`);
+			continue;
+		}
+
+		logger.verbose(`Parsing data/sound/${songID}/${songID}.1`);
+		try {
+			const charts = await ParseDotOneFile(dotOnePath, {
+				genre,
+				marquee,
+				songArtist: artist,
+				songTitle: title,
+			});
+
+			for (const chart of charts) {
+				const notecount = chart.notecount;
+
+				if (chart.difficulty === "LEGGENDARIA") {
+					if (notecount !== 0 && notecounts[`${chart.playtype}-LEGGENDARIA`]) {
+						logger.warn(
+							`${chart.artist} - ${chart.title} has conflicting ${
+								chart.playtype
+							} LEGGENDARIAs.
+INLINE: ${notecount} notes, p0: ${notecounts[`${chart.playtype}-LEGGENDARIA`]} notes.
+Picking the INLINE version, as it's likely to be the correct one, but confirm this manually.`
+						);
+					}
+				}
+
+				notecounts[`${chart.playtype}-${chart.difficulty}`] = notecount;
+			}
+		} catch (err) {
+			logger.error(err);
+			curLoc += structSize;
+
+			if (buffer.length <= curLoc) {
+				moreData = false;
+			}
+
+			continue;
+		}
+
+		parsedData.push({
+			title,
+			artist,
+			marquee,
+			folder,
+			genre,
+			levels: {
+				"SP-BEGINNER": spb,
+				"SP-NORMAL": spn,
+				"SP-HYPER": sph,
+				"SP-ANOTHER": spa,
+				"SP-LEGGENDARIA": spl,
+				"DP-NORMAL": dpn,
+				"DP-HYPER": dph,
+				"DP-ANOTHER": dpa,
+				"DP-LEGGENDARIA": dpl,
+			},
+			notecounts,
+			songID,
+		});
+
+		curLoc += structSize;
+
+		if (buffer.length <= curLoc) {
+			moreData = false;
+		}
+	}
+
+	logger.info(`Done parsing.`);
+
+	return parsedData;
+}

database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts

@@ -0,0 +1,516 @@
+import { integer } from "tachi-common";
+import {
+	BGMObject,
+	BPMObject,
+	BSSObject,
+	CNObject,
+	EOCObject,
+	IIDXChartEvents,
+	IIDXParserContext,
+	IIDXParserOptions,
+	MeasureBarObject,
+	MeterObject,
+	NoteObject,
+	ParsedIIDXChart,
+	SampleObject,
+	ScratchObject,
+	TimingWindowsObject,
+} from "./types";
+import fs from "fs/promises";
+import logger from "../../../logger";
+
+interface EventParserContext {
+	isDP?: boolean;
+	songTitle?: string;
+}
+
+interface Directory {
+	offset: number;
+	length: number;
+}
+
+function ParseDirectoriesHeader(fileData: Buffer): Directory[] {
+	// .1 format has 12 directory entries at the start
+	// of the file.
+	// these are all 8 bytes long, and all next to
+	// eachother.
+	const dirsBuffer = fileData.subarray(0, 12 * 8);
+
+	const dirs: Array<{ offset: integer; length: integer }> = [];
+
+	for (let i = 0; i < 12; i++) {
+		const dirBuffer = dirsBuffer.subarray(i * 8, (i + 1) * 8);
+
+		// the first 4 bytes indicate the offset
+		// the other 4 indicate how many bytes
+		// this chart takes up.
+		const dir = {
+			offset: dirBuffer.readIntLE(0, 4),
+			length: dirBuffer.readIntLE(4, 4),
+		};
+
+		dirs.push(dir);
+	}
+
+	return dirs;
+}
+
+type Event =
+	| { type: "NOTE"; body: NoteObject }
+	| { type: "BGM"; body: BGMObject }
+	| { type: "BPM"; body: BPMObject }
+	| { type: "BSS"; body: BSSObject }
+	| { type: "CN"; body: CNObject }
+	| { type: "HBSS"; body: BSSObject }
+	| { type: "HCN"; body: CNObject }
+	| { type: "SCRATCH"; body: ScratchObject }
+	| { type: "EOC"; body: EOCObject }
+	| { type: "SAMPLE"; body: SampleObject }
+	| { type: "TIMINGWINDOW"; body: TimingWindowsObject }
+	| { type: "MEASUREBAR"; body: MeasureBarObject }
+	| { type: "METER"; body: MeterObject };
+
+const IIDX_TYPE_TO_RGC: Record<number, string> = {
+	0: "NOTE",
+	1: "NOTE",
+	2: "SAMPLE",
+	3: "SAMPLE",
+	4: "BPM",
+	5: "METER",
+	6: "EOC",
+	7: "BGM",
+	8: "TIMINGWINDOW",
+	12: "MEASUREBAR",
+	13: "HCN_INDICATOR",
+	16: "NOTECOUNT",
+};
+
+const IIDX_TIMING_TO_RGC: Record<
+	number,
+	"lateBad" | "lateGood" | "lateGreat" | "earlyGreat" | "earlyGood" | "earlyBad"
+> = {
+	0: "lateBad",
+	1: "lateGood",
+	2: "lateGreat",
+	3: "earlyGreat",
+	4: "earlyGood",
+	5: "earlyBad",
+};
+
+function ParseEvent(
+	noteData: Buffer,
+	parseContext: EventParserContext,
+	options: IIDXParserOptions
+): Event | null {
+	const data = {
+		ticks: noteData.readIntLE(0, 4),
+		type: noteData.readIntLE(4, 1),
+		param: noteData.readIntLE(5, 1),
+		val: noteData.readIntLE(6, 2),
+	};
+
+	// A second EOF indicator (the real one) is that there is an event placed at -1 miliseconds.
+	// return null, and DO NOT add this as an object.
+	if (data.ticks < 0 || data.ticks >= 2147483647) {
+		return null;
+	}
+
+	let type = IIDX_TYPE_TO_RGC[data.type];
+
+	let event: Event;
+
+	const ms = data.ticks * ((options.tickrate || 1000) / 1000);
+
+	// Disambiguate NOTE type, which may indicate other RGC types.
+	// THIS IS DIFFERENT TO THE IMMEDIATELY FOLLOWING STATEMENT!
+	if (type === "NOTE") {
+		// if val is nonzero, this note is a hold note
+		// with ticks of (length);
+		if (data.val !== 0) {
+			type = options.isHellCharge ? "HCN" : "CN";
+		}
+
+		// scratch overrides
+		if (data.param === 7) {
+			// if hold on scratch column, we have BSS.
+			if (data.val !== 0) {
+				type = options.isHellCharge ? "HBSS" : "BSS";
+			} else {
+				type = "SCRATCH";
+			}
+		}
+	}
+
+	if (type === "NOTE") {
+		const obj: NoteObject = {
+			ms,
+			col: data.param,
+		};
+
+		// if this note was on P2 side, we need to
+		// add 7 to its column.
+		if (data.type === 1) {
+			obj.col += 7;
+		}
+
+		if (data.type === 1) {
+			if (obj.col > 13 || obj.col < 0) {
+				throw TypeError(`Invalid column of ${obj.col}. Must be between 0 and 13.`);
+			}
+		} else {
+			if (obj.col > 6 || obj.col < 0) {
+				throw TypeError(`Invalid column of ${obj.col}. Must be between 0 and 6.`);
+			}
+		}
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "CN" || type === "HCN") {
+		if (type === "CN" && options.errOnCN) {
+			throw new TypeError(
+				`CN found inside ${
+					parseContext.songTitle || "UNKNOWN CHART"
+				}, but errOnCN was enabled.`
+			);
+		} else if (type === "HCN" && options.errOnHCN) {
+			throw new TypeError(
+				`HCN found inside ${
+					parseContext.songTitle || "UNKNOWN CHART"
+				}, but errOnHCN was enabled.`
+			);
+		}
+
+		const obj: CNObject = {
+			ms,
+			col: data.param,
+			msEnd: data.val + ms,
+		};
+
+		if (data.type === 1) {
+			obj.col += 7;
+		}
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "SCRATCH") {
+		const obj: ScratchObject = {
+			ms,
+		};
+
+		if (parseContext.isDP) {
+			if (data.param === 7) {
+				obj.col = 0;
+			} else if (data.param === 14) {
+				obj.col = 1;
+			} else {
+				return null;
+			}
+		}
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "BSS" || type === "HBSS") {
+		const obj: BSSObject = {
+			ms,
+			msEnd: ms + data.val,
+		};
+
+		if (parseContext.isDP) {
+			if (data.param === 7) {
+				obj.col = 0;
+			} else if (data.param === 14) {
+				obj.col = 1;
+			} else {
+				logger.warn(
+					`Unknown column of scratchObj sent to EventParser: ${data.param}, expected 7 or 14. Skipping.`
+				);
+				return null;
+			}
+		}
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "SAMPLE") {
+		const obj: SampleObject = {
+			ms,
+			col: data.param,
+			sound: data.val,
+		};
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "BPM") {
+		if (data.param === 0) {
+			logger.warn("Invalid value of 0 for denominator in BPM event. Skipping.");
+			return null;
+		}
+
+		const obj: BPMObject = {
+			ms,
+			bpm: data.val / data.param,
+		};
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "METER") {
+		const obj: MeterObject = {
+			ms,
+			num: data.val,
+			denom: data.param,
+		};
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "BGM") {
+		const obj: BGMObject = {
+			ms,
+			pan: data.param,
+			sound: data.val,
+		};
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "TIMINGWINDOW") {
+		const window = IIDX_TIMING_TO_RGC[data.param];
+
+		if (!window) {
+			logger.warn(
+				`Unknown type of timingWindow: ${window}, from .1 param ${data.param}. Skipping.`
+			);
+			return null;
+		}
+
+		const obj: TimingWindowsObject = {
+			ms,
+			window,
+			// for whatever reason, the value for TIMINGWINDOW is a signed int8,
+			// rather than the int16 every other thing expects.
+			// this means, suboptimally, we are reading noteData twice.
+			// performance is hardly a concern though, as you can blast 100k
+			// charts with this in a couple seconds.
+			val: noteData.readIntLE(6, 1),
+		};
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "EOC") {
+		const obj: EOCObject = {
+			ms,
+		};
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "MEASUREBAR") {
+		const obj: MeasureBarObject = {
+			ms,
+		};
+
+		if (parseContext.isDP) {
+			if (data.param !== 0 && data.param !== 1) {
+				logger.warn(
+					`Invalid parameter for measurebar of ${data.param}. Defaulting to 0 (left hand side).`
+				);
+				obj.side = 0;
+			} else {
+				obj.side = data.param;
+			}
+		}
+
+		event = {
+			type,
+			body: obj,
+		};
+	} else if (type === "NOTECOUNT") {
+		// You might think: Hey! This is super convenient for just quickly getting
+		// the notecount of a chart, right?
+		// Well, uh, no. It's not correct. CNs are treated as one note, which is
+		// *absolutely* not the case in game.
+		// This thing is **utterly** useless, as you have to count the amount of CNs
+		// anyway. At that point, why not count the notes?
+
+		return null;
+	} else if (type === "HCN_INDICATOR") {
+		// this chart is using HCNs.
+		options.isHellCharge = true;
+		return null;
+	} else {
+		logger.warn(
+			`(${parseContext.songTitle ?? "UNKNOWN"}) Unknown event type in .1: ${
+				data.type
+			} at ${ms.toFixed(2)}ms. Ignoring.${
+				data.type === 11
+					? " (It's BGA Related. Not sure what it does, but don't worry.)"
+					: ""
+			}`
+		);
+		return null;
+	}
+
+	return event;
+}
+
+function ParseEvents(
+	fileData: Buffer,
+	dir: Directory,
+	context: EventParserContext,
+	options: IIDXParserOptions
+) {
+	const events: IIDXChartEvents = {
+		BGM: [],
+		BPM: [],
+		BSS: [],
+		HCN: [],
+		CN: [],
+		HBSS: [],
+		NOTE: [],
+		SAMPLE: [],
+		METER: [],
+		TIMINGWINDOW: [],
+		MEASUREBAR: [],
+		EOC: [],
+		SCRATCH: [],
+	};
+
+	const data = fileData.subarray(dir.offset, dir.offset + dir.length);
+
+	// charts are a series of 8 byte "events".
+	// we are going to iterate through [data], pulling
+	// said 8 bytes, and parsing them.
+	// i am assuming that dir.length is always
+	// a multiple of 8.
+	for (let i = 0; i < dir.length; i += 8) {
+		const noteBuffer = data.subarray(i, i + 8);
+
+		const event = ParseEvent(noteBuffer, context, options);
+
+		if (!event) {
+			continue;
+		}
+
+		// @ts-expect-error invariant error but we're right.
+		events[event.type].push(event.body);
+	}
+
+	return events;
+}
+
+/**
+ * Converts the index of a directory to the corresponding difficulty name.
+ */
+const IIDX_DIFFICULTY_TO_RGC: Record<
+	number,
+	"BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA" | null
+> = {
+	0: "HYPER",
+	1: "NORMAL",
+	2: "ANOTHER",
+	3: "BEGINNER",
+	4: "LEGGENDARIA", // ^^ SP ^^
+	5: null, // vv DP vv
+	6: "HYPER",
+	7: "NORMAL",
+	8: "ANOTHER",
+	9: null,
+	10: "LEGGENDARIA",
+	11: null,
+};
+
+export function ParseDotOne(
+	fileData: Buffer,
+	context: IIDXParserContext = {},
+	options: IIDXParserOptions = {}
+) {
+	const charts: Array<ParsedIIDXChart> = [];
+
+	const dirs = ParseDirectoriesHeader(fileData);
+
+	// with all our dirs we now need to parse each chart.
+	for (let i = 0; i < 12; i++) {
+		const dir = dirs[i];
+
+		if (!dir || (dir.offset === 0 && dir.length === 0)) {
+			logger.verbose(`Skipped directory ${i}, as there was no data.`);
+			continue; // no data.
+		}
+
+		const difficulty = IIDX_DIFFICULTY_TO_RGC[i];
+
+		if (!difficulty) {
+			logger.warn(
+				`Data present inside directory ${i}, but this corresponds to an unknown difficulty. Ignoring.`
+			);
+			continue;
+		}
+
+		const eventContext: EventParserContext = {
+			isDP: i > 5,
+			songTitle: context?.songTitle,
+		};
+
+		if (!eventContext.songTitle) {
+			logger.warn(
+				"No filename provided in context.filename to IIDXParser, subsequent warnings will have no way of referring to the chart."
+			);
+		}
+
+		const events = ParseEvents(fileData, dir, eventContext, options);
+
+		const chart: ParsedIIDXChart = {
+			playtype: i > 5 ? "DP" : "SP",
+			title: context?.songTitle ?? null,
+			artist: context?.songArtist ?? null,
+			genre: context?.genre ?? null,
+			level: context?.level ?? null,
+			marquee: context?.marquee ?? null,
+			difficulty,
+			notecount: getNotecount(events),
+			events,
+		};
+
+		charts.push(chart);
+	}
+
+	return charts;
+}
+
+function getNotecount(events: IIDXChartEvents) {
+	// note: account for HMSS at some point
+	return (
+		events.NOTE.length +
+		events.SCRATCH.length +
+		events.CN.length * 2 +
+		events.HCN.length * 2 +
+		events.BSS.length * 2 +
+		events.HBSS.length * 2
+	);
+}
+
+export async function ParseDotOneFile(
+	fileath: string,
+	context: IIDXParserContext = {},
+	options: IIDXParserOptions = {}
+) {
+	const data = await fs.readFile(fileath);
+
+	return ParseDotOne(data, context, options);
+}

database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/types.ts

@@ -0,0 +1,148 @@
+import { integer } from "tachi-common";
+
+export interface ChartObject {
+	ms: number;
+}
+
+/**
+ * Note Object: Indicates that there is a playable note here.
+ * For SP, this goes from 0 to 6.
+ * For DP, this goes from 0 to 13.
+ */
+export interface NoteObject extends ChartObject {
+	col: number;
+}
+
+export interface ScratchObject extends ChartObject {
+	col?: 0 | 1;
+}
+
+export interface CNObject extends NoteObject {
+	msEnd: number;
+}
+
+export interface BSSObject extends ScratchObject {
+	msEnd: number;
+}
+
+/**
+ * Sample Object: Indicates that there is a sample (keysound) here.
+ * Regardless of SP or DP, the game stores samples that can happen on
+ * any of the 14 columns.
+ */
+export interface SampleObject extends ChartObject {
+	col: number;
+	sound: number;
+}
+
+/**
+ * BGM Object: Indicates that there is a background sample here.
+ * Unlike sample objects, these do not happen on columns, and instead use
+ * their two parameters to express panning and sound.
+ */
+export interface BGMObject extends ChartObject {
+	pan: number;
+	sound: number;
+}
+
+/**
+ * BPM Object: Indicates that there is a change of BPM here.
+ * BPM is expected to be a float.
+ */
+export interface BPMObject extends ChartObject {
+	bpm: number;
+}
+
+/**
+ * Meter Object: Indicates that there is a change of time signature here.
+ * Most charts never use this, but it does exist in a lot of old TaQ songs.
+ * It is expected that num and denom are integers.
+ */
+export interface MeterObject extends ChartObject {
+	num: number;
+	denom: number;
+}
+
+/**
+ * Measure Bar Object: Indicates that there is a measure bar here.
+ * Unlike other games, IIDX has these baked into the chart format, presumably to allow for
+ * charts such as 100%-minimoo G.
+ * For some reason, this has an additional property which determines what
+ * side it renders for.
+ */
+export interface MeasureBarObject extends ChartObject {
+	side?: 0 | 1;
+}
+
+/**
+ * End Of Chart object, Indicates that this is where the chart should end,
+ * and the game should switch to fadeout / display splashes.
+ */
+export type EOCObject = ChartObject;
+
+export interface TimingWindowsObject extends ChartObject {
+	window: "lateBad" | "lateGood" | "lateGreat" | "earlyGreat" | "earlyGood" | "earlyBad";
+	val: number;
+}
+
+export interface IIDXChartEvents {
+	SAMPLE: SampleObject[];
+	BPM: BPMObject[];
+	METER: MeterObject[];
+	BGM: BGMObject[];
+	TIMINGWINDOW: TimingWindowsObject[];
+	MEASUREBAR: MeasureBarObject[];
+	EOC: EOCObject[];
+	SCRATCH: ScratchObject[];
+	NOTE: NoteObject[];
+	BSS: BSSObject[];
+	HBSS: BSSObject[];
+	CN: CNObject[];
+	HCN: CNObject[];
+}
+
+export interface IIDXParserContext {
+	filename?: string;
+	songTitle?: string;
+	songArtist?: string;
+	charter?: string;
+	genre?: string;
+	level?: "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" | "11" | "12" | null;
+	marquee?: string;
+}
+
+export interface IIDXParserOptions {
+	errOnCN?: boolean;
+	errOnHCN?: boolean;
+	isHellCharge?: boolean;
+	tickrate?: number;
+}
+
+export interface ParsedIIDXChart {
+	playtype: "SP" | "DP";
+	title: string | null;
+	artist: string | null;
+	genre: string | null;
+	notecount: integer;
+	level: string | null;
+	marquee: string | null;
+	difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
+	events: IIDXChartEvents;
+}
+
+// no such thing as a DP-BEGINNER.
+type Difficulties = Exclude<
+	`${"SP" | "DP"}-${"BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"}`,
+	"DP-BEGINNER"
+>;
+
+export interface IIDXConvertOutput {
+	title: string;
+	artist: string;
+	marquee: string;
+	folder: number;
+	genre: string;
+	levels: Record<Difficulties, number>;
+	notecounts: Record<Difficulties, number>;
+	songID: integer;
+}

database-seeds/scripts/rerunners/iidx-mdb-parse/merge-mdb.ts

@@ -0,0 +1,281 @@
+import { Command } from "commander";
+import {
+	ChartDocument,
+	Difficulties,
+	GetGamePTConfig,
+	GPTSupportedVersions,
+	integer,
+	SongDocument,
+} from "tachi-common";
+import logger from "../../logger";
+import {
+	CreateChartID,
+	GetFreshScoreIDGenerator,
+	ReadCollection,
+	WriteCollection,
+} from "../../util";
+import { ParseIIDXData } from "./convert";
+import fs from "fs";
+import path from "path";
+
+if (require.main !== module) {
+	throw new Error(
+		`This is a script. It should be ran directly from the command line with ts-node.`
+	);
+}
+
+const program = new Command();
+program
+	.requiredOption("-b, --basedir <dir>", "The base of this IIDX install.")
+	.requiredOption("-i, --index <val>", "Whether this game uses 0 or 1 for the mdb.")
+	.requiredOption("-v, --version <This .bin version name>")
+	.option("-o, --omni", "Whether to fetch omnimix or not.")
+	.option("-f --force", "Forces overwrites when they shouldn't be automatically done.")
+	.option(
+		"--always-extract",
+		"Always re-extract IFS files even when they exist in ifs-output. Useful if the IFS files have changed."
+	);
+
+program.parse(process.argv);
+const options = program.opts() as {
+	version: GPTSupportedVersions["iidx:SP"];
+	index: "0" | "1";
+	basedir: string;
+	omni: boolean;
+	force: boolean;
+	alwaysExtract: boolean;
+};
+
+const iidxConfig = GetGamePTConfig("iidx", "SP");
+
+if (!iidxConfig.supportedVersions.includes(options.version)) {
+	throw new Error(
+		`Invalid version of '${
+			options.version
+		}'. Expected any of ${iidxConfig.supportedVersions.join(
+			", "
+		)}. If you're adding a new version, go update common/src/config.ts.`
+	);
+}
+
+if (options.index !== "0" && options.index !== "1") {
+	throw new Error(`Expected an --index of 0 or 1. Got ${options.index}.`);
+}
+
+const existingCharts: ChartDocument<"iidx:SP" | "iidx:DP">[] = ReadCollection("charts-iidx.json");
+
+const existingSongs: SongDocument<"iidx">[] = ReadCollection("songs-iidx.json");
+
+const blacklist = fs
+	.readFileSync(path.join(__dirname, "blacklist.txt"), "utf-8")
+	.split("\n")
+	.filter((e) => !e.startsWith("#") && e.trim() !== "")
+	.map((e) => new RegExp(e, "u"));
+
+function isInBlacklist(str: string) {
+	for (const regex of blacklist) {
+		if (regex.exec(str)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+async function ParseIIDXMDB() {
+	const mdbCharts = await ParseIIDXData(
+		options.basedir,
+		options.index,
+		options.omni,
+		options.alwaysExtract
+	);
+
+	const chartMap = new Map<integer, ChartDocument<"iidx:SP" | "iidx:DP">>();
+	const songMap = new Map<integer, SongDocument<"iidx">>();
+	const chartDiffMap = new Map<string, ChartDocument<"iidx:SP" | "iidx:DP">>();
+
+	for (const song of existingSongs) {
+		songMap.set(song.id, song);
+	}
+
+	for (const chart of existingCharts) {
+		// what, you thought this was easy?
+		if (Array.isArray(chart.data.inGameID)) {
+			for (const igid of chart.data.inGameID) {
+				chartMap.set(igid, chart);
+			}
+		} else {
+			chartMap.set(chart.data.inGameID, chart);
+		}
+	}
+
+	for (const chart of existingCharts) {
+		if (Array.isArray(chart.data.inGameID)) {
+			for (const igid of chart.data.inGameID) {
+				chartDiffMap.set(`${igid}-${chart.playtype}-${chart.difficulty}`, chart);
+			}
+		} else {
+			chartDiffMap.set(`${chart.data.inGameID}-${chart.playtype}-${chart.difficulty}`, chart);
+		}
+	}
+
+	const getFreeScoreID = GetFreshScoreIDGenerator("iidx");
+
+	for (const inp of mdbCharts) {
+		if (isInBlacklist(`S${inp.songID}`)) {
+			logger.verbose(
+				`Skipped ${inp.artist} - ${inp.title} (${inp.songID}) as it was in the blacklist.`
+			);
+			continue;
+		}
+
+		const anySongIDMatch = chartMap.get(inp.songID);
+		let song: SongDocument<"iidx">;
+
+		if (!anySongIDMatch) {
+			// new song?
+
+			const searchTerms = [inp.genre];
+			if (inp.marquee.toLowerCase() !== inp.title.toLowerCase()) {
+				searchTerms.push(inp.marquee);
+			}
+
+			const tachiSong: SongDocument<"iidx"> = {
+				id: getFreeScoreID(),
+				artist: inp.artist,
+				title: inp.title,
+				data: {
+					genre: inp.genre,
+					displayVersion: inp.folder.toString(),
+				},
+				searchTerms: [],
+				altTitles: [],
+			};
+
+			logger.info(`Added new song ${inp.title}.`);
+
+			if (inp.title.match(/\?/gu)) {
+				logger.warn(
+					`${inp.title} has a potentially konami-screwed title. Investigate it manually.`
+				);
+			}
+
+			if (inp.artist.match(/\?/gu)) {
+				logger.warn(
+					`${inp.artist} - ${inp.title} has a potentially konami-screwed title. Investigate it manually.`
+				);
+			}
+
+			existingSongs.push(tachiSong);
+
+			song = tachiSong;
+		} else {
+			const sxng = songMap.get(anySongIDMatch.songID);
+
+			if (!sxng) {
+				logger.error(`Song ${anySongIDMatch.songID} has charts but no song?`);
+				throw new Error(`Song ${anySongIDMatch.songID} has charts but no song?`);
+			}
+			song = sxng;
+		}
+
+		const diffNames = Object.keys(inp.levels) as (keyof typeof inp.levels)[];
+
+		for (const diffName of diffNames) {
+			if (isInBlacklist(`C${inp.songID}-${diffName}`)) {
+				logger.verbose(
+					`Ignored ${song.title} (${inp.songID}) ${diffName} as it was in the blacklist.`
+				);
+				continue;
+			}
+
+			const chart = chartDiffMap.get(`${inp.songID}-${diffName}`);
+
+			const notecount = inp.notecounts[diffName];
+			const level = inp.levels[diffName];
+
+			if (level === 0 && notecount && notecount > 0) {
+				logger.info(
+					`Chart ${song.title} ${diffName} has notecount ${notecount}, but has no level assigned. Skipping.`
+				);
+				continue;
+			}
+			if (level === 0) {
+				continue;
+			}
+
+			if (!chart) {
+				// no chart && no notecount => chart has never existed
+				// and still doesnt.
+				if (!notecount) {
+					continue;
+				}
+
+				// otherwise, make new chart?
+				const tachiChart: ChartDocument<"iidx:SP" | "iidx:DP"> = {
+					chartID: CreateChartID(),
+					difficulty: diffName.split("-")[1] as Difficulties["iidx:SP" | "iidx:DP"],
+					level: level.toString(),
+					levelNum: level,
+					isPrimary: true,
+					playtype: diffName.split("-")[0] as "SP" | "DP",
+					rgcID: null,
+					songID: song.id,
+					tierlistInfo: {},
+					versions: [options.version],
+					data: {
+						inGameID: inp.songID,
+						arcChartID: null,
+						notecount,
+						hashSHA256: null,
+						"2dxtraSet": null,
+						bpiCoefficient: null,
+						kaidenAverage: null,
+						worldRecord: null,
+					},
+				};
+
+				logger.info(`Inserting new chart ${inp.title} ${diffName}.`);
+				existingCharts.push(tachiChart);
+			} else {
+				if (!notecount) {
+					logger.warn(
+						`Chart ${inp.title} ${diffName} already exists, but has no notecount anymore. Not marking it as part of this version.`
+					);
+					continue;
+				}
+
+				// chart already exists, diff notecounts.
+				if (chart.data.notecount !== notecount) {
+					logger.warn(
+						`Chart ${inp.title} ${diffName} has a different notecount in the JSON to the data just parsed. Has this chart been edited? OLD: ${chart.data.notecount} -> NEW: ${notecount}.`
+					);
+					if (!options.force) {
+						logger.warn(
+							`Must be resolved manually. Use --force to blindly overwrite it anyway.`
+						);
+						continue;
+					}
+					chart.data.notecount = notecount;
+				}
+
+				if (chart.levelNum !== level) {
+					logger.info(
+						`Chart ${inp.title} ${diffName} has had a level change. ${chart.level} -> ${level}. Updating this.`
+					);
+					chart.level = level.toString();
+					chart.levelNum = level;
+				}
+
+				if (!chart.versions.includes(options.version)) {
+					chart.versions.push(options.version);
+				}
+			}
+		}
+	}
+
+	WriteCollection("songs-iidx.json", existingSongs);
+	WriteCollection("charts-iidx.json", existingCharts);
+}
+
+ParseIIDXMDB();

pnpm-lock.yaml

@@ -256,6 +256,7 @@ importers:
       fast-xml-parser: ^4.0.2
       glob: ^7.2.0
       html-entities: ^2.3.2
+      iconv-lite: ^0.6.3
       lodash.get: ^4.4.2
       mei-logger: ^1.0.1
       monk: ^7.3.4
@@ -274,6 +275,7 @@ importers:
       fast-xml-parser: 4.0.2
       glob: 7.2.3
       html-entities: 2.3.3
+      iconv-lite: 0.6.3
       lodash.get: 4.4.2
       mei-logger: 1.0.3
       monk: 7.3.4