Add inGameID matching support for maimai DX and support nullable inGameIDs in general (#1114)

* warn about null/missing inGameIDs

* add support for inGameID matching in maimai DX

* add indexes for maimai classic charts

* fix variable ordering

less diff

* Make custom tests work so we can avoid some footguns

* linting

* add inGameID to maimai DX test data

* eslint why u no work

* more readable tests

* clean up scripts

* [maimaidx] Splice 1

* Rising in the Sun SPH's levelNum should be 8

common/src/config/game-support/maimai-dx.ts

@@ -211,10 +211,12 @@ export const MAIMAI_DX_SINGLE_CONF = {
 		buddiesplus: "BUDDiES PLUS",
 	},
 
-	chartData: z.strictObject({}),
+	chartData: z.strictObject({
+		inGameID: z.number().int().nonnegative().nullable(),
+	}),
 
 	preferences: z.strictObject({}),
 	scoreMeta: z.strictObject({}),
 
-	supportedMatchTypes: ["songTitle", "tachiSongID"],
+	supportedMatchTypes: ["songTitle", "tachiSongID", "inGameID"],
 } as const satisfies INTERNAL_GAME_PT_CONFIG;

database-seeds/collections/charts-iidx.json

@@ -162258,7 +162258,7 @@
 		"difficulty": "HYPER",
 		"isPrimary": true,
 		"level": "8",
-		"levelNum": 9,
+		"levelNum": 8,
 		"playtype": "SP",
 		"songID": 761,
 		"versions": [
@@ -1406404,4 +1406404,4 @@
 			"30-2dxtra"
 		]
 	}
-]
+]

database-seeds/collections/songs-maimaidx.json

@@ -15569,4 +15569,4 @@
 		"searchTerms": [],
 		"title": "Bloody Trail"
 	}
-]
+]

database-seeds/scripts/rerunners/maimaidx/merge-options.ts

@@ -10,6 +10,9 @@ import {
 	WriteCollection,
 } from "../../util";
 import { ChartDocument, Difficulties, GetGamePTConfig, SongDocument } from "tachi-common";
+import { CreateLogger } from "mei-logger";
+
+const logger = CreateLogger("merge-options");
 
 const VERSION_DISPLAY_NAMES = [
 	"maimai",
@@ -36,6 +39,13 @@ const VERSION_DISPLAY_NAMES = [
 	"BUDDiES",
 	"BUDDiES PLUS",
 ];
+const DIFFICULTIES = [
+	"Basic",
+	"Advanced",
+	"Expert",
+	"Master",
+	"Re:Master",
+];
 
 interface IDWithDisplayName {
 	id: number;
@@ -77,36 +87,51 @@ function calculateLevelNum(data: Pick<NotesData, "level" | "levelDecimal">) {
 	return Number(`${data.level}.${data.levelDecimal}`);
 }
 
-const program = new Command();
-program.requiredOption("-i, --input <OPTION FOLDERS...>");
-program.requiredOption("-v, --version <VERSION>");
+if (require.main !== module) {
+	throw new Error(
+		`This is a script. It should be ran directly from the command line with ts-node.`
+	);
+}
 
+const program = new Command();
+program
+	.requiredOption("-i, --input <OPTIONS DIRS...>", "The options directories of your maimai DX install.")
+	.requiredOption("-v, --version <VERSION>", "The version of this maimai DX install.")
+	.option("-f, --force", "Forces inGameID overwrites where it shouldn't be automatically done.");
 program.parse(process.argv);
 const options = program.opts();
 
 const versions = Object.keys(GetGamePTConfig("maimaidx", "Single").versions);
+
+if (versions.indexOf(options.version) === -1) {
+	throw new Error(
+		`Invalid version of '${options.version}'. Expected any of ${
+			versions.join(",")
+		}. If you're adding a new version, go update common/src/config/game-config/maimai-dx.ts.`
+	);
+}
+
 const isLatestVersion =
 	versions.indexOf(options.version.replace(/(-intl|-omni)$/u, "")) === versions.length - 1;
-
-const existingSongDocs: Array<SongDocument<"maimaidx">> = ReadCollection("songs-maimaidx.json");
-const existingChartDocs: Array<ChartDocument<"maimaidx:Single">> =
+const existingSongs: Array<SongDocument<"maimaidx">> = ReadCollection("songs-maimaidx.json");
+const existingCharts: Array<ChartDocument<"maimaidx:Single">> =
 	ReadCollection("charts-maimaidx.json");
-const existingSongs = new Map(existingSongDocs.map((s) => [s.id, s]));
-const existingCharts = new Map<string, ChartDocument<"maimaidx:Single">>();
-const titleArtistToSongIDMap = new Map<string, number>();
+const songMap = new Map(existingSongs.map((s) => [s.id, s]));
+const chartMap = new Map<string, ChartDocument<"maimaidx:Single">>();
+const songTitleArtistMap = new Map<string, number>();
 
-for (const chart of existingChartDocs) {
-	const song = existingSongs.get(chart.songID);
+for (const chart of existingCharts) {
+	const song = songMap.get(chart.songID);
 
 	if (song === undefined) {
-		console.error(
+		logger.error(
 			`CONSISTENCY ERROR: Chart ID ${chart.chartID} does not belong to any songs! (songID was ${chart.songID})`
 		);
 		process.exit(1);
 	}
 
-	existingCharts.set(`${song.title}-${song.artist}-${chart.difficulty}`, chart);
-	titleArtistToSongIDMap.set(`${song.title}-${song.artist}`, song.id);
+	chartMap.set(`${song.title}-${song.artist}-${chart.difficulty}`, chart);
+	songTitleArtistMap.set(`${song.title}-${song.artist}`, song.id);
 }
 
 const parser = new XMLParser();
@@ -116,154 +141,207 @@ const newCharts: Array<ChartDocument<"maimaidx:Single">> = [];
 
 const songIDGenerator = GetFreshSongIDGenerator("maimaidx");
 
-for (const optionFolder of options.input) {
-	const musicFolder = path.join(optionFolder, "music");
-
-	if (!existsSync(musicFolder)) {
-		console.warn(`Option at ${optionFolder} does not have a "music" folder.`);
-		continue;
-	}
-
-	for (const song of readdirSync(musicFolder)) {
-		if (!song.startsWith("music")) {
+for (const optionsDir of options.input) {
+	for (const option of readdirSync(optionsDir)) {
+		if (!option.match(/[A-Z]\d{3}/u)) {
 			continue;
 		}
 
-		const songFolder = path.join(musicFolder, song);
+		const optionDir = path.join(optionsDir, option);
+		const musicsDir = path.join(optionsDir, option, "music");
 
-		if (!statSync(songFolder).isDirectory()) {
+		if (!existsSync(musicsDir)) {
+			logger.warn(`Option at ${optionDir} does not have a "music" folder.`);
 			continue;
 		}
 
-		const musicXmlLocation = path.join(songFolder, "Music.xml");
+		logger.info(`Scanning music directory ${musicsDir} for songs.`);
 
-		if (!existsSync(musicXmlLocation)) {
-			console.warn(`Music folder at ${songFolder} does not have a Music.xml file.`);
-			continue;
-		}
+		for (const music of readdirSync(musicsDir)) {
+			if (!music.match(/music\d+$/u)) {
+				continue;
+			}
 
-		const data = parser.parse(readFileSync(musicXmlLocation)) as MusicXML;
-		const musicData = data.MusicData;
+			const musicDir = path.join(musicsDir, music);
 
-		const inGameID = musicData.name.id;
+			if (!statSync(musicDir).isDirectory()) {
+				continue;
+			}
 
-		if (inGameID >= 100000) {
-			// 100000 and above is reserved for WORLD'S END songs, don't care.
-			continue;
-		}
+			const musicXmlLocation = path.join(musicDir, "Music.xml");
 
-		let songID: number | undefined;
+			if (!existsSync(musicXmlLocation)) {
+				logger.warn(`Music folder at ${musicDir} does not have a Music.xml file.`);
+				continue;
+			}
+
+			const data = parser.parse(readFileSync(musicXmlLocation)) as MusicXML;
+			const musicData = data.MusicData;
+			const inGameID = musicData.name.id;
+
+			if (inGameID >= 100000) {
+				// Ignore UTAGE charts, which are not supported by Tachi.
+				continue;
+			}
 
-		if (inGameID === 11422) {
 			// Manual override since the song's title is empty in the dataset and not
 			// IDEOGRAPHIC SPACE (U+3000).
-			songID = 959;
-		} else {
-			songID = titleArtistToSongIDMap.get(
-				`${musicData.name.str}-${musicData.artistName.str}`
-			);
-		}
-
-		if (songID === undefined) {
+			let tachiSongID = inGameID === 11422
+				? 959
+				: songTitleArtistMap.get(`${musicData.name.str}-${musicData.artistName.str}`);
+			
+			// Has this song been disabled in-game?
 			if (musicData.disable) {
+				if (tachiSongID !== undefined) {
+					logger.info(
+						`Removing charts of song ID ${tachiSongID} from version ${
+							options.version
+						}, because the disable flag in Music.xml is enabled.`
+					);
+
+					// Songs are removed mid-version by marking the `disable` flag,
+					// since option data can only add or overwrite, never remove.
+					existingCharts
+						.filter((c) => c.songID === tachiSongID)
+						.forEach((c) => {
+							const versionIndex = c.versions.indexOf(options.version);
+
+							if (versionIndex !== -1) {
+								c.versions.splice(versionIndex, 1);
+							}
+						});
+				}
+				
 				continue;
 			}
+			
+			// New song?
+			if (tachiSongID === undefined) {
+				const displayVersion = VERSION_DISPLAY_NAMES[musicData.AddVersion.id];
 
-			songID = songIDGenerator();
+				if (!displayVersion) {
+					throw new Error(
+						`Unknown version ID ${
+							musicData.AddVersion.id
+						}. Update database-seeds/scripts/rerunners/maimaidx/merge-options.ts.`
+					);
+				}
 
-			const displayVersion = VERSION_DISPLAY_NAMES[musicData.AddVersion.id];
+				tachiSongID = songIDGenerator();
 
-			if (!displayVersion) {
-				throw new Error(
-					`Unknown version ID ${musicData.AddVersion.id}. Update merge-options.ts.`
-				);
+				const songDoc: SongDocument<"maimaidx"> = {
+					title: musicData.name.str,
+					altTitles: [],
+					searchTerms: [],
+					artist: musicData.artistName.str,
+					id: tachiSongID,
+					data: {
+						displayVersion,
+						genre: musicData.genreName.str,
+					},
+				};
+
+				newSongs.push(songDoc);
+				songTitleArtistMap.set(`${songDoc.title}-${songDoc.artist}`, tachiSongID);
+
+				logger.info(`Added new song ${songDoc.artist} - ${songDoc.title}.`);
 			}
 
-			const songDoc: SongDocument<"maimaidx"> = {
-				title: musicData.name.str,
-				altTitles: [],
-				searchTerms: [],
-				artist: musicData.artistName.str,
-				id: songID,
-				data: {
-					displayVersion,
-					genre: musicData.genreName.str,
-				},
-			};
+			for (const [index, difficulty] of musicData.notesData.Notes.entries()) {
+				if (!difficulty.isEnable || !existsSync(path.join(musicDir, difficulty.file.path))) {
+					continue;
+				}
 
-			newSongs.push(songDoc);
-			titleArtistToSongIDMap.set(`${songDoc.title}-${songDoc.artist}`, songID);
-		} else if (musicData.disable) {
-			// Sometimes, removed songs slip into the base game,
-			// and are "removed" later by marking the disable flag.
-			existingChartDocs
-				.filter((c) => c.songID === songID)
-				.forEach((c) => {
-					const versionIndex = c.versions.indexOf(options.version);
+				let difficultyName = DIFFICULTIES[index];
 
-					if (versionIndex !== -1) {
-						c.versions.splice(versionIndex);
+				if (difficultyName === undefined) {
+					throw new Error(
+						`Unknown difficulty ID ${
+							index
+						}. Update database-seeds/scripts/rerunners/maimaidx/merge-options.ts and possibly common/src/config/game-support/maimai-dx.ts.`
+					);
+				}
+	
+				if (inGameID > 10000) {
+					difficultyName = `DX ${difficultyName}`;
+				}
+	
+				let exists: ChartDocument<"maimaidx:Single"> | undefined;
+	
+				if (inGameID === 11422) {
+					exists = chartMap.get(`-x0o0x_-${difficultyName}`);
+				} else {
+					exists = chartMap.get(
+						`${musicData.name.str}-${musicData.artistName.str}-${difficultyName}`
+					);
+				}
+	
+				const level = calculateLevel(difficulty);
+				const levelNum = calculateLevelNum(difficulty);
+	
+				if (exists) {
+					const displayName = `${musicData.artistName.str} - ${musicData.name.str} [${exists.difficulty}] (${exists.chartID})`;
+
+					if (exists.data.inGameID === null) {
+						logger.info(`Adding inGameID ${inGameID} for chart ${displayName}.`);
+						exists.data.inGameID = inGameID;
+					} else if (exists.data.inGameID !== inGameID) {
+						logger.warn(
+							`The chart ${displayName} already exists in charts-maimaidx under a different inGameID (${
+								exists.data.inGameID
+							} != ${inGameID}). Is this a duplicate with a different inGameID?`
+						);
+
+						if (options.force) {
+							logger.warn("Overwriting anyways, because --force has been requested.");
+							exists.data.inGameID = inGameID;
+						} else {
+							logger.warn("Must be resolved manually. Use --force to overwrite anyways.");
+						}
 					}
-				});
-			continue;
-		}
+					
+					const versionIndex = exists.versions.indexOf(options.version);
+	
+					if (versionIndex === -1) {
+						exists.versions.push(options.version);
+					}
+	
+					if (isLatestVersion) {
+						if (exists.level !== level) {
+							logger.info(`Chart ${displayName} has had a level change: ${exists.level} -> ${level}.`);
+							exists.level = level;
+						}
 
-		for (const [index, difficulty] of musicData.notesData.Notes.entries()) {
-			if (!difficulty.isEnable || !existsSync(path.join(songFolder, difficulty.file.path))) {
-				continue;
-			}
-
-			// There's also a sixth difficulty, "Strong", but it has always stayed disabled.
-			// Tachi doesn't support it anyways.
-			let difficultyName = ["Basic", "Advanced", "Expert", "Master", "Re:Master"][index];
-
-			if (inGameID > 10000) {
-				difficultyName = `DX ${difficultyName}`;
-			}
-
-			let exists: ChartDocument<"maimaidx:Single"> | undefined;
-
-			if (inGameID === 11422) {
-				exists = existingCharts.get(`-x0o0x_-${difficultyName}`);
-			} else {
-				exists = existingCharts.get(
-					`${musicData.name.str}-${musicData.artistName.str}-${difficultyName}`
-				);
-			}
-
-			const level = calculateLevel(difficulty);
-			const levelNum = calculateLevelNum(difficulty);
-
-			if (exists) {
-				const versionIndex = exists.versions.indexOf(options.version);
-
-				if (versionIndex === -1) {
-					exists.versions.push(options.version);
+						if (exists.levelNum !== levelNum) {
+							logger.info(`Chart ${displayName} has had a levelNum change: ${exists.levelNum} -> ${levelNum}.`);
+							exists.levelNum = levelNum;
+						}
+					}
+	
+					continue;
 				}
+	
+				const chartDoc: ChartDocument<"maimaidx:Single"> = {
+					chartID: CreateChartID(),
+					songID: tachiSongID,
+					difficulty: difficultyName as Difficulties["maimaidx:Single"],
+					isPrimary: true,
+					level,
+					levelNum,
+					versions: [options.version],
+					playtype: "Single",
+					data: {
+						inGameID,
+					},
+				};
+				
+				newCharts.push(chartDoc);
 
-				if (isLatestVersion) {
-					exists.level = level;
-					exists.levelNum = levelNum;
-				}
-
-				continue;
+				logger.info(`Inserted new chart ${musicData.artistName.str} - ${musicData.name.str} [${chartDoc.difficulty}] (${chartDoc.chartID}).`);
 			}
-
-			const chartDoc: ChartDocument<"maimaidx:Single"> = {
-				chartID: CreateChartID(),
-				songID,
-				difficulty: difficultyName as Difficulties["maimaidx:Single"],
-				isPrimary: true,
-				level,
-				levelNum,
-				versions: [options.version],
-				playtype: "Single",
-				data: {},
-			};
-			newCharts.push(chartDoc);
 		}
 	}
-}
+} 
 
 MutateCollection("songs-maimaidx.json", (songs: Array<SongDocument<"maimaidx">>) => [
 	...songs,
@@ -273,4 +351,4 @@ MutateCollection("songs-maimaidx.json", (songs: Array<SongDocument<"maimaidx">>)
 // overwrite this collection instead of mutating it
 // we already know the existing chart docs and might have mutated them to
 // declare the new versions, or update chart constants.
-WriteCollection("charts-maimaidx.json", [...existingChartDocs, ...newCharts]);
+WriteCollection("charts-maimaidx.json", [...existingCharts, ...newCharts]);

database-seeds/scripts/rerunners/maimaidx/parse-maimaidx-dataset.js

@@ -1,8 +1,11 @@
 const fetch = require("node-fetch");
-const { CreateChartID, ReadCollection, WriteCollection } = require("../../util");
+const { CreateChartID, ReadCollection, WriteCollection, GetFreshSongIDGenerator } = require("../../util");
+const { CreateLogger } = require("mei-logger");
 
-const CURRENT_VERSION = "buddies";
-const CURRENT_VERSION_NUM = 240;
+const logger = CreateLogger("parse-maimaidx-dataset");
+
+const CURRENT_VERSION = "buddiesplus";
+const CURRENT_VERSION_NUM = 245;
 const DATA_URL = "https://maimai.sega.jp/data/maimai_songs.json";
 const ALIAS_URL =
 	"https://raw.githubusercontent.com/lomotos10/GCM-bot/main/data/aliases/en/maimai.tsv";
@@ -57,12 +60,18 @@ const versionOverrides = {
 	const songs = ReadCollection("songs-maimaidx.json");
 	const charts = ReadCollection("charts-maimaidx.json");
 
-	const existingSongs = new Map(songs.map((e) => [e.title, e.id]));
-	const existingCharts = new Map(charts.map((e) => [`${e.songID} ${e.difficulty}`, e.chartID]));
+	const existingSongs = new Map(songs.map((e) => [`${e.title}-${e.artist}`, e.id]));
+	const existingCharts = new Map(charts.map((e) => [`${e.songID}-${e.difficulty}`, e.chartID]));
+
+	logger.info(`Fetching official song information from ${DATA_URL}...`);
 
 	const datum = await fetch(DATA_URL).then((r) => r.json());
 	const aliases = new Map();
+
+	logger.info(`Fetching aliases from ${ALIAS_URL}...`);
+
 	const aliasesRaw = await fetch(ALIAS_URL).then((r) => r.text());
+	
 	for (const line of aliasesRaw.split("\n")) {
 		const [title, ...alias] = line.split("\t");
 		aliases.set(title, alias);
@@ -72,7 +81,8 @@ const versionOverrides = {
 	// japanese titled songs
 	const altTitles = [];
 
-	let songID = Math.max(Math.max(...existingSongs.values()), 0) + 1;
+	const getFreshSongID = GetFreshSongIDGenerator("maimaidx");
+	let chartsWithLevelChanges = 0;
 
 	for (const data of datum) {
 		// We don't support UTAGE charts, which are similar to CHUNITHM's WORLD'S END.
@@ -80,26 +90,30 @@ const versionOverrides = {
 			continue;
 		}
 
-		let thisSongID = songID;
-
 		const version = versionOverrides[data.title] ?? Number(data.version.substring(0, 3));
+		
 		if (version > CURRENT_VERSION_NUM && !Object.keys(versionOverrides).includes(data.title)) {
-			// Skipping songs that are newer than currently supported version (FESTiVAL PLUS).
+			// Skipping songs that are newer than currently supported version.
+			logger.warn(`Ignoring song ${data.artist} - ${data.title}, which is newer than CURRENT_VERSION_NUM.`);
 			continue;
 		}
+
+		let tachiSongID = existingSongs.get(`${data.title}-${data.artist}`);
+
 		if (data.title === "　" && data.artist === "x0o0x_") {
 			// Manual override since the song's title is "" (empty) in the dataset.
-			continue;
+			tachiSongID = 959;
 		}
 
-		if (existingSongs.has(data.title)) {
-			thisSongID = existingSongs.get(data.title);
-			const song = songs.find((e) => e.id === thisSongID);
+		if (tachiSongID !== undefined) {
+			const song = songs.find((e) => e.id === tachiSongID);
+			
 			song.searchTerms = aliases.get(data.title.trim()) ?? [];
 		} else {
-			songID++;
+			tachiSongID = getFreshSongID();
+
 			songs.push({
-				id: thisSongID,
+				id: tachiSongID,
 				title: data.title.trim(),
 				artist: data.artist.trim(),
 				searchTerms: aliases.get(data.title.trim()) ?? [],
@@ -109,7 +123,8 @@ const versionOverrides = {
 					genre: data.catcode.trim(),
 				},
 			});
-			console.log(`New song: ${data.artist.trim()} - ${data.title.trim()}`);
+
+			logger.info(`Added new song ${data.artist.trim()} - ${data.title.trim()}.`);
 		}
 
 		diffNames.forEach((diff) => {
@@ -118,28 +133,42 @@ const versionOverrides = {
 				return;
 			}
 
-			const chartID = existingCharts.get(`${thisSongID} DX ${diffMap.get(diff)}`);
+			const chartID = existingCharts.get(`${tachiSongID}-DX ${diffMap.get(diff)}`);
 			const chart = charts.find((e) => e.chartID === chartID);
-			if (chartID) {
+			const lvNum = Number(data[`dx_lev_${diff}`].replace(/\+/u, ".6"));
+			
+			if (chartID) {	
 				if (!chart.versions.includes(CURRENT_VERSION)) {
 					chart.versions.push(CURRENT_VERSION);
 				}
+
+				if (chart.level !== data[`dx_lev_${diff}`]) {
+					logger.info(`Chart ${data.artist} - ${data.title} [DX ${diffMap.get(diff)}] (${chartID}) has had a level change: ${chart.level} -> ${data[`dx_lev_${diff}`]}.`);
+
+					chart.level = data[`dx_lev_${diff}`];
+					chart.levelNum = lvNum;
+					
+					chartsWithLevelChanges++;
+				}
+
 				return;
 			}
 
-			const lvNum = Number(data[`dx_lev_${diff}`].replace(/\+/u, ".7"));
-
 			charts.push({
-				songID: thisSongID,
+				songID: tachiSongID,
 				chartID: CreateChartID(),
 				level: data[`dx_lev_${diff}`],
 				levelNum: lvNum,
 				isPrimary: true,
 				difficulty: `DX ${diffMap.get(diff)}`,
 				playtype: "Single",
-				data: {},
+				data: {
+					inGameID: null,
+				},
 				versions: [CURRENT_VERSION],
 			});
+
+			logger.info(`Added new chart ${data.artist} - ${data.title} [DX ${diffMap.get(diff)}].`);
 		});
 
 		diffNames.forEach((diff) => {
@@ -148,19 +177,30 @@ const versionOverrides = {
 				return;
 			}
 
-			const chartID = existingCharts.get(`${thisSongID} ${diffMap.get(diff)}`);
+			const chartID = existingCharts.get(`${tachiSongID}-${diffMap.get(diff)}`);
 			const chart = charts.find((e) => e.chartID === chartID);
+
+			const lvNum = Number(data[`lev_${diff}`].replace(/\+/u, ".6"));
+
 			if (chartID) {
 				if (!chart.versions.includes(CURRENT_VERSION)) {
 					chart.versions.push(CURRENT_VERSION);
 				}
+
+				if (chart.level !== data[`lev_${diff}`]) {
+					logger.info(`Chart ${data.artist} - ${data.title} [${diffMap.get(diff)}] (${chartID}) has had a level change: ${chart.level} -> ${data[`lev_${diff}`]}.`);
+
+					chart.level = data[`lev_${diff}`];
+					chart.levelNum = lvNum;
+
+					chartsWithLevelChanges++;
+				}
+
 				return;
 			}
 
-			const lvNum = Number(data[`lev_${diff}`].replace(/\+/u, ".7"));
-
 			charts.push({
-				songID: thisSongID,
+				songID: tachiSongID,
 				chartID: CreateChartID(),
 				level: data[`lev_${diff}`],
 				levelNum: lvNum,
@@ -170,9 +210,15 @@ const versionOverrides = {
 				data: {},
 				versions: [CURRENT_VERSION],
 			});
+
+			logger.info(`Added new chart ${data.artist} - ${data.title} [${diffMap.get(diff)}].`);
 		});
 	}
 
+	if (chartsWithLevelChanges > 0) {
+		logger.warn(`${chartsWithLevelChanges} chart(s) has had level changes, and their levelNum has been adjusted to default values. You should obtain the new levelNum for this chart using merge-options.ts or add-maimaidx-internal-levels.ts.`);
+	}
+
 	WriteCollection("charts-maimaidx.json", charts);
 	WriteCollection("songs-maimaidx.json", songs);
 })();

database-seeds/scripts/test/custom.test.ts

@@ -18,33 +18,68 @@ function test<T>(desc: string, fn: TestFn<T>): Test<T> {
 const CHART_CHECKS: { [G in Game]?: Array<Test<ChartDocument<GPTStrings[G]>>> } = {
 	iidx: [
 		test("Level should not be 0", (c) => c.level !== "0"),
-		test("LevelNum should be an integer greater than 0", (c) =>
-			c.levelNum > 0 && Number.isInteger(c.levelNum)),
-		test("Level and LevelNum should align", (c) => c.level === c.levelNum.toString()),
+		test(
+			"LevelNum should be an integer greater than 0 if level is known",
+			(c) =>	c.level === "?" || c.levelNum > 0
+		),
+		test(
+			"Level and LevelNum should align",
+			(c) => (c.level === "?" && c.levelNum === 0) || c.level === c.levelNum.toString()
+		),
 	],
 	chunithm: [
 		test("Level should not be 0", (c) => c.level !== "0"),
-		test("LevelNum should be an integer greater than 0", (c) =>
-			c.levelNum > 0 && Number.isInteger(c.levelNum)),
+		test("LevelNum should be a number greater than 0", (c) => c.levelNum > 0),
+		test(
+			"Level and LevelNum should align (X+ should be X.5 or higher)",
+			(c) => {
+				if (c.level.endsWith("+")) {
+					return c.levelNum * 10 % 10 >= 5
+				} else {
+					return c.levelNum * 10 % 10 < 5
+				}
+			},
+		),
+		test(
+			"inGameID should not exceed 8000, which is reserved for WORLD'S END charts.",
+			(c) => c.data.inGameID === null || c.data.inGameID < 8000,
+		)
 	],
 	maimaidx: [
 		test("Level should not be 0", (c) => c.level !== "0"),
-		test("LevelNum should be an integer greater than 0", (c) =>
-			c.levelNum > 0 && Number.isInteger(c.levelNum)),
+		test("LevelNum should be an number greater than 0", (c) => c.levelNum > 0),
+		test("inGameID for ST charts should be smaller than 10000",
+			(c) => {
+				if (c.difficulty.startsWith("DX")) {
+					return true;
+				}
+
+				return c.data.inGameID === null || c.data.inGameID < 10000;
+			},
+		),
+		test(
+			"inGameID for DX charts should be between 10000 (inclusive) and 20000 (exclusive)",
+			(c) => {
+				if (!c.difficulty.startsWith("DX")) {
+					return true;
+				}
+
+				if (c.data.inGameID === null) {
+					return true;
+				}
+
+				return c.data.inGameID >= 10000 && c.data.inGameID < 20000;
+			},
+		),
+		test(
+			"inGameID should not exceed 100000, which is reserved for UTAGE charts.",
+			(c) => c.data.inGameID === null || c.data.inGameID < 100000,
+		),
 	],
 };
 
 const SONG_CHECKS: { [G in Game]?: Array<Test<SongDocument<G>>> } = {};
 
-function FormatPrError(err, foreword = "Error") {
-	const receivedText =
-		typeof err.userVal === "object" && err.userVal !== null
-			? ""
-			: ` | Received ${err.userVal} [${err.userVal === null ? "null" : typeof err.userVal}]`;
-
-	return `${foreword}: ${err.keychain} | ${err.message}${receivedText}.`;
-}
-
 let exitCode = 0;
 const suites: Array<{ name: string; good: boolean; report: unknown }> = [];
 
@@ -89,14 +124,10 @@ for (const collection of collections) {
 		const pretty = formatFn(d, game as Game);
 
 		for (const check of checks[game as Game] ?? []) {
-			try {
-				check.fn(d);
-			} catch (err) {
+			if (!check.fn(d)) {
 				console.error(
 					chalk.red(
-						`[ERR] ${collectionName} | ${check.desc} | ${pretty} | ${FormatPrError(
-							err
-						)}.`
+						`[ERR] ${collectionName} | ${check.desc} | ${pretty}`,
 					)
 				);
 				failed = true;

database-seeds/scripts/test/match-type.test.ts

@@ -92,6 +92,36 @@ for (const { game, matchType, playtype } of uniquenessChecks) {
 			continue;
 		}
 
+		if (
+			(matchType === "inGameID" || matchType === "sdvxInGameID")
+			&& (el.data.inGameID === null || el.data.inGameID === undefined)
+		) {
+			console.log(
+				chalk.yellow(
+					`Chart ID ${el.chartID} (song ID ${el.songID}) cannot be matched using matchType=${
+						matchType
+					} because its inGameID is unknown.`,
+				)
+			);
+			warns++;
+			continue;
+		}
+
+		if (
+			matchType === "inGameStrID"
+			&& (el.data.inGameStrID === null && el.data.inGameStrID === undefined)
+		) {
+			console.log(
+				chalk.yellow(
+					`Chart ID ${el.chartID} (song ID ${el.songID}) cannot be matched using matchType=${
+						matchType
+					} because its inGameStrID is unknown.`,
+				)
+			);
+			warns++;
+			continue;
+		}
+
 		let newUniqueThingies = handler.fn(el);
 
 		// make single returns into arrays. convenient.

server/src/external/mongo/indexes.ts

@@ -127,6 +127,10 @@ const staticIndexes: Partial<Record<Databases, Array<Index>>> = {
 		index({ "data.hashSHA256": 1, playtype: 1 }, UNIQUE),
 		index({ "data.hashMD5": 1, playtype: 1 }, UNIQUE),
 	],
+	"charts-maimai": [
+		index({ "data.inGameID": 1, difficulty: 1 }, UNIQUE),
+		index({ "data.inGameStrID": 1, difficulty: 1 }, UNIQUE),
+	],
 	"import-locks": [index({ userID: 1 }, UNIQUE)],
 	"score-blacklist": [index({ scoreID: 1 }, UNIQUE)],
 	migrations: [index({ migrationID: 1 }, UNIQUE)],

server/src/test-utils/test-data.ts

@@ -387,7 +387,9 @@ export const TestingMaimaiChart: ChartDocument<"maimai:Single"> = {
 
 export const TestingMaimaiDXChart: ChartDocument<"maimaidx:Single"> = {
 	chartID: "fbde73d543f93cbad9ec67052a7165700c5c8df6",
-	data: {},
+	data: {
+		inGameID: 1,
+	},
 	difficulty: "Advanced",
 	isPrimary: true,
 	level: "7",