[maimaidx] add search terms, chart constants and new songs (#888)

database-seeds/scripts/rerunners/maimaidx/add-maimaidx-internal-levels.ts

@@ -0,0 +1,381 @@
+/**
+ * Run parse-maimaidx-dataset.js first.
+ *
+ * Internal levels for FESTiVAL songs are at
+ * https://docs.google.com/spreadsheets/d/1xbDMo-36bGL_d435Oy8TTVq4ADFmxl9sYFqhTXiJYRg/edit
+ *
+ * Download sheets as CSV and use `-f <CSV filename>`.
+ *
+ * If you have a `music` folder from a current maimai DX dump, pass it with `-d path/to/music`.
+ */
+import fs from "fs";
+import path from "path";
+import { Command } from "commander";
+import { parse } from "csv-parse/sync";
+import { XMLParser } from "fast-xml-parser";
+import { ReadCollection, MutateCollection } from "../../util";
+import { ChartDocument, SongDocument } from "tachi-common";
+
+interface XmlNotes {
+	file: { path: string };
+
+	level: number;
+	levelDecimal: number;
+
+	notesDesigner: { id: number; str: string };
+	notesType: number;
+
+	musicLevelID: number;
+	maxNotes: number;
+	isEnable: boolean;
+}
+
+const diffMap = new Map([
+	["BAS", "Basic"],
+	["ADV", "Advanced"],
+	["EXP", "Expert"],
+	["MAS", "Master"],
+	["ReMAS", "Re:Master"],
+]);
+
+const categoryMap = new Map([
+	["POPS&アニメ", "POPS＆アニメ"],
+	["niconico", "niconico＆ボーカロイド"],
+	["東方", "東方Project"],
+	["maimaiオリジナル", "maimai"],
+	["ゲーム&Variety", "ゲーム＆バラエティ"],
+	["ゲキチュウ", "オンゲキ＆CHUNITHM"],
+]);
+
+const manualTitleMap = new Map([
+	// 14 and higher
+	["Excalibur ～Revived Resolution～", "Excalibur ～Revived resolution～"],
+	["Sqlupp(Camellia's Sqleipd*Hiytex Remix)", 'Sqlupp (Camellia\'s "Sqleipd*Hiytex" Remix)'],
+	["≠彡゛/了→", '≠彡"/了→'],
+	["Mjolnir", "Mjölnir"],
+	["FREEDOM DiVE(tpz Overcute Remix)", "FREEDOM DiVE (tpz Overcute Remix)"],
+
+	// 13+
+	["GRANDIR", "GRÄNDIR"],
+	["D✪N’T ST✪P R✪CKIN’", "D✪N’T  ST✪P  R✪CKIN’"],
+	["Seclet Sleuth", "Secret Sleuth"],
+	["L'epilogue", "L'épilogue"],
+
+	// 13
+	[
+		"REVIVER オルタンシア･サーガ-蒼の騎士団- オリジナルVer.",
+		"REVIVER オルタンシア・サーガ -蒼の騎士団- オリジナルVer.",
+	],
+	[
+		"チルノのパーフェクトさんすう教室6 ⑨周年バージョン",
+		"チルノのパーフェクトさんすう教室　⑨周年バージョン",
+	],
+	[
+		"【東方ニコカラ】秘神マターラfeat.魂音泉【IOSYS】",
+		"【東方ニコカラ】秘神マターラ feat.魂音泉【IOSYS】",
+	],
+	["Save This World νMix", "Save This World νMIX"],
+	["ウッーウッーウマウマ( ﾟ∀ﾟ)", "ウッーウッーウマウマ(ﾟ∀ﾟ)"],
+	["砂の惑星 feat.HATSUNE MIKU", "砂の惑星 feat. HATSUNE MIKU"],
+	[
+		"Seyana.～何でも言うことを聞いてくれるアカネチャン～",
+		"Seyana. ～何でも言うことを聞いてくれるアカネチャン～",
+	],
+	["レッツゴー！陰陽師", "レッツゴー!陰陽師"],
+	["L4TS:2018(feat.あひる＆KTA)", "L4TS:2018 (feat. あひる & KTA)"],
+	["曖昧Mind", "曖昧mind"],
+	["紅星ミゼラブル〜廃憶編", "紅星ミゼラブル～廃憶編"],
+	["ファンタジーゾーンOPA!-OPA! -GMT remix-", "ファンタジーゾーン OPA-OPA! -GMT remix-"],
+	["Turn Around", "Turn around"],
+	["ぼくたちいつでもしゅわっしゅわ！", "ぼくたちいつでも　しゅわっしゅわ！"],
+	["God Knows…", "God knows..."],
+	["Jorqer", "Jörqer"],
+	["スカーレット警察のゲットーパトロール２４時", "スカーレット警察のゲットーパトロール24時"],
+]);
+
+function normalizeTitle(title: string): string {
+	return (
+		title
+			.toLowerCase()
+			.replace(/ /gu, "")
+			// ideographic space is used in some titles
+			// eslint-disable-next-line no-irregular-whitespace
+			.replace(/　/gu, "")
+			// so is nbsp I think?
+			// eslint-disable-next-line no-irregular-whitespace
+			.replace(/ /gu, "")
+			.replace(/：/gu, ":")
+			.replace(/（/gu, "(")
+			.replace(/）/gu, ")")
+			.replace(/！/gu, "!")
+			.replace(/？/gu, "?")
+			.replace(/`/gu, "'")
+			.replace(/’/gu, "'")
+			.replace(/”/gu, '"')
+			.replace(/“/gu, '"')
+			.replace(/～/gu, "~")
+			.replace(/－/gu, "-")
+			.replace(/＠/gu, "@")
+	);
+}
+
+function findSong(
+	songs: SongDocument<"maimaidx">[],
+	title: string,
+	category: string
+): SongDocument<"maimaidx"> | undefined {
+	// There are two songs with the exact same title and that only differs
+	// by category:
+	// - Link (maimai) is 68
+	// - Link (niconico) is 244
+	if (title === "Link") {
+		return songs.find((s) => s.id === (category === "maimai" ? 68 : 244));
+	}
+
+	// These songs will return the same result if normalized
+	if (title === "Heartbeats") {
+		return songs.find((s) => s.id === 131);
+	}
+	if (title === "Heart Beats") {
+		return songs.find((s) => s.id === 211);
+	}
+
+	return songs.find(
+		(s) =>
+			normalizeTitle(s.title) === normalizeTitle(title) ||
+			s.title === manualTitleMap.get(title)
+	);
+}
+
+function calculateDisplayLevel(internalLevel: number): string {
+	const plusDifficulty = (internalLevel * 10) % 10 >= 7;
+	const level = `${Math.floor(internalLevel)}${plusDifficulty && internalLevel >= 7 ? "+" : ""}`;
+	return level;
+}
+
+function calculateDifficulty(style: string, sheetDifficulty: string): string {
+	return `${style === "DX" ? `${style} ` : ""}${diffMap.get(sheetDifficulty)}`;
+}
+
+function addTmaiSheet(csvData: string[][]) {
+	const songs = ReadCollection("songs-maimaidx.json");
+
+	MutateCollection("charts-maimaidx.json", (charts: ChartDocument<"maimaidx:Single">[]) => {
+		for (let rowNumber = 1; rowNumber < csvData.length; rowNumber++) {
+			const row = csvData[rowNumber]!;
+			const title = row[1];
+			if (!title) {
+				break;
+			}
+
+			const song = songs.find((s) => s.title === title);
+			if (!song) {
+				console.log(`Could not find song ${title}`);
+				continue;
+			}
+
+			const difficulty = calculateDifficulty(row[2]!, row[3]!);
+			const chart = charts.find((c) => c.songID === song.id && c.difficulty === difficulty);
+			if (!chart) {
+				console.log(`Could not find chart ${difficulty} for ${title}`);
+				continue;
+			}
+
+			const internalLevel = Number(row[7]!);
+			const level = calculateDisplayLevel(internalLevel);
+			if (chart.level !== level) {
+				console.log(
+					`Overwriting level for ${song.title} [${chart.difficulty}]: ${chart.level} -> ${level}`
+				);
+				chart.level = level;
+			}
+			if (chart.levelNum !== internalLevel) {
+				console.log(
+					`Overwriting levelNum for ${song.title} [${chart.difficulty}]: ${chart.levelNum} -> ${internalLevel}`
+				);
+				chart.levelNum = internalLevel;
+			}
+		}
+		return charts;
+	});
+}
+
+/**
+ * Adds internal levels to charts from a CSV file.
+ * @param csvData raw CSV data
+ * @param parseCategory whether to include category header in parsing
+ * @param headerRow the index (starting from 0) of the first row with a song
+ * 	(or with category if `parseCategory` is `true`)
+ *
+ *  **To be considered a category only the first column can have text.**
+ * @param hasCategoryColumn whether there is a category column for each song
+ * @param markLatest whether to mark chart as latest
+ */
+function addOtherSheet(
+	csvData: string[][],
+	parseCategory: boolean,
+	headerRow: number,
+	hasCategoryColumn: boolean,
+	markLatest: boolean
+) {
+	const songs = ReadCollection("songs-maimaidx.json");
+	const categoryColumnOffset = hasCategoryColumn ? 1 : 0;
+	let currentCategory = "";
+
+	MutateCollection("charts-maimaidx.json", (charts) => {
+		for (
+			let colNumber = 0;
+			colNumber + 4 + categoryColumnOffset < csvData[0]!.length;
+			colNumber += 6 + categoryColumnOffset
+		) {
+			for (let rowNumber = headerRow; rowNumber < csvData.length; rowNumber++) {
+				const row = csvData[rowNumber]!;
+				const title = row[colNumber]!;
+
+				if (
+					parseCategory &&
+					title &&
+					[1, 2, 3, 4].every((i) => !row[colNumber + categoryColumnOffset + i])
+				) {
+					currentCategory = categoryMap.get(title) ?? "";
+					continue;
+				}
+
+				const style = row[colNumber + categoryColumnOffset + 1];
+				if (style !== "DX" && style !== "STD") {
+					continue;
+				}
+
+				const sheetDifficulty = row[colNumber + categoryColumnOffset + 2]!;
+				const internalLevelString = row[colNumber + categoryColumnOffset + 4];
+				if (!internalLevelString || internalLevelString === "#N/A") {
+					continue;
+				}
+				const internalLevel = Number(internalLevelString.match(/\d+\.\d+/u)?.[0]);
+				const level = calculateDisplayLevel(internalLevel);
+
+				const song = findSong(songs, title, currentCategory);
+				if (!song) {
+					console.log(`Could not find song ${title}`);
+					continue;
+				}
+
+				const difficulty = calculateDifficulty(style, sheetDifficulty);
+				const chart = charts.find(
+					(c) => c.songID === song.id && c.difficulty === difficulty
+				);
+				if (!chart) {
+					console.log(`Could not find chart ${difficulty} for ${title}`);
+					continue;
+				}
+				if (chart.level !== level) {
+					console.log(
+						`Overwriting level for ${song.title} [${chart.difficulty}]: ${chart.level} -> ${level}`
+					);
+					chart.level = level;
+				}
+				if (chart.levelNum !== internalLevel) {
+					console.log(
+						`Overwriting levelNum for ${song.title} [${chart.difficulty}]: ${chart.levelNum} -> ${internalLevel}`
+					);
+					chart.levelNum = internalLevel;
+				}
+				if (chart.data.isLatest !== markLatest) {
+					console.log(
+						`Marking ${song.title} [${chart.difficulty}]'s isLatest to ${markLatest}`
+					);
+					chart.data.isLatest = markLatest;
+				}
+			}
+		}
+		return charts;
+	});
+}
+
+const program = new Command();
+program.option("-f, --file <filename>", "CSV file to read from");
+program.option("-d, --directory <music>", "path to A000/music directory");
+program.parse(process.argv);
+const options = program.opts();
+
+if (options.directory) {
+	const songs = ReadCollection("songs-maimaidx.json");
+	const parser = new XMLParser({ ignoreAttributes: false });
+
+	MutateCollection("charts-maimaidx.json", (charts: ChartDocument<"maimaidx:Single">[]) => {
+		const items = fs.readdirSync(options.directory);
+		items.forEach((item) => {
+			const fullPath = path.join(options.directory, item);
+			if (
+				!fs.lstatSync(fullPath).isDirectory() ||
+				!fs.existsSync(path.join(fullPath, "Music.xml"))
+			) {
+				return;
+			}
+			const musicData = parser.parse(
+				fs.readFileSync(path.join(fullPath, "Music.xml"))
+			).MusicData;
+			const title = `${musicData.name.str}`; // The song "39" is treated as a number by the XML parser
+			const category = musicData.genreName.str;
+			const song = findSong(songs, title, category);
+			if (!song) {
+				console.log(`Could not find song ${title}`);
+				return;
+			}
+
+			// DX song IDs start from 10000
+			const style = Math.floor(musicData.name.id / 10000) === 1 ? "DX " : "";
+
+			for (const [idx, notes] of Object.entries(musicData.notesData.Notes as XmlNotes[])) {
+				if (!notes.isEnable) {
+					continue;
+				}
+				const difficulty = `${style}${[...diffMap.values()][Number(idx)]}`;
+				const chart = charts.find(
+					(c) => c.songID === song.id && c.difficulty === difficulty
+				);
+				if (!chart) {
+					console.log(`Could not find chart ${difficulty} for ${title}`);
+					continue;
+				}
+				const internalLevel = Number((notes.level + notes.levelDecimal / 10).toFixed(1));
+				let level = notes.level.toString();
+				if (notes.level >= 7 && notes.levelDecimal >= 7) {
+					level += "+";
+				}
+
+				if (chart.level !== level) {
+					console.log(
+						`Overwriting level for ${song.title} [${chart.difficulty}]: ${chart.level} -> ${level}`
+					);
+					chart.level = level;
+				}
+				if (chart.levelNum !== internalLevel) {
+					console.log(
+						`Overwriting levelNum for ${song.title} [${chart.difficulty}]: ${chart.levelNum} -> ${internalLevel}`
+					);
+					chart.levelNum = internalLevel;
+				}
+			}
+		});
+		return charts;
+	});
+} else if (options.file) {
+	const csvData = parse(fs.readFileSync(options.file));
+	const newSongsSheet = options.file.includes("新曲.csv");
+	const tmaiSheet = options.file.includes(" - Tmai.csv");
+	const highLevelSheet = / - (14以上|13+|13).csv$/u.test(options.file);
+
+	if (tmaiSheet) {
+		addTmaiSheet(csvData);
+	} else if (newSongsSheet) {
+		addOtherSheet(csvData, false, 7, false, true);
+	} else if (highLevelSheet) {
+		addOtherSheet(csvData, false, 3, true, false);
+	} else {
+		addOtherSheet(csvData, true, 2, true, false);
+	}
+} else {
+	console.error("Must specify either a file or a directory");
+	process.exit(1);
+}

database-seeds/scripts/rerunners/maimaidx/parse-maimaidx-dataset.js

@@ -1,7 +1,11 @@
 const fetch = require("node-fetch");
 const { CreateChartID, ReadCollection, WriteCollection } = require("../../util");
 
+const CURRENT_VERSION = "festival";
+const CURRENT_VERSION_NUM = 230;
 const DATA_URL = "https://maimai.sega.jp/data/maimai_songs.json";
+const ALIAS_URL =
+	"https://raw.githubusercontent.com/lomotos10/GCM-bot/main/data/aliases/en/maimai.tsv";
 
 const versionMap = new Map([
 	[0, null],
@@ -37,6 +41,9 @@ const diffMap = new Map([
 	["remas", "Re:Master"],
 ]);
 
+// maimai DX FESTiVAL+ songs that are in maimai DX FESTiVAL International Version
+const versionOverrides = ["INTERNET OVERDOSE", "Knight Rider", "Let you DIVE!", "Trrricksters!!"];
+
 (async () => {
 	const songs = ReadCollection("songs-maimaidx.json");
 	const charts = ReadCollection("charts-maimaidx.json");
@@ -45,10 +52,15 @@ const diffMap = new Map([
 	const existingCharts = new Map(charts.map((e) => [`${e.songID} ${e.difficulty}`, e.chartID]));
 
 	const datum = await fetch(DATA_URL).then((r) => r.json());
+	const aliases = new Map();
+	const aliasesRaw = await fetch(ALIAS_URL).then((r) => r.text());
+	for (const line of aliasesRaw.split("\n")) {
+		const [title, ...alias] = line.split("\t");
+		aliases.set(title, alias);
+	}
 
 	// not properly implemented yet as i can't find romaji titles for
 	// japanese titled songs
-	const searchTerms = [];
 	const altTitles = [];
 
 	let songID = Math.max(Math.max(...existingSongs.values()), 0) + 1;
@@ -56,22 +68,36 @@ const diffMap = new Map([
 	for (const data of datum) {
 		let thisSongID = songID;
 
-		const version = Number(data.version.substring(0, 3));
+		let version = Number(data.version.substring(0, 3));
+		if (version > CURRENT_VERSION_NUM && !versionOverrides.includes(data.title)) {
+			// Skipping songs that are newer than currently supported version (FESTiVAL).
+			continue;
+		}
+		if (data.title === "　" && data.artist === "x0o0x_") {
+			// Manual override since the song's title is "" (empty) in the dataset.
+			continue;
+		}
+		if (versionOverrides.includes(data.title)) {
+			version = CURRENT_VERSION_NUM;
+		}
 
 		if (existingSongs.has(data.title)) {
 			thisSongID = existingSongs.get(data.title);
+			const song = songs.find((e) => e.id === thisSongID);
+			song.searchTerms = aliases.get(data.title.trim()) ?? [];
 		} else {
 			songID++;
 			songs.push({
 				id: thisSongID,
 				title: data.title.trim(),
 				artist: data.artist.trim(),
-				searchTerms,
+				searchTerms: aliases.get(data.title.trim()) ?? [],
 				altTitles,
 				data: {
 					displayVersion: versionMap.get(version),
 				},
 			});
+			console.log(`New song: ${data.artist.trim()} - ${data.title.trim()}`);
 		}
 
 		diffNames.forEach((diff) => {
@@ -80,18 +106,22 @@ const diffMap = new Map([
 				return;
 			}
 
-			if (existingCharts.get(`${thisSongID} DX ${diffMap.get(diff)}`)) {
+			const chartID = existingCharts.get(`${thisSongID} DX ${diffMap.get(diff)}`);
+			const chart = charts.find((e) => e.chartID === chartID);
+			if (chartID) {
+				if (!chart.versions.includes(CURRENT_VERSION)) {
+					chart.versions.push(CURRENT_VERSION);
+				}
 				return;
 			}
 
-			const chartID = CreateChartID();
-			const isLatest = Number(version) === 225;
+			const isLatest = version >= CURRENT_VERSION_NUM;
 
 			const lvNum = Number(data[`dx_lev_${diff}`].replace(/\+/u, ".7"));
 
 			charts.push({
 				songID: thisSongID,
-				chartID: chartID,
+				chartID: CreateChartID(),
 				level: data[`dx_lev_${diff}`],
 				levelNum: lvNum,
 				isPrimary: true,
@@ -100,7 +130,7 @@ const diffMap = new Map([
 				data: {
 					isLatest,
 				},
-				versions: ["universeplus"],
+				versions: [CURRENT_VERSION],
 			});
 		});
 
@@ -110,19 +140,22 @@ const diffMap = new Map([
 				return;
 			}
 
-			if (existingCharts.get(`${thisSongID} ${diffMap.get(diff)}`)) {
+			const chartID = existingCharts.get(`${thisSongID} ${diffMap.get(diff)}`);
+			const chart = charts.find((e) => e.chartID === chartID);
+			if (chartID) {
+				if (!chart.versions.includes(CURRENT_VERSION)) {
+					chart.versions.push(CURRENT_VERSION);
+				}
 				return;
 			}
 
-			const chartID = CreateChartID();
-			const isLatest = Number(version) === 225;
+			const isLatest = version >= CURRENT_VERSION_NUM;
 
 			const lvNum = Number(data[`lev_${diff}`].replace(/\+/u, ".7"));
 
 			charts.push({
 				songID: thisSongID,
-				chartID: chartID,
-				rgcID: null,
+				chartID: CreateChartID(),
 				level: data[`lev_${diff}`],
 				levelNum: lvNum,
 				isPrimary: true,
@@ -131,8 +164,7 @@ const diffMap = new Map([
 				data: {
 					isLatest,
 				},
-				tierlistInfo: {},
-				versions: ["universeplus"],
+				versions: [CURRENT_VERSION],
 			});
 		});
 	}