add cloud IX, treat CSV titles as officially correct

database-seeds/collections/charts-wacca.json

@@ -1549,7 +1549,7 @@
 	{
 		"chartID": "d60eac20879de2f749e02d8c36aa226c1f1d4717",
 		"data": {
-			"isHot": false
+			"isHot": true
 		},
 		"difficulty": "EXPERT",
 		"isPrimary": true,
@@ -1566,7 +1566,7 @@
 	{
 		"chartID": "f4bd38d4a9a5a8cc589b0e409c4e1d56c12b4900",
 		"data": {
-			"isHot": false
+			"isHot": true
 		},
 		"difficulty": "HARD",
 		"isPrimary": true,
@@ -1583,7 +1583,7 @@
 	{
 		"chartID": "6633efa8325221bdc884eb0f19454feb81d5d066",
 		"data": {
-			"isHot": false
+			"isHot": true
 		},
 		"difficulty": "NORMAL",
 		"isPrimary": true,
@@ -18188,5 +18188,56 @@
 		"versions": [
 			"reverse"
 		]
+	},
+	{
+		"chartID": "be9aa7d0d5289e4d3ceec959572921c75fcf71e7",
+		"data": {
+			"isHot": true
+		},
+		"difficulty": "EXPERT",
+		"isPrimary": true,
+		"level": "13",
+		"levelNum": 13.6,
+		"playtype": "Single",
+		"rgcID": null,
+		"songID": 351,
+		"tierlistInfo": {},
+		"versions": [
+			"reverse"
+		]
+	},
+	{
+		"chartID": "b596170daf333e4686250c8edeaffca010ebe2be",
+		"data": {
+			"isHot": true
+		},
+		"difficulty": "HARD",
+		"isPrimary": true,
+		"level": "10",
+		"levelNum": 10.5,
+		"playtype": "Single",
+		"rgcID": null,
+		"songID": 351,
+		"tierlistInfo": {},
+		"versions": [
+			"reverse"
+		]
+	},
+	{
+		"chartID": "d7e1af40ad8a4916646a7da373f70db1f131d360",
+		"data": {
+			"isHot": true
+		},
+		"difficulty": "NORMAL",
+		"isPrimary": true,
+		"level": "4",
+		"levelNum": 4,
+		"playtype": "Single",
+		"rgcID": null,
+		"songID": 351,
+		"tierlistInfo": {},
+		"versions": [
+			"reverse"
+		]
 	}
 ]

database-seeds/collections/songs-wacca.json

@@ -13,7 +13,9 @@
 		"title": "ヒトガタ"
 	},
 	{
-		"altTitles": [],
+		"altTitles": [
+			"叩ケ　叩ケ　手ェ叩ケ"
+		],
 		"artist": "SIRO",
 		"data": {
 			"artistJP": "シロ",
@@ -3056,18 +3058,18 @@
 	},
 	{
 		"altTitles": [
-			"ナイト・オブ・ナイツ (かめりあ’s“ワンス・アポン・ア・ナイト”Remix)"
+			"ナイト・オブ・ナイツ (かめりあ's\"ワンス・アポン・ア・ナイト\"Remix)"
 		],
 		"artist": "かめりあ",
 		"data": {
 			"artistJP": "カメリア",
 			"displayVersion": "s",
 			"genre": "東方アレンジ",
-			"titleJP": "ナイト・オブ・ナイツ (カメリア'ズ“ワンス・アポン・ア・ナイト”リミックス)"
+			"titleJP": "ナイト・オブ・ナイツ (カメリア'ズ“ワンス・アポン・ア・ナイト”リミックス)"
 		},
 		"id": 236,
 		"searchTerms": [],
-		"title": "ナイト・オブ・ナイツ (かめりあ's\"ワンス・アポン・ア・ナイト\"Remix)"
+		"title": "ナイト・オブ・ナイツ (かめりあ’s“ワンス・アポン・ア・ナイト”Remix)"
 	},
 	{
 		"altTitles": [],
@@ -3642,7 +3644,9 @@
 		"title": "Galaxy Friends"
 	},
 	{
-		"altTitles": [],
+		"altTitles": [
+			"13 DONKEYS"
+		],
 		"artist": "Massive New Krew",
 		"data": {
 			"artistJP": "マッシブニュークルー",
@@ -4550,5 +4554,20 @@
 		"id": 350,
 		"searchTerms": [],
 		"title": "Dimension Hacker"
+	},
+	{
+		"altTitles": [
+			"cloud Ⅸ"
+		],
+		"artist": "NOMA",
+		"data": {
+			"artistJP": "ノマ",
+			"displayVersion": "reverse",
+			"genre": "オリジナル",
+			"titleJP": "クラウドナイン"
+		},
+		"id": 351,
+		"searchTerms": [],
+		"title": "cloud IX"
 	}
-]
+]

database-seeds/scripts/rerunners/parse-wacca-csv.js

@@ -2,8 +2,7 @@ const { parse } = require("csv-parse/sync");
 const fs = require("fs");
 const fetch = require("node-fetch");
 const { Command } = require("commander");
-const { CreateChartID, ReadCollection } = require("../util");
-const path = require("path");
+const { CreateChartID, ReadCollection, WriteCollection } = require("../util");
 const { decode } = require("html-entities");
 const logger = require("../logger");
 
@@ -37,10 +36,34 @@ const dirtyRecords = parse(fs.readFileSync(options.file), {});
 
 const dataMap = new Map();
 
+const titleMap = {
+	"13 DONKEYS": "13 Donkeys",
+	"cloud Ⅸ": "cloud IX",
+};
+
+// Converts the title on the site to a noramlized version that we
+// can match against the CSV.
+function siteTitleNormalize(siteTitle) {
+	let normalized = decode(siteTitle.replace(/　/g, " ")).trim();
+
+	if (normalized in titleMap) {
+		normalized = titleMap[normalized];
+	}
+
+	return normalized;
+}
+
+// Note that the title in the CSV is the one we want - it's what's
+// used on the actual site. However, this normalizes it so we can
+// match against the broken titles on the music search site.
+function csvTitleNormalize(csvTitle) {
+	return csvTitle.replace(/”|“/gu, '"').replace(/’/gu, "'");
+}
+
 // we have to skip the first record because its the headers,
 // and there's literally no way to change this behaviour.
 for (const record of dirtyRecords.slice(1)) {
-	dataMap.set(record[0].replace(/”|“/gu, '"').replace(/’/gu, "'"), record);
+	dataMap.set(csvTitleNormalize(record[0]), record);
 }
 
 const STARTS = {
@@ -74,13 +97,7 @@ const STARTS = {
 	let songID = Math.max(...existingSongs.values()) + 1;
 
 	for (const data of datum) {
-		let prettiedTitle = decode(data.title.display.replace(/　/g, " ")).trim();
-
-		// This song got its title changed.
-		// No, I don't know why.
-		if (prettiedTitle === "13 DONKEYS") {
-			prettiedTitle = "13 Donkeys";
-		}
+		let siteTitleNormalized = siteTitleNormalize(data.title.display);
 
 		const time = Date.parse(data.release_date);
 		let ver;
@@ -101,35 +118,44 @@ const STARTS = {
 
 		// re-screw "'s to their shift-jis equivalent, because it seems like decoding
 		// " is locale specific. Thanks.
-		const record = dataMap.get(prettiedTitle);
+		const record = dataMap.get(siteTitleNormalized);
 
 		if (!record) {
 			logger.warn(
-				`Can't find record with title ${prettiedTitle}. Dumping potentially similar titles.\n${[
+				`Can't find record with title ${siteTitleNormalized}. Dumping potentially similar titles.\n${[
 					...dataMap.keys(),
 				]
-					.filter((e) => e.startsWith(prettiedTitle[0]))
+					.filter((e) => e.startsWith(siteTitleNormalized[0]))
 					.join("\n")}`
 			);
 			continue;
 		}
 
+		// Use the CSV title.
+		const title = record[0];
+		const siteTitle = decode(data.title.display).trim();
+		const altTitles = [];
+		if (title !== siteTitle) {
+			// Include the title on the music site just in case.
+			altTitles.push(siteTitle);
+		}
+
 		let thisSongID = songID;
-		if (existingSongs.has(prettiedTitle)) {
-			thisSongID = existingSongs.get(prettiedTitle);
+		if (existingSongs.has(title)) {
+			thisSongID = existingSongs.get(title);
 		} else {
 			songID++;
 		}
 		songs.push({
 			id: thisSongID,
-			title: prettiedTitle,
+			title,
 			artist: decode(data.artist.display).trim(),
 			searchTerms: [],
-			altTitles: [],
+			altTitles,
 			data: {
-				titleJP: data.title.ruby,
-				artistJP: data.artist.ruby,
-				genre: data.category,
+				titleJP: decode(data.title.ruby),
+				artistJP: decode(data.artist.ruby),
+				genre: decode(data.category),
 				displayVersion: ver,
 			},
 		});
@@ -164,12 +190,6 @@ const STARTS = {
 		}
 	}
 
-	fs.writeFileSync(
-		path.resolve(__dirname, "../../collections/charts-wacca.json"),
-		JSON.stringify(charts, null, "\t")
-	);
-	fs.writeFileSync(
-		path.resolve(__dirname, "../../collections/songs-wacca.json"),
-		JSON.stringify(songs, null, "\t")
-	);
+	WriteCollection("charts-wacca.json", charts);
+	WriteCollection("songs-wacca.json", songs);
 })();