From 858377e5682efc36a1bd2a0df90efeff7cddac31 Mon Sep 17 00:00:00 2001 From: Christopher Cooper Date: Sun, 13 Feb 2022 17:42:26 -0800 Subject: [PATCH] add cloud IX, treat CSV titles as officially correct --- database-seeds/collections/charts-wacca.json | 57 +++++++++++++- database-seeds/collections/songs-wacca.json | 31 ++++++-- .../scripts/rerunners/parse-wacca-csv.js | 76 ++++++++++++------- 3 files changed, 127 insertions(+), 37 deletions(-) diff --git a/database-seeds/collections/charts-wacca.json b/database-seeds/collections/charts-wacca.json index 2d277c97c..8c0337f83 100644 --- a/database-seeds/collections/charts-wacca.json +++ b/database-seeds/collections/charts-wacca.json @@ -1549,7 +1549,7 @@ { "chartID": "d60eac20879de2f749e02d8c36aa226c1f1d4717", "data": { - "isHot": false + "isHot": true }, "difficulty": "EXPERT", "isPrimary": true, @@ -1566,7 +1566,7 @@ { "chartID": "f4bd38d4a9a5a8cc589b0e409c4e1d56c12b4900", "data": { - "isHot": false + "isHot": true }, "difficulty": "HARD", "isPrimary": true, @@ -1583,7 +1583,7 @@ { "chartID": "6633efa8325221bdc884eb0f19454feb81d5d066", "data": { - "isHot": false + "isHot": true }, "difficulty": "NORMAL", "isPrimary": true, @@ -18188,5 +18188,56 @@ "versions": [ "reverse" ] + }, + { + "chartID": "be9aa7d0d5289e4d3ceec959572921c75fcf71e7", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "13", + "levelNum": 13.6, + "playtype": "Single", + "rgcID": null, + "songID": 351, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "b596170daf333e4686250c8edeaffca010ebe2be", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "10", + "levelNum": 10.5, + "playtype": "Single", + "rgcID": null, + "songID": 351, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "d7e1af40ad8a4916646a7da373f70db1f131d360", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "4", + "levelNum": 4, + "playtype": "Single", + "rgcID": null, + "songID": 351, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] } ] \ No newline at end of file diff --git a/database-seeds/collections/songs-wacca.json b/database-seeds/collections/songs-wacca.json index 949f1555d..412f6c861 100644 --- a/database-seeds/collections/songs-wacca.json +++ b/database-seeds/collections/songs-wacca.json @@ -13,7 +13,9 @@ "title": "ヒトガタ" }, { - "altTitles": [], + "altTitles": [ + "叩ケ　叩ケ　手ェ叩ケ" + ], "artist": "SIRO", "data": { "artistJP": "シロ", @@ -3056,18 +3058,18 @@ }, { "altTitles": [ - "ナイト・オブ・ナイツ (かめりあ’s“ワンス・アポン・ア・ナイト”Remix)" + "ナイト・オブ・ナイツ (かめりあ's\"ワンス・アポン・ア・ナイト\"Remix)" ], "artist": "かめりあ", "data": { "artistJP": "カメリア", "displayVersion": "s", "genre": "東方アレンジ", - "titleJP": "ナイト・オブ・ナイツ (カメリア'ズ“ワンス・アポン・ア・ナイト”リミックス)" + "titleJP": "ナイト・オブ・ナイツ (カメリア'ズ“ワンス・アポン・ア・ナイト”リミックス)" }, "id": 236, "searchTerms": [], - "title": "ナイト・オブ・ナイツ (かめりあ's\"ワンス・アポン・ア・ナイト\"Remix)" + "title": "ナイト・オブ・ナイツ (かめりあ’s“ワンス・アポン・ア・ナイト”Remix)" }, { "altTitles": [], @@ -3642,7 +3644,9 @@ "title": "Galaxy Friends" }, { - "altTitles": [], + "altTitles": [ + "13 DONKEYS" + ], "artist": "Massive New Krew", "data": { "artistJP": "マッシブニュークルー", @@ -4550,5 +4554,20 @@ "id": 350, "searchTerms": [], "title": "Dimension Hacker" + }, + { + "altTitles": [ + "cloud Ⅸ" + ], + "artist": "NOMA", + "data": { + "artistJP": "ノマ", + "displayVersion": "reverse", + "genre": "オリジナル", + "titleJP": "クラウドナイン" + }, + "id": 351, + "searchTerms": [], + "title": "cloud IX" } -] +] \ No newline at end of file diff --git a/database-seeds/scripts/rerunners/parse-wacca-csv.js b/database-seeds/scripts/rerunners/parse-wacca-csv.js index 87957abe2..399617feb 100644 --- a/database-seeds/scripts/rerunners/parse-wacca-csv.js +++ b/database-seeds/scripts/rerunners/parse-wacca-csv.js @@ -2,8 +2,7 @@ const { parse } = require("csv-parse/sync"); const fs = require("fs"); const fetch = require("node-fetch"); const { Command } = require("commander"); -const { CreateChartID, ReadCollection } = require("../util"); -const path = require("path"); +const { CreateChartID, ReadCollection, WriteCollection } = require("../util"); const { decode } = require("html-entities"); const logger = require("../logger"); @@ -37,10 +36,34 @@ const dirtyRecords = parse(fs.readFileSync(options.file), {}); const dataMap = new Map(); +const titleMap = { + "13 DONKEYS": "13 Donkeys", + "cloud Ⅸ": "cloud IX", +}; + +// Converts the title on the site to a noramlized version that we +// can match against the CSV. +function siteTitleNormalize(siteTitle) { + let normalized = decode(siteTitle.replace(/　/g, " ")).trim(); + + if (normalized in titleMap) { + normalized = titleMap[normalized]; + } + + return normalized; +} + +// Note that the title in the CSV is the one we want - it's what's +// used on the actual site. However, this normalizes it so we can +// match against the broken titles on the music search site. +function csvTitleNormalize(csvTitle) { + return csvTitle.replace(/”|“/gu, '"').replace(/’/gu, "'"); +} + // we have to skip the first record because its the headers, // and there's literally no way to change this behaviour. for (const record of dirtyRecords.slice(1)) { - dataMap.set(record[0].replace(/”|“/gu, '"').replace(/’/gu, "'"), record); + dataMap.set(csvTitleNormalize(record[0]), record); } const STARTS = { @@ -74,13 +97,7 @@ const STARTS = { let songID = Math.max(...existingSongs.values()) + 1; for (const data of datum) { - let prettiedTitle = decode(data.title.display.replace(/　/g, " ")).trim(); - - // This song got its title changed. - // No, I don't know why. - if (prettiedTitle === "13 DONKEYS") { - prettiedTitle = "13 Donkeys"; - } + let siteTitleNormalized = siteTitleNormalize(data.title.display); const time = Date.parse(data.release_date); let ver; @@ -101,35 +118,44 @@ const STARTS = { // re-screw "'s to their shift-jis equivalent, because it seems like decoding // " is locale specific. Thanks. - const record = dataMap.get(prettiedTitle); + const record = dataMap.get(siteTitleNormalized); if (!record) { logger.warn( - `Can't find record with title ${prettiedTitle}. Dumping potentially similar titles.\n${[ + `Can't find record with title ${siteTitleNormalized}. Dumping potentially similar titles.\n${[ ...dataMap.keys(), ] - .filter((e) => e.startsWith(prettiedTitle[0])) + .filter((e) => e.startsWith(siteTitleNormalized[0])) .join("\n")}` ); continue; } + // Use the CSV title. + const title = record[0]; + const siteTitle = decode(data.title.display).trim(); + const altTitles = []; + if (title !== siteTitle) { + // Include the title on the music site just in case. + altTitles.push(siteTitle); + } + let thisSongID = songID; - if (existingSongs.has(prettiedTitle)) { - thisSongID = existingSongs.get(prettiedTitle); + if (existingSongs.has(title)) { + thisSongID = existingSongs.get(title); } else { songID++; } songs.push({ id: thisSongID, - title: prettiedTitle, + title, artist: decode(data.artist.display).trim(), searchTerms: [], - altTitles: [], + altTitles, data: { - titleJP: data.title.ruby, - artistJP: data.artist.ruby, - genre: data.category, + titleJP: decode(data.title.ruby), + artistJP: decode(data.artist.ruby), + genre: decode(data.category), displayVersion: ver, }, }); @@ -164,12 +190,6 @@ const STARTS = { } } - fs.writeFileSync( - path.resolve(__dirname, "../../collections/charts-wacca.json"), - JSON.stringify(charts, null, "\t") - ); - fs.writeFileSync( - path.resolve(__dirname, "../../collections/songs-wacca.json"), - JSON.stringify(songs, null, "\t") - ); + WriteCollection("charts-wacca.json", charts); + WriteCollection("songs-wacca.json", songs); })();