diff --git a/database-seeds/collections/charts-wacca.json b/database-seeds/collections/charts-wacca.json index 15390ca43..2d277c97c 100644 --- a/database-seeds/collections/charts-wacca.json +++ b/database-seeds/collections/charts-wacca.json @@ -17661,5 +17661,532 @@ "versions": [ "reverse" ] + }, + { + "chartID": "77edf990ef8e7e3280459cad2d459df5b7b7c034", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "11", + "levelNum": 11.1, + "playtype": "Single", + "rgcID": null, + "songID": 342, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "e4d13130b92d1e7c8a6f6bed4970d7f8667e19aa", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "8", + "levelNum": 8, + "playtype": "Single", + "rgcID": null, + "songID": 342, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "55c6bb6dbfe3571fbf19adfacce57c3aeaa1c7a4", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "3", + "levelNum": 3, + "playtype": "Single", + "rgcID": null, + "songID": 342, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "1de2f7d2b3b11e9a77c9694a88c6a4b5d7ea68e1", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "13", + "levelNum": 13.5, + "playtype": "Single", + "rgcID": null, + "songID": 343, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "46d05a10c25829abe5af400dfb3581762171b08b", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "10", + "levelNum": 10.5, + "playtype": "Single", + "rgcID": null, + "songID": 343, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "851aa89816d2ef3a7a9ffb9518429970df73299e", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "4", + "levelNum": 4, + "playtype": "Single", + "rgcID": null, + "songID": 343, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "6e9563006e087c39da9148a43733181403e7a59e", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "12+", + "levelNum": 12.7, + "playtype": "Single", + "rgcID": null, + "songID": 344, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "cecc3acc951afd92dd46f3d9c9e6d9d43b1abd33", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "8", + "levelNum": 8.6, + "playtype": "Single", + "rgcID": null, + "songID": 344, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "9b9e955f4ee55e12c30146da526aebd0f890ccfc", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "3", + "levelNum": 3, + "playtype": "Single", + "rgcID": null, + "songID": 344, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "502ab5f1c1a5647aa6539f5e25862f5f6e28eb60", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "12", + "levelNum": 12.6, + "playtype": "Single", + "rgcID": null, + "songID": 345, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "b693625992ae937272984d96ada6966fd55419c0", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "8+", + "levelNum": 8.7, + "playtype": "Single", + "rgcID": null, + "songID": 345, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "ab2837bcf61022204c034b222184e4bede46b5d2", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "3", + "levelNum": 3, + "playtype": "Single", + "rgcID": null, + "songID": 345, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "290ca37ff14eb6d840e24e579cad33ab2476df49", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "12", + "levelNum": 12.4, + "playtype": "Single", + "rgcID": null, + "songID": 346, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "629b18ed41270f44bd8c20d51f378e23f9be836a", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "8", + "levelNum": 8.3, + "playtype": "Single", + "rgcID": null, + "songID": 346, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "385d60dcfde335a831672d5a2768dddf2c687899", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "3", + "levelNum": 3, + "playtype": "Single", + "rgcID": null, + "songID": 346, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "6df01a0b4b12dfd490b6773de9f127bad0b70a8f", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "13+", + "levelNum": 13.7, + "playtype": "Single", + "rgcID": null, + "songID": 347, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "1a3d9c2ef04d645e65411d2482e0790eed94fd18", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "11+", + "levelNum": 11.7, + "playtype": "Single", + "rgcID": null, + "songID": 347, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "60b50db4aac2185da5be2958a2d06d455438b2bf", + "data": { + "isHot": true + }, + "difficulty": "INFERNO", + "isPrimary": true, + "level": "14", + "levelNum": 14.2, + "playtype": "Single", + "rgcID": null, + "songID": 347, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "99a9418b2656f664aed6374731aac60a625ec891", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "8", + "levelNum": 8, + "playtype": "Single", + "rgcID": null, + "songID": 347, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "646deb78aced40cae47a8844feead5fc2504532c", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "13", + "levelNum": 13.2, + "playtype": "Single", + "rgcID": null, + "songID": 348, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "356ca41312750ffd7d5572505b339de0119457e4", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "10+", + "levelNum": 10.7, + "playtype": "Single", + "rgcID": null, + "songID": 348, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "7e122f2fce0b362808b8614c48438ff3ab7cd256", + "data": { + "isHot": true + }, + "difficulty": "INFERNO", + "isPrimary": true, + "level": "14", + "levelNum": 14.1, + "playtype": "Single", + "rgcID": null, + "songID": 348, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "a3a3d55693f2ceb25296dae8563ed086e7636ab4", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "4", + "levelNum": 4, + "playtype": "Single", + "rgcID": null, + "songID": 348, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "4c7eb4525c496743d038ea74e538871560920cb1", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "13", + "levelNum": 13.4, + "playtype": "Single", + "rgcID": null, + "songID": 349, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "a33cb31903cb4348da34f27675d609c6ce8bb90b", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "10", + "levelNum": 10.6, + "playtype": "Single", + "rgcID": null, + "songID": 349, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "2af6942f94f9f3ebc43e2e2312db0bfe44d2303d", + "data": { + "isHot": true + }, + "difficulty": "INFERNO", + "isPrimary": true, + "level": "14", + "levelNum": 14.1, + "playtype": "Single", + "rgcID": null, + "songID": 349, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "d2340de1b9b6e5e3c1ace41dde588ea316811943", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "3", + "levelNum": 3, + "playtype": "Single", + "rgcID": null, + "songID": 349, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "eda02d064b2f67f37648c971eadec3a05f026107", + "data": { + "isHot": true + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "13", + "levelNum": 13.4, + "playtype": "Single", + "rgcID": null, + "songID": 350, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "bf8da4215fdcbeb68c34ee186c770251dbab1bf3", + "data": { + "isHot": true + }, + "difficulty": "HARD", + "isPrimary": true, + "level": "10+", + "levelNum": 10.8, + "playtype": "Single", + "rgcID": null, + "songID": 350, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "7e0cbfbb24bff0b8a61a5d131a6699d9f9b0fb4e", + "data": { + "isHot": true + }, + "difficulty": "INFERNO", + "isPrimary": true, + "level": "14", + "levelNum": 14.1, + "playtype": "Single", + "rgcID": null, + "songID": 350, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + }, + { + "chartID": "57d4197ba25853a3fad5841294b78c5253a0cfeb", + "data": { + "isHot": true + }, + "difficulty": "NORMAL", + "isPrimary": true, + "level": "4", + "levelNum": 4, + "playtype": "Single", + "rgcID": null, + "songID": 350, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] } ] \ No newline at end of file diff --git a/database-seeds/collections/songs-wacca.json b/database-seeds/collections/songs-wacca.json index af576bdea..ff19a1cd6 100644 --- a/database-seeds/collections/songs-wacca.json +++ b/database-seeds/collections/songs-wacca.json @@ -4431,5 +4431,122 @@ "id": 341, "searchTerms": [], "title": "Rainbow Dream" + }, + { + "altTitles": [], + "artist": "ギルガメッシュ（CV：大塚剛央）、千極兆司（CV：立花慎之介）、緋野天魔（CV：小野賢章）、神水鶴久遠（CV：広瀬裕也）、皇麗夢（CV：豊永利行）", + "data": { + "artistJP": "ギルガメッシュ（オオツカタケオ）、センゴクチョウジ（タチバナシンノスケ）、ヒノテンマ（オノケンショウ）、カミツルクオン（ヒロセユウヤ）、スメラキレイム（トヨナガトシユキ）", + "displayVersion": "reverse", + "genre": "バラエティ", + "titleJP": "ファムファタール" + }, + "id": 342, + "searchTerms": [], + "title": "ファムファタール" + }, + { + "altTitles": [], + "artist": "餓鬼レンジャー feat. 梅田サイファー", + "data": { + "artistJP": "ガキレンジャーフィートウメダサイファー", + "displayVersion": "reverse", + "genre": "バラエティ", + "titleJP": "アゲナゲン" + }, + "id": 343, + "searchTerms": [], + "title": "アゲナゲン" + }, + { + "altTitles": [], + "artist": "ゆうゆ", + "data": { + "artistJP": "ユウユ", + "displayVersion": "reverse", + "genre": "ボカロ", + "titleJP": "テンガク" + }, + "id": 344, + "searchTerms": [], + "title": "天樂" + }, + { + "altTitles": [], + "artist": "supercell", + "data": { + "artistJP": "スーパーセル", + "displayVersion": "reverse", + "genre": "ボカロ", + "titleJP": "メルト" + }, + "id": 345, + "searchTerms": [], + "title": "メルト" + }, + { + "altTitles": [], + "artist": "亜沙", + "data": { + "artistJP": "アサ", + "displayVersion": "reverse", + "genre": "ボカロ", + "titleJP": "ヨシワララメント" + }, + "id": 346, + "searchTerms": [], + "title": "吉原ラメント" + }, + { + "altTitles": [], + "artist": "REDALiCE", + "data": { + "artistJP": "レッドアリス", + "displayVersion": "reverse", + "genre": "TANO*C（オリジナル）", + "titleJP": "クリムゾンテイオウ" + }, + "id": 347, + "searchTerms": [], + "title": "クリムゾン帝王" + }, + { + "altTitles": [], + "artist": "aran", + "data": { + "artistJP": "アラン", + "displayVersion": "reverse", + "genre": "TANO*C（オリジナル）", + "titleJP": "エフェメラ" + }, + "id": 348, + "searchTerms": [], + "title": "EPHMR" + }, + { + "altTitles": [], + "artist": "Laur", + "data": { + "artistJP": "ラウル", + "displayVersion": "reverse", + "genre": "TANO*C（オリジナル）", + "titleJP": "デンジャラアアアアス　ジャングル" + }, + "id": 349, + "searchTerms": [], + "title": "DANGEROOOOUS JUNGLE" + }, + { + "altTitles": [], + "artist": "Kobaryo", + "data": { + "artistJP": "コバリョー", + "displayVersion": "reverse", + "genre": "TANO*C（オリジナル）", + "titleJP": "ディメンション　ハッカー" + }, + "id": 350, + "searchTerms": [], + "title": "Dimension Hacker" } ] \ No newline at end of file diff --git a/database-seeds/scripts/package.json b/database-seeds/scripts/package.json index f7ae231cf..a089346f3 100644 --- a/database-seeds/scripts/package.json +++ b/database-seeds/scripts/package.json @@ -30,8 +30,10 @@ "deepmerge": "^4.2.2", "fast-json-stable-hash": "^1.0.3", "glob": "^7.2.0", + "html-entities": "^2.3.2", "mei-logger": "^1.0.1", "monk": "^7.3.4", + "node-fetch": "2.6.7", "prudence": "^0.9.7", "tachi-common": "0.5.12", "ts-node": "^10.4.0", diff --git a/database-seeds/scripts/pnpm-lock.yaml b/database-seeds/scripts/pnpm-lock.yaml index df417a724..ebf386b98 100644 --- a/database-seeds/scripts/pnpm-lock.yaml +++ b/database-seeds/scripts/pnpm-lock.yaml @@ -14,8 +14,10 @@ specifiers: eslint-plugin-prettier: ^4.0.0 fast-json-stable-hash: ^1.0.3 glob: ^7.2.0 + html-entities: ^2.3.2 mei-logger: ^1.0.1 monk: ^7.3.4 + node-fetch: 2.6.7 prettier: ^2.5.1 prudence: ^0.9.7 tachi-common: 0.5.12 @@ -31,8 +33,10 @@ dependencies: deepmerge: 4.2.2 fast-json-stable-hash: 1.0.3 glob: 7.2.0 + html-entities: 2.3.2 mei-logger: 1.0.1_ts-node@10.4.0 monk: 7.3.4 + node-fetch: 2.6.7 prudence: 0.9.7 tachi-common: 0.5.12 ts-node: 10.4.0_40146b36d18138e3202fbb722e5f65e4 @@ -1603,6 +1607,10 @@ packages: type-fest: 0.8.1 dev: false + /html-entities/2.3.2: + resolution: {integrity: sha512-c3Ab/url5ksaT0WyleslpBEthOzWhrjQbg75y7XUsfSzi3Dgzt0l8w5e7DylRn15MTlMMD58dTfzddNS2kcAjQ==} + dev: false + /html-escaper/2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} dev: false @@ -2188,6 +2196,18 @@ packages: resolution: {integrity: sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=} dev: true + /node-fetch/2.6.7: + resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + whatwg-url: 5.0.0 + dev: false + /node-preload/0.2.1: resolution: {integrity: sha512-RM5oyBy45cLEoHqCeh+MNuFAxO0vTFBLskvQbOKnEE7YTTSN4tbN8QWDIPQ6L+WvKsB/qLEGpYe2ZZ9d4W9OIQ==} engines: {node: '>=8'} @@ -2954,6 +2974,10 @@ packages: punycode: 2.1.1 dev: false + /tr46/0.0.3: + resolution: {integrity: sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=} + dev: false + /triple-beam/1.3.0: resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} dev: false @@ -3097,6 +3121,17 @@ packages: extsprintf: 1.3.0 dev: false + /webidl-conversions/3.0.1: + resolution: {integrity: sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=} + dev: false + + /whatwg-url/5.0.0: + resolution: {integrity: sha1-lmRU6HZUYuN2RNNib2dCzotwll0=} + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + dev: false + /which-boxed-primitive/1.0.2: resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} dependencies: diff --git a/database-seeds/scripts/rerunners/parse-wacca-csv.js b/database-seeds/scripts/rerunners/parse-wacca-csv.js index 2dd0c29ba..c626805e8 100644 --- a/database-seeds/scripts/rerunners/parse-wacca-csv.js +++ b/database-seeds/scripts/rerunners/parse-wacca-csv.js @@ -2,7 +2,7 @@ const { parse } = require("csv-parse/sync"); const fs = require("fs"); const fetch = require("node-fetch"); const { Command } = require("commander"); -const { CreateChartID } = require("../util"); +const { CreateChartID, ReadCollection } = require("../util"); const path = require("path"); const { decode } = require("html-entities"); const logger = require("../logger"); @@ -51,6 +51,13 @@ const STARTS = { }; (async () => { + const existingSongs = new Map(ReadCollection("songs-wacca.json").map(e => + [e.title, e.id])); + const existingCharts = new Map(ReadCollection("charts-wacca.json").map(e => + // We need to combine songID and difficulty for the key, + // this seems like the simplest way. + [`${e.songID} ${e.difficulty}`, e.chartID])); + const datum = await fetch("https://wacca.marv.jp/music/search.php", { method: "POST", headers: { @@ -62,7 +69,7 @@ const STARTS = { const songs = []; const charts = []; - let songID = 1; + let songID = Math.max(...existingSongs.values()) + 1; for (const data of datum) { let prettiedTitle = decode(data.title.display.replace(/　/g, " ")).trim(); @@ -105,8 +112,14 @@ const STARTS = { continue; } + let thisSongID = songID; + if (existingSongs.has(prettiedTitle)) { + thisSongID = existingSongs.get(prettiedTitle); + } else { + songID++; + } songs.push({ - id: songID, + id: thisSongID, title: prettiedTitle, artist: decode(data.artist.display).trim(), searchTerms: [], @@ -129,9 +142,11 @@ const STARTS = { continue; } + const chartID = existingCharts.get(`${thisSongID} ${diffName}`) || CreateChartID(); + charts.push({ - songID, - chartID: CreateChartID(), + songID: thisSongID, + chartID: chartID, rgcID: null, level, levelNum, @@ -145,16 +160,9 @@ const STARTS = { versions: ["reverse"], }); } - - songID++; } - fs.writeFileSync( - path.resolve(__dirname, "../../collections/charts-wacca.json"), - JSON.stringify(charts, null, "\t") - ); - fs.writeFileSync( - path.resolve(__dirname, "../../collections/songs-wacca.json"), - JSON.stringify(songs, null, "\t") - ); + fs.writeFileSync(path.resolve(__dirname, "../../collections/charts-wacca.json"), JSON.stringify(charts, null, "\t")); + fs.writeFileSync(path.resolve(__dirname, "../../collections/songs-wacca.json"), JSON.stringify(songs, null, "\t")); + })();