diff --git a/database-seeds/scripts/finders.js b/database-seeds/scripts/finders.js new file mode 100644 index 000000000..1c01944b6 --- /dev/null +++ b/database-seeds/scripts/finders.js @@ -0,0 +1,12 @@ +function FindChartWithPTDFVersion(collection, songID, playtype, difficulty, version) { + return collection.find(chart => chart.songID === songID && chart.playtype === playtype && chart.difficulty === difficulty && chart.versions.includes(version)); +} + +function FindSongWithTitle(collection, title) { + return collection.find(e => e.title === title || e.altTitles.includes(title)); +} + +module.exports = { + FindSongWithTitle, + FindChartWithPTDFVersion, +} \ No newline at end of file diff --git a/database-seeds/scripts/mutations.js b/database-seeds/scripts/mutations.js new file mode 100644 index 000000000..6d6d5ae4a --- /dev/null +++ b/database-seeds/scripts/mutations.js @@ -0,0 +1,30 @@ +const { MutateCollection, EfficientInPlaceDeepmerge } = require("./util"); +const deepmerge = require("deepmerge"); + +function DoesMatchCriteria(element, mutation) { + for (const [key, value] of Object.entries(mutation.match)) { + if (element[key] !== value) { + return false; + } + } + + return true; +} + +function ApplyMutations(name, mutations) { + MutateCollection(name, (collection) => { + for (const element of collection) { + for (const mutation of mutations) { + if (DoesMatchCriteria(element, mutation)) { + EfficientInPlaceDeepmerge(element, mutation.data); + } + } + } + + return collection; + }); +} + +module.exports = { + ApplyMutations +} \ No newline at end of file diff --git a/database-seeds/scripts/package.json b/database-seeds/scripts/package.json index b71a5f186..5964eff76 100644 --- a/database-seeds/scripts/package.json +++ b/database-seeds/scripts/package.json @@ -24,6 +24,7 @@ "homepage": "https://github.com/TeamNewGuys/tachi-database-seeds#readme", "dependencies": { "commander": "^8.2.0", + "deepmerge": "^4.2.2", "fast-json-stable-hash": "^1.0.3", "glob": "^7.2.0", "mei-logger": "^1.0.1", diff --git a/database-seeds/scripts/pnpm-lock.yaml b/database-seeds/scripts/pnpm-lock.yaml index 622df6ecd..8c40f632f 100644 --- a/database-seeds/scripts/pnpm-lock.yaml +++ b/database-seeds/scripts/pnpm-lock.yaml @@ -2,6 +2,7 @@ lockfileVersion: 5.3 specifiers: commander: ^8.2.0 + deepmerge: ^4.2.2 fast-json-stable-hash: ^1.0.3 glob: ^7.2.0 mei-logger: ^1.0.1 @@ -11,6 +12,7 @@ specifiers: dependencies: commander: 8.2.0 + deepmerge: 4.2.2 fast-json-stable-hash: 1.0.3 glob: 7.2.0 mei-logger: 1.0.1 @@ -760,6 +762,11 @@ packages: engines: {node: '>=0.10.0'} dev: false + /deepmerge/4.2.2: + resolution: {integrity: sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==} + engines: {node: '>=0.10.0'} + dev: false + /default-require-extensions/3.0.0: resolution: {integrity: sha512-ek6DpXq/SCpvjhpFsLFRVtIxJCRw6fUR42lYMVZuUMK7n8eMz4Uh5clckdBjEpLhn/gEBZo7hDJnJcwdKLKQjg==} engines: {node: '>=8'} diff --git a/database-seeds/scripts/rerunners/iidx-pe-kr/_README b/database-seeds/scripts/rerunners/iidx-pe-kr/_README new file mode 100644 index 000000000..6bbda2ac6 --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-pe-kr/_README @@ -0,0 +1,5 @@ +get the data by going to https://iidx.insane.pe.kr/!/sp10/ and typing copy(tabledata_orig). + +parsers here handle the rest. + +the sp8h, 9 and 10 tierlists *suck*, so, we're ignoring them. \ No newline at end of file diff --git a/database-seeds/scripts/rerunners/iidx-pe-kr/parse-iidxkr-data.js b/database-seeds/scripts/rerunners/iidx-pe-kr/parse-iidxkr-data.js new file mode 100644 index 000000000..6d64c2291 --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-pe-kr/parse-iidxkr-data.js @@ -0,0 +1,123 @@ +/* eslint-disable no-await-in-loop */ +const fs = require("fs"); +const path = require("path"); +const { FindChartWithPTDFVersion, FindSongWithTitle } = require("../../finders"); +const { ApplyMutations } = require("../../mutations"); +const { ReadCollection } = require("../../util"); + +const songs = ReadCollection("songs-iidx.json"); + +async function parseKr( + file, + mode, + catVals +) { + let charts = ReadCollection("charts-iidx.json"); + + const krdata = JSON.parse(fs.readFileSync(path.join(__dirname, file), "utf-8")); + const mutations = []; + + for (let i = 0; i < krdata.categories.length; i++) { + const data = krdata.categories[i]; + if (data.sortindex < 0) { + continue; + } + + const cv = catVals[i]; + + if (!cv) { + throw new Error(`cv krdata mismatch ${krdata.categories.length} ${catVals.length}?`); + } + + for (const item of data.items) { + const formattedTitle = item.data.title.replace(/†$/u, "").trim(); + const song = FindSongWithTitle(songs, formattedTitle); + + if (!song) { + console.warn(`Could not find song with title ${item.data.title}`); + continue; + } + + const t = item.data.type; + + let diff; + if (t === "A") { + diff = "ANOTHER"; + } else if (t === "L") { + diff = "LEGGENDARIA"; + } else if (t === "H") { + diff = "HYPER"; + } else if (t === "N") { + diff = "NORMAL"; + } else { + console.warn(`${song.title} Unknown difficulty ${t}.`); + continue; + } + + const chart = await FindChartWithPTDFVersion(charts, song.id, "SP", diff, "29"); + + if (!chart) { + console.warn(`${song.title} ${diff} - Couldn't find chart?`); + continue; + } + + // Update this part of the tierlist, we've got everything. + + mutations.push({ + match: { + chartID: chart.chartID, + }, + data: { + tierlistInfo: { + [`kt-${mode}`]: { + text: cv.text, + value: cv.value, + individualDifference: cv.idv, + } + } + } + }) + + console.log(`Updated ${chart.chartID} to value ${cv.value}.`); + } + } + + console.info(`Finished parsing ${file}`); + + delete charts; + + ApplyMutations("charts-iidx.json", mutations); +} + +function h(text, value, idv = false) { + return { text, value, idv }; +} + +parseKr("sp11N.json", "NC", [ + h("11S+", 12.2), + h("11S", 12), + h("11A+", 11.9), + h("11A", 11.8), + h("11B", 11.6), + h("11C", 11.4), + h("11D", 11.2), + h("11E", 11.0), + h("11F", 10.8), +]); + +parseKr("sp11H.json", "HC", [ + h("11S+", 12.2), + h("11S+", 12.2, true), // 個人差 + h("11S", 12), + h("11S", 12, true), // 個人差 + h("11A+", 11.9), + h("11A+", 11.9, true), // 個人差 + h("11A", 11.8), + h("11A", 11.8, true), // 個人差 + h("11B+", 11.7), + h("11B", 11.6), + h("11C", 11.4), + h("11D", 11.2), + h("11E", 11.0), + h("11F", 10.8), +]); diff --git a/database-seeds/scripts/util.js b/database-seeds/scripts/util.js index d21e48939..d19b412ac 100644 --- a/database-seeds/scripts/util.js +++ b/database-seeds/scripts/util.js @@ -6,23 +6,26 @@ const fjsh = require("fast-json-stable-hash"); const glob = require("glob"); function IterateCollections(cb) { - const main = path.join(__dirname, "../collections"); + for (const collection of fs.readdirSync(COLLECTIONS_DIR)) { + const data = cb(JSON.parse(fs.readFileSync(path.join(COLLECTIONS_DIR, collection))), collection); - for (const collection of fs.readdirSync(main)) { - const data = cb(JSON.parse(fs.readFileSync(path.join(main, collection))), collection); - - fs.writeFileSync(path.join(main, collection), JSON.stringify(data, null, "\t")); + fs.writeFileSync(path.join(COLLECTIONS_DIR, collection), JSON.stringify(data, null, "\t")); } DeterministicCollectionSort(); } +const COLLECTIONS_DIR = path.join(__dirname, "../collections"); + +function ReadCollection(name) { + return JSON.parse(fs.readFileSync(path.join(COLLECTIONS_DIR, name))); +} + function MutateCollection(name, cb) { - const main = path.join(__dirname, "../collections"); - const data = cb(JSON.parse(fs.readFileSync(path.join(main, name)))); + const data = cb(ReadCollection(name)); - fs.writeFileSync(path.join(main, name), JSON.stringify(data, null, "\t")); + fs.writeFileSync(path.join(COLLECTIONS_DIR, name), JSON.stringify(data, null, "\t")); DeterministicCollectionSort(); } @@ -39,9 +42,23 @@ function CreateFolderID( return `F${fjsh.hash(Object.assign({ game, playtype }, query), "SHA256")}`; } +// quick inplace deepmerge hack +// probably doesn't work for arrays, i don't care though. +function EfficientInPlaceDeepmerge(ref, apply) { + for (const key in apply) { + if (typeof apply[key] === "object" && apply[key]) { + EfficientInPlaceDeepmerge(ref[key], apply[key]); + } else { + ref[key] = apply[key] + } + } +} + module.exports = { IterateCollections, MutateCollection, CreateChartID, CreateFolderID, + ReadCollection, + EfficientInPlaceDeepmerge } \ No newline at end of file