Add entire workflow for mutating, finding and efficiently dispatching changes to data

database-seeds/scripts/finders.js

@@ -0,0 +1,12 @@
+function FindChartWithPTDFVersion(collection, songID, playtype, difficulty, version) {
+	return collection.find(chart => chart.songID === songID && chart.playtype === playtype && chart.difficulty === difficulty && chart.versions.includes(version));
+}
+
+function FindSongWithTitle(collection, title) {
+	return collection.find(e => e.title === title || e.altTitles.includes(title));
+}
+
+module.exports = {
+	FindSongWithTitle,
+	FindChartWithPTDFVersion,
+}

database-seeds/scripts/mutations.js

@@ -0,0 +1,30 @@
+const { MutateCollection, EfficientInPlaceDeepmerge } = require("./util");
+const deepmerge = require("deepmerge");
+
+function DoesMatchCriteria(element, mutation) {
+	for (const [key, value] of Object.entries(mutation.match)) {
+		if (element[key] !== value) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+function ApplyMutations(name, mutations) {
+	MutateCollection(name, (collection) => {
+		for (const element of collection) {
+			for (const mutation of mutations) {
+				if (DoesMatchCriteria(element, mutation)) {
+					EfficientInPlaceDeepmerge(element, mutation.data);
+				}
+			}
+		}
+
+		return collection;
+	});
+}
+
+module.exports = {
+	ApplyMutations
+}

database-seeds/scripts/package.json

@@ -24,6 +24,7 @@
 	"homepage": "https://github.com/TeamNewGuys/tachi-database-seeds#readme",
 	"dependencies": {
 		"commander": "^8.2.0",
+		"deepmerge": "^4.2.2",
 		"fast-json-stable-hash": "^1.0.3",
 		"glob": "^7.2.0",
 		"mei-logger": "^1.0.1",

database-seeds/scripts/pnpm-lock.yaml

@@ -2,6 +2,7 @@ lockfileVersion: 5.3
 
 specifiers:
   commander: ^8.2.0
+  deepmerge: ^4.2.2
   fast-json-stable-hash: ^1.0.3
   glob: ^7.2.0
   mei-logger: ^1.0.1
@@ -11,6 +12,7 @@ specifiers:
 
 dependencies:
   commander: 8.2.0
+  deepmerge: 4.2.2
   fast-json-stable-hash: 1.0.3
   glob: 7.2.0
   mei-logger: 1.0.1
@@ -760,6 +762,11 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /deepmerge/4.2.2:
+    resolution: {integrity: sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
   /default-require-extensions/3.0.0:
     resolution: {integrity: sha512-ek6DpXq/SCpvjhpFsLFRVtIxJCRw6fUR42lYMVZuUMK7n8eMz4Uh5clckdBjEpLhn/gEBZo7hDJnJcwdKLKQjg==}
     engines: {node: '>=8'}

database-seeds/scripts/rerunners/iidx-pe-kr/_README

@@ -0,0 +1,5 @@
+get the data by going to https://iidx.insane.pe.kr/!/sp10/ and typing copy(tabledata_orig).
+
+parsers here handle the rest.
+
+the sp8h, 9 and 10 tierlists *suck*, so, we're ignoring them.

database-seeds/scripts/rerunners/iidx-pe-kr/parse-iidxkr-data.js

@@ -0,0 +1,123 @@
+/* eslint-disable no-await-in-loop */
+const fs = require("fs");
+const path = require("path");
+const { FindChartWithPTDFVersion, FindSongWithTitle } = require("../../finders");
+const { ApplyMutations } = require("../../mutations");
+const { ReadCollection } = require("../../util");
+
+const songs = ReadCollection("songs-iidx.json");
+
+async function parseKr(
+	file,
+	mode,
+	catVals
+) {
+	let charts = ReadCollection("charts-iidx.json");
+
+	const krdata = JSON.parse(fs.readFileSync(path.join(__dirname, file), "utf-8"));
+	const mutations = [];
+
+	for (let i = 0; i < krdata.categories.length; i++) {
+		const data = krdata.categories[i];
+		if (data.sortindex < 0) {
+			continue;
+		}
+
+		const cv = catVals[i];
+
+		if (!cv) {
+			throw new Error(`cv krdata mismatch ${krdata.categories.length} ${catVals.length}?`);
+		}
+
+		for (const item of data.items) {
+			const formattedTitle = item.data.title.replace(/†$/u, "").trim();
+			const song = FindSongWithTitle(songs, formattedTitle);
+
+			if (!song) {
+				console.warn(`Could not find song with title ${item.data.title}`);
+				continue;
+			}
+
+			const t = item.data.type;
+
+			let diff;
+			if (t === "A") {
+				diff = "ANOTHER";
+			} else if (t === "L") {
+				diff = "LEGGENDARIA";
+			} else if (t === "H") {
+				diff = "HYPER";
+			} else if (t === "N") {
+				diff = "NORMAL";
+			} else {
+				console.warn(`${song.title} Unknown difficulty ${t}.`);
+				continue;
+			}
+
+			const chart = await FindChartWithPTDFVersion(charts, song.id, "SP", diff, "29");
+
+			if (!chart) {
+				console.warn(`${song.title} ${diff} - Couldn't find chart?`);
+				continue;
+			}
+
+			// Update this part of the tierlist, we've got everything.
+
+			mutations.push({
+				match: {
+					chartID: chart.chartID,
+				},
+				data: {
+					tierlistInfo: {
+						[`kt-${mode}`]: {
+							text: cv.text,
+							value: cv.value,
+							individualDifference: cv.idv,
+						}
+					}
+				}
+			})
+
+			console.log(`Updated ${chart.chartID} to value ${cv.value}.`);
+		}
+	}
+
+	console.info(`Finished parsing ${file}`);
+
+	delete charts;
+
+	ApplyMutations("charts-iidx.json", mutations);
+}
+
+function h(text, value, idv = false) {
+	return { text, value, idv };
+}
+
+parseKr("sp11N.json", "NC", [
+	h("11S+", 12.2),
+	h("11S", 12),
+	h("11A+", 11.9),
+	h("11A", 11.8),
+	h("11B", 11.6),
+	h("11C", 11.4),
+	h("11D", 11.2),
+	h("11E", 11.0),
+	h("11F", 10.8),
+]);
+
+parseKr("sp11H.json", "HC", [
+	h("11S+", 12.2),
+	h("11S+", 12.2, true), // 個人差
+	h("11S", 12),
+	h("11S", 12, true), // 個人差
+	h("11A+", 11.9),
+	h("11A+", 11.9, true), // 個人差
+	h("11A", 11.8),
+	h("11A", 11.8, true), // 個人差
+	h("11B+", 11.7),
+	h("11B", 11.6),
+	h("11C", 11.4),
+	h("11D", 11.2),
+	h("11E", 11.0),
+	h("11F", 10.8),
+]);

database-seeds/scripts/util.js

@@ -6,23 +6,26 @@ const fjsh = require("fast-json-stable-hash");
 const glob = require("glob");
 
 function IterateCollections(cb) {
-	const main = path.join(__dirname, "../collections");
+	for (const collection of fs.readdirSync(COLLECTIONS_DIR)) {
+		const data = cb(JSON.parse(fs.readFileSync(path.join(COLLECTIONS_DIR, collection))), collection);
 
-	for (const collection of fs.readdirSync(main)) {
-		const data = cb(JSON.parse(fs.readFileSync(path.join(main, collection))), collection);
-
-		fs.writeFileSync(path.join(main, collection), JSON.stringify(data, null, "\t"));
+		fs.writeFileSync(path.join(COLLECTIONS_DIR, collection), JSON.stringify(data, null, "\t"));
 	}
 
 	DeterministicCollectionSort();
 }
 
+const COLLECTIONS_DIR = path.join(__dirname, "../collections");
+
+function ReadCollection(name) {
+	return JSON.parse(fs.readFileSync(path.join(COLLECTIONS_DIR, name)));
+}
+
 function MutateCollection(name, cb) {
-	const main = path.join(__dirname, "../collections");
 
-	const data = cb(JSON.parse(fs.readFileSync(path.join(main, name))));
+	const data = cb(ReadCollection(name));
 
-	fs.writeFileSync(path.join(main, name), JSON.stringify(data, null, "\t"));
+	fs.writeFileSync(path.join(COLLECTIONS_DIR, name), JSON.stringify(data, null, "\t"));
 
 	DeterministicCollectionSort();
 }
@@ -39,9 +42,23 @@ function CreateFolderID(
 	return `F${fjsh.hash(Object.assign({ game, playtype }, query), "SHA256")}`;
 }
 
+// quick inplace deepmerge hack
+// probably doesn't work for arrays, i don't care though.
+function EfficientInPlaceDeepmerge(ref, apply) {
+	for (const key in apply) {
+		if (typeof apply[key] === "object" && apply[key]) {
+			EfficientInPlaceDeepmerge(ref[key], apply[key]);
+		} else {
+			ref[key] = apply[key]
+		}
+	}
+}
+
 module.exports = {
 	IterateCollections,
 	MutateCollection,
 	CreateChartID,
 	CreateFolderID,
+	ReadCollection,
+	EfficientInPlaceDeepmerge
 }