seeds: update arcaea to 6.14.10 (#1653)

db/seeds/charts-arcaea.json

@@ -19261,7 +19261,7 @@
 		"isPrimary": true,
 		"legacyChartID": "ac4a98cff77a547af921b8c2400a7c3bf5639723",
 		"level": "3",
-		"levelNum": 3.5,
+		"levelNum": 3,
 		"songID": "S19e3cb7a00757b082b1",
 		"versions": [
 			"mobile"
@@ -27863,7 +27863,7 @@
 		"isPrimary": true,
 		"legacyChartID": "9cf84e6b509735cbd94a30ee12e86b655a266ac9",
 		"level": "4",
-		"levelNum": 4,
+		"levelNum": 4.5,
 		"songID": "S19e3cb7a00b4d0bb64a",
 		"versions": [
 			"mobile"
@@ -28866,7 +28866,7 @@
 		"isPrimary": true,
 		"legacyChartID": "80d977d82e427b0c1950ff3e1d7078be0dde0645",
 		"level": "3",
-		"levelNum": 3,
+		"levelNum": 3.5,
 		"songID": "S19e3cb7a00bad6bc309",
 		"versions": [
 			"mobile"
@@ -29852,5 +29852,260 @@
 		"versions": [
 			"mobile"
 		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "altersist",
+			"notecount": 1077
+		},
+		"difficulty": "Eternal",
+		"id": "C19ea8d88f71f69fefcf",
+		"isPrimary": true,
+		"legacyChartID": "11ae68e1786ffcb5401394475f0685d2780583b6",
+		"level": "9+",
+		"levelNum": 9.8,
+		"songID": "S19ea8d88f710ccfdd65",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "altersist",
+			"notecount": 803
+		},
+		"difficulty": "Future",
+		"id": "C19ea8d88f712a49e583",
+		"isPrimary": true,
+		"legacyChartID": "06e29f0d9117c818d16ddcb73d5ccfb86d3cab64",
+		"level": "8",
+		"levelNum": 8.4,
+		"songID": "S19ea8d88f710ccfdd65",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "altersist",
+			"notecount": 454
+		},
+		"difficulty": "Past",
+		"id": "C19ea8d88f71fcc46092",
+		"isPrimary": true,
+		"legacyChartID": "d062d0eca887a24b30c82c93617f21cc7585e319",
+		"level": "2",
+		"levelNum": 2.5,
+		"songID": "S19ea8d88f710ccfdd65",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "altersist",
+			"notecount": 621
+		},
+		"difficulty": "Present",
+		"id": "C19ea8d88f716d6c9095",
+		"isPrimary": true,
+		"legacyChartID": "d9f0ce13bff21807f60f765301a6af066f1209c7",
+		"level": "6",
+		"levelNum": 6,
+		"songID": "S19ea8d88f710ccfdd65",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "flexidefine",
+			"notecount": 1055
+		},
+		"difficulty": "Future",
+		"id": "C19ea8d88f71d2c0ca86",
+		"isPrimary": true,
+		"legacyChartID": "e76e031652b7bf5e4fcd58104b06a78780d7ccdc",
+		"level": "9+",
+		"levelNum": 9.9,
+		"songID": "S19ea8d88f717283de9a",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "flexidefine",
+			"notecount": 644
+		},
+		"difficulty": "Past",
+		"id": "C19ea8d88f71caf536d1",
+		"isPrimary": true,
+		"legacyChartID": "df9f639bbc34fae9e5863d52733af18180d3027a",
+		"level": "2",
+		"levelNum": 2,
+		"songID": "S19ea8d88f717283de9a",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "flexidefine",
+			"notecount": 842
+		},
+		"difficulty": "Present",
+		"id": "C19ea8d88f71c1027e7a",
+		"isPrimary": true,
+		"legacyChartID": "8316ea7db38f120582681fc83f480d2cc3add78d",
+		"level": "6",
+		"levelNum": 6,
+		"songID": "S19ea8d88f717283de9a",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "csqn",
+			"notecount": 1071
+		},
+		"difficulty": "Eternal",
+		"id": "C19ea8d88f71a8aabdc3",
+		"isPrimary": true,
+		"legacyChartID": "da9082af0083a2f710f581467d8c58806d13f45e",
+		"level": "10",
+		"levelNum": 10.3,
+		"songID": "S19ea8d88f71945526c5",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "csqn",
+			"notecount": 923
+		},
+		"difficulty": "Future",
+		"id": "C19ea8d88f71d98e4bff",
+		"isPrimary": true,
+		"legacyChartID": "a69df633656825eedb1497a375e9d43752bc40ce",
+		"level": "9",
+		"levelNum": 9.6,
+		"songID": "S19ea8d88f71945526c5",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "csqn",
+			"notecount": 418
+		},
+		"difficulty": "Past",
+		"id": "C19ea8d88f71e07fa0ba",
+		"isPrimary": true,
+		"legacyChartID": "502c41a87e012e47fd0699c69d224e04193355d2",
+		"level": "4",
+		"levelNum": 4,
+		"songID": "S19ea8d88f71945526c5",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "csqn",
+			"notecount": 625
+		},
+		"difficulty": "Present",
+		"id": "C19ea8d88f7107d87b10",
+		"isPrimary": true,
+		"legacyChartID": "50fe18d394f8e74b9ea986cba000080323127974",
+		"level": "7",
+		"levelNum": 7.5,
+		"songID": "S19ea8d88f71945526c5",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "cosmogyral",
+			"notecount": 1049
+		},
+		"difficulty": "Eternal",
+		"id": "C19ea8d88f7144208ddd",
+		"isPrimary": true,
+		"legacyChartID": "8037a5ebdc4d4c8fc3743f51a23e0b6a0246c2f2",
+		"level": "9+",
+		"levelNum": 9.8,
+		"songID": "S19ea8d88f71ca081a55",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "cosmogyral",
+			"notecount": 916
+		},
+		"difficulty": "Future",
+		"id": "C19ea8d88f71d98d33ee",
+		"isPrimary": true,
+		"legacyChartID": "7be8b469b627f3320d5429997b5cb444d4202de1",
+		"level": "8",
+		"levelNum": 8.6,
+		"songID": "S19ea8d88f71ca081a55",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "cosmogyral",
+			"notecount": 547
+		},
+		"difficulty": "Past",
+		"id": "C19ea8d88f71dae1e660",
+		"isPrimary": true,
+		"legacyChartID": "0b63ee133167a7fd0b7d866de5b2330bb4a450a7",
+		"level": "2",
+		"levelNum": 2.5,
+		"songID": "S19ea8d88f71ca081a55",
+		"versions": [
+			"mobile"
+		]
+	},
+	{
+		"data": {
+			"displayVersion": "6.14",
+			"inGameStrID": "cosmogyral",
+			"notecount": 687
+		},
+		"difficulty": "Present",
+		"id": "C19ea8d88f71646e5d02",
+		"isPrimary": true,
+		"legacyChartID": "85b13e8e5abc7c27a2e0c8604b34bce6da3fc54f",
+		"level": "5",
+		"levelNum": 5.5,
+		"songID": "S19ea8d88f71ca081a55",
+		"versions": [
+			"mobile"
+		]
 	}
 ]

db/seeds/songs-arcaea.json

@@ -8200,5 +8200,63 @@
 		"legacySongID": 543,
 		"searchTerms": [],
 		"title": "Mistempered Malignance"
+	},
+	{
+		"altTitles": [],
+		"artist": "void (Mournfinale)",
+		"data": {
+			"songPack": "Memory Archive"
+		},
+		"id": "S19ea8d88f710ccfdd65",
+		"legacySongID": 547,
+		"searchTerms": [
+			"おるたーしすと",
+			"알터시스트"
+		],
+		"title": "Altersist"
+	},
+	{
+		"altTitles": [],
+		"artist": "Lamanya",
+		"data": {
+			"songPack": "World Extend 4"
+		},
+		"id": "S19ea8d88f717283de9a",
+		"legacySongID": 546,
+		"searchTerms": [
+			"ふれきしでぃふぁいん",
+			"플렉시디파인"
+		],
+		"title": "flexidefine"
+	},
+	{
+		"altTitles": [],
+		"artist": "Aoi",
+		"data": {
+			"songPack": "Memory Archive"
+		},
+		"id": "S19ea8d88f71945526c5",
+		"legacySongID": 548,
+		"searchTerms": [
+			"しーえすきゅーえぬ",
+			"시 에스 큐 엔",
+			"csqn"
+		],
+		"title": "c.s.q.n."
+	},
+	{
+		"altTitles": [],
+		"artist": "Darren + Altermis",
+		"data": {
+			"songPack": "World Extend 4"
+		},
+		"id": "S19ea8d88f71ca081a55",
+		"legacySongID": 545,
+		"searchTerms": [
+			"こすもじゃいらる",
+			"코스모자이럴",
+			"코스모자이랄"
+		],
+		"title": "Cosmogyral"
 	}
 ]

typescript/seeds-scripts/rerunners/arcaea/README.md

@@ -0,0 +1,8 @@
+1. Have a working Tachi Docker container
+2. Parse music data
+   1. Get the latest APK and extract `songlist` and `packlist` from `assets/songs`
+   2. `bun run merge-songlist.ts -i <path/to/songlist> -p <path/to/packlist>`
+3. Fetch notecounts/internal levels
+   1. `bun run notes-and-constant.ts`
+   2. There is also the backup option of scraping wikiwiki (not recommended)
+4. Run `just db-load-seeds`

typescript/seeds-scripts/rerunners/arcaea/merge-songlist.ts

@@ -233,7 +233,6 @@ for (const entry of data.songs) {
 		if (exists) {
 			// update chart levels
 			exists.level = `${chart.rating}${chart.ratingPlus ? "+" : ""}`;
-			exists.levelNum = chart.rating + (chart.ratingPlus ? 0.7 : 0);
 			continue;
 		}
 

typescript/seeds-scripts/rerunners/arcaea/notes-and-constants.ts

@@ -0,0 +1,160 @@
+import { ReadCollection, WriteCollection } from "../../util";
+import { type ChartDocument } from "tachi-common";
+
+const SONG_BLACKLIST = [
+	"particlearts",
+	"ignotusafterburn",
+	"redandblueandgreen",
+	"singularityvvvip",
+	"overdead",
+	"mismal",
+	"ifirmx",
+	"hivemindrmx",
+	"lfdyrmx",
+	"unknownrmx",
+];
+
+const logNotecountChanges: [string, number, number][] = [];
+const logLevelChanges: [string, number, number][] = [];
+const logToolbeltErrors: Set<string> = new Set();
+const logArcsongErrors: Set<string> = new Set();
+const logErrors: string[] = [];
+
+const inverseConvertDifficulty = (input: string) => {
+	switch (input) {
+		case "Past":
+			return 0;
+		case "Present":
+			return 1;
+		case "Future":
+			return 2;
+		case "Beyond":
+			return 3;
+		case "Eternal":
+			return 4;
+		default:
+			throw new Error(
+				`Unknown difficulty ${input}, can't convert this into one of Arcaea's difficulty values. Consider updating the script.`,
+			);
+	}
+};
+
+type DataRow = {
+	levelNum: number;
+	notecount: number;
+};
+
+const getToolbeltData = (toolbeltData: any, chart: ChartDocument<"arcaea">): DataRow | null => {
+	const toolbeltSong = toolbeltData.find((s) => s.id === chart.data.inGameStrID);
+
+	if (toolbeltSong === undefined) {
+		logToolbeltErrors.add(`Unknown song: ${chart.data.inGameStrID}`);
+		return null;
+	}
+
+	const toolbeltChart = toolbeltSong.charts[inverseConvertDifficulty(chart.difficulty)];
+
+	if (toolbeltChart === undefined) {
+		logToolbeltErrors.add(`Unknown chart: ${chart.data.inGameStrID} ${chart.difficulty}`);
+		return null;
+	}
+
+	return { notecount: toolbeltChart.notes, levelNum: toolbeltChart.constant };
+};
+
+const getArcsongData = (arcsongData: any, chart: ChartDocument<"arcaea">): DataRow | null => {
+	const arcsongChart = arcsongData.find(
+		(c) =>
+			c.song_id === chart.data.inGameStrID &&
+			c.rating_class === inverseConvertDifficulty(chart.difficulty),
+	);
+
+	if (arcsongChart === undefined) {
+		logArcsongErrors.add(
+			`[Arcsong] Unknown chart: ${chart.data.inGameStrID} ${chart.difficulty}`,
+		);
+		return null;
+	}
+
+	return { notecount: arcsongChart.note, levelNum: arcsongChart.rating / 10 };
+};
+
+const syncNotesAndConstants = async () => {
+	const rawToolbeltData = await fetch(
+		"https://raw.githubusercontent.com/DarrenDanielDay/arcaea-toolbelt-data/refs/heads/main/src/data/notes-and-constants.json",
+	);
+	const toolbeltData = await rawToolbeltData.json();
+	const rawArcsongData = await fetch(
+		"https://raw.githubusercontent.com/CuSO4Deposit/ArcaeaSongDatabase/refs/heads/main/arcsong.json",
+	);
+	const arcsongData = await rawArcsongData.json();
+
+	const charts = ReadCollection("charts-arcaea.json");
+
+	for (const chart of charts) {
+		if (SONG_BLACKLIST.includes(chart.data.inGameStrID)) {
+			continue;
+		}
+
+		const toolbelt = getToolbeltData(toolbeltData, chart);
+		const arcsong = getArcsongData(arcsongData, chart);
+		const chartName = `${chart.data.inGameStrID} ${chart.difficulty}`;
+
+		if (toolbelt === null && arcsong === null) {
+			logErrors.push(`No data for ${chartName}`);
+			continue;
+		}
+
+		if (toolbelt && arcsong && toolbelt.notecount !== arcsong.notecount) {
+			logErrors.push(
+				`notecount mismatch on ${chartName}: T=${toolbelt.notecount} A=${arcsong.notecount}`,
+			);
+		} else {
+			const notecount = toolbelt?.notecount ?? arcsong!.notecount;
+			if (chart.data.notecount !== notecount) {
+				logNotecountChanges.push([chartName, chart.data.notecount, notecount]);
+				chart.data.notecount = notecount;
+			}
+		}
+
+		if (toolbelt && arcsong && toolbelt.levelNum !== arcsong.levelNum) {
+			logErrors.push(
+				`levelNum mismatch on ${chartName}: T=${toolbelt.levelNum} A=${arcsong.levelNum}`,
+			);
+		} else {
+			const levelNum = toolbelt?.levelNum ?? arcsong!.levelNum;
+			if (chart.levelNum !== levelNum) {
+				logLevelChanges.push([chartName, chart.levelNum, levelNum]);
+				chart.levelNum = levelNum;
+			}
+		}
+	}
+
+	console.log("Notecount changes:");
+	for (const [name, old, updated] of logNotecountChanges) {
+		if (old === undefined) {
+			console.log(`\t${name}: ${updated}`);
+		} else {
+			console.error(`\t${name}: ${old} -> ${updated} THIS SHOULD NOT HAVE HAPPENED`);
+		}
+	}
+	console.log("\nLevel changes:");
+	for (const [name, old, updated] of logLevelChanges) {
+		console.log(`\t${name}: ${old} -> ${updated}`);
+	}
+	console.log("\nData problems:");
+	for (const row of logToolbeltErrors) {
+		console.warn(`\t[Toolbelt] ${row}`);
+	}
+	for (const row of logArcsongErrors) {
+		console.warn(`\t[Arcsong] ${row}`);
+	}
+	console.log("\nErrors:");
+	for (const row of logErrors) {
+		console.error("\t" + row.split("\n").join("\n\t"));
+	}
+
+	WriteCollection("charts-arcaea.json", charts);
+};
+
+syncNotesAndConstants();

typescript/seeds-scripts/rerunners/ongeki/README.md

@@ -10,4 +10,4 @@
    1. Run `bun scrape-sdvx-in.ts`.
    2. Fix any missing entries manually in `seeds/collections/charts-ongeki.json` (refer to `songs-ongeki.json` for `songID`s).
 5. Manually add search terms in `songs-ongeki.json`, especially romanizations (optional)
-6. In the root project folder, run `just seeds test`.
+6. Run `just db-load-seeds`.