[popn] Add Unilab seeds (2023-09-05) & mdb-to-mxb script (#1108)

* Add Unilab seeds (2023-09-05)

* Add mdb to mxb rerunner & fix old scripts

* Add missing comma to make the linter happy

* fix: typo

* i have no reason to believe this code is broken

* feat: get rid of some old crap

---------

Co-authored-by: zkldi <20380519+zkldi@users.noreply.github.com>
This commit is contained in:
blobdash
2024-06-11 12:50:57 +01:00
committed by GitHub
co-authored by zkldi
parent 37b804ecdf
commit 7b6f4b50ab
14 changed files with 20336 additions and 6373 deletions
+1
View File
@@ -136,6 +136,7 @@ export const POPN_9B_CONF = {
versions: {
peace: "peace",
kaimei: "Kaimei Riddles",
unilab: "Unilab",
},
chartData: z.strictObject({
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+95
View File
@@ -4565,6 +4565,101 @@
"tableID": "popn-9B-peace-alllevels",
"title": "Pop'n Music peace Levels (All)"
},
{
"default": false,
"description": "All pop'n Unilab levels individually.",
"folders": [
"Fc16a3e59241c973c696ae94da201a3a2f7598ff6a4b9db5c95a285aa24c01908",
"F7257b2f3d4c5a16198c5f4cefcd5e6ec97372ecf835ec2f601a09147cfa4a737",
"F7274d96cc1d75a6133ffee349fcee08b99c14d562cf5ecb6f02f86089af6185d",
"F919477b4cc85b3762047d2601185c5e388cadc71c50a4fac0fc69d015d6e3bc3",
"F312b52c5605a49fdd03c93ffac75da00b59a3a68a5a187dd8819ed21537cbb3a",
"F25d5dc7f32117f7bb41e14f057fbd5fc6113581eb6401bc0e32fd039bd5201d7",
"F0957ad19cfa9f391beacc97f546376d4383696f4c022d8934d83ea738d9d5a92",
"Fd3b7141270ce0faf99499e5e3094fe15b6de553a4d03f336e44fd0c22c33b867",
"Fcd89aeb62fdf0be581e01b045a6c8b690c40a864d9d5b1bb343f7dd340c3eb23",
"Fa6865793a37f4c61b24aa3dad173a796e58ecec93efa2863b601f7bc71694e86",
"F3d5cae48b934aeda8006a99f006a5e3c0874eb538248b6faf2f2e09c1c7e8c8d",
"Fe0818e5d452d78d63a11b5136c3f8409294e5a8b02ade4403b0ba4f3a983fdd8",
"F924c37c9bce17b0f22383b2651db6619516e959356debf98084aef772977567f",
"F58e2c8776005494db5e710bc980962a5682f6e39e5582200ac9d6e4b48efc9b0",
"F52a8d173b8cc44cd0765a3880c0f612c0a0ee15762bf4fef1e14cf9dcf5325de",
"F1bcd22810a7be3b85ac8f1823913e16a0417b823de2cf6cde5d2e7b614c2f250",
"F39f4c3cafa0dec47b17c1cf9307a0bea6b3db96ba7d4b493d0c5931d8d0e2a41",
"Ffb0814ebaca88c589f361adb58064fdc47201792e6c410804d3061e566f304ea",
"F1dd648ab8c58821c708845a8d63b6ce50f13a723ac6a32fa93d69c917f2a8de8",
"Fa627b5f49062a79c891c694623028a30b18f34b667e35848ad10f8035710156f",
"F746cf6442a767afc5a311b3cc3332e543b047cbb3ca2465c57ac74fe0d5695e3",
"F69bd925ff234b217837333d9f47a526c57a1f5a428be72e1b4fe9008096a4082",
"Fa8bd04ab0d6eed2609eccc895fe626a4c64d91d90d44d861028085dccfac71a5",
"F5be142a4a5b0cf0b007ec0283360cb50b1133f8d8871d2aa50ba00202a5636fa",
"F25ab4b290aec401544c14c2d7f497d70dfee30e8b117096c40db5c8047b1c803",
"F06c969775c7def45e3ad7519ac5ad6fced7f58266e4bd31a396d7db0c9ea7af6",
"Feb46faa22291f0388420978e354e053b75d9fa43b62c275e49bef6cec4e043fb",
"F3a7ccc7b59b6945ddaf408490e74342bde48a7599bbb5a0c21f24f02d1633d86",
"F80c26535bc3f9efe9254f6adc2b55f601d1280d47d675b6695fd5e17e666a72b",
"Ffc79257b4b5c0fa030e46289276524b72a34a5b0e69b9f7237e7c084a115cb0f",
"F2fcd0312e82917e2d76d9494b653180c2b04ac2112d10b51c67c61f6cd10bfa1",
"Fcac66115d41f12c288ac34d609b55cb2e5e64922f3c11393ffe45f30902cc1d8",
"F706fdfee9b49ac263f4d5ce760f5176d2ad97f16fd753130c7341a7e226422aa",
"F57a4649078946be4e71b08ad8cd27c1aa7cdfb61e1648aa9745b995cf643e92b",
"F5569ae60b72e068ee393278ae51415e0452f52f1454643253f79ddb5d9f57fa4",
"F2a145eb30b866e48c2f58c4f5b0079bd660897241d4d47a883edf30428e1ea75",
"F73220404ce13eb5bd14dbee365cf172fcd0d173957e696ef8a0dbd8b34a38005",
"F780ed319938778a4c8cb3e3364758e31f5036453594e2c2a9b76334ca82b2740",
"F8d2662aeb238fd62461dd74ddd39a6d70e95fcf1cea7a4e7e5786d140dfa6283",
"F59dabb6f9f17396cdf22023f9fdb0b198a7aec198dd35e649201d6b28b686bfc",
"Fd1eca971ea91ccd67d642f570b35894ab562706e9856d612a87b1cec826f09a9",
"F6a0c2280b9d0c813043f1925e7ae66ed9471147a2ec33a51ad2ed332a5fab39d",
"F5913ba6cb0923f40128aa1556ce3c8149eba9a685492695f74275bd8e8efaab8",
"Fc6c828a87fbbebd63ba80de259d54b5b58df9fb06e8ac1c363ffc0497bcdf3d3",
"F2eb5c86a59d01b81713f01a832bd8daf1fac51b32bc2059a5a4bb7d9ab0749be",
"Fec2b66031fca71f4999684492fd4f5736f5a867b589d31259a18809692ca0e10",
"F346ab8a5e2825f59145fcba95561f3f959566ffcd96e192824b6a908d053b392",
"F33f7883915460f55cc003214fa949b1662b54de7ba850d8e6d5d19d8ced91654",
"Fef381544ff909f7a1c2fe188bed5ded71f42bc876176754c76e9dc5bc6e981cf",
"F9a91dc3605c53e940d0ffc893c4c12dff6bc07d7e9ea13b1cf6b2b52917c5da4"
],
"game": "popn",
"inactive": false,
"playtype": "9B",
"tableID": "popn-9B-unilab-alllevels",
"title": "Pop'n Music Unilab All Levels"
},
{
"default": false,
"description": "All pop'n Unilab levels, with some folders joined together.",
"folders": [
"F31bb32091f07c65b233c022a1703e31dbab62c9f880502858891ff00888c48b7",
"F82fb4cb2f4368273899b23794712588f41c5b6facd212eec54e3a27c462ef758",
"F76a6ad8cc6142ad0b4796c3fd1f94332569a4b048af639da46e83e7ef0945bc6",
"F2fcd0312e82917e2d76d9494b653180c2b04ac2112d10b51c67c61f6cd10bfa1",
"Fcac66115d41f12c288ac34d609b55cb2e5e64922f3c11393ffe45f30902cc1d8",
"F706fdfee9b49ac263f4d5ce760f5176d2ad97f16fd753130c7341a7e226422aa",
"F57a4649078946be4e71b08ad8cd27c1aa7cdfb61e1648aa9745b995cf643e92b",
"F5569ae60b72e068ee393278ae51415e0452f52f1454643253f79ddb5d9f57fa4",
"F2a145eb30b866e48c2f58c4f5b0079bd660897241d4d47a883edf30428e1ea75",
"F73220404ce13eb5bd14dbee365cf172fcd0d173957e696ef8a0dbd8b34a38005",
"F780ed319938778a4c8cb3e3364758e31f5036453594e2c2a9b76334ca82b2740",
"F8d2662aeb238fd62461dd74ddd39a6d70e95fcf1cea7a4e7e5786d140dfa6283",
"F59dabb6f9f17396cdf22023f9fdb0b198a7aec198dd35e649201d6b28b686bfc",
"Fd1eca971ea91ccd67d642f570b35894ab562706e9856d612a87b1cec826f09a9",
"F6a0c2280b9d0c813043f1925e7ae66ed9471147a2ec33a51ad2ed332a5fab39d",
"F5913ba6cb0923f40128aa1556ce3c8149eba9a685492695f74275bd8e8efaab8",
"Fc6c828a87fbbebd63ba80de259d54b5b58df9fb06e8ac1c363ffc0497bcdf3d3",
"F2eb5c86a59d01b81713f01a832bd8daf1fac51b32bc2059a5a4bb7d9ab0749be",
"Fec2b66031fca71f4999684492fd4f5736f5a867b589d31259a18809692ca0e10",
"F346ab8a5e2825f59145fcba95561f3f959566ffcd96e192824b6a908d053b392",
"F33f7883915460f55cc003214fa949b1662b54de7ba850d8e6d5d19d8ced91654",
"Fef381544ff909f7a1c2fe188bed5ded71f42bc876176754c76e9dc5bc6e981cf",
"F9a91dc3605c53e940d0ffc893c4c12dff6bc07d7e9ea13b1cf6b2b52917c5da4"
],
"game": "popn",
"inactive": false,
"playtype": "9B",
"tableID": "popn-9B-unilab-levels",
"title": "Pop'n Music Unilab Levels"
},
{
"default": true,
"description": "Levels for SOUND VOLTEX in EXCEED GEAR.",
@@ -1,291 +0,0 @@
const fetch = require("node-fetch");
const { MutateCollection, ReadCollection, CreateChartID } = require("../../util");
const logger = require("../../logger");
const tableInfos = [
{
name: "発狂PMSデータベース(lv46~)",
symbol: "P●",
data_url: "https://pmsdatabase.github.io/insane_pmsdatabase/insane_pmsdatabase_score.json",
level_order: [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
"!i",
"?",
"◆",
"~",
].map((e) => e.toString()),
},
{
name: "PMSデータベース(Lv1~45)",
symbol: "PLv",
data_url: "https://pmsdatabase.github.io/pmsdatabase/pmsdatabase_score.json",
level_order: [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
"45+",
"?",
].map((e) => e.toString()),
},
];
const hasRandom = [
"769aac4e9bad64547438251f860a0aea",
"633943accbfaf1e782e9aa13229093c3",
"2ff8c8b0a5b356e99a9f8a2bf9da2832",
"836313ed8fc678a72b7870704c1238dd",
"be5fa66cebdc93fecbe24d8374bded0d",
"df81bc8022bd51e2e17dd9738b452041",
"a8caeb99af594f40f7fff48b32066f89",
"608bf3db51cd9150e2299ec51f4100ca",
"1b823369559fe3222934b016b661ae12",
"74307878f06f97e56d6a0d70cdc2caaf",
"ab5a3e7023a00d484c72d371954bfec7",
"7706336dab51e727bfc8c39f15b9b8c4",
"4f5956a6334e809492ca242e9a42c716",
"7272f671003e0192cf8e6402dbb4c92b",
"11a9e65457a9d0ce86a7d2fcb92f20a0",
"97e2781c785e228ec1d8dd936aefeb26",
"a653d6f6cbaf5b250e9e9da81397f2e5",
"e5576c4ae3faf95b4f77ec48032c7bb1",
"c032c1c45a25ffbb3996d46b978228e7",
"3064f6f5b04b9d8c45be299963de41dd",
"8c16736ecc39c3b94b01163bf15a45d0",
"5b314833ec377a259f4b9c48a1b5c85b",
"8c7940010a72f0a48a037fe8391cd2fb",
"a3da3ef759095cc21e14422e2a14f797",
"0dc76d5e5fddc6768b1a2090a4afe35f",
"c09e67e807d3be0d14695c3060ad7956",
"aaa6a9021bffee4f930ae91f32774113",
"bb9072b6b0460a5feb96a6a5dad45875",
"b5e112417d68ab9bf9bd31655b383318",
"b40d9e637e3ce07a3b4054ae7b453e8a",
"725276e227259ff5613b7cba51903662",
"0624e226b1341db8457aaa3f608370c5",
"620c162aec244bc3a42e6daf3786cc18",
"138dc09661f886a51b210af6a551487e",
"e9a266efc4ec71fddbc4639ac626f23a",
"59718af97aa91967dde2285c5aa4d736",
"994fabe27b2d38152d5a1bb6d5e8de39",
"aa271d610c9fbfda90456e83014a1219",
"7df2d0d36b5a6445909430c96c54137f",
"adc50317bed7fc028d3f8606653b692e",
"ae044a61dbff57608bf4d7796b093246",
"2bcbf73992b52b6ec3d5ee2f42242710",
"2532497bca563bd2aa7887616e49cd5b",
"5d34e01aa0f25ed30278518756f14c5f",
"6a8223c80f24b9c2056dc23e093d36dd",
"a524ca620432264e8f3343ef14062dea",
"9c742c873dce4df4a949930238d3cbde",
"df2f90e09069dd13d87da3ba152123aa",
"4010caafe71464630760391f1652046b",
"2d2d3268e83a8caccada4c0cd60a66ad",
"8a54a27a27bba973b1dbee516de4a9f2",
"7f89e8ee473955c9e7606eeb9415181f",
"1c0adfdbdcda89c4c0472be4a6b36f0b",
"d89148dce284949777d4704c68142cc4",
"ceffedee6eab4dae4cced8a499df1aac",
"48be2d81abb4aece3334a88fc47e90ad",
"0d01d78db2588007f0efbc7ca567dd6d",
"60db41bda92d0998668ab30aa6ee412b",
"bc45e15c9e6f86dbd378575d5e421b86",
"343d2073a79be1562e8f706150c387c1",
];
function FormatBMSTables(bmsTables) {
return bmsTables.map((e) => `${e.table}${e.level}`).join(", ");
}
if (require.main === module) {
(async () => {
const charts = [];
const songs = [];
const existingMD5s = new Set(ReadCollection("charts-pms.json").map((e) => e.data.hashMD5));
let i = Math.max(...ReadCollection("songs-pms.json").map((s) => s.id)) + 1;
for (const tableInfo of tableInfos) {
// THE JSON SENT OVER THE WIRE IS INVALID!
// We have to strip a Byte Order Mark ourselves. Why? No idea.
// This is illegal JSON. How do people keep messing up such a simple format?
const text = await fetch(tableInfo.data_url).then((r) => r.text());
const data = JSON.parse(text.split(/^\uFEFF/u)[1]);
for (const d of data) {
if (existingMD5s.has(d.md5)) {
continue;
}
if (hasRandom.includes(d.md5)) {
continue;
}
if (!tableInfo.level_order.includes(d.level)) {
logger.info(`Skipping unknown level ${d.level}`);
continue;
}
const song = {
id: i,
title: d.title,
artist: d.artist,
searchTerms: [],
altTitles: [],
data: {
genre: null,
subtitle: null,
subartist: null,
tableString: FormatBMSTables([
{
table: tableInfo.symbol,
level: d.level,
},
]),
},
};
const l = Number(d.level);
const tierlistInfo = Number.isNaN(l)
? {}
: {
"sgl-EC": MakeTierlistStuff(l, tableInfo.symbol),
"sgl-HC": MakeTierlistStuff(l, tableInfo.symbol),
};
const chart = {
songID: i,
chartID: CreateChartID(),
rgcID: null,
level: "?",
levelNum: 0,
playtype: "Controller",
difficulty: "CHART",
isPrimary: true,
tierlistInfo,
data: {
notecount: Number(d.notes),
hashMD5: d.md5,
hashSHA256: d.sha256,
tableFolders: [
{
table: tableInfo.symbol,
level: d.level,
},
],
},
versions: [],
};
i++;
songs.push(song);
charts.push(chart);
charts.push(
Object.assign({}, chart, { chartID: CreateChartID(), playtype: "Keyboard" })
);
}
}
MutateCollection("songs-pms.json", (currentSongs) => currentSongs.concat(songs));
MutateCollection("charts-pms.json", (currentCharts) => currentCharts.concat(charts));
})();
}
function MakeTierlistStuff(l, symbol) {
let value = l;
if (symbol === "P●") {
value = ConvertLevel(l);
}
return {
value,
text: value <= 45 ? `○${value.toString()}` : `●${l}`,
individualDifference: false,
};
}
function ConvertLevel(i) {
return {
1: 46,
2: 46.5,
3: 47,
4: 47.25,
5: 47.5,
6: 48,
7: 48.5,
8: 49,
9: 49.25,
10: 49.5,
11: 50,
12: 50.5,
13: 51,
14: 51.5,
15: 52,
16: 52.5,
17: 53,
18: 53.5,
19: 55,
20: 55.5,
}[i];
}
@@ -1,10 +1,10 @@
const { MutateCollection, CreateFolderID } = require("../../util");
const { GetGamePTConfig } = require("tachi-common");
const tableMainFolders = [];
const tableAllFolders = [];
const { Command } = require("commander");
const { PrettyFormatGameVersion } = require("tachi-common/config/static-config");
const program = new Command();
program.requiredOption("-v, --version <version>");
@@ -12,7 +12,7 @@ program.requiredOption("-v, --version <version>");
program.parse(process.argv);
const options = program.opts();
const fmtVersion = PrettyFormatGameVersion("popn:9B", options.version);
const fmtVersion = GetGamePTConfig("popn", "9B").versions[options.version];
MutateCollection("folders.json", (folders) => {
for (let i = 0; i < 3; i++) {
@@ -78,6 +78,30 @@ for (const song of data) {
continue;
}
// try to find the chart anyway (match inGameID, difficulty and level)
const existingChartManual = curCharts.find(
(curchart) =>
curchart.data.inGameID === song.music_entry &&
curchart.difficulty === chart.difficulty &&
curchart.levelNum === chart.level
);
if (existingChartManual) {
// it means the chart hash has changed, but the chart did not get rerated.
console.warn(
`${song.title} [${chart.difficulty} ${chart.level}] has changed hash but has not been rerated. Make sure to check if the chart has changed or not!`
);
if (!existingChartManual.versions.includes(options.version)) {
existingChartManual.versions.push(options.version);
}
if (Array.isArray(existingChartManual.data.hashSHA256)) {
existingChartManual.data.hashSHA256.push(chart.hash);
} else {
const newHashes = [existingChartManual.data.hashSHA256, chart.hash];
existingChartManual.data.hashSHA256 = newHashes;
}
continue;
}
charts.push({
songID,
chartID: CreateChartID(),
@@ -89,13 +113,11 @@ for (const song of data) {
playtype: "9B",
level: chart.level.toString(),
levelNum: chart.level,
rgcID: null,
isPrimary: true,
data: {
hashSHA256: chart.hash,
inGameID: index,
},
tierlistInfo: {},
versions: [options.version],
});
}
@@ -0,0 +1,107 @@
const { Command } = require("commander");
const { XMLParser } = require("fast-xml-parser");
const iconv = require("iconv-lite");
const fs = require("fs");
const path = require("path");
const crypto = require("crypto");
const program = new Command();
program
.option("-m, --mdb <mdb folder>") // refers to the multixml mdb exported with popnhax_tools
.option("-d, --data <root of popn install>"); // refers to the "contents" folder of your install. you MUST have unpacked every ifs file inside of contents/data/sd with ifstools before running this script.
program.parse(process.argv);
const options = program.opts();
if (!options.mdb || !options.data) {
throw new Error("Missing options --mdb or --data.");
}
const parser = new XMLParser({ ignoreAttributes: false });
const mdbs = [];
const files = fs.readdirSync(options.mdb);
for (const file of files) {
const mdb = fs.readFileSync(path.join(options.mdb, file));
const decr = iconv.decode(Buffer.from(mdb), "Shift_JIS");
const parsed = parser.parse(decr);
mdbs.push(parsed);
}
const mxb = [];
function getDiffForIdx(idx) {
switch (idx) {
case "ep":
return "Easy";
case "np":
return "Normal";
case "hp":
return "Hyper";
case "op":
return "EX";
default:
throw new Error("Unknown idx parsed");
}
}
const valid_diffs = ["ep", "np", "hp", "op"];
function getSha256ForChart(filename, folder, idx) {
let filepath = path.join(
options.data,
"data",
"sd",
folder,
`${filename}_ifs`,
`${filename}_${idx}.bin`
);
if (filename === "tour" && idx === "op") {
// silly workaround for one song which is in another directory for no fucking reason
filepath = path.join(
options.data,
"data",
"sd",
folder,
`${filename}_diff_ifs`,
`${filename}_${idx}.bin`
);
}
const input = fs.readFileSync(filepath);
const hash = crypto.createHash("sha256");
hash.update(input);
return hash.digest("hex");
}
for (const mdb of mdbs) {
for (const song of mdb.database.music) {
if (song.charts.chart && song.charts.chart.length > 0) {
// skip entries with no charts
const charts = [];
for (const chart of song.charts.chart) {
if (valid_diffs.includes(chart["@_idx"])) {
// shouldn't include battle charts
const chartHash = getSha256ForChart(
chart.filename["#text"],
chart.folder["#text"],
chart["@_idx"]
);
charts.push({
filename: `${chart.filename["#text"]}`,
hash: chartHash,
difficulty: getDiffForIdx(chart["@_idx"]),
level: Number(chart.diff["#text"]),
});
}
}
mxb.push({
music_entry: Number(song["@_id"]),
title: `${song.title["#text"]}`,
artist: `${song.artist["#text"]}`,
genre: `${song.genre["#text"]}`,
charts,
});
}
}
}
fs.writeFileSync("mxb.json", JSON.stringify(mxb));
@@ -104,8 +104,6 @@ for (const music of xmlData.mdb.music) {
},
isPrimary: true,
versions: ["konaste"],
tierlistInfo: {},
rgcID: null,
});
if (!newSong) {
@@ -101,14 +101,12 @@ for (const [key, d] of Object.entries(data)) {
charts.push({
chartID: CreateChartID(),
rgcID: null,
songID: ids[maybeUniqueSongID],
level: d.level.toString(),
levelNum: d.level,
isPrimary: true,
difficulty: d.chartHash,
playtype: "Stamina",
tierlistInfo: {},
versions: [],
data: {
difficultyTag: difficulty,
@@ -93,9 +93,7 @@ for (const chart of dbRows) {
level: chart.level.toString(),
levelNum: chart.level,
playtype,
rgcID: null,
songID,
tierlistInfo: {},
versions: [],
});
}
@@ -45,19 +45,12 @@ chart, `songID` refers to the above song document:
```json
{
"rgcID": null,
"chartID": "c2311194e3897ddb5745b1760d2c0141f933e683",
"difficulty": "ANOTHER",
"songID": 1,
"playtype": "SP",
"levelNum": 10,
"level": "10",
"flags": {
"IN BASE GAME": true,
"OMNIMIX": true,
"N-1": true,
"2dxtra": false
},
"data": {
"inGameID": 1000,
"notecount": 786,
@@ -100,19 +93,12 @@ For example:
```json
{
"rgcID": null,
"chartID": "103ff8bb004e1a8a005f808c025c3feb",
"difficulty": "ANOTHER",
"songID": 1,
"playtype": "SP",
"levelNum": 5,
"level": "5",
"flags": {
"IN BASE GAME": true,
"OMNIMIX": true,
"N-1": true,
"2dxtra": false
},
"data": {
"inGameID": 1000,
"notecount": 433
@@ -90,6 +90,8 @@ function ConvertDifficulty(diff: number): Difficulties["popn:9B"] {
function ConvertVersion(ver: number): Versions["popn:9B"] {
switch (ver) {
case 27:
return "unilab";
case 26:
return "kaimei";
case 25: