diff --git a/client/src/util/seeds.ts b/client/src/util/seeds.ts index ca1504a21..6e60bdb89 100644 --- a/client/src/util/seeds.ts +++ b/client/src/util/seeds.ts @@ -15,7 +15,7 @@ import { SongDocument, TableDocument, } from "tachi-common"; -import { GitCommit } from "types/git"; +import { Branch, GitCommit } from "types/git"; import { BMSCourseWithRelated, ChartWithRelated, @@ -83,8 +83,42 @@ export async function LoadSeeds(repo: string, ref: string): Promise { if (repo === "local") { + if (sha === "WORKING_DIRECTORY") { + const branchRes = await APIFetchV1<{ current: Branch }>(`/seeds/branches`); + + if (!branchRes.success) { + throw new Error(`Failed to fetch branches, but was loading LOCAL_DIRECTORY?`); + } + + const params = new URLSearchParams({ branch: branchRes.body.current.name }); + + const res = await APIFetchV1>(`/seeds/commits?${params.toString()}`); + + if (!res.success) { + throw new Error(`Failed to fetch commits, but was loading LOCAL_DIRECTORY?`); + } + + return { + sha: "WORKING_DIRECTORY", + commit: { + author: { + name: "Not Committed Yet", + date: "1970-01-01", + email: "null@example.com", + }, + committer: { + name: "Not Committed Yet", + date: "1970-01-01", + email: "null@example.com", + }, + message: "Uncommitted changes on your local disk.", + }, + parents: [{ sha: res.body[0].sha! }], + }; + } + const params = new URLSearchParams({ sha }); const res = await APIFetchV1(`/seeds/commit?${params.toString()}`); diff --git a/database-seeds/collections/charts-iidx.json b/database-seeds/collections/charts-iidx.json index 7cb90816b..5782bf634 100644 --- a/database-seeds/collections/charts-iidx.json +++ b/database-seeds/collections/charts-iidx.json @@ -455417,8 +455417,8 @@ }, "difficulty": "HYPER", "isPrimary": true, - "level": "8", - "levelNum": 8, + "level": "9", + "levelNum": 9, "playtype": "SP", "rgcID": null, "songID": 701, @@ -459430,8 +459430,8 @@ }, "difficulty": "NORMAL", "isPrimary": true, - "level": "4", - "levelNum": 4, + "level": "5", + "levelNum": 5, "playtype": "DP", "rgcID": null, "songID": 705, @@ -459616,8 +459616,8 @@ }, "difficulty": "ANOTHER", "isPrimary": true, - "level": "7", - "levelNum": 7, + "level": "8", + "levelNum": 8, "playtype": "SP", "rgcID": null, "songID": 705, @@ -459649,8 +459649,8 @@ }, "difficulty": "HYPER", "isPrimary": true, - "level": "5", - "levelNum": 5, + "level": "6", + "levelNum": 6, "playtype": "SP", "rgcID": null, "songID": 705, @@ -1191878,8 +1191878,8 @@ }, "difficulty": "ANOTHER", "isPrimary": true, - "level": "9", - "levelNum": 9, + "level": "11", + "levelNum": 11, "playtype": "DP", "rgcID": null, "songID": 1822, diff --git a/database-seeds/scripts/package.json b/database-seeds/scripts/package.json index 6dee1cdcb..db0b3d693 100644 --- a/database-seeds/scripts/package.json +++ b/database-seeds/scripts/package.json @@ -21,12 +21,12 @@ "fast-xml-parser": "^4.0.2", "glob": "^7.2.0", "html-entities": "^2.3.2", + "iconv-lite": "^0.6.3", "lodash.get": "^4.4.2", "mei-logger": "^1.0.1", "monk": "^7.3.4", "node-fetch": "2.6.7", "prudence": "^0.9.7", "xml2js": "^0.4.23" - }, - "devDependencies": {} + } } \ No newline at end of file diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore b/database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore new file mode 100644 index 000000000..eb80f3b1e --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/.gitignore @@ -0,0 +1,2 @@ +# this gets flooded with extracted .ifses +ifs-output/* \ No newline at end of file diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/HOWTO.md b/database-seeds/scripts/rerunners/iidx-mdb-parse/HOWTO.md new file mode 100644 index 000000000..76f16e095 --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/HOWTO.md @@ -0,0 +1,11 @@ +# How do I use this? + +[Make sure you have the repository set up.](https://docs.bokutachi.xyz/contributing/setup/) + +With that done, run `pnpx ts-node merge-mdb.ts --help`. Provide the right data for `--basedir, --index and --version`. + +Make sure you have `ifstools` installed in your PATH. You'll need python3 installed. + +Once you have `python3` installed, run `pip install ifstools`. + +The script should now "just work"(TM). diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt b/database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt new file mode 100644 index 000000000..087523a93 --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/blacklist.txt @@ -0,0 +1,16 @@ +# songID + playtype + difficulty combos to ignore +# these are regexes, and parse things in the format of +# C$songid-$playtype-$difficulty +# they also receive strings in the format of +# S$songid +# this is to support features like ignoring entire songs, with something like +# S1234$ +# lines starting with '#' are ignored + +# we don't care about beginners +BEGINNER + +S16072$ +S16080$ +S16081$ +S16082$ diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/convert.ts b/database-seeds/scripts/rerunners/iidx-mdb-parse/convert.ts new file mode 100644 index 000000000..4d8e007b2 --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/convert.ts @@ -0,0 +1,219 @@ +import fs, { mkdirSync } from "fs"; +import path from "path"; +import iconv from "iconv-lite"; +import { execSync } from "child_process"; +import { ParseDotOneFile } from "./dot-one-parser/parser"; +import logger from "../../logger"; +import { IIDXConvertOutput } from "./dot-one-parser/types"; +import { integer } from "tachi-common"; + +function sjisHack(b: Buffer, i: number) { + return iconv.decode(b.slice(i, i + 64).filter((e) => e !== 0) as Buffer, "shift_jis"); +} + +// where to store extracted IFS output +const ifsOUT = path.join(__dirname, "./ifs-output"); +mkdirSync(ifsOUT, { recursive: true }); + +/** + * Extracts an IFS file into the ./ifs-output cache directory. + */ +function ifsExtract(ifsPath: string) { + logger.info(`Extracting ${ifsPath}...`); + execSync(`ifstools "${ifsPath}" -o "${ifsOUT}" -y`, { stdio: "ignore" }); +} + +// @optimisable +// this can be processed in parallel +// but like. this basically only has to be done once a year. +// https://xkcd.com/1205/ +export async function ParseIIDXData( + basedir: string, + index: "0" | "1", + omni: boolean, + alwaysExtract: boolean +) { + logger.info(`Parsing from ${basedir}.`); + + const mdbPath = path.join( + basedir, + "data/info", + index, + omni ? "music_omni.bin" : "music_data.bin" + ); + + logger.info(`Parsing MDB ${mdbPath}.`); + + const buffer = fs.readFileSync(mdbPath); + + if (buffer.slice(0, 4).toString() !== "IIDX") { + throw new Error("Invalid MDB File."); + } + + const start = buffer.readInt16LE(0xa) * 2 + 0x10; + + let structSize: integer; + + switch (buffer[4]) { + case 27: + case 28: + case 29: + structSize = 0x52c; + break; + default: + throw new Error("Unknown version of MDB."); + } + + let moreData = true; + + let curLoc = start; + + const parsedData: Array = []; + + while (moreData) { + const struct = buffer.slice(curLoc, curLoc + structSize); + + // laziest hack + const levels = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].map((e) => struct.readUInt8(288 + e)); + + // remember that i get paid real money to program in this language + const [spb, spn, sph, spa, spl, _dpb, dpn, dph, dpa, dpl] = levels as [ + number, + number, + number, + number, + number, + number, // dpb has nothing defined, or maybe it does, nobody knows, or cares + number, + number, + number, + number + ]; + + const songID = struct.readUInt16LE(944); + + const title = sjisHack(struct, 0); + const marquee = sjisHack(struct, 64); + const genre = sjisHack(struct, 128); + const artist = sjisHack(struct, 192); + const folder = struct.readUInt8(280); + + const notecounts: Record = {}; + + // internal SongID - songIDs are not really an int, and are sometimes prefixed with a 0 + // to fit as a 5 digit string. + const iSongID = songID < 10_000 ? `0${songID}` : songID.toString(); + + // where the IFS for this song might be located + const ifsPath = path.join(basedir, `data/sound/${iSongID}.ifs`); + const ifsLeggPath = path.join(basedir, `data/sound/${iSongID}-p0.ifs`); + + // where the dotOne for this file might be located. + let dotOnePath = path.join(basedir, `data/sound/${iSongID}/${iSongID}.1`); + + // where the dotone file will be after extraction (if necessary) + const extractedIFSLoc = path.join(ifsOUT, `${iSongID}_ifs/${iSongID}/${iSongID}.1`); + + const extractedIFSLeggLoc = path.join(ifsOUT, `${iSongID}-p0_ifs/${iSongID}/${iSongID}.1`); + + if (fs.existsSync(ifsLeggPath)) { + if (!fs.existsSync(extractedIFSLeggLoc)) { + ifsExtract(ifsLeggPath); + } + + const charts = await ParseDotOneFile(extractedIFSLeggLoc, { + genre, + marquee, + songArtist: artist, + songTitle: title, + }); + + for (const chart of charts) { + notecounts[`${chart.playtype}-${chart.difficulty}`] = chart.notecount; + } + } + + // if we've already extracted this, and --always-extract isn't set + if (fs.existsSync(extractedIFSLoc) && !alwaysExtract) { + dotOnePath = extractedIFSLoc; + } else if (fs.existsSync(ifsPath)) { + ifsExtract(ifsPath); + + dotOnePath = extractedIFSLoc; + } + + if (!fs.existsSync(dotOnePath)) { + curLoc += structSize; + logger.error(`${dotOnePath} cannot find file. neither exist.`); + continue; + } + + logger.verbose(`Parsing data/sound/${songID}/${songID}.1`); + try { + const charts = await ParseDotOneFile(dotOnePath, { + genre, + marquee, + songArtist: artist, + songTitle: title, + }); + + for (const chart of charts) { + const notecount = chart.notecount; + + if (chart.difficulty === "LEGGENDARIA") { + if (notecount !== 0 && notecounts[`${chart.playtype}-LEGGENDARIA`]) { + logger.warn( + `${chart.artist} - ${chart.title} has conflicting ${ + chart.playtype + } LEGGENDARIAs. +INLINE: ${notecount} notes, p0: ${notecounts[`${chart.playtype}-LEGGENDARIA`]} notes. +Picking the INLINE version, as it's likely to be the correct one, but confirm this manually.` + ); + } + } + + notecounts[`${chart.playtype}-${chart.difficulty}`] = notecount; + } + } catch (err) { + logger.error(err); + curLoc += structSize; + + if (buffer.length <= curLoc) { + moreData = false; + } + + continue; + } + + parsedData.push({ + title, + artist, + marquee, + folder, + genre, + levels: { + "SP-BEGINNER": spb, + "SP-NORMAL": spn, + "SP-HYPER": sph, + "SP-ANOTHER": spa, + "SP-LEGGENDARIA": spl, + "DP-NORMAL": dpn, + "DP-HYPER": dph, + "DP-ANOTHER": dpa, + "DP-LEGGENDARIA": dpl, + }, + notecounts, + songID, + }); + + curLoc += structSize; + + if (buffer.length <= curLoc) { + moreData = false; + } + } + + logger.info(`Done parsing.`); + + return parsedData; +} diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts b/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts new file mode 100644 index 000000000..4b792baa8 --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/parser.ts @@ -0,0 +1,516 @@ +import { integer } from "tachi-common"; +import { + BGMObject, + BPMObject, + BSSObject, + CNObject, + EOCObject, + IIDXChartEvents, + IIDXParserContext, + IIDXParserOptions, + MeasureBarObject, + MeterObject, + NoteObject, + ParsedIIDXChart, + SampleObject, + ScratchObject, + TimingWindowsObject, +} from "./types"; +import fs from "fs/promises"; +import logger from "../../../logger"; + +interface EventParserContext { + isDP?: boolean; + songTitle?: string; +} + +interface Directory { + offset: number; + length: number; +} + +function ParseDirectoriesHeader(fileData: Buffer): Directory[] { + // .1 format has 12 directory entries at the start + // of the file. + // these are all 8 bytes long, and all next to + // eachother. + const dirsBuffer = fileData.subarray(0, 12 * 8); + + const dirs: Array<{ offset: integer; length: integer }> = []; + + for (let i = 0; i < 12; i++) { + const dirBuffer = dirsBuffer.subarray(i * 8, (i + 1) * 8); + + // the first 4 bytes indicate the offset + // the other 4 indicate how many bytes + // this chart takes up. + const dir = { + offset: dirBuffer.readIntLE(0, 4), + length: dirBuffer.readIntLE(4, 4), + }; + + dirs.push(dir); + } + + return dirs; +} + +type Event = + | { type: "NOTE"; body: NoteObject } + | { type: "BGM"; body: BGMObject } + | { type: "BPM"; body: BPMObject } + | { type: "BSS"; body: BSSObject } + | { type: "CN"; body: CNObject } + | { type: "HBSS"; body: BSSObject } + | { type: "HCN"; body: CNObject } + | { type: "SCRATCH"; body: ScratchObject } + | { type: "EOC"; body: EOCObject } + | { type: "SAMPLE"; body: SampleObject } + | { type: "TIMINGWINDOW"; body: TimingWindowsObject } + | { type: "MEASUREBAR"; body: MeasureBarObject } + | { type: "METER"; body: MeterObject }; + +const IIDX_TYPE_TO_RGC: Record = { + 0: "NOTE", + 1: "NOTE", + 2: "SAMPLE", + 3: "SAMPLE", + 4: "BPM", + 5: "METER", + 6: "EOC", + 7: "BGM", + 8: "TIMINGWINDOW", + 12: "MEASUREBAR", + 13: "HCN_INDICATOR", + 16: "NOTECOUNT", +}; + +const IIDX_TIMING_TO_RGC: Record< + number, + "lateBad" | "lateGood" | "lateGreat" | "earlyGreat" | "earlyGood" | "earlyBad" +> = { + 0: "lateBad", + 1: "lateGood", + 2: "lateGreat", + 3: "earlyGreat", + 4: "earlyGood", + 5: "earlyBad", +}; + +function ParseEvent( + noteData: Buffer, + parseContext: EventParserContext, + options: IIDXParserOptions +): Event | null { + const data = { + ticks: noteData.readIntLE(0, 4), + type: noteData.readIntLE(4, 1), + param: noteData.readIntLE(5, 1), + val: noteData.readIntLE(6, 2), + }; + + // A second EOF indicator (the real one) is that there is an event placed at -1 miliseconds. + // return null, and DO NOT add this as an object. + if (data.ticks < 0 || data.ticks >= 2147483647) { + return null; + } + + let type = IIDX_TYPE_TO_RGC[data.type]; + + let event: Event; + + const ms = data.ticks * ((options.tickrate || 1000) / 1000); + + // Disambiguate NOTE type, which may indicate other RGC types. + // THIS IS DIFFERENT TO THE IMMEDIATELY FOLLOWING STATEMENT! + if (type === "NOTE") { + // if val is nonzero, this note is a hold note + // with ticks of (length); + if (data.val !== 0) { + type = options.isHellCharge ? "HCN" : "CN"; + } + + // scratch overrides + if (data.param === 7) { + // if hold on scratch column, we have BSS. + if (data.val !== 0) { + type = options.isHellCharge ? "HBSS" : "BSS"; + } else { + type = "SCRATCH"; + } + } + } + + if (type === "NOTE") { + const obj: NoteObject = { + ms, + col: data.param, + }; + + // if this note was on P2 side, we need to + // add 7 to its column. + if (data.type === 1) { + obj.col += 7; + } + + if (data.type === 1) { + if (obj.col > 13 || obj.col < 0) { + throw TypeError(`Invalid column of ${obj.col}. Must be between 0 and 13.`); + } + } else { + if (obj.col > 6 || obj.col < 0) { + throw TypeError(`Invalid column of ${obj.col}. Must be between 0 and 6.`); + } + } + + event = { + type, + body: obj, + }; + } else if (type === "CN" || type === "HCN") { + if (type === "CN" && options.errOnCN) { + throw new TypeError( + `CN found inside ${ + parseContext.songTitle || "UNKNOWN CHART" + }, but errOnCN was enabled.` + ); + } else if (type === "HCN" && options.errOnHCN) { + throw new TypeError( + `HCN found inside ${ + parseContext.songTitle || "UNKNOWN CHART" + }, but errOnHCN was enabled.` + ); + } + + const obj: CNObject = { + ms, + col: data.param, + msEnd: data.val + ms, + }; + + if (data.type === 1) { + obj.col += 7; + } + + event = { + type, + body: obj, + }; + } else if (type === "SCRATCH") { + const obj: ScratchObject = { + ms, + }; + + if (parseContext.isDP) { + if (data.param === 7) { + obj.col = 0; + } else if (data.param === 14) { + obj.col = 1; + } else { + return null; + } + } + + event = { + type, + body: obj, + }; + } else if (type === "BSS" || type === "HBSS") { + const obj: BSSObject = { + ms, + msEnd: ms + data.val, + }; + + if (parseContext.isDP) { + if (data.param === 7) { + obj.col = 0; + } else if (data.param === 14) { + obj.col = 1; + } else { + logger.warn( + `Unknown column of scratchObj sent to EventParser: ${data.param}, expected 7 or 14. Skipping.` + ); + return null; + } + } + + event = { + type, + body: obj, + }; + } else if (type === "SAMPLE") { + const obj: SampleObject = { + ms, + col: data.param, + sound: data.val, + }; + + event = { + type, + body: obj, + }; + } else if (type === "BPM") { + if (data.param === 0) { + logger.warn("Invalid value of 0 for denominator in BPM event. Skipping."); + return null; + } + + const obj: BPMObject = { + ms, + bpm: data.val / data.param, + }; + + event = { + type, + body: obj, + }; + } else if (type === "METER") { + const obj: MeterObject = { + ms, + num: data.val, + denom: data.param, + }; + + event = { + type, + body: obj, + }; + } else if (type === "BGM") { + const obj: BGMObject = { + ms, + pan: data.param, + sound: data.val, + }; + + event = { + type, + body: obj, + }; + } else if (type === "TIMINGWINDOW") { + const window = IIDX_TIMING_TO_RGC[data.param]; + + if (!window) { + logger.warn( + `Unknown type of timingWindow: ${window}, from .1 param ${data.param}. Skipping.` + ); + return null; + } + + const obj: TimingWindowsObject = { + ms, + window, + // for whatever reason, the value for TIMINGWINDOW is a signed int8, + // rather than the int16 every other thing expects. + // this means, suboptimally, we are reading noteData twice. + // performance is hardly a concern though, as you can blast 100k + // charts with this in a couple seconds. + val: noteData.readIntLE(6, 1), + }; + + event = { + type, + body: obj, + }; + } else if (type === "EOC") { + const obj: EOCObject = { + ms, + }; + + event = { + type, + body: obj, + }; + } else if (type === "MEASUREBAR") { + const obj: MeasureBarObject = { + ms, + }; + + if (parseContext.isDP) { + if (data.param !== 0 && data.param !== 1) { + logger.warn( + `Invalid parameter for measurebar of ${data.param}. Defaulting to 0 (left hand side).` + ); + obj.side = 0; + } else { + obj.side = data.param; + } + } + + event = { + type, + body: obj, + }; + } else if (type === "NOTECOUNT") { + // You might think: Hey! This is super convenient for just quickly getting + // the notecount of a chart, right? + // Well, uh, no. It's not correct. CNs are treated as one note, which is + // *absolutely* not the case in game. + // This thing is **utterly** useless, as you have to count the amount of CNs + // anyway. At that point, why not count the notes? + + return null; + } else if (type === "HCN_INDICATOR") { + // this chart is using HCNs. + options.isHellCharge = true; + return null; + } else { + logger.warn( + `(${parseContext.songTitle ?? "UNKNOWN"}) Unknown event type in .1: ${ + data.type + } at ${ms.toFixed(2)}ms. Ignoring.${ + data.type === 11 + ? " (It's BGA Related. Not sure what it does, but don't worry.)" + : "" + }` + ); + return null; + } + + return event; +} + +function ParseEvents( + fileData: Buffer, + dir: Directory, + context: EventParserContext, + options: IIDXParserOptions +) { + const events: IIDXChartEvents = { + BGM: [], + BPM: [], + BSS: [], + HCN: [], + CN: [], + HBSS: [], + NOTE: [], + SAMPLE: [], + METER: [], + TIMINGWINDOW: [], + MEASUREBAR: [], + EOC: [], + SCRATCH: [], + }; + + const data = fileData.subarray(dir.offset, dir.offset + dir.length); + + // charts are a series of 8 byte "events". + // we are going to iterate through [data], pulling + // said 8 bytes, and parsing them. + // i am assuming that dir.length is always + // a multiple of 8. + for (let i = 0; i < dir.length; i += 8) { + const noteBuffer = data.subarray(i, i + 8); + + const event = ParseEvent(noteBuffer, context, options); + + if (!event) { + continue; + } + + // @ts-expect-error invariant error but we're right. + events[event.type].push(event.body); + } + + return events; +} + +/** + * Converts the index of a directory to the corresponding difficulty name. + */ +const IIDX_DIFFICULTY_TO_RGC: Record< + number, + "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA" | null +> = { + 0: "HYPER", + 1: "NORMAL", + 2: "ANOTHER", + 3: "BEGINNER", + 4: "LEGGENDARIA", // ^^ SP ^^ + 5: null, // vv DP vv + 6: "HYPER", + 7: "NORMAL", + 8: "ANOTHER", + 9: null, + 10: "LEGGENDARIA", + 11: null, +}; + +export function ParseDotOne( + fileData: Buffer, + context: IIDXParserContext = {}, + options: IIDXParserOptions = {} +) { + const charts: Array = []; + + const dirs = ParseDirectoriesHeader(fileData); + + // with all our dirs we now need to parse each chart. + for (let i = 0; i < 12; i++) { + const dir = dirs[i]; + + if (!dir || (dir.offset === 0 && dir.length === 0)) { + logger.verbose(`Skipped directory ${i}, as there was no data.`); + continue; // no data. + } + + const difficulty = IIDX_DIFFICULTY_TO_RGC[i]; + + if (!difficulty) { + logger.warn( + `Data present inside directory ${i}, but this corresponds to an unknown difficulty. Ignoring.` + ); + continue; + } + + const eventContext: EventParserContext = { + isDP: i > 5, + songTitle: context?.songTitle, + }; + + if (!eventContext.songTitle) { + logger.warn( + "No filename provided in context.filename to IIDXParser, subsequent warnings will have no way of referring to the chart." + ); + } + + const events = ParseEvents(fileData, dir, eventContext, options); + + const chart: ParsedIIDXChart = { + playtype: i > 5 ? "DP" : "SP", + title: context?.songTitle ?? null, + artist: context?.songArtist ?? null, + genre: context?.genre ?? null, + level: context?.level ?? null, + marquee: context?.marquee ?? null, + difficulty, + notecount: getNotecount(events), + events, + }; + + charts.push(chart); + } + + return charts; +} + +function getNotecount(events: IIDXChartEvents) { + // note: account for HMSS at some point + return ( + events.NOTE.length + + events.SCRATCH.length + + events.CN.length * 2 + + events.HCN.length * 2 + + events.BSS.length * 2 + + events.HBSS.length * 2 + ); +} + +export async function ParseDotOneFile( + fileath: string, + context: IIDXParserContext = {}, + options: IIDXParserOptions = {} +) { + const data = await fs.readFile(fileath); + + return ParseDotOne(data, context, options); +} diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/types.ts b/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/types.ts new file mode 100644 index 000000000..f5632ca80 --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/dot-one-parser/types.ts @@ -0,0 +1,148 @@ +import { integer } from "tachi-common"; + +export interface ChartObject { + ms: number; +} + +/** + * Note Object: Indicates that there is a playable note here. + * For SP, this goes from 0 to 6. + * For DP, this goes from 0 to 13. + */ +export interface NoteObject extends ChartObject { + col: number; +} + +export interface ScratchObject extends ChartObject { + col?: 0 | 1; +} + +export interface CNObject extends NoteObject { + msEnd: number; +} + +export interface BSSObject extends ScratchObject { + msEnd: number; +} + +/** + * Sample Object: Indicates that there is a sample (keysound) here. + * Regardless of SP or DP, the game stores samples that can happen on + * any of the 14 columns. + */ +export interface SampleObject extends ChartObject { + col: number; + sound: number; +} + +/** + * BGM Object: Indicates that there is a background sample here. + * Unlike sample objects, these do not happen on columns, and instead use + * their two parameters to express panning and sound. + */ +export interface BGMObject extends ChartObject { + pan: number; + sound: number; +} + +/** + * BPM Object: Indicates that there is a change of BPM here. + * BPM is expected to be a float. + */ +export interface BPMObject extends ChartObject { + bpm: number; +} + +/** + * Meter Object: Indicates that there is a change of time signature here. + * Most charts never use this, but it does exist in a lot of old TaQ songs. + * It is expected that num and denom are integers. + */ +export interface MeterObject extends ChartObject { + num: number; + denom: number; +} + +/** + * Measure Bar Object: Indicates that there is a measure bar here. + * Unlike other games, IIDX has these baked into the chart format, presumably to allow for + * charts such as 100%-minimoo G. + * For some reason, this has an additional property which determines what + * side it renders for. + */ +export interface MeasureBarObject extends ChartObject { + side?: 0 | 1; +} + +/** + * End Of Chart object, Indicates that this is where the chart should end, + * and the game should switch to fadeout / display splashes. + */ +export type EOCObject = ChartObject; + +export interface TimingWindowsObject extends ChartObject { + window: "lateBad" | "lateGood" | "lateGreat" | "earlyGreat" | "earlyGood" | "earlyBad"; + val: number; +} + +export interface IIDXChartEvents { + SAMPLE: SampleObject[]; + BPM: BPMObject[]; + METER: MeterObject[]; + BGM: BGMObject[]; + TIMINGWINDOW: TimingWindowsObject[]; + MEASUREBAR: MeasureBarObject[]; + EOC: EOCObject[]; + SCRATCH: ScratchObject[]; + NOTE: NoteObject[]; + BSS: BSSObject[]; + HBSS: BSSObject[]; + CN: CNObject[]; + HCN: CNObject[]; +} + +export interface IIDXParserContext { + filename?: string; + songTitle?: string; + songArtist?: string; + charter?: string; + genre?: string; + level?: "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" | "11" | "12" | null; + marquee?: string; +} + +export interface IIDXParserOptions { + errOnCN?: boolean; + errOnHCN?: boolean; + isHellCharge?: boolean; + tickrate?: number; +} + +export interface ParsedIIDXChart { + playtype: "SP" | "DP"; + title: string | null; + artist: string | null; + genre: string | null; + notecount: integer; + level: string | null; + marquee: string | null; + difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"; + events: IIDXChartEvents; +} + +// no such thing as a DP-BEGINNER. +type Difficulties = Exclude< + `${"SP" | "DP"}-${"BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"}`, + "DP-BEGINNER" +>; + +export interface IIDXConvertOutput { + title: string; + artist: string; + marquee: string; + folder: number; + genre: string; + levels: Record; + notecounts: Record; + songID: integer; +} diff --git a/database-seeds/scripts/rerunners/iidx-mdb-parse/merge-mdb.ts b/database-seeds/scripts/rerunners/iidx-mdb-parse/merge-mdb.ts new file mode 100644 index 000000000..02cc0ff7e --- /dev/null +++ b/database-seeds/scripts/rerunners/iidx-mdb-parse/merge-mdb.ts @@ -0,0 +1,281 @@ +import { Command } from "commander"; +import { + ChartDocument, + Difficulties, + GetGamePTConfig, + GPTSupportedVersions, + integer, + SongDocument, +} from "tachi-common"; +import logger from "../../logger"; +import { + CreateChartID, + GetFreshScoreIDGenerator, + ReadCollection, + WriteCollection, +} from "../../util"; +import { ParseIIDXData } from "./convert"; +import fs from "fs"; +import path from "path"; + +if (require.main !== module) { + throw new Error( + `This is a script. It should be ran directly from the command line with ts-node.` + ); +} + +const program = new Command(); +program + .requiredOption("-b, --basedir ", "The base of this IIDX install.") + .requiredOption("-i, --index ", "Whether this game uses 0 or 1 for the mdb.") + .requiredOption("-v, --version ") + .option("-o, --omni", "Whether to fetch omnimix or not.") + .option("-f --force", "Forces overwrites when they shouldn't be automatically done.") + .option( + "--always-extract", + "Always re-extract IFS files even when they exist in ifs-output. Useful if the IFS files have changed." + ); + +program.parse(process.argv); +const options = program.opts() as { + version: GPTSupportedVersions["iidx:SP"]; + index: "0" | "1"; + basedir: string; + omni: boolean; + force: boolean; + alwaysExtract: boolean; +}; + +const iidxConfig = GetGamePTConfig("iidx", "SP"); + +if (!iidxConfig.supportedVersions.includes(options.version)) { + throw new Error( + `Invalid version of '${ + options.version + }'. Expected any of ${iidxConfig.supportedVersions.join( + ", " + )}. If you're adding a new version, go update common/src/config.ts.` + ); +} + +if (options.index !== "0" && options.index !== "1") { + throw new Error(`Expected an --index of 0 or 1. Got ${options.index}.`); +} + +const existingCharts: ChartDocument<"iidx:SP" | "iidx:DP">[] = ReadCollection("charts-iidx.json"); + +const existingSongs: SongDocument<"iidx">[] = ReadCollection("songs-iidx.json"); + +const blacklist = fs + .readFileSync(path.join(__dirname, "blacklist.txt"), "utf-8") + .split("\n") + .filter((e) => !e.startsWith("#") && e.trim() !== "") + .map((e) => new RegExp(e, "u")); + +function isInBlacklist(str: string) { + for (const regex of blacklist) { + if (regex.exec(str)) { + return true; + } + } + + return false; +} + +async function ParseIIDXMDB() { + const mdbCharts = await ParseIIDXData( + options.basedir, + options.index, + options.omni, + options.alwaysExtract + ); + + const chartMap = new Map>(); + const songMap = new Map>(); + const chartDiffMap = new Map>(); + + for (const song of existingSongs) { + songMap.set(song.id, song); + } + + for (const chart of existingCharts) { + // what, you thought this was easy? + if (Array.isArray(chart.data.inGameID)) { + for (const igid of chart.data.inGameID) { + chartMap.set(igid, chart); + } + } else { + chartMap.set(chart.data.inGameID, chart); + } + } + + for (const chart of existingCharts) { + if (Array.isArray(chart.data.inGameID)) { + for (const igid of chart.data.inGameID) { + chartDiffMap.set(`${igid}-${chart.playtype}-${chart.difficulty}`, chart); + } + } else { + chartDiffMap.set(`${chart.data.inGameID}-${chart.playtype}-${chart.difficulty}`, chart); + } + } + + const getFreeScoreID = GetFreshScoreIDGenerator("iidx"); + + for (const inp of mdbCharts) { + if (isInBlacklist(`S${inp.songID}`)) { + logger.verbose( + `Skipped ${inp.artist} - ${inp.title} (${inp.songID}) as it was in the blacklist.` + ); + continue; + } + + const anySongIDMatch = chartMap.get(inp.songID); + let song: SongDocument<"iidx">; + + if (!anySongIDMatch) { + // new song? + + const searchTerms = [inp.genre]; + if (inp.marquee.toLowerCase() !== inp.title.toLowerCase()) { + searchTerms.push(inp.marquee); + } + + const tachiSong: SongDocument<"iidx"> = { + id: getFreeScoreID(), + artist: inp.artist, + title: inp.title, + data: { + genre: inp.genre, + displayVersion: inp.folder.toString(), + }, + searchTerms: [], + altTitles: [], + }; + + logger.info(`Added new song ${inp.title}.`); + + if (inp.title.match(/\?/gu)) { + logger.warn( + `${inp.title} has a potentially konami-screwed title. Investigate it manually.` + ); + } + + if (inp.artist.match(/\?/gu)) { + logger.warn( + `${inp.artist} - ${inp.title} has a potentially konami-screwed title. Investigate it manually.` + ); + } + + existingSongs.push(tachiSong); + + song = tachiSong; + } else { + const sxng = songMap.get(anySongIDMatch.songID); + + if (!sxng) { + logger.error(`Song ${anySongIDMatch.songID} has charts but no song?`); + throw new Error(`Song ${anySongIDMatch.songID} has charts but no song?`); + } + song = sxng; + } + + const diffNames = Object.keys(inp.levels) as (keyof typeof inp.levels)[]; + + for (const diffName of diffNames) { + if (isInBlacklist(`C${inp.songID}-${diffName}`)) { + logger.verbose( + `Ignored ${song.title} (${inp.songID}) ${diffName} as it was in the blacklist.` + ); + continue; + } + + const chart = chartDiffMap.get(`${inp.songID}-${diffName}`); + + const notecount = inp.notecounts[diffName]; + const level = inp.levels[diffName]; + + if (level === 0 && notecount && notecount > 0) { + logger.info( + `Chart ${song.title} ${diffName} has notecount ${notecount}, but has no level assigned. Skipping.` + ); + continue; + } + if (level === 0) { + continue; + } + + if (!chart) { + // no chart && no notecount => chart has never existed + // and still doesnt. + if (!notecount) { + continue; + } + + // otherwise, make new chart? + const tachiChart: ChartDocument<"iidx:SP" | "iidx:DP"> = { + chartID: CreateChartID(), + difficulty: diffName.split("-")[1] as Difficulties["iidx:SP" | "iidx:DP"], + level: level.toString(), + levelNum: level, + isPrimary: true, + playtype: diffName.split("-")[0] as "SP" | "DP", + rgcID: null, + songID: song.id, + tierlistInfo: {}, + versions: [options.version], + data: { + inGameID: inp.songID, + arcChartID: null, + notecount, + hashSHA256: null, + "2dxtraSet": null, + bpiCoefficient: null, + kaidenAverage: null, + worldRecord: null, + }, + }; + + logger.info(`Inserting new chart ${inp.title} ${diffName}.`); + existingCharts.push(tachiChart); + } else { + if (!notecount) { + logger.warn( + `Chart ${inp.title} ${diffName} already exists, but has no notecount anymore. Not marking it as part of this version.` + ); + continue; + } + + // chart already exists, diff notecounts. + if (chart.data.notecount !== notecount) { + logger.warn( + `Chart ${inp.title} ${diffName} has a different notecount in the JSON to the data just parsed. Has this chart been edited? OLD: ${chart.data.notecount} -> NEW: ${notecount}.` + ); + if (!options.force) { + logger.warn( + `Must be resolved manually. Use --force to blindly overwrite it anyway.` + ); + continue; + } + chart.data.notecount = notecount; + } + + if (chart.levelNum !== level) { + logger.info( + `Chart ${inp.title} ${diffName} has had a level change. ${chart.level} -> ${level}. Updating this.` + ); + chart.level = level.toString(); + chart.levelNum = level; + } + + if (!chart.versions.includes(options.version)) { + chart.versions.push(options.version); + } + } + } + } + + WriteCollection("songs-iidx.json", existingSongs); + WriteCollection("charts-iidx.json", existingCharts); +} + +ParseIIDXMDB(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4596f7f54..6e27a4bdb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -256,6 +256,7 @@ importers: fast-xml-parser: ^4.0.2 glob: ^7.2.0 html-entities: ^2.3.2 + iconv-lite: ^0.6.3 lodash.get: ^4.4.2 mei-logger: ^1.0.1 monk: ^7.3.4 @@ -274,6 +275,7 @@ importers: fast-xml-parser: 4.0.2 glob: 7.2.3 html-entities: 2.3.3 + iconv-lite: 0.6.3 lodash.get: 4.4.2 mei-logger: 1.0.3 monk: 7.3.4