* checkpoint

* server compiles again

* fmt

* checkpoint

* last one out, get the lights

* euston station

* like, mostly good. I'm under the ocean right now.

* fix: some nonsense AI stuff

* fix: new eslint rules, and cleanup imports across the board

* feat: folder slug rewrite, email logic change, too

* fix: register must use txn not db

* checkpoint
This commit is contained in:
zk
2026-04-20 23:11:16 +01:00
committed by GitHub
parent edda03e916
commit 2bc4c3a628
1073 changed files with 52259 additions and 56015 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
function FindChartWithPTDFVersion(collection, songID, playtype, difficulty, version) {
return collection.find(
(chart) =>
chart.songID === songID &&
chart.song.id === songID &&
chart.playtype === playtype &&
chart.difficulty === difficulty &&
chart.versions.includes(version),
@@ -2,13 +2,18 @@ import { Command } from "commander";
import fjsh from "fast-json-stable-hash";
import fs from "fs";
import {
type ChartDocument,
type Difficulties,
type integer,
type MONGO_ChartDocument,
type MONGO_SongDocument,
type SongDocument,
} from "tachi-common";
import { CreateChartID, ReadCollection, WriteCollection } from "../../util";
import {
CreateChartID,
GetFreshSongIDGenerator,
ReadCollection,
WriteCollection,
} from "../../util";
type LocalizedText = { en: string } & Record<string, string>;
type LocalizedSearchTerms = Record<string, Array<string>>;
@@ -85,7 +90,7 @@ class MultiMapUniqueValues<K, V> {
}
}
function convertDifficulty(input: integer): Difficulties["arcaea:Touch"] {
function convertDifficulty(input: integer): Difficulties["arcaea"] {
switch (input) {
case 0:
return "Past";
@@ -142,31 +147,30 @@ const packlistContent = fs.readFileSync(options.packlist, { encoding: "utf-8" })
const packlistData: { packs: Array<PacklistEntry> } = JSON.parse(packlistContent);
const packsByID = Object.fromEntries(packlistData.packs.map((p) => [p.id, p]));
const existingSongDocsById: Map<number, MONGO_SongDocument<"arcaea">> = new Map(
ReadCollection("songs-arcaea.json").map((e: MONGO_SongDocument<"arcaea">) => [e.id, e]),
const existingSongDocsById: Map<number, SongDocument<"arcaea">> = new Map(
ReadCollection("songs-arcaea.json").map((e: SongDocument<"arcaea">) => [e.id, e]),
);
const existingChartDocs: Array<MONGO_ChartDocument<"arcaea:Touch">> =
ReadCollection("charts-arcaea.json");
const existingChartDocs: Array<ChartDocument<"arcaea">> = ReadCollection("charts-arcaea.json");
const inGameIDToSongsMap: MultiMapUniqueValues<
string,
MONGO_SongDocument<"arcaea">
SongDocument<"arcaea">
> = new MultiMapUniqueValues();
const existingCharts: Map<string, MONGO_ChartDocument<"arcaea:Touch">> = new Map();
const existingCharts: Map<string, ChartDocument<"arcaea">> = new Map();
for (const chart of existingChartDocs) {
const song = existingSongDocsById.get(chart.songID);
const song = existingSongDocsById.get(chart.song.id);
if (!song) {
console.warn(`Chart ${chart.songID} does not belong to any song?`);
console.warn(`Chart ${chart.song.id} does not belong to any song?`);
continue;
}
inGameIDToSongsMap.set(chart.data.inGameID, song);
existingCharts.set(`${chart.data.inGameID}-${chart.difficulty}`, chart);
inGameIDToSongsMap.set(chart.data.inGameStrID, song);
existingCharts.set(`${chart.data.inGameStrID}-${chart.difficulty}`, chart);
}
const getNewSongID = GetFreshSongIDGenerator("arcaea");
const newSongs: Array<MONGO_SongDocument<"arcaea">> = [];
const newCharts: Array<MONGO_ChartDocument<"arcaea:Touch">> = [];
const newSongs: Array<SongDocument<"arcaea">> = [];
const newCharts: Array<ChartDocument<"arcaea">> = [];
for (const entry of data.songs) {
const inGameID = entry.id;
@@ -186,7 +190,7 @@ for (const entry of data.songs) {
...new Set(Object.values(entry.title_localized).filter((t) => t !== title)),
];
const songDoc: MONGO_SongDocument<"arcaea"> = {
const songDoc: SongDocument<"arcaea"> = {
title,
artist: entry.artist,
altTitles,
@@ -204,7 +208,7 @@ for (const entry of data.songs) {
}
for (const chart of entry.difficulties) {
let song: MONGO_SongDocument<"arcaea">;
let song: SongDocument<"arcaea">;
if (chart.hidden_until_unlocked && chart.hidden_until === "always") {
// Deactivated difficulty
@@ -238,7 +242,7 @@ for (const entry of data.songs) {
...new Set(Object.values(chart.title_localized).filter((t) => t !== title)),
];
const songDoc: MONGO_SongDocument<"arcaea"> = {
const songDoc: SongDocument<"arcaea"> = {
title,
artist: chart.artist ?? entry.artist,
altTitles,
@@ -265,7 +269,8 @@ for (const entry of data.songs) {
song = possibleSong;
}
const chartDoc: MONGO_ChartDocument<"arcaea:Touch"> = {
const chartDoc: ChartDocument<"arcaea"> = {
game: "arcaea",
chartID: CreateChartID(),
songID: song.id,
difficulty,
@@ -275,7 +280,7 @@ for (const entry of data.songs) {
versions: ["mobile"],
playtype: "Touch",
data: {
inGameID,
inGameStrID: inGameID,
// Filled in later, but not by this script
notecount: 0,
},
@@ -37,7 +37,7 @@ if (
courses.push({
md5sums,
playtype: options.playtype,
game: options.playtype === "7K" ? "bms-7k" : "bms-14k",
set: options.set,
title: d.name,
value: Number(options.index) + i,
@@ -47,7 +47,7 @@ MutateCollection("bms-course-lookup.json", (courses) => {
courses.push({
md5sums,
playtype: options.playtype,
game: options.playtype === "7K" ? "bms-7k" : "bms-14k",
set: options.set,
title: d.title,
value: dans[i].id,
@@ -3,17 +3,17 @@ import { LoadBMSTable } from "bms-table-loader";
import {
BMS_TABLES,
type BMSTableInfo,
type MONGO_FolderDocument,
type MONGO_TableDocument,
type FolderDocument,
type TableDocument,
} from "tachi-common";
import { MutateCollection, ReadCollection } from "../../util";
import { CreateLegacyFolderIDFromFolder, MutateCollection, ReadCollection } from "../../util";
const existsTables = ReadCollection("tables.json").map((e) => e.tableID);
const existsFolders = ReadCollection("folders.json").map((e) => e.folderID);
async function UpdateTable(tableInfo: BMSTableInfo) {
const tableID = `bms-${tableInfo.playtype}-${tableInfo.asciiPrefix}`;
const tableID = `bms-${tableInfo.game}-${tableInfo.asciiPrefix}`;
if (existsTables.includes(tableID)) {
return;
@@ -25,12 +25,12 @@ async function UpdateTable(tableInfo: BMSTableInfo) {
const levels = table.getLevelOrder();
const folders: Array<MONGO_FolderDocument> = [];
const folders: Array<FolderDocument> = [];
for (const level of levels) {
const f: Omit<MONGO_FolderDocument, "folderID"> = {
const f: Omit<FolderDocument, "folderID"> = {
title: `${tableInfo.prefix}${level}`,
playtype: tableInfo.playtype,
playtype: tableInfo.game === "bms-7k" ? "7K" : "14K",
game: "bms",
searchTerms: [],
type: "charts",
@@ -45,12 +45,12 @@ async function UpdateTable(tableInfo: BMSTableInfo) {
inactive: false,
};
const folderID = CreateFolderIDFromFolder(f);
const folderID = CreateLegacyFolderIDFromFolder(f);
const realFolder = {
...f,
folderID,
} as MONGO_FolderDocument;
} as FolderDocument;
if (existsFolders.includes(folderID)) {
continue;
@@ -66,12 +66,12 @@ async function UpdateTable(tableInfo: BMSTableInfo) {
return f;
});
MutateCollection("tables.json", (t: Array<MONGO_TableDocument>) => {
MutateCollection("tables.json", (t: Array<TableDocument>) => {
t.push({
folders: folders.map((e) => e.folderID),
game: "bms",
default: false,
playtype: tableInfo.playtype,
playtype: tableInfo.game === "bms-7k" ? "7K" : "14K",
inactive: false,
description: tableInfo.description,
title: tableInfo.name,
@@ -86,7 +86,7 @@ async function UpdateTable(tableInfo: BMSTableInfo) {
const f = {
title: tableInfo.name,
playtype: tableInfo.playtype,
playtype: tableInfo.game === "bms-7k" ? "7K" : "14K",
game: "bms",
searchTerms: [tableInfo.asciiPrefix],
type: "charts",
@@ -96,18 +96,18 @@ async function UpdateTable(tableInfo: BMSTableInfo) {
inactive: false,
};
const folderID = CreateFolderIDFromFolder(f);
const folderID = CreateLegacyFolderIDFromFolder(f);
const realFolder = {
...f,
folderID,
} as MONGO_FolderDocument;
} as FolderDocument;
// add this to meta table.
if (!existsFolders.includes(folderID)) {
MutateCollection("tables.json", (tables) => {
for (const table of tables) {
if (table.tableID === `bms-${tableInfo.playtype}-meta`) {
if (table.tableID === `bms-${tableInfo.game}-meta`) {
table.folders.push(folderID);
}
}
@@ -3,11 +3,11 @@ import { XMLParser } from "fast-xml-parser";
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
import path from "path";
import {
type ChartDocument,
type Difficulties,
GetGamePTConfig,
type integer,
type MONGO_ChartDocument,
type MONGO_SongDocument,
LEGACY_GetGamePTConfig,
type SongDocument,
} from "tachi-common";
import { log } from "../../log";
@@ -99,7 +99,7 @@ const program = new Command()
const options = program.opts();
const baseVersion = options.version.replace(/(-intl|-omni)$/u, "");
const tachiVersions = Object.keys(GetGamePTConfig("chunithm", "Single").versions);
const tachiVersions = Object.keys(LEGACY_GetGamePTConfig("chunithm", "Single").versions);
if (!VERSIONS.includes(baseVersion)) {
throw new Error(
@@ -120,18 +120,16 @@ if (!tachiVersions.includes(options.version)) {
const isOmnimixVersion = /-omni$/u.test(options.version);
const isLatestVersion = VERSIONS.indexOf(baseVersion) === VERSIONS.length - 1;
const existingSongDocs: Array<MONGO_SongDocument<"chunithm">> =
ReadCollection("songs-chunithm.json");
const existingChartDocs: Array<MONGO_ChartDocument<"chunithm:Single">> =
ReadCollection("charts-chunithm.json");
const existingSongDocs: Array<SongDocument<"chunithm">> = ReadCollection("songs-chunithm.json");
const existingChartDocs: Array<ChartDocument<"chunithm">> = ReadCollection("charts-chunithm.json");
const songMap = new Map(existingSongDocs.map((s) => [s.id, s]));
const songTitleMap = new Map(existingSongDocs.map((s) => [s.title, s]));
const inGameIDToSongIDMap = new Map<number, number>();
const existingCharts = new Map<string, MONGO_ChartDocument<"chunithm:Single">>();
const existingCharts = new Map<string, ChartDocument<"chunithm">>();
for (const chart of existingChartDocs) {
inGameIDToSongIDMap.set(chart.data.inGameID, chart.songID);
inGameIDToSongIDMap.set(chart.data.inGameID, chart.song.id);
existingCharts.set(`${chart.data.inGameID}-${chart.difficulty}`, chart);
}
@@ -145,8 +143,8 @@ const parser = new XMLParser({
},
});
const newSongs: Array<MONGO_SongDocument<"chunithm">> = [];
const newCharts: Array<MONGO_ChartDocument<"chunithm:Single">> = [];
const newSongs: Array<SongDocument<"chunithm">> = [];
const newCharts: Array<ChartDocument<"chunithm">> = [];
for (const optionsDir of options.input) {
for (const option of readdirSync(optionsDir)) {
@@ -253,7 +251,7 @@ for (const optionsDir of options.input) {
);
}
const songDoc: MONGO_SongDocument<"chunithm"> = {
const songDoc: SongDocument<"chunithm"> = {
title: musicData.name.str,
altTitles: [],
searchTerms: [],
@@ -349,10 +347,11 @@ for (const optionsDir of options.input) {
continue;
}
const chartDoc: MONGO_ChartDocument<"chunithm:Single"> = {
const chartDoc: ChartDocument<"chunithm"> = {
game: "chunithm",
chartID: CreateChartID(),
songID: tachiSongID,
difficulty: difficultyName as Difficulties["chunithm:Single"],
difficulty: difficultyName as Difficulties["chunithm"],
isPrimary: true,
level,
levelNum,
@@ -1,5 +1,5 @@
import fetch from "node-fetch";
import { type Difficulties, type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type Difficulties, type SongDocument } from "tachi-common";
import { CreateChartID, MutateCollection, ReadCollection, WriteCollection } from "../../util";
@@ -14,7 +14,7 @@ if (!CHUNIREC_TOKEN) {
type ShortDifficultyNames = "adv" | "bas" | "exp" | "mas" | "ult";
const DIFFICULTY_MAP = new Map<ShortDifficultyNames, Difficulties["chunithm:Single"]>([
const DIFFICULTY_MAP = new Map<ShortDifficultyNames, Difficulties["chunithm"]>([
["adv", "ADVANCED"],
["bas", "BASIC"],
["exp", "EXPERT"],
@@ -138,24 +138,24 @@ function releaseDateToVersion(date: Date): string {
}
(async () => {
const chunithmSongs: ChunithmSong[] = await fetch(DATA_URL, {}).then((r: Response) => r.json());
const chunithmSongs: ChunithmSong[] = await fetch(DATA_URL, {}).then((r) => r.json());
const chunirecSongs: ChunirecSong[] = await fetch(
`https://api.chunirec.net/2.0/music/showall.json?token=${CHUNIREC_TOKEN}&region=jp2`,
{},
).then((r: Response) => r.json());
).then((r) => r.json());
const existingChartDocs = ReadCollection("charts-chunithm.json");
const inGameIDToSongIDMap = new Map<number, number>();
const existingCharts = new Map<string, MONGO_ChartDocument<"chunithm:Single">>();
const existingCharts = new Map<string, ChartDocument<"chunithm">>();
for (const chart of existingChartDocs) {
inGameIDToSongIDMap.set(chart.data.inGameID, chart.songID);
inGameIDToSongIDMap.set(chart.data.inGameID, chart.song.id);
existingCharts.set(`${chart.data.inGameID}-${chart.difficulty}`, chart);
}
const newSongs: Array<MONGO_SongDocument<"chunithm">> = [];
const newCharts: Array<MONGO_ChartDocument<"chunithm:Single">> = [];
const newSongs: Array<SongDocument<"chunithm">> = [];
const newCharts: Array<ChartDocument<"chunithm">> = [];
for (const chunithmSong of chunithmSongs) {
const inGameID = Number(chunithmSong.id);
@@ -224,6 +224,7 @@ function releaseDateToVersion(date: Date): string {
}
newCharts.push({
game: "chunithm" as const,
chartID: CreateChartID(),
data: {
inGameID,
@@ -239,7 +240,7 @@ function releaseDateToVersion(date: Date): string {
}
}
MutateCollection("songs-chunithm.json", (songs: Array<MONGO_SongDocument<"chunithm">>) => [
MutateCollection("songs-chunithm.json", (songs: Array<SongDocument<"chunithm">>) => [
...songs,
...newSongs,
]);
@@ -1,6 +1,6 @@
import { Parser } from "binary-parser";
import fs from "fs";
import { type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type SongDocument } from "tachi-common";
import { ReadCollection, WriteCollection } from "../../util";
@@ -169,8 +169,7 @@ class SSQParser {
}
class DataUpdater {
existingChartDocs: MONGO_ChartDocument<"ddr:DP" | "ddr:SP">[] =
ReadCollection("charts-ddr.json");
existingChartDocs: ChartDocument<"ddr-dp" | "ddr-sp">[] = ReadCollection("charts-ddr.json");
songs = ReadCollection("songs-ddr.json");
saveCharts() {
@@ -182,15 +181,15 @@ class DataUpdater {
updateStepCounts(basename: string, stepDataArray: StepData[]) {
for (const stepData of stepDataArray) {
const song: MONGO_SongDocument<"ddr"> = this.songs.find(
(s: MONGO_SongDocument<"ddr">) => s.data.basename === basename,
const song: SongDocument<"ddr"> = this.songs.find(
(s: SongDocument<"ddr">) => s.data.basename === basename,
);
if (!song) {
break;
}
for (const chart of this.existingChartDocs) {
if (
song.id === chart.songID &&
song.id === chart.song.id &&
chart.playtype === stepData.style &&
chart.difficulty === stepData.difficulty
) {
@@ -1,17 +1,17 @@
import {
type ChartDocument,
type Difficulties,
type GameConfig,
type GameGroupConfig,
GetGameGroupConfig,
type MONGO_ChartDocument,
type MONGO_SongDocument,
type Playtypes,
type LEGACY_Playtypes,
type SongDocument,
type Versions,
} from "tachi-common";
import { DDR_FLARE_CATEGORIES } from "tachi-common/config/game-support/ddr";
import { CreateChartID, MutateCollection, ReadCollection, WriteCollection } from "../../util";
import { CreateChartID, ReadCollection, WriteCollection } from "../../util";
const gameConfig = GetGameGroupConfig("ddr") as GameConfig<"ddr">;
const gameGroupConfig = GetGameGroupConfig("ddr") as GameGroupConfig<"ddr">;
export interface Music {
mcode: number;
@@ -26,7 +26,7 @@ export interface Music {
diffLv: string;
}
const DIFFICULTIES: Array<Difficulties["ddr:DP" | "ddr:SP"]> = [
const DIFFICULTIES: Array<Difficulties["ddr-dp" | "ddr-sp"]> = [
"BEGINNER",
"BASIC",
"DIFFICULT",
@@ -53,7 +53,7 @@ function seriesToFlareCategory(series: number) {
return DDR_FLARE_CATEGORIES.enum.NONE;
}
function buildSong(music: Music): MONGO_SongDocument<"ddr"> {
function buildSong(music: Music): SongDocument<"ddr"> {
return {
artist: `${music.artist}`,
title: `${music.title}`,
@@ -70,13 +70,15 @@ function buildSong(music: Music): MONGO_SongDocument<"ddr"> {
function buildChart(
music: Music,
playtype: Playtypes["ddr"],
difficulty: Difficulties["ddr:DP" | "ddr:SP"],
version: Versions["ddr:DP" | "ddr:SP"],
): MONGO_ChartDocument<"ddr:DP" | "ddr:SP"> {
playtype: LEGACY_Playtypes["ddr"],
difficulty: Difficulties["ddr-dp" | "ddr-sp"],
version: Versions["ddr-dp" | "ddr-sp"],
): ChartDocument<"ddr-dp" | "ddr-sp"> {
const splitDiff = music.diffLv.split(" ");
const diffIndex = (playtype === "SP" ? 0 : 5) + DIFFICULTIES.indexOf(difficulty);
const game: "ddr-dp" | "ddr-sp" = playtype === "SP" ? "ddr-sp" : "ddr-dp";
return {
game,
chartID: CreateChartID(),
songID: music.mcode,
difficulty: difficulty,
@@ -91,18 +93,18 @@ function buildChart(
};
}
export function parseGameData(version: Versions["ddr:DP" | "ddr:SP"], gameData: Music[]) {
export function parseGameData(version: Versions["ddr-dp" | "ddr-sp"], gameData: Music[]) {
const songs = ReadCollection("songs-ddr.json");
const existingChartDocs = ReadCollection("charts-ddr.json");
const existingCharts = new Map<string, MONGO_ChartDocument<"ddr:DP" | "ddr:SP">>();
const existingCharts = new Map<string, ChartDocument<"ddr-dp" | "ddr-sp">>();
for (const chart of existingChartDocs) {
existingCharts.set(`${chart.data.inGameID}-${chart.difficulty}-${chart.playtype}`, chart);
}
const newSongs: MONGO_SongDocument<"ddr">[] = [];
const newCharts: MONGO_ChartDocument<"ddr:DP" | "ddr:SP">[] = [];
const newSongs: SongDocument<"ddr">[] = [];
const newCharts: ChartDocument<"ddr-dp" | "ddr-sp">[] = [];
for (const music of gameData) {
const song = songs.find((s: MONGO_SongDocument<"ddr">) => s.data.inGameID === music.mcode);
const song = songs.find((s: SongDocument<"ddr">) => s.data.inGameID === music.mcode);
if (!song) {
const newSong = buildSong(music);
console.log(
@@ -116,7 +118,7 @@ export function parseGameData(version: Versions["ddr:DP" | "ddr:SP"], gameData:
for (let i = 0; i < 10; i++) {
const diff = parseInt(splitDiff[i]!, 10);
if (diff > 0 && diff <= 19) {
const playtype = gameConfig.playtypes.at(Math.floor(i / 5))!;
const playtype = gameGroupConfig.playtypes.at(Math.floor(i / 5))!;
const difficulty = DIFFICULTIES.at(i % 5)!;
const exists = existingCharts.get(`${music.mcode}-${difficulty}-${playtype}`);
if (exists) {
@@ -0,0 +1,71 @@
/**
* Assigns `slug` on folder seed rows in `db/seeds/folders.json`.
* Rules live in `./folder-slug.ts` (`computeFolderSlug`).
*
* Default is dry-run. Pass `--write` to persist.
*
* @example
* bun rerunners/folder-slugs/add-folder-slugs.ts
* bun rerunners/folder-slugs/add-folder-slugs.ts --write
* bun rerunners/folder-slugs/add-folder-slugs.ts --only iidx-sp,iidx-dp
*/
import { Command } from "commander";
import { MutateCollection, ReadCollection } from "../../util";
import { applyFolderSlugs } from "./apply-folder-slugs";
import { type SeedFolderRow } from "./folder-slug";
const program = new Command();
program
.name("add-folder-slugs")
.description("Set folder.slug for all games in folders.json")
.option("--write", "Persist changes to db/seeds/folders.json", false)
.option(
"--only <games>",
"Comma-separated `game` values (e.g. iidx-sp,chunithm). Default: all games.",
)
.parse(process.argv);
const opts = program.opts() as { only?: string; write: boolean };
const onlyGames =
opts.only === undefined || opts.only === ""
? null
: new Set(
opts.only
.split(",")
.map((s) => s.trim())
.filter((s) => s.length > 0),
);
if (opts.write) {
MutateCollection("folders.json", (rows: Array<SeedFolderRow>) => {
const out = applyFolderSlugs(rows, onlyGames);
console.log(
`[add-folder-slugs] wrote: updated ${out.updated}, already had slug ${out.skippedAlreadySet}`,
);
console.log(`[add-folder-slugs] updated by game: ${JSON.stringify(out.updatedByGame)}`);
return rows;
});
} else {
const folders = ReadCollection("folders.json") as Array<SeedFolderRow>;
const clone = structuredClone(folders) as Array<SeedFolderRow>;
const out = applyFolderSlugs(clone, onlyGames);
const n =
onlyGames === null ? folders.length : folders.filter((f) => onlyGames.has(f.game)).length;
console.log(`[add-folder-slugs] dry-run (pass --write to save)`);
if (onlyGames !== null) {
console.log(`[add-folder-slugs] --only ${[...onlyGames].join(",")}`);
}
console.log(`[add-folder-slugs] folder rows in scope: ${n}`);
console.log(
`[add-folder-slugs] would set slug on ${out.updated} rows; ${out.skippedAlreadySet} already correct`,
);
console.log(`[add-folder-slugs] would update by game: ${JSON.stringify(out.updatedByGame)}`);
}
@@ -0,0 +1,50 @@
/**
* IIDX-only convenience wrapper. Prefer `add-folder-slugs.ts --only iidx-sp,iidx-dp`.
*/
import { Command } from "commander";
import { MutateCollection, ReadCollection } from "../../util";
import { applyFolderSlugs } from "./apply-folder-slugs";
import { type SeedFolderRow } from "./folder-slug";
const IIDX_ONLY = new Set(["iidx-dp", "iidx-sp"]);
const program = new Command();
program
.name("add-iidx-folder-slugs")
.description("Set folder.slug for iidx-sp / iidx-dp rows in folders.json")
.option("--write", "Persist changes to db/seeds/folders.json", false)
.parse(process.argv);
const opts = program.opts() as { write: boolean };
if (opts.write) {
MutateCollection("folders.json", (rows: Array<SeedFolderRow>) => {
const out = applyFolderSlugs(rows, IIDX_ONLY);
console.log(
`[add-iidx-folder-slugs] wrote: updated ${out.updated}, already had slug ${out.skippedAlreadySet}`,
);
console.log(
`[add-iidx-folder-slugs] updated by game: ${JSON.stringify(out.updatedByGame)}`,
);
return rows;
});
} else {
const folders = ReadCollection("folders.json") as Array<SeedFolderRow>;
const clone = structuredClone(folders) as Array<SeedFolderRow>;
const out = applyFolderSlugs(clone, IIDX_ONLY);
const iidxCount = folders.filter((f) => IIDX_ONLY.has(f.game)).length;
console.log(`[add-iidx-folder-slugs] dry-run (pass --write to save)`);
console.log(`[add-iidx-folder-slugs] IIDX folder rows: ${iidxCount}`);
console.log(
`[add-iidx-folder-slugs] would set slug on ${out.updated} rows; ${out.skippedAlreadySet} already correct`,
);
console.log(
`[add-iidx-folder-slugs] would update by game: ${JSON.stringify(out.updatedByGame)}`,
);
}
@@ -0,0 +1,44 @@
import { computeFolderSlug, type SeedFolderRow } from "./folder-slug";
export function applyFolderSlugs(
folders: Array<SeedFolderRow>,
onlyGames: ReadonlySet<string> | null,
): {
skippedAlreadySet: number;
updated: number;
updatedByGame: Record<string, number>;
} {
let updated = 0;
let skippedAlreadySet = 0;
const updatedByGame: Record<string, number> = {};
const slugKeys = new Map<string, string>();
for (const folder of folders) {
if (onlyGames !== null && !onlyGames.has(folder.game)) {
continue;
}
const slug = computeFolderSlug(folder);
const key = `${folder.game}\0${slug}`;
if (slugKeys.has(key)) {
throw new Error(
`Duplicate slug "${slug}" for game ${folder.game}: "${slugKeys.get(key)}" vs "${folder.title}"`,
);
}
slugKeys.set(key, folder.title);
if (folder.slug === slug) {
skippedAlreadySet++;
continue;
}
folder.slug = slug;
updated++;
updatedByGame[folder.game] = (updatedByGame[folder.game] ?? 0) + 1;
}
return { updated, skippedAlreadySet, updatedByGame };
}
@@ -0,0 +1,6 @@
/**
* Re-exported from `tachi-common` so rerunner paths stay stable.
* @see {@link computeFolderSlug} in `tachi-common`
*/
export * from "tachi-common/lib/folder-slug";
@@ -2,17 +2,17 @@ import { Command } from "commander";
import { XMLParser } from "fast-xml-parser";
import { readFileSync } from "fs";
import {
type ChartDocument,
type Difficulties,
type MONGO_ChartDocument,
type MONGO_SongDocument,
type Playtypes,
type LEGACY_Playtypes,
type SongDocument,
type Versions,
} from "tachi-common";
import { GITADORA_GITA_CONF } from "tachi-common/config/game-support/gitadora";
import { GAME_GITADORA_GITA_CONF } from "tachi-common/config/game-support/gitadora";
import { CreateChartID, MutateCollection, ReadCollection, WriteCollection } from "../../util";
const supportedVersions = Object.keys(GITADORA_GITA_CONF.versions);
const supportedVersions = Object.keys(GAME_GITADORA_GITA_CONF.versions);
const program = new Command();
program.requiredOption("-i, --input <music_db.xml>");
@@ -30,7 +30,7 @@ if (!supportedVersions.includes(options.version)) {
);
throw new Error(`Invalid version ${options.version}.`);
}
const version: Versions["gitadora:Dora" | "gitadora:Gita"] = options.version;
const version: Versions["gitadora-dora" | "gitadora-gita"] = options.version;
interface XMLText<T> {
"#text": T;
@@ -86,7 +86,7 @@ interface Entry {
}
// Used for destructuring the difficulty byte array - see implementation for mapping
const DIFFICULTIES: (Difficulties["gitadora:Dora" | "gitadora:Gita"] | null)[] = [
const DIFFICULTIES: (Difficulties["gitadora-dora" | "gitadora-gita"] | null)[] = [
null,
"BASIC",
"ADVANCED",
@@ -103,7 +103,7 @@ const DIFFICULTIES: (Difficulties["gitadora:Dora" | "gitadora:Gita"] | null)[] =
"BASS EXTREME",
"BASS MASTER",
];
const PLAYTYPE_DIFFICULTY_MAP: (Playtypes["gitadora"] | null)[] = [
const PLAYTYPE_DIFFICULTY_MAP: (LEGACY_Playtypes["gitadora"] | null)[] = [
null,
"Gita",
"Gita",
@@ -121,7 +121,7 @@ const PLAYTYPE_DIFFICULTY_MAP: (Playtypes["gitadora"] | null)[] = [
"Gita",
];
function buildSong(entry: Entry): MONGO_SongDocument<"gitadora"> {
function buildSong(entry: Entry): SongDocument<"gitadora"> {
return {
artist: entry.artist_title_ascii["#text"] || "", // Sometimes artist is just not there - this is valid!
title: entry.title_name["#text"],
@@ -135,14 +135,17 @@ function buildSong(entry: Entry): MONGO_SongDocument<"gitadora"> {
const buildChart = (
entry: Entry,
difficulty: number,
difficultyString: Difficulties["gitadora:Dora" | "gitadora:Gita"],
playType: Playtypes["gitadora"],
): MONGO_ChartDocument<"gitadora:Dora" | "gitadora:Gita"> => {
difficultyString: Difficulties["gitadora-dora" | "gitadora-gita"],
playType: LEGACY_Playtypes["gitadora"],
): ChartDocument<"gitadora-dora" | "gitadora-gita"> => {
// Comes in a 435, needs to become "4.35"
const convertedDifficultyAsFloat = difficulty / 100;
const convertedDifficultyAsString = convertedDifficultyAsFloat.toFixed(2);
const game: "gitadora-dora" | "gitadora-gita" =
playType === "Gita" ? "gitadora-gita" : "gitadora-dora";
return {
game,
chartID: CreateChartID(),
data: {
inGameID: entry.music_id["#text"],
@@ -159,20 +162,20 @@ const buildChart = (
const songs = ReadCollection("songs-gitadora.json");
const existingChartDocs = ReadCollection("charts-gitadora.json");
const existingCharts = new Map<string, MONGO_ChartDocument<"gitadora:Dora" | "gitadora:Gita">>();
const existingCharts = new Map<string, ChartDocument<"gitadora-dora" | "gitadora-gita">>();
for (const chart of existingChartDocs) {
existingCharts.set(`${chart.data.inGameID}-${chart.difficulty}-${chart.playtype}`, chart);
}
const newSongs: MONGO_SongDocument<"gitadora">[] = [];
const newCharts: MONGO_ChartDocument<"gitadora:Dora" | "gitadora:Gita">[] = [];
const newSongs: SongDocument<"gitadora">[] = [];
const newCharts: ChartDocument<"gitadora-dora" | "gitadora-gita">[] = [];
for (const entry of data.mdb.mdb_data as Entry[]) {
// For logging purposes - song may not exist, not safe to access for logging
const mid = entry.music_id["#text"];
const title = entry.title_name["#text"];
const artist = entry.artist_title_ascii["#text"] || "";
const song = songs.find((s: MONGO_SongDocument<"gitadora">) => s.id === mid);
const song = songs.find((s: SongDocument<"gitadora">) => s.id === mid);
if (!song) {
const newSong = buildSong(entry);
console.log(`New song: ${newSong.artist} - ${newSong.title} (id ${newSong.id})`);
@@ -216,7 +219,7 @@ for (const entry of data.mdb.mdb_data as Entry[]) {
}
}
MutateCollection("songs-gitadora.json", (songs: Array<MONGO_SongDocument<"gitadora">>) => [
MutateCollection("songs-gitadora.json", (songs: Array<SongDocument<"gitadora">>) => [
...songs,
...newSongs,
]);
@@ -1,15 +1,15 @@
import { readdirSync, readFileSync } from "fs";
import { join } from "path";
import {
type ChartDocument,
type Difficulties,
type GPTStrings,
type GamesForGroup,
type integer,
type MONGO_ChartDocument,
type MONGO_SongDocument,
type Playtypes,
type LEGACY_Playtypes,
type SongDocument,
} from "tachi-common";
import { CreateChartID, GetFreshScoreIDGenerator, MutateCollection } from "../../util";
import { CreateChartID, GetFreshSongIDGenerator, MutateCollection } from "../../../util";
interface AixChart {
bpm_max: integer;
@@ -36,7 +36,7 @@ function ConvertAixStuff(d: AixData, songID: integer) {
searchTerms.push(d.title_ascii);
}
const song: MONGO_SongDocument<"iidx"> = {
const song: SongDocument<"iidx"> = {
title: d.title,
artist: d.artist,
altTitles: [],
@@ -48,7 +48,7 @@ function ConvertAixStuff(d: AixData, songID: integer) {
id: songID,
};
const charts: Array<MONGO_ChartDocument<GPTStrings["iidx"]>> = [];
const charts: Array<ChartDocument<GamesForGroup["iidx"]>> = [];
for (const [diff, c] of Object.entries(d.charts)) {
// wonderful ts oddity
charts.push(ParseAixChart(d, c, diff as ChartStrings, songID));
@@ -66,8 +66,8 @@ const DIFF_MAP = {
} as const;
function SplitAixDiff(diff: ChartStrings): {
difficulty: Difficulties["iidx:DP" | "iidx:SP"];
playtype: Playtypes["iidx"];
difficulty: Difficulties[GamesForGroup["iidx"]];
playtype: LEGACY_Playtypes["iidx"];
} {
const difficulty = DIFF_MAP[diff[2]];
@@ -84,10 +84,11 @@ function SplitAixDiff(diff: ChartStrings): {
function ParseAixChart(d: AixData, c: AixChart, diff: ChartStrings, songID: integer) {
const { difficulty, playtype } = SplitAixDiff(diff);
let chart: MONGO_ChartDocument<GPTStrings["iidx"]>;
let chart: ChartDocument<GamesForGroup["iidx"]>;
if (playtype === "SP") {
const temp: MONGO_ChartDocument<"iidx:SP"> = {
const temp: ChartDocument<"iidx-sp"> = {
game: "iidx-sp",
chartID: CreateChartID(),
data: {
notecount: c.note_count,
@@ -107,12 +108,13 @@ function ParseAixChart(d: AixData, c: AixChart, diff: ChartStrings, songID: inte
level: c.rating.toString(),
levelNum: c.rating,
songID,
versions: ["INFINITAS"],
versions: ["inf"],
};
chart = temp;
} else {
const temp: MONGO_ChartDocument<"iidx:DP"> = {
const temp: ChartDocument<"iidx-dp"> = {
game: "iidx-dp",
chartID: CreateChartID(),
data: {
notecount: c.note_count,
@@ -130,7 +132,7 @@ function ParseAixChart(d: AixData, c: AixChart, diff: ChartStrings, songID: inte
level: c.rating.toString(),
levelNum: c.rating,
songID,
versions: ["INFINITAS"],
versions: ["inf"],
};
chart = temp;
@@ -142,10 +144,10 @@ function ParseAixChart(d: AixData, c: AixChart, diff: ChartStrings, songID: inte
if (require.main === module) {
const files = readdirSync(__dirname);
const getSongID = GetFreshScoreIDGenerator("iidx");
const getSongID = GetFreshSongIDGenerator("iidx");
const newSongs: Array<MONGO_SongDocument<"iidx">> = [];
const newCharts: Array<MONGO_ChartDocument<GPTStrings["iidx"]>> = [];
const newSongs: Array<SongDocument<"iidx">> = [];
const newCharts: Array<ChartDocument<GamesForGroup["iidx"]>> = [];
for (const file of files) {
if (file.endsWith(".json")) {
const songID = getSongID();
@@ -3,11 +3,12 @@ import { Command } from "commander";
import fs from "fs";
import path from "path";
import {
type ChartDocument,
type Difficulties,
GetGamePTConfig,
type GamesForGroup,
type integer,
type MONGO_ChartDocument,
type MONGO_SongDocument,
LEGACY_GetGamePTConfig,
type SongDocument,
type Versions,
} from "tachi-common";
@@ -44,10 +45,10 @@ const options = program.opts() as {
force: boolean;
index: "0" | "1";
omni: boolean;
version: Versions["iidx:SP"];
version: Versions["iidx-sp"];
};
const iidxConfig = GetGamePTConfig("iidx", "SP");
const iidxConfig = LEGACY_GetGamePTConfig("iidx", "SP");
if (!Object.keys(iidxConfig.versions).includes(options.version)) {
throw new Error(
@@ -63,10 +64,9 @@ if (options.index !== "0" && options.index !== "1") {
throw new Error(`Expected an --index of 0 or 1. Got ${options.index}.`);
}
const existingCharts: MONGO_ChartDocument<"iidx:DP" | "iidx:SP">[] =
ReadCollection("charts-iidx.json");
const existingCharts: ChartDocument<GamesForGroup["iidx"]>[] = ReadCollection("charts-iidx.json");
const existingSongs: MONGO_SongDocument<"iidx">[] = ReadCollection("songs-iidx.json");
const existingSongs: SongDocument<"iidx">[] = ReadCollection("songs-iidx.json");
const blacklist = fs
.readFileSync(path.join(__dirname, "blacklist.txt"), "utf-8")
@@ -101,10 +101,10 @@ async function ParseIIDXMDB() {
options.alwaysExtract,
);
const chartMap = new Map<integer, MONGO_ChartDocument<"iidx:DP" | "iidx:SP">>();
const songMap = new Map<integer, MONGO_SongDocument<"iidx">>();
const songTitleMap = new Map<string, MONGO_SongDocument<"iidx">>();
const chartDiffMap = new Map<string, MONGO_ChartDocument<"iidx:DP" | "iidx:SP">>();
const chartMap = new Map<integer, ChartDocument<GamesForGroup["iidx"]>>();
const songMap = new Map<integer, SongDocument<"iidx">>();
const songTitleMap = new Map<string, SongDocument<"iidx">>();
const chartDiffMap = new Map<string, ChartDocument<GamesForGroup["iidx"]>>();
for (const song of existingSongs) {
songMap.set(song.id, song);
@@ -143,7 +143,7 @@ async function ParseIIDXMDB() {
}
const anySongIDMatch = chartMap.get(inp.songID);
let song: MONGO_SongDocument<"iidx">;
let song: SongDocument<"iidx">;
if (!anySongIDMatch) {
// new song?
@@ -170,7 +170,7 @@ async function ParseIIDXMDB() {
searchTerms.push(inp.marquee);
}
const tachiSong: MONGO_SongDocument<"iidx"> = {
const tachiSong: SongDocument<"iidx"> = {
id: getFreeSongID(),
artist: inp.artist,
title: inp.title,
@@ -242,13 +242,16 @@ async function ParseIIDXMDB() {
}
// otherwise, make new chart?
const tachiChart: MONGO_ChartDocument<"iidx:DP" | "iidx:SP"> = {
const playtypeStr = diffName.split("-")[0] as "DP" | "SP";
const game: GamesForGroup["iidx"] = playtypeStr === "SP" ? "iidx-sp" : "iidx-dp";
const tachiChart: ChartDocument<GamesForGroup["iidx"]> = {
game,
chartID: CreateChartID(),
difficulty: diffName.split("-")[1] as Difficulties["iidx:DP" | "iidx:SP"],
difficulty: diffName.split("-")[1] as Difficulties[GamesForGroup["iidx"]],
level: level.toString(),
levelNum: level,
isPrimary: true,
playtype: diffName.split("-")[0] as "DP" | "SP",
playtype: playtypeStr,
songID: song.id,
versions: [options.version],
@@ -105,7 +105,7 @@ function FindSongOnTitle(title) {
function FindChartWithPTDFVersion(songID, playtype, diff, version) {
for (const chart of chartData) {
if (
chart.songID === songID &&
chart.song.id === songID &&
chart.playtype === playtype &&
chart.difficulty === diff &&
chart.versions.includes(version)
@@ -1,6 +1,6 @@
import { Command } from "commander";
import fs from "fs";
import { type MONGO_GoalDocument, type MONGO_QuestDocument } from "tachi-common";
import { type GoalDocument, type QuestDocument } from "tachi-common";
import { CreateGoalID, CreateQuestID, MutateCollection } from "../util";
@@ -13,7 +13,7 @@ const options = program.opts();
// stolen from client/src/types/tachi.ts
type RawQuestDocument = {
rawQuestData: Array<RawQuestSection>;
} & Omit<MONGO_QuestDocument, "questData" | "questID">;
} & Omit<QuestDocument, "questData" | "questID">;
type RawQuestSection = {
desc: string;
@@ -22,16 +22,16 @@ type RawQuestSection = {
};
type RawQuestGoal = {
goal: Pick<MONGO_GoalDocument, "charts" | "criteria" | "name">;
goal: Pick<GoalDocument, "charts" | "criteria" | "name">;
note?: string;
};
const data = JSON.parse(fs.readFileSync(options.file, "utf-8")) as Array<RawQuestDocument>;
const newGoals: Array<MONGO_GoalDocument> = [];
const newGoals: Array<GoalDocument> = [];
function HydrateQuest(raw: RawQuestDocument): MONGO_QuestDocument {
const questData: MONGO_QuestDocument["questData"] = [];
function HydrateQuest(raw: RawQuestDocument): QuestDocument {
const questData: QuestDocument["questData"] = [];
const { game, playtype } = raw;
@@ -39,16 +39,16 @@ function HydrateQuest(raw: RawQuestDocument): MONGO_QuestDocument {
const goals: Array<{ goalID: string; note?: string }> = [];
for (const rawGoal of rawQuest.rawGoals) {
const goalID = CreateGoalID(rawGoal.goal.charts, rawGoal.goal.criteria, game, playtype);
const goalID = CreateGoalID(rawGoal.goal.charts, rawGoal.goal.criteria, game);
const newGoal: MONGO_GoalDocument = {
const newGoal: GoalDocument = {
charts: rawGoal.goal.charts,
criteria: rawGoal.goal.criteria,
game,
playtype,
goalID,
name: rawGoal.goal.name,
} as MONGO_GoalDocument;
} as GoalDocument;
newGoals.push(newGoal);
@@ -1,5 +1,5 @@
import { type MONGO_FolderDocument } from "tachi-common";
import { JUBEAT_SINGLE_CONF } from "tachi-common/config/game-support/jubeat";
import { type FolderDocument } from "tachi-common";
import { GAME_JUBEAT_SINGLE_CONF } from "tachi-common/config/game-support/jubeat";
import { CreateFolderID, MutateCollection } from "../../util";
@@ -26,9 +26,9 @@ const versions = ["ave"];
for (const version of versions) {
const shouldAddDecimals = true;
const versionPretty = JUBEAT_SINGLE_CONF.versions[version];
const levelFolders: Array<MONGO_FolderDocument> = [];
const levelHardFolders: Array<MONGO_FolderDocument> = [];
const versionPretty = GAME_JUBEAT_SINGLE_CONF.versions[version];
const levelFolders: Array<FolderDocument> = [];
const levelHardFolders: Array<FolderDocument> = [];
for (const level of ["1", "2", "3", "4", "5", "6", "7", "8"]) {
const f = CreateFolder(
@@ -82,8 +82,8 @@ for (const version of versions) {
),
);
const detailFolders: Array<MONGO_FolderDocument> = [];
const detailHardFolders: Array<MONGO_FolderDocument> = [];
const detailFolders: Array<FolderDocument> = [];
const detailHardFolders: Array<FolderDocument> = [];
if (shouldAddDecimals) {
for (const level of [
@@ -11,7 +11,7 @@ import { XMLParser } from "fast-xml-parser";
*/
import fs from "fs";
import path from "path";
import { type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type SongDocument } from "tachi-common";
import { MutateCollection, ReadCollection } from "../../util";
@@ -156,10 +156,10 @@ function normalizeTitle(title: string): string {
}
function findSong(
songs: MONGO_SongDocument<"maimaidx">[],
songs: SongDocument<"maimaidx">[],
title: string,
category: string,
): MONGO_SongDocument<"maimaidx"> | undefined {
): SongDocument<"maimaidx"> | undefined {
// There are two songs with the exact same title and that only differs
// by category:
// - Link (maimai) is 68
@@ -196,7 +196,7 @@ function calculateDifficulty(style: string, sheetDifficulty: string): string {
function addTmaiSheet(csvData: string[][]) {
const songs = ReadCollection("songs-maimaidx.json");
MutateCollection("charts-maimaidx.json", (charts: MONGO_ChartDocument<"maimaidx:Single">[]) => {
MutateCollection("charts-maimaidx.json", (charts: ChartDocument<"maimaidx">[]) => {
for (let rowNumber = 1; rowNumber < csvData.length; rowNumber++) {
const row = csvData[rowNumber]!;
const title = row[1];
@@ -257,7 +257,7 @@ function addOtherSheet(csvData: string[][], headerRow: number, options: AddOther
const categoryColumnOffset = hasCategoryColumn ? 1 : 0;
let currentCategory = "";
MutateCollection("charts-maimaidx.json", (charts: MONGO_ChartDocument<"maimaidx:Single">[]) => {
MutateCollection("charts-maimaidx.json", (charts: ChartDocument<"maimaidx">[]) => {
for (
let colNumber = 0;
colNumber + 4 + categoryColumnOffset < csvData[0]!.length;
@@ -335,7 +335,7 @@ if (options.directory) {
const songs = ReadCollection("songs-maimaidx.json");
const parser = new XMLParser({ ignoreAttributes: false });
MutateCollection("charts-maimaidx.json", (charts: MONGO_ChartDocument<"maimaidx:Single">[]) => {
MutateCollection("charts-maimaidx.json", (charts: ChartDocument<"maimaidx">[]) => {
const items = fs.readdirSync(options.directory);
for (const item of items) {
@@ -5,13 +5,18 @@ import { XMLParser } from "fast-xml-parser";
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
import path from "path";
import {
type ChartDocument,
type Difficulties,
GetGamePTConfig,
type MONGO_ChartDocument,
type MONGO_SongDocument,
LEGACY_GetGamePTConfig,
type SongDocument,
} from "tachi-common";
import { CreateChartID, ReadCollection, WriteCollection } from "../../util";
import {
CreateChartID,
GetFreshSongIDGenerator,
ReadCollection,
WriteCollection,
} from "../../util";
const VERSION_DISPLAY_NAMES = [
"maimai",
@@ -121,7 +126,7 @@ if (!options.vgmsBinary) {
}
}
const versions = Object.keys(GetGamePTConfig("maimaidx", "Single").versions);
const versions = Object.keys(LEGACY_GetGamePTConfig("maimaidx", "Single").versions);
if (versions.indexOf(options.version) === -1) {
throw new Error(
@@ -133,20 +138,19 @@ if (versions.indexOf(options.version) === -1) {
const isLatestVersion =
versions.indexOf(options.version.replace(/(-intl|-omni)$/u, "")) === versions.length - 1;
const existingSongs: Array<MONGO_SongDocument<"maimaidx">> = ReadCollection("songs-maimaidx.json");
const existingCharts: Array<MONGO_ChartDocument<"maimaidx:Single">> =
ReadCollection("charts-maimaidx.json");
const existingSongs: Array<SongDocument<"maimaidx">> = ReadCollection("songs-maimaidx.json");
const existingCharts: Array<ChartDocument<"maimaidx">> = ReadCollection("charts-maimaidx.json");
const songMap = new Map(existingSongs.map((s) => [s.id, s]));
const chartMap = new Map<string, MONGO_ChartDocument<"maimaidx:Single">>();
const chartMap = new Map<string, ChartDocument<"maimaidx">>();
const songTitleArtistMap = new Map<string, number>();
const durationMap = new Map<string, number>();
for (const chart of existingCharts) {
const song = songMap.get(chart.songID);
const song = songMap.get(chart.song.id);
if (song === undefined) {
log.error(
`CONSISTENCY ERROR: Chart ID ${chart.chartID} does not belong to any songs! (songID was ${chart.songID})`,
`CONSISTENCY ERROR: Chart ID ${chart.chartID} does not belong to any songs! (songID was ${chart.song.id})`,
);
process.exit(1);
}
@@ -227,8 +231,8 @@ const parser = new XMLParser({
},
});
const newSongs: Array<MONGO_SongDocument<"maimaidx">> = [];
const newCharts: Array<MONGO_ChartDocument<"maimaidx:Single">> = [];
const newSongs: Array<SongDocument<"maimaidx">> = [];
const newCharts: Array<ChartDocument<"maimaidx">> = [];
const songIDGenerator = GetFreshSongIDGenerator("maimaidx");
@@ -333,14 +337,16 @@ for (const optionsDir of options.input) {
// New song?
if (tachiSongID === undefined) {
tachiSongID = songIDGenerator();
const newSongID = songIDGenerator();
const songDoc: MONGO_SongDocument<"maimaidx"> = {
tachiSongID = newSongID;
const songDoc: SongDocument<"maimaidx"> = {
title: musicData.name.str,
altTitles: [],
searchTerms: [],
artist: musicData.artistName.str,
id: tachiSongID,
id: newSongID,
data: {
genre,
duration,
@@ -348,8 +354,8 @@ for (const optionsDir of options.input) {
};
newSongs.push(songDoc);
songTitleArtistMap.set(`${songDoc.title}-${songDoc.artist}`, tachiSongID);
songMap.set(tachiSongID, songDoc);
songTitleArtistMap.set(`${songDoc.title}-${songDoc.artist}`, newSongID);
songMap.set(newSongID, songDoc);
log.info(
`Added new song ${songDoc.artist} - ${songDoc.title} (inGameID ${inGameID}, tachiSongID ${tachiSongID}).`,
@@ -389,7 +395,7 @@ for (const optionsDir of options.input) {
difficultyName = `DX ${difficultyName}`;
}
let exists: MONGO_ChartDocument<"maimaidx:Single"> | undefined;
let exists: ChartDocument<"maimaidx"> | undefined;
if (inGameID === 11422) {
exists = chartMap.get(`-x0o0x_-${difficultyName}`);
@@ -450,10 +456,11 @@ for (const optionsDir of options.input) {
continue;
}
const chartDoc: MONGO_ChartDocument<"maimaidx:Single"> = {
const chartDoc: ChartDocument<"maimaidx"> = {
chartID: CreateChartID(),
songID: tachiSongID,
difficulty: difficultyName as Difficulties["maimaidx:Single"],
songID: tachiSongID!,
game: "maimaidx",
difficulty: difficultyName as Difficulties["maimaidx"],
isPrimary: true,
level,
levelNum,
@@ -1,5 +1,5 @@
import { log } from "#log";
import { type Difficulties, type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type Difficulties, type SongDocument } from "tachi-common";
import {
CreateChartID,
@@ -71,19 +71,18 @@ const VERSION_MAP = new Map([
]);
const CURRENT_VERSION_NUM = 250;
const CURRENT_VERSION = "prism";
const DIFFICULTY_TO_TACHI_DIFFICULTY: Record<MaimaiDXDifficulty, Difficulties["maimaidx:Single"]> =
{
lev_bas: "Basic",
lev_adv: "Advanced",
lev_exp: "Expert",
lev_mas: "Master",
lev_remas: "Re:Master",
dx_lev_bas: "DX Basic",
dx_lev_adv: "DX Advanced",
dx_lev_exp: "DX Expert",
dx_lev_mas: "DX Master",
dx_lev_remas: "DX Re:Master",
} as const;
const DIFFICULTY_TO_TACHI_DIFFICULTY: Record<MaimaiDXDifficulty, Difficulties["maimaidx"]> = {
lev_bas: "Basic",
lev_adv: "Advanced",
lev_exp: "Expert",
lev_mas: "Master",
lev_remas: "Re:Master",
dx_lev_bas: "DX Basic",
dx_lev_adv: "DX Advanced",
dx_lev_exp: "DX Expert",
dx_lev_mas: "DX Master",
dx_lev_remas: "DX Re:Master",
} as const;
const DX_REGEX = /dx\s*:\s*([01])/u;
const LV_REGEX = /lv\s*:\s*(\[(?:-?[\d.]+\s*,?\s*)+\])/u;
@@ -151,9 +150,8 @@ async function ParseInternalLevelData(
}
async function ParseMaimaiDXDataset() {
const songs: Array<MONGO_SongDocument<"maimaidx">> = ReadCollection("songs-maimaidx.json");
const charts: Array<MONGO_ChartDocument<"maimaidx:Single">> =
ReadCollection("charts-maimaidx.json");
const songs: Array<SongDocument<"maimaidx">> = ReadCollection("songs-maimaidx.json");
const charts: Array<ChartDocument<"maimaidx">> = ReadCollection("charts-maimaidx.json");
const existingSongMap = new Map(songs.map((s) => [`${s.title}-${s.artist}`, s]));
const existingChartMap = new Map(charts.map((c) => [`${c.songID}-${c.difficulty}`, c]));
@@ -206,14 +204,13 @@ async function ParseMaimaiDXDataset() {
if (!tachiSongID) {
tachiSongID = getFreeSongID();
const songDoc: MONGO_SongDocument<"maimaidx"> = {
const songDoc: SongDocument<"maimaidx"> = {
id: tachiSongID,
title: data.title.trim(),
artist: data.artist.trim(),
searchTerms: [],
altTitles: [],
data: {
displayVersion,
genre: data.catcode.trim(),
},
};
@@ -225,7 +222,7 @@ async function ParseMaimaiDXDataset() {
for (const [key, difficulty] of Object.entries(DIFFICULTY_TO_TACHI_DIFFICULTY) as [
MaimaiDXDifficulty,
Difficulties["maimaidx:Single"],
Difficulties["maimaidx"],
][]) {
const level = data[key];
@@ -276,15 +273,17 @@ async function ParseMaimaiDXDataset() {
continue;
}
const chartDoc: MONGO_ChartDocument<"maimaidx:Single"> = {
songID: tachiSongID,
const chartDoc: ChartDocument<"maimaidx"> = {
songID: tachiSongID!,
chartID: CreateChartID(),
game: "maimaidx",
level,
levelNum,
isPrimary: true,
difficulty,
playtype: "Single",
data: {
displayVersion,
inGameID: null,
},
versions: [CURRENT_VERSION],
@@ -1,7 +1,7 @@
import { spawnSync } from "child_process";
import { Command } from "commander";
import fs from "fs";
import { type MONGO_QuestDocument, type MONGO_QuestlineDocument } from "tachi-common";
import { type QuestDocument, type QuestlineDocument } from "tachi-common";
import { MutateCollection, ReadCollection } from "../util";
@@ -25,7 +25,7 @@ const { game, playtype } = options;
const quests = ReadCollection("quests.json").filter(
(e) => e.game === game && e.playtype === playtype,
) as Array<MONGO_QuestDocument>;
) as Array<QuestDocument>;
if (quests.length === 0) {
console.log(`No quests found for game ${game} and playtype ${playtype}.`);
@@ -109,7 +109,7 @@ if (questIDs.length === 0) {
process.exit(1);
}
const questline: MONGO_QuestlineDocument = {
const questline: QuestlineDocument = {
game,
playtype,
desc,
@@ -1,7 +1,7 @@
import { Command } from "commander";
import { GetGamePTConfig, type MONGO_FolderDocument } from "tachi-common";
import { type FolderDocument, GetGameConfig } from "tachi-common";
import { CreateFolderID, MutateCollection } from "../../util";
import { CreateLegacyFolderID, MutateCollection } from "../../util";
const LEVELS = [
"0",
@@ -36,7 +36,7 @@ const command = new Command().requiredOption("-v, --version <version>").parse(pr
const options = command.opts();
const version = options.version;
const tachiVersions = GetGamePTConfig("ongeki", "Single").versions;
const tachiVersions = GetGameConfig("ongeki").versions;
const versionName = tachiVersions[version];
if (!versionName) {
@@ -45,7 +45,7 @@ if (!versionName) {
);
}
const newFolders: MONGO_FolderDocument[] = [];
const newFolders: FolderDocument[] = [];
const levelFolderIDs: string[] = [];
const difficultyFolderIDs: string[] = [];
@@ -62,7 +62,7 @@ for (const level of LEVELS) {
},
};
}
const folderID = CreateFolderID(data, "ongeki", "Single");
const folderID = CreateLegacyFolderID(data, "ongeki", "Single");
levelFolderIDs.push(folderID);
@@ -92,7 +92,7 @@ for (const difficulty of DIFFICULTIES) {
};
}
const folderID = CreateFolderID(data, "ongeki", "Single");
const folderID = CreateLegacyFolderID(data, "ongeki", "Single");
difficultyFolderIDs.push(folderID);
@@ -1,13 +1,13 @@
// usage: cd rerunners && npx ts-node ongeki/parse-music-data.ts
import fs from "fs/promises";
import { type Difficulties, type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type Difficulties, type SongDocument } from "tachi-common";
import { CreateChartID, ReadCollection, WriteCollection } from "../../util";
type OngekiChart = MONGO_ChartDocument<"ongeki:Single">;
type OngekiSong = MONGO_SongDocument<"ongeki">;
type Difficulty = Difficulties["ongeki:Single"];
type OngekiChart = ChartDocument<"ongeki">;
type OngekiSong = SongDocument<"ongeki">;
type Difficulty = Difficulties["ongeki"];
const CURRENT_VERSION = "refresh";
const CURRENT_OMNIMIX = "refreshOmni";
@@ -149,9 +149,10 @@ const main = async () => {
if (!ver.startsWith("オンゲキ")) {
ver = `オンゲキ ${ver}`;
}
chart = {
const newChart: OngekiChart = {
chartID: CreateChartID(),
songID: song.id,
game: "ongeki",
data: {
displayVersion: ver as any,
inGameID: inputSong.id,
@@ -164,9 +165,10 @@ const main = async () => {
playtype: "Single",
versions: [CURRENT_VERSION, CURRENT_OMNIMIX],
};
if (chart.difficulty === "LUNATIC") {
chart.data.isReMaster = inputSong.isReMaster;
if (newChart.difficulty === "LUNATIC") {
newChart.data.isReMaster = inputSong.isReMaster;
}
chart = newChart;
changes.charts.push(`${song.title} ${chart.difficulty} ${chart.level}`);
charts.push(chart);
}
@@ -6,7 +6,7 @@ import { execFile } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type SongDocument } from "tachi-common";
import { ReadCollection, WriteCollection } from "../../util";
@@ -25,8 +25,8 @@ const game = options.game;
const readOpt = async (
musicPath: string,
charts: MONGO_ChartDocument<"chunithm:Single" | "ongeki:Single">[],
songs: MONGO_SongDocument<"chunithm" | "ongeki">[],
charts: ChartDocument<"chunithm" | "ongeki">[],
songs: SongDocument<"chunithm" | "ongeki">[],
) => {
let musicDir: string[];
try {
@@ -65,7 +65,7 @@ const readOpt = async (
}
continue;
}
const song = songs.find((s) => s.id === chart.songID);
const song = songs.find((s) => s.id === chart.song.id);
if (song === undefined) {
console.error(`Song #${id}: orphan`);
continue;
@@ -1,12 +1,12 @@
/* eslint-disable no-await-in-loop */
import readline from "readline";
import { type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type SongDocument } from "tachi-common";
import { ReadCollection, WriteCollection } from "../../util";
type OngekiChart = MONGO_ChartDocument<"ongeki:Single">;
type OngekiSong = MONGO_SongDocument<"ongeki">;
type OngekiChart = ChartDocument<"ongeki">;
type OngekiSong = SongDocument<"ongeki">;
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
const prompt = (query: string) => new Promise((resolve) => rl.question(query, resolve));
@@ -136,7 +136,7 @@ const listMissing = async (charts: OngekiChart[], songs: OngekiSong[], minLevel:
continue;
}
if (chart.levelNum >= minLevel && !chart.data.chartViewURL) {
const song = songs.find((s) => s.id === chart.songID);
const song = songs.find((s) => s.id === chart.song.id);
// @ts-expect-error The linter disagrees with the compiler
chart.data.chartViewURL = await prompt(
`Missing: ${song?.artist} ${song?.title} ${chart.difficulty} ${chart.level}\n`,
@@ -497,14 +497,14 @@ function addTiers(levelNum, csvData, headerRow, leftOffset, simple) {
let chart = charts.find(
(chart) =>
chart.songID === song.id &&
chart.song.id === song.id &&
chart.levelNum === levelNum &&
chart.difficulty === difficulty,
);
if (!chart) {
// Sometimes the difficulty is missing, or straight up wrong (e.g. MXM instead of VVD) so just try without.
chart = charts.find(
(chart) => chart.songID === song.id && chart.levelNum === levelNum,
(chart) => chart.song.id === song.id && chart.levelNum === levelNum,
);
}
@@ -564,7 +564,7 @@ function addTiers(levelNum, csvData, headerRow, leftOffset, simple) {
if (missingTiers.length > 0) {
console.log(`\nThe following lv${levelNum} charts are still missing a tier:`);
for (const chart of missingTiers) {
const song = songs.find((song) => song.id === chart.songID);
const song = songs.find((song) => song.id === chart.song.id);
console.log(
`${song.id}: ${song.title} [${chart.difficulty}] (displayVersion: ${song.data.displayVersion})`,
);
@@ -90,7 +90,7 @@ for (const music of xmlData.mdb.music) {
const difficulty = getDifficulty(diffKey, music.info.inf_ver["#text"]);
const chartIndex = charts.findIndex(
(chart) => chart.songID === id && chart.difficulty === difficulty,
(chart) => chart.song.id === id && chart.difficulty === difficulty,
);
if (chartIndex === -1) {
charts.push({
@@ -3,10 +3,10 @@ import { XMLParser } from "fast-xml-parser";
import fs from "fs";
import { decode } from "iconv-lite";
import {
type ChartDocument,
type Difficulties,
type integer,
type MONGO_ChartDocument,
type MONGO_SongDocument,
type SongDocument,
} from "tachi-common";
import {
@@ -66,10 +66,10 @@ interface MDBEntry {
const existingChartDocs = ReadCollection("charts-sdvx.json");
const inGameIDToSongIDMap = new Map<number, number>();
const existingCharts = new Map<string, MONGO_ChartDocument<"sdvx:Single">>();
const existingCharts = new Map<string, ChartDocument<"sdvx">>();
for (const chart of existingChartDocs) {
inGameIDToSongIDMap.set(chart.data.inGameID, chart.songID);
inGameIDToSongIDMap.set(chart.data.inGameID, chart.song.id);
existingCharts.set(`${chart.data.inGameID}-${chart.difficulty}`, chart);
}
@@ -99,7 +99,7 @@ function convertVersion(input: number) {
function convertDiff(
diffString: keyof MDBEntry["difficulty"],
infVer: integer,
): Difficulties["sdvx:Single"] {
): Difficulties["sdvx"] {
switch (diffString) {
case "novice":
return "NOV";
@@ -130,8 +130,8 @@ function convertDiff(
}
}
const newSongs: Array<MONGO_SongDocument<"sdvx">> = [];
const newCharts: Array<MONGO_ChartDocument<"sdvx:Single">> = [];
const newSongs: Array<SongDocument<"sdvx">> = [];
const newCharts: Array<ChartDocument<"sdvx">> = [];
// anything we don't want to include?
const blacklist = [1259, 1491, 1438, 1490];
@@ -217,7 +217,7 @@ for (const entry of data.mdb.music as Array<MDBEntry>) {
}
// new song, add to seeds.
const songDoc: MONGO_SongDocument<"sdvx"> = {
const songDoc: SongDocument<"sdvx"> = {
title: fixedTitle,
artist: fixString(entry.info.artist_name),
searchTerms: [entry.info.ascii],
@@ -277,9 +277,10 @@ for (const entry of data.mdb.music as Array<MDBEntry>) {
continue;
}
const chartDoc: MONGO_ChartDocument<"sdvx:Single"> = {
const chartDoc: ChartDocument<"sdvx"> = {
chartID: CreateChartID(),
songID,
game: "sdvx",
difficulty,
isPrimary: true,
level: maybeEntry.difnum["#text"].toString(),
@@ -2,12 +2,12 @@
import * as cheerio from "cheerio";
import fetch from "node-fetch";
import { type MONGO_ChartDocument, type MONGO_SongDocument } from "tachi-common";
import { type ChartDocument, type SongDocument } from "tachi-common";
import { ReadCollection, WriteCollection } from "../../util";
type SDVXChart = MONGO_ChartDocument<"sdvx:Single">;
type SDVXSong = MONGO_SongDocument<"sdvx">;
type SDVXChart = ChartDocument<"sdvx">;
type SDVXSong = SongDocument<"sdvx">;
// Internal data for PUC tierlists are inconsistent
const MANUAL_PUC_TIERS: { [level: number]: { [tier: string]: string } } = {
@@ -92,7 +92,7 @@ async function scrape(
level: number,
tierlistType: "PUC_LAMP" | "S_RANK",
) {
const html = await fetch(url).then((r: Response) => r.text());
const html = await fetch(url).then((r) => r.text());
const $ = cheerio.load(html);
for (const tierBox of $(".tier_box")) {
@@ -26,7 +26,12 @@ if (!table) {
MutateCollection("folders.json", (folders) => {
for (const folder of folders) {
if (table.folders.includes(folder.folderID)) {
const id = folder.folderID ?? folder.id;
const matchesRef =
(folder.slug !== undefined && table.folders.includes(folder.slug)) ||
(id !== undefined && table.folders.includes(id));
if (matchesRef) {
folder.inactive = !folder.inactive;
}
}
@@ -1,6 +1,6 @@
import fs from "fs";
import path from "path";
import { GAME_GROUP_CONFIGS, GAME_PT_CONFIGS, GPTStringToV3 } from "tachi-common";
import { ALL_GAMES, GAME_GROUP_CONFIGS } from "tachi-common";
import { ReadCollection, WriteCollection } from "../../util";
@@ -58,9 +58,7 @@ for (const game of Object.keys(GAME_GROUP_CONFIGS)) {
}
// now, for each collection remove the playtype field.
for (const gptString of Object.keys(GAME_PT_CONFIGS)) {
const nugame = GPTStringToV3(gptString as any);
for (const nugame of ALL_GAMES) {
const data = ReadCollection(`charts-${nugame}.json`);
for (const entry of data) {
delete entry.playtype;
@@ -1,6 +1,6 @@
import fs from "fs";
import path from "path";
import { GAME_GROUP_CONFIGS, GAME_PT_CONFIGS, GPTStringToV3, V3ToGameGroup } from "tachi-common";
import { ALL_GAMES, GAME_GROUP_CONFIGS, GameToGameGroup } from "tachi-common";
import { CreateChartID, CreateSongID, ReadCollection, WriteCollection } from "../../util";
@@ -72,9 +72,8 @@ for (const gameGroup of Object.keys(GAME_GROUP_CONFIGS)) {
// ── Pass 2: charts ────────────────────────────────────────────────────────────
for (const oldGpt of Object.keys(GAME_PT_CONFIGS)) {
const game = GPTStringToV3(oldGpt as any);
const gameGroup = V3ToGameGroup(game);
for (const game of ALL_GAMES) {
const gameGroup = GameToGameGroup(game);
const collection = `charts-${game}.json`;
const data = ReadCollection(collection);
let modified = 0;
@@ -1,6 +1,17 @@
/**
* Legacy Mongo → v3 hex ids for folders/tables. After this, run `4-tablefolders-to-object.ts`,
* `5-folders-to-sql-queries.ts`, then `6-tables-folder-refs-to-slugs.ts` so `tables.json`
* references folders by slug.
*/
import fs from "fs";
import path from "path";
import { type GameGroup, GetGPTString, GPTStringToV3, type Playtype } from "tachi-common";
import {
type GameGroup,
GPTStringToGame,
LEGACY_GetGPTString,
type LEGACY_Playtype,
} from "tachi-common";
import { CreateFolderID, CreateTableID, ReadCollection, WriteCollection } from "../../util";
@@ -47,7 +58,9 @@ for (const entry of folders) {
delete entry.folderID;
}
entry.game = GPTStringToV3(GetGPTString(entry.game as GameGroup, entry.playtype as Playtype));
entry.game = GPTStringToGame(
LEGACY_GetGPTString(entry.game as GameGroup, entry.playtype as LEGACY_Playtype),
);
delete entry.playtype;
}
@@ -69,7 +82,9 @@ for (const entry of tables) {
delete entry.tableID;
}
entry.game = GPTStringToV3(GetGPTString(entry.game as GameGroup, entry.playtype as Playtype));
entry.game = GPTStringToGame(
LEGACY_GetGPTString(entry.game as GameGroup, entry.playtype as LEGACY_Playtype),
);
delete entry.playtype;
// Rewrite folder references to new hex ids.
@@ -1,7 +1,16 @@
import fs from "fs";
import path from "path";
import { computeFolderSlug, type SeedFolderRow } from "tachi-common";
import { fileURLToPath } from "url";
/**
* Converts legacy Mongo-style folder `data` into SQL `where` strings, then assigns
* `slug` per {@link computeFolderSlug} (required for Postgres `folder.slug`).
*
* After this, run `6-tables-folder-refs-to-slugs.ts` so `tables.json` lists folder
* slugs instead of ids (see `importSeeds` in the server).
*/
// ── Helpers ───────────────────────────────────────────────────────────────────
function sqlStr(value: string): string {
@@ -424,5 +433,22 @@ for (const entry of folders) {
delete entry.type;
}
const slugKeys = new Map<string, string>();
for (const entry of folders) {
const slug = computeFolderSlug(entry as SeedFolderRow);
const game = String(entry.game);
const key = `${game}\0${slug}`;
if (slugKeys.has(key)) {
throw new Error(
`Duplicate slug "${slug}" for game ${game}: "${slugKeys.get(key)}" vs "${String(entry.title)}"`,
);
}
slugKeys.set(key, String(entry.title));
entry.slug = slug;
}
fs.writeFileSync(FOLDERS_JSON, `${JSON.stringify(folders, null, "\t")}\n`);
console.log(`Wrote ${FOLDERS_JSON} (${folders.length} folders).`);
console.log(`Wrote ${FOLDERS_JSON} (${folders.length} folders, with slugs).`);
@@ -0,0 +1,21 @@
import { ReadCollection, WriteCollection } from "../../util";
const courses = ReadCollection("bms-course-lookup.json");
let modified = 0;
for (const entry of courses) {
if (entry.game !== undefined && entry.playtype === undefined) {
continue;
}
if (entry.playtype === undefined) {
throw new Error(`Entry missing both playtype and game: ${JSON.stringify(entry)}`);
}
entry.game = entry.playtype === "7K" ? "bms-7k" : "bms-14k";
delete entry.playtype;
modified++;
}
WriteCollection("bms-course-lookup.json", courses);
console.log(`bms-course-lookup.json: migrated ${modified} entries`);
@@ -0,0 +1,96 @@
/**
* Rewrites `tables.json` so each `folders[]` entry is a folder **slug** (per game),
* not a hex folder id. Run after `5-folders-to-sql-queries.ts` (folders must have
* `where` and `slug`).
*
* Idempotent: if an entry is already a slug for that game, it is left unchanged.
*/
import fs from "fs";
import path from "path";
import { computeFolderSlug, type SeedFolderRow } from "tachi-common";
import { fileURLToPath } from "url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const SEEDS = path.join(__dirname, "../../../../db/seeds");
function resolvedFolderSlug(f: {
game: string;
id: string;
slug?: string;
title: string;
versionFilter?: Array<string>;
where: string;
}): string {
if (f.slug !== undefined && f.slug !== "") {
return f.slug;
}
return computeFolderSlug({
game: f.game,
id: f.id,
title: f.title,
where: f.where,
versionFilter: f.versionFilter,
} as SeedFolderRow);
}
const foldersPath = path.join(SEEDS, "folders.json");
const tablesPath = path.join(SEEDS, "tables.json");
const folders = JSON.parse(fs.readFileSync(foldersPath, "utf-8")) as Array<{
game: string;
id: string;
slug?: string;
title: string;
versionFilter?: Array<string>;
where: string;
}>;
const idToSlug = new Map<string, string>();
const slugByGame = new Map<string, Set<string>>();
for (const f of folders) {
const slug = resolvedFolderSlug(f);
idToSlug.set(f.id, slug);
const set = slugByGame.get(f.game) ?? new Set<string>();
set.add(slug);
slugByGame.set(f.game, set);
}
const tables = JSON.parse(fs.readFileSync(tablesPath, "utf-8")) as Array<{
folders: Array<string>;
game: string;
id: string;
title: string;
}>;
for (const t of tables) {
const validSlugs = slugByGame.get(t.game);
if (validSlugs === undefined) {
throw new Error(`Table "${t.title}": no folders for game ${JSON.stringify(t.game)}`);
}
t.folders = t.folders.map((ref, i) => {
const asSlug = idToSlug.get(ref);
if (asSlug !== undefined) {
return asSlug;
}
if (validSlugs.has(ref)) {
return ref;
}
throw new Error(
`Table "${t.title}" (${t.id}): folders[${i}] = ${JSON.stringify(ref)} is neither a known folder id nor a slug for game ${JSON.stringify(t.game)}`,
);
});
}
fs.writeFileSync(tablesPath, `${JSON.stringify(tables, null, "\t")}\n`);
console.log(`Wrote ${tablesPath} (${tables.length} tables; folder refs are slugs).`);
@@ -35,7 +35,7 @@ const existingChartDocs = ReadCollection("charts-wacca.json");
const inGameIDToSongIDMap = new Map();
for (const chart of existingChartDocs) {
inGameIDToSongIDMap.set(chart.data.inGameID, chart.songID);
inGameIDToSongIDMap.set(chart.data.inGameID, chart.song.id);
}
for (const song of songdata) {
+2 -2
View File
@@ -53,8 +53,8 @@ function TableSort(a, b) {
}
function BMSCourseSort(a, b) {
if (a.playtype !== b.playtype) {
return a.playtype.localeCompare(b.playtype);
if (a.game !== b.game) {
return a.game.localeCompare(b.game);
}
if (a.set !== b.set) {
@@ -1,7 +1,7 @@
import chalk from "chalk";
import fs from "fs";
import path from "path";
import { type GameGroup, V3ToGameGroup } from "tachi-common";
import { type GameGroup, GameToGameGroup } from "tachi-common";
import { z } from "zod";
import { GetChartCollectionGame, GetSongCollectionGame, ReadCollection } from "../util";
@@ -54,7 +54,7 @@ for (const collection of collections) {
throw new Error(`Collection passed was literally ${collection}, why?`);
}
gameGroup = V3ToGameGroup(maybeGame);
gameGroup = GameToGameGroup(maybeGame);
}
for (const d of data) {
+5 -5
View File
@@ -2,10 +2,10 @@ import chalk from "chalk";
import fs from "fs";
import path from "path";
import {
type ChartDocument,
type GameGroup,
type GPTStrings,
type MONGO_ChartDocument,
type MONGO_SongDocument,
type GamesForGroup,
type SongDocument,
} from "tachi-common";
import { ReadCollection } from "../util";
@@ -26,7 +26,7 @@ function test<T>(desc: string, fn: TestFn<T>, reason?: FmtFn<T>): Test<T> {
// TODO(zk): This is stratified on GameGroup when it
// should really be on Game, alas, poor yorick, I knew
// him well.
const CHART_CHECKS: { [G in GameGroup]?: Array<Test<MONGO_ChartDocument<GPTStrings[G]>>> } = {
const CHART_CHECKS: { [G in GameGroup]?: Array<Test<ChartDocument<GamesForGroup[G]>>> } = {
iidx: [
test("Level should not be 0", (c) => c.level !== "0"),
test("LevelNum should be an integer greater than 0 if level is known", (c) =>
@@ -110,7 +110,7 @@ const CHART_CHECKS: { [G in GameGroup]?: Array<Test<MONGO_ChartDocument<GPTStrin
],
};
const SONG_CHECKS: { [G in GameGroup]?: Array<Test<MONGO_SongDocument<G>>> } = {};
const SONG_CHECKS: { [G in GameGroup]?: Array<Test<SongDocument<G>>> } = {};
let exitCode = 0;
const suites: Array<{ good: boolean; name: string; report: unknown }> = [];
@@ -1,42 +1,3 @@
import chalk from "chalk";
import { type MONGO_GoalDocument } from "tachi-common";
import { CreateGoalID, ReadCollection } from "../util";
import { FormatFunctions } from "./test-utils";
// disabled for now
console.log("Disabled for now");
process.exit(0);
let success = 0;
let fails = 0;
const goals = ReadCollection("goals.json", true) as MONGO_GoalDocument[];
for (const goal of goals) {
const pretty = FormatFunctions["goals.json"]!(goal, null);
const expectedGoalID = CreateGoalID(goal.charts, goal.criteria, goal.game);
if (expectedGoalID !== goal.goalID) {
console.error(
chalk.red(
`[ERR] | ${pretty} | Expected goalID to be ${expectedGoalID}. Got ${goal.goalID}.`,
),
);
fails++;
} else {
success++;
}
}
console.log(chalk[fails === 0 ? "green" : "red"](`[GOAL_ID]: ${success} good, ${fails} bad.`));
if (fails !== 0) {
console.log(
chalk.yellow("Run scripts/rerunners/fix-goal-ids.js to automatically fix these issues."),
);
}
process.exit(fails !== 0 ? 1 : 0);
+5 -5
View File
@@ -2,12 +2,12 @@ import chalk from "chalk";
import fjsh from "fast-json-stable-hash";
import get from "lodash.get";
import {
ALL_GAMES,
allSupportedGameGroups,
type GameGroup,
GameToGameGroup,
type integer,
v3AllGames,
type V3Game,
V3ToGameGroup,
} from "tachi-common";
import { GetChartCollectionGame, GetSongCollectionGame, ReadCollection } from "../util";
@@ -32,12 +32,12 @@ for (const gameGroup of allSupportedGameGroups) {
SongChartKeys[`songs-${gameGroup}`] = ["id", "legacySongID"];
}
for (const v3Game of v3AllGames) {
for (const v3Game of ALL_GAMES) {
SongChartKeys[`charts-${v3Game}`] = chartDupKeyDecls();
}
const UniqueKeys: Record<string, DuplicateKeyDecl[]> = {
"bms-course-lookup": [["set", "playtype", "value"]],
"bms-course-lookup": [["set", "game", "value"]],
folders: ["id", "legacyFolderID"],
tables: ["id", "legacyTableID"],
questlines: ["questlineID"],
@@ -110,7 +110,7 @@ function gameGroupForFormat(collection: string): GameGroup {
if (collection.startsWith("charts-")) {
const v3Game = GetChartCollectionGame(`${collection}.json`) as V3Game;
return V3ToGameGroup(v3Game);
return GameToGameGroup(v3Game);
}
throw new Error(`Expected songs-* or charts-* collection, got ${collection}`);
@@ -1,14 +1,13 @@
import chalk from "chalk";
import {
GetGPTConfig,
ALL_GAMES,
type ChartDocument,
FormatGame,
GameToGameGroup,
GetGameConfig,
type MatchTypes,
type MONGO_ChartDocument,
type MONGO_SongDocument,
v3AllGames,
V3FormatGame,
type SongDocument,
type V3Game,
V3ToGameGroup,
V3ToGPTString,
} from "tachi-common";
import { ReadCollection } from "../util";
@@ -16,10 +15,10 @@ import { ReadCollection } from "../util";
// check that a given matchType works for a given game.
const uniquenessChecks: Array<{ game: V3Game; matchType: MatchTypes }> = [];
for (const game of v3AllGames) {
const gptConfig = GetGPTConfig(V3ToGPTString(game));
for (const game of ALL_GAMES) {
const gameConfig = GetGameConfig(game);
for (const matchType of gptConfig.supportedMatchTypes) {
for (const matchType of gameConfig.supportedMatchTypes) {
uniquenessChecks.push({ game, matchType });
}
}
@@ -44,7 +43,7 @@ const MATCH_TYPE_CHECKS: Record<
},
bmsChartHash: {
type: "CHARTS",
fn: (c: MONGO_ChartDocument<"bms:7K" | "bms:14K">) => [c.data.hashMD5, c.data.hashSHA256],
fn: (c: ChartDocument<"bms-7k" | "bms-14k">) => [c.data.hashMD5, c.data.hashSHA256],
},
inGameID: { type: "CHARTS", fn: (c) => `${c.data.inGameID}-${c.difficulty}` },
inGameStrID: { type: "CHARTS", fn: (c) => `${c.data.inGameStrID}-${c.difficulty}` },
@@ -65,7 +64,7 @@ const MATCH_TYPE_CHECKS: Record<
uscChartHash: { type: "CHARTS", fn: (c) => c.data.hashSHA1 },
ddrSongHash: {
type: "SONGS",
fn: (s: MONGO_SongDocument<"ddr">) => {
fn: (s: SongDocument<"ddr">) => {
// if there's no ddrSongHash then it's a konaste song / we're missing seed data
// so just use the inGameID
if (s.data.ddrSongHash === undefined) {
@@ -80,7 +79,7 @@ let exitCode = 0;
const suites: Array<{ good: boolean; name: string; report: unknown }> = [];
for (const { game, matchType } of uniquenessChecks) {
const name = `${V3FormatGame(game)} ${matchType}`;
const name = `${FormatGame(game)} ${matchType}`;
console.log(`[CHECKING MATCHTYPE] ${name}.`);
const handler = MATCH_TYPE_CHECKS[matchType];
@@ -92,7 +91,7 @@ for (const { game, matchType } of uniquenessChecks) {
const data =
handler.type === "CHARTS"
? ReadCollection(`charts-${game}.json`)
: ReadCollection(`songs-${V3ToGameGroup(game)}.json`);
: ReadCollection(`songs-${GameToGameGroup(game)}.json`);
const uniqueIDs = new Set();
for (const el of data) {
@@ -140,7 +139,7 @@ for (const { game, matchType } of uniquenessChecks) {
if (matchType === "songTitle") {
console.log(
chalk.yellow(
`Song title ${maybeUnique} wasn't case-insensitively unique in ${V3FormatGame(
`Song title ${maybeUnique} wasn't case-insensitively unique in ${FormatGame(
game,
)}. Imports using this song title *will* have their scores rejected.`,
),
@@ -149,7 +148,7 @@ for (const { game, matchType } of uniquenessChecks) {
} else {
console.log(
chalk.red(
`ID ${maybeUnique} wasn't unique in ${V3FormatGame(
`ID ${maybeUnique} wasn't unique in ${FormatGame(
game,
)} (matchType=${matchType}). It needs to be for this matchType to be legal.`,
),
@@ -2,11 +2,28 @@
// and no quest-goal desync, etc.
import chalk from "chalk";
import { type GameGroup, v3AllGames, type V3Game, V3ToGameGroup } from "tachi-common";
import {
ALL_GAMES,
computeFolderSlug,
type GameGroup,
GameToGameGroup,
type SeedFolderRow,
type V3Game,
} from "tachi-common";
import { ReadCollection } from "../util";
import { FormatFunctions } from "./test-utils";
function folderSeedSlug(folder: Record<string, unknown>): string {
const f = folder as SeedFolderRow;
if (typeof f.slug === "string" && f.slug !== "") {
return f.slug;
}
return computeFolderSlug(f);
}
interface ReferenceCheck {
base: string;
parent: string;
@@ -17,12 +34,12 @@ interface ReferenceCheck {
}
const refChecks: Array<ReferenceCheck> = [
...v3AllGames.map((game) => ({
...ALL_GAMES.map((game) => ({
base: `charts-${game}`,
parent: `songs-${V3ToGameGroup(game)}`,
parent: `songs-${GameToGameGroup(game)}`,
baseKey: "songID",
parentKey: "id",
gameGroup: V3ToGameGroup(game),
gameGroup: GameToGameGroup(game),
game,
})),
{
@@ -35,9 +52,9 @@ const refChecks: Array<ReferenceCheck> = [
},
{
base: "tables",
baseKey: (table) => table.folders,
baseKey: (table) => table.folders.map((slug: string) => `${table.game}:${slug}`),
parent: "folders",
parentKey: "id",
parentKey: (folder) => `${folder.game}:${folderSeedSlug(folder)}`,
gameGroup: null,
game: null,
},
+6 -7
View File
@@ -1,11 +1,10 @@
import {
ALL_GAMES,
allSupportedGameGroups,
GAME_CONFIGS,
GAME_GROUP_CONFIGS,
GAME_PT_CONFIGS,
type GameGroup,
v3AllGames,
type V3Game,
V3ToGPTString,
} from "tachi-common";
import { z, type ZodType } from "zod";
@@ -19,7 +18,7 @@ export type AllCollections =
| `charts-${V3Game}.json`
| `songs-${GameGroup}.json`;
export const V3_GAME_SCHEMA = z.enum(v3AllGames as [V3Game, ...Array<V3Game>]);
export const V3_GAME_SCHEMA = z.enum(ALL_GAMES as [V3Game, ...Array<V3Game>]);
export const V3_GAME_GROUP_SCHEMA = z.enum(
allSupportedGameGroups as [GameGroup, ...Array<GameGroup>],
);
@@ -55,7 +54,7 @@ export const V3_BMS_COURSE_LOOKUP_SCHEMA = z.strictObject({
md5sums: z.string(),
title: z.string(),
set: z.string(),
playtype: z.string(),
game: z.enum(["bms-7k", "bms-14k"]),
value: z.string(),
});
@@ -82,7 +81,7 @@ export const V3_SONG_SCHEMAS: Record<`songs-${GameGroup}.json`, ZodType> = Objec
// even when k is a subset of string
export const V3_CHART_SCHEMAS: Record<`charts-${V3Game}.json`, ZodType> = Object.fromEntries(
v3AllGames.map((game) => [
ALL_GAMES.map((game) => [
`charts-${game}.json` as const,
z.strictObject({
id: V3_TACHI_ID("C"),
@@ -94,7 +93,7 @@ export const V3_CHART_SCHEMAS: Record<`charts-${V3Game}.json`, ZodType> = Object
levelNum: z.number(),
songID: V3_TACHI_ID("S"),
versions: z.array(z.string()),
data: GAME_PT_CONFIGS[V3ToGPTString(game)].chartData,
data: GAME_CONFIGS[game].chartData,
}),
]),
) as unknown as Record<`charts-${V3Game}.json`, ZodType>;
@@ -1,7 +1,7 @@
import type { z } from "zod";
import chalk from "chalk";
import { v3AllGames, V3FormatGame } from "tachi-common";
import { ALL_GAMES, FormatGame } from "tachi-common";
import { ReadCollection } from "../util";
import { type V3_TABLE_SCHEMA } from "./schemas";
@@ -16,7 +16,7 @@ for (const t of tables) {
if (t.default && t.inactive) {
console.log(
chalk.red(
`[TABLE-DEFAULT] The default table for ${V3FormatGame(t.game)} '${
`[TABLE-DEFAULT] The default table for ${FormatGame(t.game)} '${
t.title
}' is inactive. This is not legal.`,
),
@@ -28,7 +28,7 @@ for (const t of tables) {
if (defaultTableMap[t.game]) {
console.log(
chalk.red(
`[TABLE-DEFAULT] There are multiple default tables for ${V3FormatGame(
`[TABLE-DEFAULT] There are multiple default tables for ${FormatGame(
t.game,
)}. This is not legal.`,
),
@@ -39,7 +39,7 @@ for (const t of tables) {
}
}
for (const game of v3AllGames) {
for (const game of ALL_GAMES) {
if (!defaultTableMap[game]) {
console.log(chalk.red(`[TABLE-DEFAULT] There is no default table for ${game}.`));
errs += 1;
+2 -2
View File
@@ -1,4 +1,4 @@
import { allSupportedGameGroups, type GameGroup, v3AllGames } from "tachi-common";
import { ALL_GAMES, allSupportedGameGroups, type GameGroup } from "tachi-common";
import { ReadCollection } from "../util";
import { type V3_SCHEMAS } from "./schemas";
@@ -37,7 +37,7 @@ for (const gameGroup of allSupportedGameGroups) {
FormatFunctions[`songs-${gameGroup}.json`] = songFormat;
}
for (const game of v3AllGames) {
for (const game of ALL_GAMES) {
// TODO(zk): again just a part of this shitty hack
// eslint-disable-next-line @typescript-eslint/no-explicit-any
FormatFunctions[`charts-${game}.json`] = chartFormat as any;
+7
View File
@@ -107,6 +107,12 @@ function GetSongCollectionGameGroup(filename) {
return result[1];
}
function GetFreshSongIDGenerator(gameGroup) {
const existing = ReadCollection(`songs-${gameGroup}.json`);
let max = existing.reduce((acc, s) => Math.max(acc, s.id ?? 0), 0);
return () => ++max;
}
module.exports = {
CreateChartID,
CreateFolderID,
@@ -118,6 +124,7 @@ module.exports = {
CreateTableID,
EfficientInPlaceDeepmerge,
GetChartCollectionGame,
GetFreshSongIDGenerator,
GetSongCollectionGame: GetSongCollectionGameGroup,
IterateCollections,
MutateCollection,