mirror of
https://github.com/zkldi/Tachi.git
synced 2026-10-05 05:18:12 +03:00
move scripts to toplevel
This commit is contained in:
zkldi
@@ -1,8 +0,0 @@
|
||||
Hey. If by any chance I'm not available, you can (and should) run these scripts with TS-Node.
|
||||
https://github.com/TypeStrong/ts-node
|
||||
|
||||
They have only been tested for ts-node.
|
||||
|
||||
npm install -g ts-node
|
||||
|
||||
it's probably installed on the machine you're seeing this on, winter(?). love ya.
|
||||
@@ -1,5 +0,0 @@
|
||||
This folder is for porting documents from the old KamaitachiDB to the newer formats.
|
||||
|
||||
The scripts here should NOT be ran unless you know *exactly* what the hell you're doing.
|
||||
|
||||
And, even then - basically only I know, so. Please be careful.
|
||||
@@ -1,53 +0,0 @@
|
||||
import { ChartDocument } from "kamaitachi-common";
|
||||
import db from "../../db/db";
|
||||
import { rootLogger } from "../../logger";
|
||||
import MigrateRecords from "./migrate";
|
||||
|
||||
function ConvertFn(c: any): ChartDocument<"iidx:SP" | "iidx:DP"> {
|
||||
const newChartDoc: ChartDocument<"iidx:SP" | "iidx:DP"> = {
|
||||
rgcID: null,
|
||||
chartID: c.chartID,
|
||||
difficulty: c.difficulty,
|
||||
songID: c.id,
|
||||
playtype: c.playtype,
|
||||
levelNum: c.levelNum,
|
||||
isRemoved: false,
|
||||
level: c.level.toString(),
|
||||
length: c.length,
|
||||
bpmString: "", // sentinel
|
||||
flags: {
|
||||
"IN BASE GAME": !!c.flags["IN BASE GAME"],
|
||||
OMNIMIX: !!c.flags.OMNIMIX,
|
||||
"N-1": !!c.flags["N-1"],
|
||||
},
|
||||
data: {
|
||||
inGameID: c.internals.inGameINTID,
|
||||
notecount: c.notedata.notecount,
|
||||
},
|
||||
};
|
||||
|
||||
if (typeof c.length === "number") {
|
||||
let m = Math.floor(c.length / 60_000);
|
||||
let s = c.length - m * 60_000;
|
||||
|
||||
newChartDoc.length = `${m}:${s}`;
|
||||
}
|
||||
|
||||
if (c.monoBPM) {
|
||||
newChartDoc.bpmString = c.bpmMin.toString();
|
||||
} else {
|
||||
newChartDoc.bpmString = `${c.bpmMin}-${c.bpmMax}`;
|
||||
}
|
||||
|
||||
if (newChartDoc.length === null) {
|
||||
rootLogger.warn(`Length of chart ${newChartDoc.chartID} is null, continuing anyway?`);
|
||||
}
|
||||
|
||||
return newChartDoc;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
await MigrateRecords(db.charts.iidx, "charts-iidx", ConvertFn);
|
||||
|
||||
process.exit(0);
|
||||
})();
|
||||
@@ -1,46 +0,0 @@
|
||||
import { ICollection } from "monk";
|
||||
import CreateLogCtx from "../../logger";
|
||||
import { oldKTDB } from "./old-db";
|
||||
|
||||
const logger = CreateLogCtx("MIGRATE.ts");
|
||||
|
||||
let newDocuments: unknown[] = [];
|
||||
|
||||
export default async function MigrateRecords(
|
||||
existingCollection: ICollection,
|
||||
collectionName: string,
|
||||
HandlerFN: (c: unknown) => unknown
|
||||
) {
|
||||
let existingRecords = await existingCollection.count({});
|
||||
logger.info(`Starting migration for ${collectionName}...`);
|
||||
|
||||
if (existingRecords > 0) {
|
||||
logger.error(`${existingRecords} documents already exist in db, terminating.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let i = 0;
|
||||
|
||||
await oldKTDB
|
||||
.get(collectionName)
|
||||
.find({})
|
||||
// @ts-expect-error it exists.
|
||||
.each((c) => {
|
||||
i++;
|
||||
|
||||
if (i % 10000 === 0) {
|
||||
logger.info(`Processed ${i} documents.`);
|
||||
}
|
||||
const newDoc = HandlerFN(c);
|
||||
|
||||
if (newDoc !== null) {
|
||||
newDocuments.push(newDoc);
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`Inserting ${newDocuments.length} documents.`);
|
||||
await existingCollection.insert(newDocuments);
|
||||
logger.info(`Done!`);
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
import monk from "monk";
|
||||
|
||||
export const oldKTDB = monk(`${process.env.MONGO_BASE_URL}/kamaitachidb`);
|
||||
@@ -1,145 +0,0 @@
|
||||
import { PrivateUserDocument, ScoreDocument } from "kamaitachi-common";
|
||||
import { grades, lamps, supportedGames, validPlaytypes } from "kamaitachi-common/js/config";
|
||||
import db from "../../db/db";
|
||||
import { rootLogger } from "../../logger";
|
||||
import { CreateScoreID } from "../../score-import/framework/core/score-id";
|
||||
import MigrateRecords from "./migrate";
|
||||
|
||||
// HERE. WE. GO!
|
||||
|
||||
function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
|
||||
if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
|
||||
base[baseProp] = other[otherProp];
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertFn(c: any): ScoreDocument | null {
|
||||
if (!supportedGames.includes(c.game)) {
|
||||
rootLogger.warn(`Ignored game ${c.game}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (["popn", "jubeat", "gitadora", "usc"].includes(c.game)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// @ts-expect-error yea
|
||||
if (validPlaytypes[c.game].includes(c.playtype)) {
|
||||
rootLogger.warn(`Ignored game pt ${c.game}, ${c.playtype}`);
|
||||
}
|
||||
|
||||
let base: Omit<ScoreDocument, "scoreID"> = {
|
||||
userID: c.userID,
|
||||
songID: c.songID,
|
||||
playtype: c.scoreData.playtype,
|
||||
chartID: c.chartID,
|
||||
game: c.game,
|
||||
timeAdded: c.timeAdded,
|
||||
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
|
||||
isScorePB: !!c.isScorePB,
|
||||
isLampPB: !!c.isLampPB,
|
||||
comment: c.comment ?? null,
|
||||
highlight: c.highlight ?? false,
|
||||
service: c.service,
|
||||
importType: null,
|
||||
calculatedData: c.calculatedData,
|
||||
scoreData: {
|
||||
esd: c.scoreData.esd ?? null,
|
||||
grade: c.scoreData.grade,
|
||||
// @ts-expect-error shut
|
||||
gradeIndex: grades[c.game].indexOf(c.scoreData.grade),
|
||||
lamp: c.scoreData.lamp,
|
||||
// @ts-expect-error shut
|
||||
lampIndex: lamps[c.game].indexOf(c.scoreData.lamp),
|
||||
percent: c.scoreData.percent,
|
||||
score: c.scoreData.score,
|
||||
hitData: c.scoreData.hitData,
|
||||
hitMeta: c.scoreData.hitMeta,
|
||||
},
|
||||
scoreMeta: {},
|
||||
};
|
||||
|
||||
if (c.scoreMeta) {
|
||||
if (base.game === "iidx") {
|
||||
ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom");
|
||||
ConditionalAssign(base.scoreMeta, "gauge", c.scoreMeta, "optionsGauge");
|
||||
ConditionalAssign(base.scoreMeta, "assist", c.scoreMeta, "optionsAssist");
|
||||
ConditionalAssign(base.scoreMeta, "range", c.scoreMeta, "optionsRange");
|
||||
ConditionalAssign(base.scoreMeta, "pacemaker", c.scoreMeta, "pacemaker");
|
||||
ConditionalAssign(base.scoreMeta, "pacemakerName", c.scoreMeta, "pacemakerName");
|
||||
ConditionalAssign(base.scoreMeta, "pacemakerTarget", c.scoreMeta, "pacemakerTarget");
|
||||
|
||||
if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
|
||||
base.scoreMeta.random = null;
|
||||
}
|
||||
|
||||
// @ts-expect-error asdf
|
||||
if (base.scoreMeta.range === "") {
|
||||
// @ts-expect-error asdf
|
||||
base.scoreMeta.range = "NONE";
|
||||
}
|
||||
} else if (base.game === "bms") {
|
||||
ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom");
|
||||
ConditionalAssign(base.scoreMeta, "inputDevice", c.scoreData.hitMeta, "inputDevice");
|
||||
}
|
||||
}
|
||||
|
||||
// @ts-expect-error shut
|
||||
if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
|
||||
// @ts-expect-error shut
|
||||
base.scoreData.hitMeta.bp = null;
|
||||
}
|
||||
|
||||
if (base.game === "iidx") {
|
||||
// @ts-expect-error fuck you
|
||||
if (base.scoreData.hitMeta.gauge > 200) {
|
||||
// @ts-expect-error fuck you
|
||||
base.scoreData.hitMeta.gauge = null;
|
||||
}
|
||||
|
||||
// @ts-expect-error fuck you
|
||||
if (base.scoreData.hitMeta.gaugeHistory) {
|
||||
// @ts-expect-error fuck you
|
||||
base.scoreData.hitMeta.gaugeHistory = base.scoreData.hitMeta.gaugeHistory.map((e) =>
|
||||
e > 200 ? null : e
|
||||
);
|
||||
}
|
||||
} else if (base.game === "bms") {
|
||||
// @ts-expect-error shut
|
||||
if (base.scoreData.hitMeta.gauge === -1) {
|
||||
// @ts-expect-error shut
|
||||
base.scoreData.hitMeta.gauge = null;
|
||||
}
|
||||
|
||||
// @ts-expect-error shut
|
||||
delete base.scoreData.hitMeta.inputDevice;
|
||||
} else if (base.game === "sdvx" || base.game === "usc") {
|
||||
ConditionalAssign(base.scoreData.hitData, "miss", base.scoreData.hitData, "error");
|
||||
|
||||
//@ts-expect-error yea
|
||||
delete base.scoreData.hitData.error;
|
||||
} else if (base.game === "chunithm") {
|
||||
if (base.scoreData.percent === 101) {
|
||||
base.scoreData.grade === "SSS";
|
||||
}
|
||||
}
|
||||
|
||||
if (base.service === "mm") {
|
||||
base.service = "maimagic";
|
||||
}
|
||||
|
||||
let scoreID = CreateScoreID(base.userID, base, base.chartID);
|
||||
|
||||
let score: ScoreDocument = {
|
||||
...base,
|
||||
scoreID,
|
||||
};
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
await MigrateRecords(db.scores, "scores", ConvertFn);
|
||||
|
||||
process.exit(0);
|
||||
})();
|
||||
@@ -1,40 +0,0 @@
|
||||
import { SongDocument } from "kamaitachi-common";
|
||||
import { gameOrders } from "kamaitachi-common/js/config";
|
||||
import db from "../../db/db";
|
||||
import { rootLogger } from "../../logger";
|
||||
import MigrateRecords from "./migrate";
|
||||
|
||||
function ConvertFn(c: any): SongDocument<"iidx"> {
|
||||
const newSongDoc: SongDocument<"iidx"> = {
|
||||
title: c.title,
|
||||
artist: c.artist,
|
||||
id: c.id,
|
||||
isAvailable: true, // NOT NECESSARILY
|
||||
firstVersion: c.firstAppearance,
|
||||
"alt-titles": c["alt-titles"].filter((e: string) => e !== c.title),
|
||||
"search-titles": c["search-titles"]
|
||||
.map((e: string) => e.toString())
|
||||
.filter((e: string) => e !== c.title),
|
||||
versions: [], // sentinel
|
||||
data: {
|
||||
genre: c.genre,
|
||||
},
|
||||
};
|
||||
|
||||
let idx = gameOrders.iidx.indexOf(c.firstAppearance);
|
||||
|
||||
if (idx === -1) {
|
||||
rootLogger.warn(`Invalid firstAppearance of ${c.firstAppearance}, running anyway.`);
|
||||
newSongDoc.versions = [c.firstAppearance];
|
||||
} else {
|
||||
newSongDoc.versions = gameOrders.iidx.slice(idx);
|
||||
}
|
||||
|
||||
return newSongDoc;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
await MigrateRecords(db.songs.iidx, "songs-iidx", ConvertFn);
|
||||
|
||||
process.exit(0);
|
||||
})();
|
||||
@@ -1,40 +0,0 @@
|
||||
import { PrivateUserDocument } from "kamaitachi-common";
|
||||
import db from "../../db/db";
|
||||
import MigrateRecords from "./migrate";
|
||||
|
||||
function ConvertFn(c: any): PrivateUserDocument {
|
||||
const newUserDoc: PrivateUserDocument = {
|
||||
username: c.username,
|
||||
usernameLowercase: c.username.toLowerCase(),
|
||||
about: c.about,
|
||||
clan: c.clan,
|
||||
customBanner: c.custombanner,
|
||||
customPfp: c.custompfp,
|
||||
email: c.email,
|
||||
friends: c.friends,
|
||||
id: c.id,
|
||||
lastSeen: c.lastSeen,
|
||||
password: c.password,
|
||||
permissions: {},
|
||||
settings: {
|
||||
invisible: !!c.settings.invisible,
|
||||
nsfwSplashes: !!c.settings.nsfwsplashes,
|
||||
},
|
||||
socialMedia: {
|
||||
discord: c.socialmedia.discord || null,
|
||||
twitter: c.socialmedia.twitter || null,
|
||||
youtube: c.socialmedia.youtube || null,
|
||||
twitch: c.socialmedia.twitch || null,
|
||||
github: c.socialmedia.github || null,
|
||||
steam: c.socialmedia.steam || null,
|
||||
},
|
||||
};
|
||||
|
||||
return newUserDoc;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
await MigrateRecords(db.users, "users", ConvertFn);
|
||||
|
||||
process.exit(0);
|
||||
})();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,22 +0,0 @@
|
||||
import db from "../db/db";
|
||||
import CreateLogCtx from "../logger";
|
||||
|
||||
const logger = CreateLogCtx("scripts/initialise-counters.ts");
|
||||
|
||||
(async () => {
|
||||
let users = await db.counters.findOne({
|
||||
counterName: "users",
|
||||
});
|
||||
|
||||
if (users) {
|
||||
throw new Error(`"users" document already exists, exiting.`);
|
||||
}
|
||||
|
||||
await db.counters.insert({
|
||||
counterName: "users",
|
||||
value: 1,
|
||||
});
|
||||
|
||||
logger.info("Successfully initialised counter documents. Exiting.");
|
||||
process.exit(0);
|
||||
})();
|
||||
@@ -1,33 +0,0 @@
|
||||
import monk from "monk";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { Command } from "commander";
|
||||
import CreateLogCtx from "../logger";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
const logger = CreateLogCtx("save-testdata-from-ktblack.ts");
|
||||
|
||||
program.option("-c, --collection <collection>", "The collection to fetch ktblack data from.");
|
||||
|
||||
program.parse(process.argv);
|
||||
const options = program.opts();
|
||||
const collection = options.collection;
|
||||
|
||||
logger.info(`Connecting to ${process.env.MONGO_BASE_URL}/ktblackdb...`);
|
||||
const ktBlackDB = monk(`${process.env.MONGO_BASE_URL}/ktblackdb`);
|
||||
|
||||
(async () => {
|
||||
logger.info(`Fetching data for ${collection}...`);
|
||||
let data = await ktBlackDB.get(collection).find({}, { projection: { _id: 0 } });
|
||||
|
||||
logger.info(`Fetched ${data.length} documents, Writing...`);
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(__dirname, `../test-utils/test-data/kamaitachi/ktblack-${collection}.json`),
|
||||
JSON.stringify(data)
|
||||
);
|
||||
|
||||
logger.info("Done!");
|
||||
process.exit(0);
|
||||
})();
|
||||
@@ -1,65 +0,0 @@
|
||||
import monk from "monk";
|
||||
import { Command } from "commander";
|
||||
import CreateLogCtx from "../logger";
|
||||
import { IndexOptions } from "mongodb";
|
||||
import { ValidDatabases } from "kamaitachi-common";
|
||||
|
||||
const logger = CreateLogCtx("set-indexes.ts");
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program.option("-d, --db <database>", "The database to index.");
|
||||
program.option(
|
||||
"-r, --reset",
|
||||
"Whether to reset all indexes on this database before indexing or not."
|
||||
);
|
||||
|
||||
program.parse(process.argv);
|
||||
const options = program.opts();
|
||||
|
||||
const db = monk(`${process.env.MONGO_BASE_URL ?? "127.0.0.1"}/${options.db}`);
|
||||
|
||||
interface Index {
|
||||
fields: Record<string, unknown>;
|
||||
options?: IndexOptions;
|
||||
}
|
||||
|
||||
function index(fields: Record<string, unknown>, options?: IndexOptions) {
|
||||
return { fields, options };
|
||||
}
|
||||
|
||||
const UNIQUE = { unique: true };
|
||||
|
||||
const indexes: Partial<Record<ValidDatabases, Index[]>> = {
|
||||
scores: [index({ scoreID: 1 }, UNIQUE)],
|
||||
"charts-iidx": [
|
||||
index({ chartID: 1 }, UNIQUE),
|
||||
index({ songID: 1, difficulty: 1, playtype: 1 }, UNIQUE),
|
||||
],
|
||||
"songs-iidx": [index({ id: 1 }, UNIQUE), index({ title: 1 })],
|
||||
sessions: [
|
||||
// lol
|
||||
index({ timeStarted: 1, timeEnded: 1, userID: 1, game: 1, playtype: 1 }),
|
||||
],
|
||||
};
|
||||
|
||||
(async () => {
|
||||
logger.info(`Starting indexing for ${options.db}...`);
|
||||
|
||||
for (const collection in indexes) {
|
||||
if (options.reset) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await db.get(collection).dropIndexes();
|
||||
}
|
||||
|
||||
// @ts-expect-error yeah blah blah
|
||||
for (const index of indexes[collection]) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
let r = await db.get(collection).createIndex(index.fields, index.options);
|
||||
|
||||
logger.info(r);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Done.");
|
||||
})();
|
||||
@@ -1,21 +0,0 @@
|
||||
/* eslint-disable no-console */
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { PRUDENCE_CHART_SCHEMAS } from "../../db/schemas";
|
||||
import p from "prudence";
|
||||
import db from "../../db/db";
|
||||
|
||||
let data = JSON.parse(fs.readFileSync(path.join(__dirname, "./hcnverdata.json"), "utf-8"));
|
||||
|
||||
for (const d of data) {
|
||||
let r = p(d, PRUDENCE_CHART_SCHEMAS.iidx);
|
||||
if (r) {
|
||||
console.dir(r);
|
||||
throw r;
|
||||
}
|
||||
}
|
||||
|
||||
db.charts.iidx.insert(data).then(() => {
|
||||
console.log("Done.");
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -1,4 +0,0 @@
|
||||
/* eslint-disable no-console */
|
||||
import bcrypt from "bcrypt";
|
||||
|
||||
console.log(bcrypt.hashSync("password", 12));
|
||||
@@ -1,242 +0,0 @@
|
||||
[
|
||||
{
|
||||
"songID": 919,
|
||||
"rgcID": null,
|
||||
"chartID": "90a2e4664a1a3f8a3a2fd2d8d6a89fd0a47ca659",
|
||||
"level": "11",
|
||||
"length": null,
|
||||
"levelNum": 11,
|
||||
"bpmString": "178",
|
||||
"playtype": "SP",
|
||||
"difficulty": "HYPER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 1146,
|
||||
"inGameID": 17100
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 919,
|
||||
"rgcID": null,
|
||||
"chartID": "360ac8bd65fe8b019d0457bd6603c2bff29d0cd8",
|
||||
"level": "7",
|
||||
"length": null,
|
||||
"levelNum": 7,
|
||||
"bpmString": "178",
|
||||
"playtype": "SP",
|
||||
"difficulty": "NORMAL",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 518,
|
||||
"inGameID": 17100
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 919,
|
||||
"rgcID": null,
|
||||
"chartID": "d7b1c2beb792de3f76a13297a02d8326367f71bc",
|
||||
"level": "12",
|
||||
"length": null,
|
||||
"levelNum": 12,
|
||||
"bpmString": "178",
|
||||
"playtype": "SP",
|
||||
"difficulty": "ANOTHER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 1629,
|
||||
"inGameID": 17100
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 919,
|
||||
"rgcID": null,
|
||||
"chartID": "e36e466c21f208b7489fadf465a257c5ab00f672",
|
||||
"level": "11",
|
||||
"length": null,
|
||||
"levelNum": 11,
|
||||
"bpmString": "178",
|
||||
"playtype": "DP",
|
||||
"difficulty": "HYPER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 1117,
|
||||
"inGameID": 17100
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 919,
|
||||
"rgcID": null,
|
||||
"chartID": "b421a3efeed03f97f5d0ab1901bf539d398755c8",
|
||||
"level": "6",
|
||||
"length": null,
|
||||
"levelNum": 6,
|
||||
"bpmString": "178",
|
||||
"playtype": "DP",
|
||||
"difficulty": "NORMAL",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 506,
|
||||
"inGameID": 17100
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 919,
|
||||
"rgcID": null,
|
||||
"chartID": "7292edfe76c7178f1f608d5e34c0fa09d8cfcc17",
|
||||
"level": "12",
|
||||
"length": null,
|
||||
"levelNum": 12,
|
||||
"bpmString": "178",
|
||||
"playtype": "DP",
|
||||
"difficulty": "ANOTHER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 1764,
|
||||
"inGameID": 17100
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 994,
|
||||
"rgcID": null,
|
||||
"chartID": "125dd4e63a8fced77696313a7a274b4e1c51df4d",
|
||||
"level": "9",
|
||||
"length": null,
|
||||
"levelNum": 9,
|
||||
"bpmString": "146",
|
||||
"playtype": "SP",
|
||||
"difficulty": "HYPER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 708,
|
||||
"inGameID": 18102
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 994,
|
||||
"rgcID": null,
|
||||
"chartID": "7de251ff40cca708de47cce82cbe58d60bc13561",
|
||||
"level": "5",
|
||||
"length": null,
|
||||
"levelNum": 5,
|
||||
"bpmString": "146",
|
||||
"playtype": "SP",
|
||||
"difficulty": "NORMAL",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 268,
|
||||
"inGameID": 18102
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 994,
|
||||
"rgcID": null,
|
||||
"chartID": "20aabe8ae38b5facc6a4f7953f6c73c4c9817840",
|
||||
"level": "11",
|
||||
"length": null,
|
||||
"levelNum": 11,
|
||||
"bpmString": "146",
|
||||
"playtype": "SP",
|
||||
"difficulty": "ANOTHER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 1125,
|
||||
"inGameID": 18102
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 994,
|
||||
"rgcID": null,
|
||||
"chartID": "fa58148d78b33fdb6bc7437065d35285a962d27d",
|
||||
"level": "9",
|
||||
"length": null,
|
||||
"levelNum": 9,
|
||||
"bpmString": "146",
|
||||
"playtype": "DP",
|
||||
"difficulty": "HYPER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 740,
|
||||
"inGameID": 18102
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 994,
|
||||
"rgcID": null,
|
||||
"chartID": "731a4407beeba780e429796ddc6d64444ea64079",
|
||||
"level": "4",
|
||||
"length": null,
|
||||
"levelNum": 4,
|
||||
"bpmString": "146",
|
||||
"playtype": "DP",
|
||||
"difficulty": "NORMAL",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 429,
|
||||
"inGameID": 18102
|
||||
}
|
||||
},
|
||||
{
|
||||
"songID": 994,
|
||||
"rgcID": null,
|
||||
"chartID": "1807545aa0e71df21c6ab518a0dccdc12aa3535e",
|
||||
"level": "12",
|
||||
"length": null,
|
||||
"levelNum": 12,
|
||||
"bpmString": "146",
|
||||
"playtype": "DP",
|
||||
"difficulty": "ANOTHER",
|
||||
"flags": {
|
||||
"IN BASE GAME": false,
|
||||
"OMNIMIX": true,
|
||||
"N-1": true
|
||||
},
|
||||
"data": {
|
||||
"notecount": 1120,
|
||||
"inGameID": 18102
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -1,9 +0,0 @@
|
||||
import fs from "fs";
|
||||
import db from "../../db/db";
|
||||
import path from "path";
|
||||
|
||||
(async () => {
|
||||
let data = await db.scores.find({ game: "iidx", playtype: "SP" }, { limit: 1000 });
|
||||
fs.writeFileSync(path.join(__dirname, "data.json"), JSON.stringify(data));
|
||||
process.exit(0);
|
||||
})();
|
||||
@@ -1,101 +0,0 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import fetch from "node-fetch";
|
||||
import { Command } from "commander";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { config, Difficulties, ESDCore, IIDXBPIData, ChartDocument } from "kamaitachi-common";
|
||||
import { FindSongOnTitleVersion } from "../score-import/database-lookup/song-title";
|
||||
import { FindChartWithPTDF } from "../score-import/database-lookup/chart-ptdf";
|
||||
import db from "../db/db";
|
||||
import CreateLogCtx from "../logger";
|
||||
const program = new Command();
|
||||
|
||||
const logger = CreateLogCtx("update-bpi-poyashi.ts");
|
||||
|
||||
program.option("-f, --fetch", "Fetch the latest data from the poyashi repo.");
|
||||
|
||||
program.parse(process.argv);
|
||||
const options = program.opts();
|
||||
|
||||
const dataLoc = path.join(__dirname, "../bpi-poyashi-data/output.json");
|
||||
|
||||
const difficultyResolve: Record<string, [string, string]> = {
|
||||
3: ["SP", "HYPER"],
|
||||
4: ["SP", "ANOTHER"],
|
||||
8: ["DP", "HYPER"],
|
||||
9: ["DP", "ANOTHER"],
|
||||
10: ["SP", "LEGGENDARIA"],
|
||||
11: ["DP", "LEGGENDARIA"],
|
||||
};
|
||||
|
||||
async function UpdatePoyashiData() {
|
||||
let data;
|
||||
if (options.fetch) {
|
||||
// lol!
|
||||
logger.info("Fetching data from github...");
|
||||
let rj = await fetch(
|
||||
"https://raw.githubusercontent.com/potakusan/bpim-score-repo/master/output/release.json"
|
||||
).then((r) => r.json());
|
||||
|
||||
logger.info("Fetched data.");
|
||||
fs.writeFileSync(dataLoc, JSON.stringify(rj));
|
||||
logger.info("Saved data to output.json.");
|
||||
|
||||
data = rj;
|
||||
} else {
|
||||
logger.info("Reading data from output.json...");
|
||||
data = JSON.parse(fs.readFileSync(dataLoc, "utf-8"));
|
||||
}
|
||||
|
||||
let realData: IIDXBPIData[] = [];
|
||||
for (const d of data) {
|
||||
const res = difficultyResolve[d.difficulty];
|
||||
|
||||
if (!res) {
|
||||
throw new Error(`Unknown difficulty ${d.difficulty}`);
|
||||
}
|
||||
|
||||
const [playtype, diff] = res;
|
||||
|
||||
let ktchiSong = await FindSongOnTitleVersion("iidx", d.title, "28");
|
||||
|
||||
if (!ktchiSong) {
|
||||
throw new Error(`Cannot find song ${d.title}?`);
|
||||
}
|
||||
|
||||
let ktchiChart = (await FindChartWithPTDF(
|
||||
"iidx",
|
||||
ktchiSong.id,
|
||||
playtype as "SP" | "DP",
|
||||
diff as Difficulties["iidx:DP" | "iidx:SP"]
|
||||
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
|
||||
|
||||
if (!ktchiChart) {
|
||||
throw new Error(
|
||||
`Cannot find chart ${ktchiSong.title} (${ktchiSong.id}) ${playtype}, ${diff}?`
|
||||
);
|
||||
}
|
||||
|
||||
let kavg = Number(d.avg);
|
||||
|
||||
let kesd = ESDCore.CalculateESD(
|
||||
config.judgementWindows.iidx.SP,
|
||||
kavg / (ktchiChart.data.notecount * 2)
|
||||
);
|
||||
|
||||
realData.push({
|
||||
coef: d.coef === -1 ? null : d.coef,
|
||||
kavg: Number(d.avg),
|
||||
wr: Number(d.wr),
|
||||
chartID: ktchiChart.chartID,
|
||||
kesd,
|
||||
});
|
||||
}
|
||||
|
||||
await db["iidx-bpi-data"].remove({});
|
||||
await db["iidx-bpi-data"].insert(realData);
|
||||
|
||||
logger.info(`Inserted ${realData.length} documents.`);
|
||||
}
|
||||
|
||||
UpdatePoyashiData();
|
||||
@@ -1,196 +0,0 @@
|
||||
import Prudence, { PrudenceSchema } from "prudence";
|
||||
import db from "../db/db";
|
||||
import { PRUDENCE_CHART_SCHEMAS, PRUDENCE_SCORE_SCHEMAS, STATIC_SCHEMAS } from "../db/schemas";
|
||||
import CreateLogCtx from "../logger";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { PrudenceError } from "prudence/js/error";
|
||||
import { supportedGames, validPlaytypes } from "kamaitachi-common/js/config";
|
||||
|
||||
const BASE_DIR = path.join(__dirname, "./validate-database-errs");
|
||||
|
||||
// note: a hell of a lot of this code is copy pasted
|
||||
// this is just a script, so, feel free to refactor it.
|
||||
|
||||
const logger = CreateLogCtx("validate-database.ts");
|
||||
|
||||
async function ValidateStaticSchemas(): Promise<void> {
|
||||
for (const x in STATIC_SCHEMAS) {
|
||||
const c = x as keyof typeof STATIC_SCHEMAS;
|
||||
|
||||
const schema = STATIC_SCHEMAS[c] as PrudenceSchema;
|
||||
|
||||
logger.info(`=== Validating Collection ${c}... ===`);
|
||||
|
||||
let successCount = 0;
|
||||
let total = 0;
|
||||
let fails: { err: PrudenceError; doc: unknown }[] = [];
|
||||
|
||||
let start = process.hrtime.bigint();
|
||||
|
||||
// @ts-expect-error shut UP
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await db[c].find({}, { projection: { _id: 0 } }).each((c: unknown) => {
|
||||
total++;
|
||||
let res = Prudence(c, schema);
|
||||
|
||||
if (res === null) {
|
||||
successCount++;
|
||||
} else {
|
||||
logger.error(res);
|
||||
fails.push({ err: res, doc: c });
|
||||
}
|
||||
});
|
||||
|
||||
let end = process.hrtime.bigint();
|
||||
logger.info(`Validated ${total} objects. Took ${Number(end - start) / 1e9} seconds.`);
|
||||
logger.info(`Success: ${successCount} (${((successCount * 100) / total).toFixed(2)}%)`);
|
||||
logger.info(`Fail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`);
|
||||
|
||||
if (fails.length !== 0) {
|
||||
logger.severe(`Invalid documents found! Please resolve them.`);
|
||||
fs.writeFileSync(path.join(BASE_DIR, `${c}.json`), JSON.stringify(fails));
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Done!");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
async function ValidateScores(): Promise<void> {
|
||||
let globalStart = process.hrtime.bigint();
|
||||
let globalTotal = 0;
|
||||
let globalSuccess = 0;
|
||||
let globalFail = 0;
|
||||
for (const game of supportedGames) {
|
||||
for (const playtype of validPlaytypes[game]) {
|
||||
// @ts-expect-error shut up
|
||||
let schema = PRUDENCE_SCORE_SCHEMAS[game][playtype];
|
||||
|
||||
let successCount = 0;
|
||||
let total = 0;
|
||||
let fails: { err: PrudenceError; doc: unknown }[] = [];
|
||||
|
||||
logger.info(`=== Validating Collection scores (${game} ${playtype}) ... ===`);
|
||||
let start = process.hrtime.bigint();
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await db.scores
|
||||
.find({ playtype, game }, { projection: { _id: 0 } })
|
||||
// @ts-expect-error shut UP
|
||||
.each((c: unknown) => {
|
||||
total++;
|
||||
let res = Prudence(c, schema);
|
||||
|
||||
if (res === null) {
|
||||
successCount++;
|
||||
} else {
|
||||
logger.error(
|
||||
`[${game} ${playtype}] [${res.keychain}] ${res.message}: ${res.userVal}`
|
||||
);
|
||||
fails.push({ err: res, doc: c });
|
||||
}
|
||||
});
|
||||
|
||||
let end = process.hrtime.bigint();
|
||||
logger.info(`Validated ${total} objects. Took ${Number(end - start) / 1e9} seconds.`);
|
||||
logger.info(`Success: ${successCount} (${((successCount * 100) / total).toFixed(2)}%)`);
|
||||
logger.info(`Fail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`);
|
||||
|
||||
if (fails.length !== 0) {
|
||||
logger.severe(`Invalid documents found! Please resolve them.`);
|
||||
fs.writeFileSync(
|
||||
path.join(BASE_DIR, `scores-${game}-${playtype}.json`),
|
||||
JSON.stringify(fails)
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(BASE_DIR, `scores-${game}-${playtype}.log`),
|
||||
`Validated ${total} objects. Took ${
|
||||
Number(end - start) / 1e9
|
||||
} seconds.\nSuccess: ${successCount} (${((successCount * 100) / total).toFixed(
|
||||
2
|
||||
)}%)\nFail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`
|
||||
);
|
||||
}
|
||||
|
||||
globalTotal += total;
|
||||
globalFail += fails.length;
|
||||
globalSuccess += successCount;
|
||||
}
|
||||
}
|
||||
|
||||
let globalEnd = process.hrtime.bigint();
|
||||
logger.info(
|
||||
`Validated ${globalTotal} objects. Took ${Number(globalEnd - globalStart) / 1e9} seconds.`
|
||||
);
|
||||
logger.info(`Success: ${globalSuccess} (${((globalSuccess * 100) / globalTotal).toFixed(2)}%)`);
|
||||
logger.info(`Fail: ${globalFail} (${((globalFail * 100) / globalTotal).toFixed(2)}%)`);
|
||||
|
||||
// process.exit(1);
|
||||
}
|
||||
|
||||
async function ValidateChartsOrSongs(type: "songs" | "charts"): Promise<void> {
|
||||
let globalStart = process.hrtime.bigint();
|
||||
let globalTotal = 0;
|
||||
let globalSuccess = 0;
|
||||
let globalFail = 0;
|
||||
for (const game of supportedGames) {
|
||||
let schema = STATIC_SCHEMAS[type]![game];
|
||||
|
||||
let successCount = 0;
|
||||
let total = 0;
|
||||
let fails: { err: PrudenceError; doc: unknown }[] = [];
|
||||
|
||||
logger.info(`=== Validating Collection ${type} (${game}) ... ===`);
|
||||
let start = process.hrtime.bigint();
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await db[type][game]
|
||||
// @ts-expect-error whatever bro
|
||||
.find({}, { projection: { _id: 0 } })
|
||||
.each((c: unknown) => {
|
||||
total++;
|
||||
let res = Prudence(c, schema);
|
||||
|
||||
if (res === null) {
|
||||
successCount++;
|
||||
} else {
|
||||
logger.error(`[${game}] [${res.keychain}] ${res.message}: ${res.userVal}`);
|
||||
fails.push({ err: res, doc: c });
|
||||
}
|
||||
});
|
||||
|
||||
let end = process.hrtime.bigint();
|
||||
logger.info(`Validated ${total} objects. Took ${Number(end - start) / 1e9} seconds.`);
|
||||
logger.info(`Success: ${successCount} (${((successCount * 100) / total).toFixed(2)}%)`);
|
||||
logger.info(`Fail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`);
|
||||
|
||||
if (fails.length !== 0) {
|
||||
logger.severe(`Invalid documents found! Please resolve them.`);
|
||||
fs.writeFileSync(path.join(BASE_DIR, `${type}-${game}.json`), JSON.stringify(fails));
|
||||
fs.writeFileSync(
|
||||
path.join(BASE_DIR, `${type}-${game}.log`),
|
||||
`Validated ${total} objects. Took ${
|
||||
Number(end - start) / 1e9
|
||||
} seconds.\nSuccess: ${successCount} (${((successCount * 100) / total).toFixed(
|
||||
2
|
||||
)}%)\nFail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`
|
||||
);
|
||||
}
|
||||
|
||||
globalTotal += total;
|
||||
globalFail += fails.length;
|
||||
globalSuccess += successCount;
|
||||
}
|
||||
|
||||
let globalEnd = process.hrtime.bigint();
|
||||
logger.info(
|
||||
`Validated ${globalTotal} objects. Took ${Number(globalEnd - globalStart) / 1e9} seconds.`
|
||||
);
|
||||
logger.info(`Success: ${globalSuccess} (${((globalSuccess * 100) / globalTotal).toFixed(2)}%)`);
|
||||
logger.info(`Fail: ${globalFail} (${((globalFail * 100) / globalTotal).toFixed(2)}%)`);
|
||||
}
|
||||
|
||||
// ValidateStaticSchemas();
|
||||
// ValidateScores();
|
||||
ValidateChartsOrSongs("songs");
|
||||
Reference in New Issue
Block a user