diff --git a/server/src/scripts/__KT_DATABASE_MIGRATION/migrate.ts b/server/src/scripts/__KT_DATABASE_MIGRATION/migrate.ts index 57e660353..bc63ecc34 100644 --- a/server/src/scripts/__KT_DATABASE_MIGRATION/migrate.ts +++ b/server/src/scripts/__KT_DATABASE_MIGRATION/migrate.ts @@ -2,7 +2,7 @@ import { ICollection } from "monk"; import CreateLogCtx from "../../logger"; import { oldKTDB } from "./old-db"; -const logger = CreateLogCtx("MIGRATE-users.ts"); +const logger = CreateLogCtx("MIGRATE.ts"); let newDocuments: unknown[] = []; @@ -18,11 +18,18 @@ export default async function MigrateRecords( process.exit(1); } + let i = 0; + await oldKTDB .get(collectionName) .find({}) // @ts-expect-error it exists. .each((c) => { + i++; + + if (i % 10000 === 0) { + logger.info(`Processed ${i} documents.`); + } const newDoc = HandlerFN(c); newDocuments.push(newDoc); diff --git a/server/src/scripts/__KT_DATABASE_MIGRATION/scores.ts b/server/src/scripts/__KT_DATABASE_MIGRATION/scores.ts index cb7a67e40..518773256 100644 --- a/server/src/scripts/__KT_DATABASE_MIGRATION/scores.ts +++ b/server/src/scripts/__KT_DATABASE_MIGRATION/scores.ts @@ -1,12 +1,19 @@ import { PrivateUserDocument, ScoreDocument } from "kamaitachi-common"; -import { grades } from "kamaitachi-common/js/config"; +import { grades, lamps } from "kamaitachi-common/js/config"; import db from "../../db/db"; +import { CreateScoreID } from "../../score-import/framework/core/score-id"; import MigrateRecords from "./migrate"; // HERE. WE. GO! +function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) { + if (Object.prototype.hasOwnProperty.call(other, otherProp)) { + base[baseProp] = other[otherProp]; + } +} + function ConvertFn(c: any): ScoreDocument { - let base: Partial = { + let base: Omit = { userID: c.userID, songID: c.songID, playtype: c.scoreData.playtype, @@ -14,7 +21,7 @@ function ConvertFn(c: any): ScoreDocument { chartID: c.chartID, game: c.game, timeAdded: c.timeAdded, - timeAchieved: c.timeAchieved, + timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved, isScorePB: c.isScorePB, isLampPB: c.isLampPB, comment: c.comment ?? null, @@ -38,19 +45,42 @@ function ConvertFn(c: any): ScoreDocument { scoreMeta: {}, }; - switch (base.game) { - case "bms": - break; - - default: - break; + if (c.scoreMeta) { + if (base.game === "iidx") { + ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom"); + ConditionalAssign(base.scoreMeta, "gauge", c.scoreMeta, "optionsGauge"); + ConditionalAssign(base.scoreMeta, "assist", c.scoreMeta, "optionsAssist"); + ConditionalAssign(base.scoreMeta, "range", c.scoreMeta, "optionsRange"); + ConditionalAssign(base.scoreMeta, "pacemaker", c.scoreMeta, "pacemaker"); + ConditionalAssign(base.scoreMeta, "pacemakerName", c.scoreMeta, "pacemakerName"); + ConditionalAssign(base.scoreMeta, "pacemakerTarget", c.scoreMeta, "pacemakerTarget"); + } else if (base.game === "bms") { + if (base.playtype !== "5K") { + if (base.playtype === "7K") { + ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom"); + ConditionalAssign( + base.scoreMeta, + "inputDevice", + c.scoreData.hitMeta, + "inputDevice" + ); + } + } + } } - return base; + let scoreID = CreateScoreID(base.userID, base, base.chartID); + + let score: ScoreDocument = { + ...base, + scoreID, + }; + + return score; } (async () => { - await MigrateRecords(db.users, "users", ConvertFn); + await MigrateRecords(db.scores, "scores", ConvertFn); process.exit(0); })(); diff --git a/server/src/scripts/single-use/sample-iidx-db.ts b/server/src/scripts/single-use/sample-iidx-db.ts new file mode 100644 index 000000000..43ce9a8ca --- /dev/null +++ b/server/src/scripts/single-use/sample-iidx-db.ts @@ -0,0 +1,9 @@ +import fs from "fs"; +import db from "../../db/db"; +import path from "path"; + +(async () => { + let data = await db.scores.find({ game: "iidx", playtype: "SP" }, { limit: 1000 }); + fs.writeFileSync(path.join(__dirname, "data.json"), JSON.stringify(data)); + process.exit(0); +})(); diff --git a/server/src/scripts/validate-database.ts b/server/src/scripts/validate-database.ts index 507ec5f86..dab262bfe 100644 --- a/server/src/scripts/validate-database.ts +++ b/server/src/scripts/validate-database.ts @@ -1,10 +1,11 @@ import Prudence, { PrudenceSchema } from "prudence"; import db from "../db/db"; -import { STATIC_SCHEMAS } from "../db/schemas"; +import { PRUDENCE_SCORE_SCHEMAS, STATIC_SCHEMAS } from "../db/schemas"; import CreateLogCtx from "../logger"; import fs from "fs"; import path from "path"; import { PrudenceError } from "prudence/js/error"; +import { supportedGames, validPlaytypes } from "kamaitachi-common/js/config"; const BASE_DIR = path.join(__dirname, "./validate-database-errs"); @@ -52,4 +53,73 @@ async function ValidateStaticSchemas(): Promise { process.exit(0); } -ValidateStaticSchemas(); +async function ValidateScores(): Promise { + let globalStart = process.hrtime.bigint(); + let globalTotal = 0; + let globalSuccess = 0; + let globalFail = 0; + for (const game of supportedGames) { + for (const playtype of validPlaytypes[game]) { + // @ts-expect-error shut up + let schema = PRUDENCE_SCORE_SCHEMAS[game][playtype]; + + let successCount = 0; + let total = 0; + let fails: { err: PrudenceError; doc: unknown }[] = []; + + logger.info(`=== Validating Collection scores (${game} ${playtype}) ... ===`); + let start = process.hrtime.bigint(); + + // eslint-disable-next-line no-await-in-loop + await db.scores + .find({ playtype, game }, { projection: { _id: 0 } }) + // @ts-expect-error shut UP + .each((c: unknown) => { + total++; + let res = Prudence(c, schema); + + if (res === null) { + successCount++; + } else { + logger.error(`[${res.keychain}] ${res.message}: ${res.userVal}`); + fails.push({ err: res, doc: c }); + } + }); + + let end = process.hrtime.bigint(); + logger.info(`Validated ${total} objects. Took ${Number(end - start) / 1e9} seconds.`); + logger.info(`Success: ${successCount} (${((successCount * 100) / total).toFixed(2)}%)`); + logger.info(`Fail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`); + + if (fails.length !== 0) { + logger.severe(`Invalid documents found! Please resolve them.`); + fs.writeFileSync( + path.join(BASE_DIR, `scores-${game}-${playtype}.json`), + JSON.stringify(fails) + ); + fs.writeFileSync( + path.join(BASE_DIR, `scores-${game}-${playtype}.log`), + `Validated ${total} objects. Took ${ + Number(end - start) / 1e9 + } seconds.\nSuccess: ${successCount} (${((successCount * 100) / total).toFixed( + 2 + )}%\nFail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%))` + ); + } + + globalTotal += total; + globalFail += fails.length; + globalSuccess += successCount; + } + } + + let globalEnd = process.hrtime.bigint(); + logger.info( + `Validated ${globalTotal} objects. Took ${Number(globalEnd - globalStart) / 1e9} seconds.` + ); + logger.info(`Success: ${globalSuccess} (${((globalSuccess * 100) / globalTotal).toFixed(2)}%)`); + logger.info(`Fail: ${globalFail} (${((globalFail * 100) / globalTotal).toFixed(2)}%)`); +} + +// ValidateStaticSchemas(); +ValidateScores();