diff --git a/server/.vscode/launch.json b/server/.vscode/launch.json index 9c0a9fe0f..1a63a0208 100644 --- a/server/.vscode/launch.json +++ b/server/.vscode/launch.json @@ -15,9 +15,17 @@ "${workspaceFolder}/js/**/*.js" ], "skipFiles": [ - "/**", - "${workspaceFolder}/node_modules/**/*.js", + "", + "node_modules", + "loader.js", + "async_hooks.js", + "**/async_hooks.js", + "**/internal/**/*", + "**/domain.js", + "**/events.js", + "*" ], + "smartStep": true, "outputCapture": "std", }, ] diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 856d5355e..48f3ce8d6 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -61,7 +61,7 @@ dependencies: express-session: 1.17.1 fast-json-stable-hash: 1.0.2 fast-xml-parser: 3.19.0 - kamaitachi-common: github.com/zkldi/Kamaitachi-common/309bc3610aca940ceeb5244ff7d337770cf27201 + kamaitachi-common: github.com/zkldi/Kamaitachi-common/63f9cae1e4e2b8a488e134712ed498a88cb4fd4f mongodb: 3.6.6 monk: 7.3.4 multer: 1.4.2 @@ -4344,8 +4344,8 @@ packages: engines: {node: '>=6'} dev: true - github.com/zkldi/Kamaitachi-common/309bc3610aca940ceeb5244ff7d337770cf27201: - resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/309bc3610aca940ceeb5244ff7d337770cf27201} + github.com/zkldi/Kamaitachi-common/63f9cae1e4e2b8a488e134712ed498a88cb4fd4f: + resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/63f9cae1e4e2b8a488e134712ed498a88cb4fd4f} name: kamaitachi-common version: 0.1.0 dependencies: diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index 1eb3ec51d..2d0c35c66 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -28,6 +28,7 @@ import { import monk from "monk"; import { MONGO_BASE_URL } from "../../lib/env/env"; import CreateLogCtx from "../../lib/logger/logger"; +import { OrphanScoreDocument } from "../../lib/score-import/import-types/common/types"; const logger = CreateLogCtx(__filename); @@ -131,6 +132,8 @@ const db = { "usc-auth-tokens": monkDB.get("usc-auth-tokens"), "beatoraja-auth-tokens": monkDB.get("beatoraja-auth-tokens"), "bms-course-lookup": monkDB.get("bms-course-lookup"), + + "orphan-scores": monkDB.get("orphan-scores"), }; export default db; diff --git a/server/src/lib/logger/logger.ts b/server/src/lib/logger/logger.ts index 92dfe71bc..f042aba8b 100644 --- a/server/src/lib/logger/logger.ts +++ b/server/src/lib/logger/logger.ts @@ -139,9 +139,12 @@ export const rootLogger = winston.createLogger({ }); function CreateLogCtx(filename: string, lg = rootLogger): KtLogger { - return lg.child({ + const logger = lg.child({ context: filename.replace(new RegExp(`^${process.cwd()}/`, "u"), ""), }) as KtLogger; + + logger.defaultMeta = { context: [filename] }; + return logger; } export function AppendLogCtx(context: string, lg: KtLogger): KtLogger { diff --git a/server/src/lib/score-import/framework/orphans/orphans.test.ts b/server/src/lib/score-import/framework/orphans/orphans.test.ts new file mode 100644 index 000000000..dc3759404 --- /dev/null +++ b/server/src/lib/score-import/framework/orphans/orphans.test.ts @@ -0,0 +1,256 @@ +import t from "tap"; +import db from "../../../../external/mongo/db"; +import { CloseAllConnections } from "../../../../test-utils/close-connections"; +import ResetDBState from "../../../../test-utils/reset-db-state"; +import CreateLogCtx from "../../../logger/logger"; +import { BatchManualContext, BatchManualScore } from "../../import-types/common/batch-manual/types"; +import { OrphanScore, ReprocessOrphan } from "./orphans"; +import fjsh from "fast-json-stable-hash"; +import { OrphanScoreDocument } from "../../import-types/common/types"; +import deepmerge from "deepmerge"; + +const logger = CreateLogCtx(__filename); + +const batchManualScore: BatchManualScore = { + score: 500, + lamp: "HARD CLEAR", + matchType: "songTitle", + identifier: "5.1.1.", + playtype: "SP", + difficulty: "ANOTHER", +}; + +const batchManualContext: BatchManualContext = { game: "iidx", service: "foo", version: "27" }; + +t.test("#OrphanScore", (t) => { + t.beforeEach(ResetDBState); + + t.test("Should orphan data and context information.", async (t) => { + const res = await OrphanScore( + "ir/direct-manual", + 1, + batchManualScore, + batchManualContext, + "Example Error Message", + logger + ); + + t.equal(res.success, true); + t.equal( + res.orphanID, + `O${fjsh.hash( + { + importType: "ir/direct-manual", + data: batchManualScore, + context: batchManualContext, + userID: 1, + }, + "sha256" + )}`, + "Should be a checksum of the orphan's contents." + ); + + const dbCheck = await db["orphan-scores"].findOne({ + orphanID: res.orphanID, + }); + + t.hasStrict(dbCheck, { + orphanID: res.orphanID, + userID: 1, + data: batchManualScore, + context: batchManualContext, + importType: "ir/direct-manual", + errMsg: "Example Error Message", + } as any); + + t.ok( + Math.abs(dbCheck!.timeInserted - Date.now()) < 10_000, + "timeInserted should be decently close to now." + ); + + t.end(); + }); + + t.test("Should skip orphans that already exist.", async (t) => { + const res1 = await OrphanScore( + "ir/direct-manual", + 1, + batchManualScore, + batchManualContext, + "Example Error Message", + logger + ); + + const res2 = await OrphanScore( + "ir/direct-manual", + 1, + batchManualScore, + batchManualContext, + "Example Error Message", + logger + ); + + t.equal(res2.success, false); + t.equal(res1.orphanID, res2.orphanID); + + const dbCheck = await db["orphan-scores"].count(); + + t.equal(dbCheck, 1); + }); + + t.end(); +}); + +t.test("#ReprocessOrphan", (t) => { + t.beforeEach(ResetDBState); + + const orphanDoc: OrphanScoreDocument = { + context: batchManualContext, + data: batchManualScore, + errMsg: "foo", + importType: "ir/direct-manual", + orphanID: "foo", + timeInserted: 0, + userID: 1, + }; + + t.test("Should convert and insert an orphan if parents are found", async (t) => { + // this orphan doc is for 5.1.1. SPA, which definitely exists in + // the test DB. + const res = await ReprocessOrphan(orphanDoc, logger); + + t.hasStrict( + res, + { + success: true, + type: "ScoreImported", + message: null, + content: { + score: { + game: "iidx", + service: "foo (DIRECT-MANUAL)", + comment: null, + importType: "ir/direct-manual", + timeAchieved: null, + scoreMeta: {}, + scoreData: { + lampIndex: 5, + gradeIndex: 1, + esd: 60.546875, + lamp: "HARD CLEAR", + score: 500, + grade: "E", + percent: 31.806615776081426, + hitData: {}, + hitMeta: {}, + }, + highlight: false, + userID: 1, + calculatedData: { + rating: 0.6840944877852578, + lampRating: 10.6, + gameSpecific: { + BPI: -15, + KESDC: -51.26832199690281, + "K%": null, + }, + }, + songID: 1, + chartID: "c2311194e3897ddb5745b1760d2c0141f933e683", + scoreID: + "R7a3a2b04bd4882ec06c198d78297fe3d56561502c7b134c067214bbfdf4f1602", + playtype: "SP", + isPrimary: true, + }, + }, + } as any, + "Should successfully import the score." + ); + + const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID }); + + t.equal(orphan, null, "Should remove the orphan document."); + + t.end(); + }); + + t.test("Should delete the orphan doc and return null if the score is invalid.", async (t) => { + await db["orphan-scores"].insert(orphanDoc); + const res = await ReprocessOrphan( + deepmerge(orphanDoc, { + data: { + score: 99999, + }, + }), + logger + ); + + t.equal(res, null); + + const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID }); + + t.equal(orphan, null, "Should remove the orphan document."); + + t.end(); + }); + + t.test( + "Should keep the orphan doc and return false if no parents could be found.", + async (t) => { + await db["orphan-scores"].insert(orphanDoc); + const res = await ReprocessOrphan( + deepmerge(orphanDoc, { + data: { + identifier: "NONSENSE CHART TITLE", + }, + }), + logger + ); + + t.equal(res, false); + + const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID }); + + t.not(orphan, null, "Should not remove the orphan document."); + + t.end(); + } + ); + + t.test( + "Should keep the orphan doc and return false if an internal failure was triggered.", + async (t) => { + await db["orphan-scores"].insert(orphanDoc); + const GAZER_SHA256 = "195fe1be5c3e74fccd04dc426e05f8a9cfa8a1059c339d0a23e99f63661f0b7d"; + + // force an internal failure by removing all bms songs and then + // leaving a valid bms chart to match with. + await db.songs.bms.remove({}); + + const res = await ReprocessOrphan( + deepmerge(orphanDoc, { + context: { + game: "bms", + }, + data: { + matchType: "bmsChartHash", + identifier: GAZER_SHA256, + }, + }), + logger + ); + + t.equal(res, false); + + const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID }); + + t.not(orphan, null, "Should not remove the orphan document."); + + t.end(); + } + ); + + t.end(); +}); + +t.teardown(CloseAllConnections); diff --git a/server/src/lib/score-import/framework/orphans/orphans.ts b/server/src/lib/score-import/framework/orphans/orphans.ts new file mode 100644 index 000000000..d2d9ff5f1 --- /dev/null +++ b/server/src/lib/score-import/framework/orphans/orphans.ts @@ -0,0 +1,117 @@ +import db from "../../../../external/mongo/db"; +import { + ConverterFunction, + ConverterFnReturnOrFailure, + ImportTypeContextMap, + ImportTypeDataMap, + OrphanScoreDocument, +} from "../../import-types/common/types"; +import { ImportTypes, integer } from "kamaitachi-common"; +import fjsh from "fast-json-stable-hash"; +import { KtLogger } from "../../../logger/logger"; +import { Converters } from "../../import-types/converters"; +import { + ConverterFailure, + InternalFailure, + KTDataNotFoundFailure, +} from "../common/converter-failures"; +import { ProcessSuccessfulConverterReturn } from "../score-importing/score-importing"; + +/** + * Creates an OrphanedScore document from the data and context, + * and inserts it into the DB if it is not already in there. + * + * @returns Returns { success: true | false, orphanID } + */ +export async function OrphanScore( + importType: T, + userID: integer, + data: ImportTypeDataMap[T], + context: ImportTypeContextMap[T], + errMsg: string | null, + logger: KtLogger +) { + const orphan: Pick = { + importType, + data, + context: context, + userID, + }; + + const orphanID = `O${fjsh.hash(orphan, "sha256")}`; + + const exists = await db["orphan-scores"].findOne({ orphanID }); + + if (exists) { + logger.debug(`Skipped orphaning score ${orphanID} because it already exists.`); + return { success: false, orphanID }; + } + + const orphanScoreDoc: OrphanScoreDocument = { + ...orphan, + orphanID, + errMsg, + timeInserted: Date.now(), + }; + + await db["orphan-scores"].insert(orphanScoreDoc); + + return { success: true, orphanID }; +} + +/** + * Takes an orphan document and re-runs the converter->scoreimport pipeline on its data. + * + * @returns False if no parent documents could be found for the score again, + * Null if the orphan document was removed, but no score was inserted (i.e. score was orphaned AND invalid, so nothing + * could be imported when parents were found). + * ImportProcessingInfo on success. + */ +export async function ReprocessOrphan(orphan: OrphanScoreDocument, logger: KtLogger) { + const ConverterFunction = Converters[orphan.importType] as ConverterFunction< + ImportTypeDataMap[ImportTypes], + ImportTypeContextMap[ImportTypes] + >; + + let res: ConverterFnReturnOrFailure; + + try { + res = await ConverterFunction(orphan.data, orphan.context, orphan.importType, logger); + } catch (err) { + // this is impossible to test, so we're going to ignore it + /* istanbul ignore next */ + if (!(err instanceof ConverterFailure)) { + logger.error(`Converter function ${orphan.importType} returned unexpected error.`, { + err, + }); + throw err; // throw this higher up, i guess. + } + + res = err; + } + + // If the data still can't be found, we do nothing about it. + if (res instanceof KTDataNotFoundFailure) { + logger.debug(`Unorphaning ${orphan.orphanID} failed. (${res.message})`); + return false; + } else if (res instanceof InternalFailure) { + logger.error(`Orphan Internal Failure - ${res.message}, OrphanID ${orphan.orphanID}`); + + return false; + } else if (res instanceof ConverterFailure) { + logger.warn( + `Recieved ConverterFailure ${res.message} on orphan ${orphan.orphanID}. Removing orphan.` + ); + + // @danger - This could go terribly, if there's a mistake in the converterFN we might accidentally + // remove a users score. + await db["orphan-scores"].remove({ orphanID: orphan.orphanID }); + + return null; + } + + await db["orphan-scores"].remove({ orphanID: orphan.orphanID }); + + // else, import the orphan. + return ProcessSuccessfulConverterReturn(orphan.userID, res, logger); +} diff --git a/server/src/lib/score-import/framework/score-import-main.ts b/server/src/lib/score-import/framework/score-import-main.ts index ed8e07ba2..22d017e64 100644 --- a/server/src/lib/score-import/framework/score-import-main.ts +++ b/server/src/lib/score-import/framework/score-import-main.ts @@ -19,8 +19,9 @@ import db from "../../../external/mongo/db"; import { GetAndUpdateUsersGoals } from "./goals/goals"; import { UpdateUsersMilestones } from "./milestones/milestones"; import { ClassHandler } from "./user-game-stats/classes"; -import { ImportInputParser } from "../import-types/common/types"; +import { ConverterFunction, ImportInputParser } from "../import-types/common/types"; import { ScorePlaytypeMap } from "./common/types"; +import { Converters } from "../import-types/converters"; /** * Performs a Kamaitachi Score Import. @@ -51,12 +52,15 @@ export default async function ScoreImportMain( // We get an iterable from the provided parser function, alongside some context and a converter function. // This iterable does not have to be an array - it's anything that's iterable, like a generator. const parseTimeStart = process.hrtime.bigint(); - const { iterable, ConverterFunction, context, game, classHandler } = await InputParser(logger); + const { iterable, context, game, classHandler } = await InputParser(logger); const parseTime = GetMilisecondsSince(parseTimeStart); logger.debug(`Parsing took ${parseTime} miliseconds.`); + // We have to cast here due to typescript generic confusions. This is guaranteed to be correct. + const ConverterFunction = (Converters[importType] as unknown) as ConverterFunction; + // --- 2. Importing --- // ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data. const importTimeStart = process.hrtime.bigint(); diff --git a/server/src/lib/score-import/framework/score-importing/score-importing.ts b/server/src/lib/score-import/framework/score-importing/score-importing.ts index 4ec64a50e..24de7ce32 100644 --- a/server/src/lib/score-import/framework/score-importing/score-importing.ts +++ b/server/src/lib/score-import/framework/score-importing/score-importing.ts @@ -5,6 +5,7 @@ import { ScoreDocument, AnySongDocument, ImportTypes, + IDStrings, } from "kamaitachi-common"; import { HydrateScore } from "./hydrate-score"; import { InsertQueue, QueueScoreInsert, ScoreIDs } from "./insert-score"; @@ -20,11 +21,12 @@ import db from "../../../../external/mongo/db"; import { AppendLogCtx, KtLogger } from "../../../logger/logger"; import { - ConverterFunctionReturns, - ConverterFnReturn, + ConverterFnReturnOrFailure, ConverterFunction, + ConverterFnSuccessReturn, } from "../../import-types/common/types"; import { DryScore } from "../common/types"; +import { OrphanScore } from "../orphans/orphans"; /** * Processes the iterable data into the Kamaitachi database. @@ -55,38 +57,19 @@ export async function ImportAllIterableData( ); } - // Due to the fact that ProcessIterableDatapoint may return an array instead of a single result - // (e-amusement is the only real example of this); - // we need to flatten out the datapoints into a single array. We also use this time - // to filter out nulls, which we don't care for (these are neither successes or failures) - const nonFlatDatapoints = await Promise.all(promises); + // We need to filter out nulls, which we don't care for (these are neither successes or failures) + const processedResults = await Promise.all(promises); logger.verbose(`Finished Importing Data (${promises.length} datapoints).`); - logger.debug(`Flattening returns...`); + logger.debug(`Removing null returns...`); - const flatDatapoints = []; + const datapoints = processedResults.filter( + (e) => e !== null + ) as ImportProcessingInfo[]; - for (const dp of nonFlatDatapoints) { - if (dp === null) { - continue; - } + logger.debug(`Removed null from results.`); - if (Array.isArray(dp)) { - for (const dpx of dp) { - if (dpx === null) { - continue; - } - - flatDatapoints.push(dpx); - } - } else { - flatDatapoints.push(dp); - } - } - - logger.debug(`Flattened returns.`); - - logger.verbose(`Recieved ${flatDatapoints.length} returns, from ${promises.length} data.`); + logger.verbose(`Recieved ${datapoints.length} returns, from ${promises.length} data.`); // Flush the score queue out after finishing most of the import. This ensures no scores get left in the // queue. @@ -96,7 +79,7 @@ export async function ImportAllIterableData( logger.verbose(`Emptied ${emptied} documents from score queue.`); } - return flatDatapoints; + return datapoints; } /** @@ -114,52 +97,56 @@ export async function ImportIterableDatapoint( ConverterFunction: ConverterFunction, context: C, logger: KtLogger -) { - let converterReturns: ConverterFunctionReturns; +): Promise { + // Converter Function Return + let cfnReturn: ConverterFnReturnOrFailure; try { - converterReturns = await ConverterFunction(data, context, importType, logger); + cfnReturn = await ConverterFunction(data, context, importType, logger); } catch (err) { - converterReturns = err; - } - - if (Array.isArray(converterReturns)) { - return Promise.all( - converterReturns.map((e) => ImportFromConverterReturn(userID, e, logger)) - ); - } - - return ImportFromConverterReturn(userID, converterReturns, logger); -} - -async function ImportFromConverterReturn( - userID: integer, - cfnReturn: ConverterFnReturn, // a single return, not an array! - logger: KtLogger -): Promise { - // null => processing didnt result in a score document, but not an error, no processing needed! - if (cfnReturn === null) { - return null; + cfnReturn = err; } // if this conversion failed, return it in the proper format if (cfnReturn instanceof ConverterFailure) { if (cfnReturn instanceof KTDataNotFoundFailure) { - logger.warn(`ConverterFailure: ${cfnReturn.message ?? "No message?"}`, { + logger.warn(`KTDataNotFoundFailure: ${cfnReturn.message ?? "No message?"}`, { cfnReturn, hideFromConsole: ["cfnReturn"], }); + + const insertOrphan = await OrphanScore( + cfnReturn.importType, + userID, + cfnReturn.data, + cfnReturn.converterContext, + cfnReturn.message, + logger + ); + + if (insertOrphan.success) { + return { + success: false, + type: "KTDataNotFound", + message: cfnReturn.message, + content: { + context: cfnReturn.converterContext, + data: cfnReturn.data, + orphanID: insertOrphan.orphanID, + }, + }; + } + return { success: false, - type: "KTDataNotFound", + type: "OrphanExists", message: cfnReturn.message, content: { - context: cfnReturn.converterContext, - data: cfnReturn.data, + orphanID: insertOrphan.orphanID, }, }; } else if (cfnReturn instanceof InvalidScoreFailure) { - logger.warn(`ConverterFailure: ${cfnReturn.message ?? "No message?"}`, { + logger.info(`InvalidScoreFailure: ${cfnReturn.message ?? "No message?"}`, { cfnReturn, hideFromConsole: ["cfnReturn"], }); @@ -205,6 +192,14 @@ async function ImportFromConverterReturn( }; } + return ProcessSuccessfulConverterReturn(userID, cfnReturn as ConverterFnSuccessReturn, logger); +} + +export async function ProcessSuccessfulConverterReturn( + userID: integer, + cfnReturn: ConverterFnSuccessReturn, + logger: KtLogger +): Promise { const result = await HydrateAndInsertScore( userID, cfnReturn.dryScore, diff --git a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts index c26fa6472..534f431d9 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts @@ -22,6 +22,5 @@ export function ParseKaiIIDX( }, classHandler: null, game: "iidx", - ConverterFunction: ConvertAPIKaiIIDX, }; } diff --git a/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts b/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts index 21b6fa7a0..cfcfb3ec1 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts @@ -22,6 +22,5 @@ export function ParseKaiSDVX( }, classHandler: null, game: "iidx", - ConverterFunction: ConvertAPIKaiSDVX, }; } diff --git a/server/src/lib/score-import/import-types/common/batch-manual/converter.ts b/server/src/lib/score-import/import-types/common/batch-manual/converter.ts index d0e330240..0e030fc96 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/converter.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/converter.ts @@ -19,7 +19,7 @@ import { import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils"; import { KtLogger } from "../../../../logger/logger"; import { DryScore } from "../../../framework/common/types"; -import { ConverterFunction, ConverterFnReturn } from "../types"; +import { ConverterFunction } from "../types"; /** * Creates a ConverterFn for the BatchManualScore format. This curries @@ -32,7 +32,7 @@ export const ConverterBatchManual: ConverterFunction => { +) => { const game = context.game; const { song, chart } = await ResolveMatchTypeToKTData(data, context, importType, logger); diff --git a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts index 080a0a0b0..f801c3019 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts @@ -172,7 +172,6 @@ export function ParseBatchManualFromObject( version: batchManual.head.version ?? null, }, iterable: batchManual.body, - ConverterFunction: ConverterBatchManual, classHandler: null, }; } diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts index 9817b0e0f..2afbf0ffe 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts @@ -2,7 +2,7 @@ import t from "tap"; import db from "../../../../../external/mongo/db"; import CreateLogCtx from "../../../../logger/logger"; import ResetDBState from "../../../../../test-utils/reset-db-state"; -import ConverterFn, { EamScoreConverter, EamScoreConverterWrapper } from "./converter"; +import ConvertEamIIDXCSV from "./converter"; import p from "prudence"; import { prAssert } from "../../../../../test-utils/asserts"; import deepmerge from "deepmerge"; @@ -63,330 +63,6 @@ const data = { title: "5.1.1", }; -// Testing floats with strictSame under tap is a pain because it uses === -// and there is no way around it. We have isApproximately for this purpose -// But it'd be easier if we could just .toFixed() the percent and compare it as -// a string. - -t.test("#EamScoreConverter", async (t) => { - t.beforeEach(ResetDBState); - - const song = { - title: "5.1.1.", - artist: "dj nagureo", - id: 1, - firstVersion: "0", - "alt-titles": [], - "search-titles": [], - data: { - genre: "PIANO AMBIENT", - }, - }; - - function EamScoreConverterAuto(score: Partial = {}) { - return EamScoreConverter( - deepmerge(valid511Score, score) as EamusementScoreData, - song, - converterContext, - data, - false, - logger - ); - } - - t.test("Valid Scores", (t) => { - t.test("Normal 511 SPA Score", async (t) => { - const res = await EamScoreConverterAuto(); - - t.not(res, null, "Should not be null."); - - t.equal( - res!.ktchiChart.chartID, - chartID511, - "Should have selected 511 SPA as the chart." - ); - - prAssert(res!.dryScore, DryScorePrudence, "Should return the expected dryscore."); - - t.end(); - }); - - t.test("--- BP Score", async (t) => { - const res = await EamScoreConverterAuto({ bp: "---" }); - - t.not(res, null, "Should not be null."); - - t.equal( - res!.ktchiChart.chartID, - chartID511, - "Should have selected 511 SPA as the chart." - ); - - prAssert( - res!.dryScore, - deepmerge(DryScorePrudence, { scoreData: { hitMeta: { bp: "undefined" } } }), - "Should return the expected dryscore." - ); - - t.end(); - }); - - t.test("Nonsense String BP Score", async (t) => { - const res = await EamScoreConverterAuto({ bp: "ASDF" }); - - t.not(res, null, "Should not be null."); - - t.equal( - res!.ktchiChart.chartID, - chartID511, - "Should have selected 511 SPA as the chart." - ); - - prAssert( - res!.dryScore, - deepmerge(DryScorePrudence, { scoreData: { hitMeta: { bp: "undefined" } } }), - "Should return the expected dryscore." - ); - - t.end(); - }); - - t.test("0 EX Score", async (t) => { - const res = await EamScoreConverterAuto({ exscore: "0", pgreat: "0", great: "0" }); - - t.equal(res, null, "Should be null."); - - t.end(); - }); - - t.test("1 EX Score", async (t) => { - const res = await EamScoreConverterAuto({ exscore: "1", pgreat: "0", great: "1" }); - - t.not(res, null, "Should not be null."); - - t.equal( - res!.ktchiChart.chartID, - chartID511, - "Should have selected 511 SPA as the chart." - ); - - prAssert( - res!.dryScore, - deepmerge(DryScorePrudence, { - scoreData: { - score: p.equalTo(1), - percent: p.aprx(0.063), - hitData: { - pgreat: p.equalTo(0), - great: p.equalTo(1), - }, - }, - }), - "Should return the expected dryscore." - ); - - t.end(); - }); - - t.test("Level 0 chart", async (t) => { - const res = await EamScoreConverterAuto({ level: "0" }); - - t.equal(res, null, "Should be null."); - - t.end(); - }); - - t.end(); - }); - - t.test("Invalid Scores", (t) => { - t.test("Chart that doesn't exist", (t) => { - EamScoreConverterAuto({ difficulty: "LEGGENDARIA" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new KTDataNotFoundFailure( - `Could not find chart for 5.1.1. (SP LEGGENDARIA [v27])`, - "file/eamusement-iidx-csv", - data, - converterContext - ), - "Should return a KTDataNotFoundFailure" - ); - - t.end(); - }); - }); - - t.test("Invalid PGreat/Great count", (t) => { - EamScoreConverterAuto({ exscore: "100", pgreat: "20", great: "5" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure( - `5.1.1. (SP ANOTHER [v27]) - PGreats * 2 + Greats did not equal EXScore (20 * 2 + 5 != 100).` - ), - "Should return an InvalidScoreFailure" - ); - - t.end(); - }); - }); - - t.test(">100%", (t) => { - EamScoreConverterAuto({ exscore: "9999" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure( - `5.1.1. (SP ANOTHER [v27]) - Invalid EX Score of 9999 (Was greater than max chart ex of 1572).` - ), - "Should return an InvalidScoreFailure" - ); - - t.end(); - }); - }); - - t.test("Nonsense Lamp", (t) => { - EamScoreConverterAuto({ lamp: "invalid" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure( - `5.1.1. (SP ANOTHER [v27]) - Invalid Lamp of invalid.` - ), - "Should return an InvalidScoreFailure" - ); - - t.end(); - }); - }); - - t.test("Invalid BP", async (t) => { - await EamScoreConverterAuto({ bp: "5.73" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of 5.73.`), - "Should return an InvalidScoreFailure" - ); - }); - - await EamScoreConverterAuto({ bp: "-5.73" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of -5.73.`), - "Should return an InvalidScoreFailure" - ); - }); - - await EamScoreConverterAuto({ bp: "-1" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of -1.`), - "Should return an InvalidScoreFailure" - ); - }); - - await EamScoreConverterAuto({ bp: "10000" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of 10000.`), - "Should return an InvalidScoreFailure" - ); - }); - - await EamScoreConverterAuto({ bp: "-10000" }) - .then(() => t.fail()) - .catch((err) => { - t.strictSame( - err, - new InvalidScoreFailure( - `5.1.1. (SP ANOTHER [v27]) - Invalid BP of -10000.` - ), - "Should return an InvalidScoreFailure" - ); - }); - - t.end(); - }); - - t.end(); - }); - - t.end(); -}); - -t.test("#EamScoreConverterWrapper", async (t) => { - t.beforeEach(ResetDBState); - - const song = { - title: "5.1.1.", - artist: "dj nagureo", - id: 1, - firstVersion: "0", - "alt-titles": [], - "search-titles": [], - data: { - genre: "PIANO AMBIENT", - }, - }; - - // wrapper wrapper, lol - function EamScoreConverterWrapperAuto(score: Partial = {}) { - return EamScoreConverterWrapper( - deepmerge(valid511Score, score) as EamusementScoreData, - song, - converterContext, - data, - false, - logger - ); - } - - t.test("Should convert thrown errors into returns", async (t) => { - const res = await EamScoreConverterWrapperAuto({ exscore: "-1" }); - - t.strictSame( - res, - new InvalidScoreFailure( - "5.1.1. (SP ANOTHER [v27]) - Invalid EX score of -1 (Was negative.)" - ) - ); - - t.end(); - }); - - t.test("Should return EamScoreConverter on success", async (t) => { - const res = await EamScoreConverterWrapperAuto(); - - if (res instanceof ConverterFailure || !res) { - return t.fail("Unexpected ConverterFailure"); - } - - t.strictSame(res.song, song, "Should return the right song"); - t.equal(res.chart.chartID, chartID511, "Should return the right chart"); - - prAssert(res.dryScore, DryScorePrudence, "Should return the right DryScore."); - - t.end(); - }); - - t.end(); -}); - t.todo("#ConverterFn", async (t) => { t.beforeEach(ResetDBState); diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts index f3620966c..bd04ef851 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts @@ -1,18 +1,19 @@ -import { Lamps, AnySongDocument, ChartDocument } from "kamaitachi-common"; +import { Lamps, ChartDocument } from "kamaitachi-common"; import { FindChartWithPTDFVersion } from "../../../../../utils/queries/charts"; -import { FindSongOnTitleInsensitive } from "../../../../../utils/queries/songs"; +import { FindSongOnTitle } from "../../../../../utils/queries/songs"; import { KTDataNotFoundFailure, - InternalFailure, InvalidScoreFailure, + SkipScoreFailure, } from "../../../framework/common/converter-failures"; -import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; -import { GetGradeFromPercent } from "../../../framework/common/score-utils"; +import { + GenericGetGradeAndPercent, + ParseDateFromString, +} from "../../../framework/common/score-utils"; import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts"; -import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; -import { KtLogger } from "../../../../logger/logger"; +import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; import { DryScore } from "../../../framework/common/types"; -import { ConverterFunction, ConverterFnReturn } from "../types"; +import { ConverterFunction } from "../types"; const EAMUSEMENT_LAMP_RESOLVER: Map = new Map([ ["NO PLAY", "NO PLAY"], @@ -25,28 +26,54 @@ const EAMUSEMENT_LAMP_RESOLVER: Map = new ["ASSIST CLEAR", "ASSIST CLEAR"], ]); -export async function EamScoreConverter( - eamScore: EamusementScoreData, - ktchiSong: AnySongDocument, - context: IIDXEamusementCSVContext, - data: IIDXEamusementCSVData, - isLegacyLeggendaria: boolean, - logger: KtLogger -) { - const HUMANISED_CHART_TITLE = `${ktchiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`; - if (!eamScore.level || eamScore.level === "0") { - // charts that dont exist in the format have a level of 0 - return null; - } +const NINE_HOURS = 1000 * 60 * 60 * 9; - if (isLegacyLeggendaria) { - eamScore.difficulty = "LEGGENDARIA"; - } +const ConvertEamIIDXCSV: ConverterFunction< + IIDXEamusementCSVData, + IIDXEamusementCSVContext +> = async (data, context, importType, logger) => { + const eamScore = data.score; if (eamScore.exscore === "0") { // skip scores with an exscore of 0 // This also skips things like score resets. - return null; + throw new SkipScoreFailure("Score has an exscore of 0."); + } + + if (!eamScore.level || eamScore.level === "0") { + // charts that dont exist in the format have a level of 0 + throw new SkipScoreFailure("Chart has a level of 0."); + } + + let isLegacyLeggendaria = false; + + // if pre-HV, leggendarias were stored in a wacky form. + if (!context.hasBeginnerAndLegg) { + // hack fix for legacy LEGGENDARIA titles + if (data.title.match(/(†|†LEGGENDARIA)$/u)) { + data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd(); + isLegacyLeggendaria = true; + } + } + + // @optimisable - This is actually a multi-fetch. Since eam-csv scores + // are batched up into (song, chart1, chart2, chart3 ...) rows + // we actually already have fetched this song a second ago. + const ktchiSong = await FindSongOnTitle("iidx", data.title); + + if (!ktchiSong) { + throw new KTDataNotFoundFailure( + `Could not find song for ${data.title}.`, + importType, + data, + context + ); + } + + const HUMANISED_CHART_TITLE = `${ktchiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`; + + if (isLegacyLeggendaria) { + eamScore.difficulty = "LEGGENDARIA"; } const ktchiChart = (await FindChartWithPTDFVersion( @@ -104,32 +131,15 @@ export async function EamScoreConverter( ); } - const percent = (100 * exscore) / MAX_EX; - const grade = GetGradeFromPercent<"iidx:SP" | "iidx:DP">("iidx", percent); + const { percent, grade } = GenericGetGradeAndPercent("iidx", exscore, ktchiChart); - if (!grade) { - logger.warn(`${HUMANISED_CHART_TITLE} - Could not resolve ${percent} into a grade?`); - throw new InternalFailure( - `${HUMANISED_CHART_TITLE} - Could not resolve ${percent} into a grade?` - ); - } - - // Now we need to figure out the timestamp for this score. - // Under, well, normal circumstances, we could figure this out quite trivially - // But e-amusement provides us the timestamp for the *song*, not the score - // - // We're going to actually ignore this. Initial drafts of this assumed we could skip the - // "epochs" (when a score reset on eamusement happened), but it turns out those are - // generated when the user triggers a score migration, NOT (as initially thought) when - // KONAMI decides. - - const timestamp = Date.parse(data.timestamp); + const timestamp = ParseDateFromString(data.timestamp); const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = { service: context.service, comment: null, game: "iidx", - importType: "file/eamusement-iidx-csv", + importType, scoreData: { score: exscore, lamp, @@ -142,7 +152,8 @@ export async function EamScoreConverter( grade, }, scoreMeta: {}, - timeAchieved: timestamp, + // japan is gmt+9 + timeAchieved: timestamp ? timestamp - NINE_HOURS : null, }; const numBP = Number(eamScore.bp); @@ -162,96 +173,8 @@ export async function EamScoreConverter( logger.info(`Skipped assigning BP for score. Had unexpected value of ${eamScore.bp}.`); } - return { ktchiChart, dryScore }; -} - -/** - * Thin wrapper around EamScoreConverter to ensure it returns the expected values - * @param eamScore - The eamusement score data to convert. - * @param song - The song this score is for. - * @param context - Context the converter may need. - * @param data - The parent data the eamScore derives from. - * @returns ConverterFnReturn - */ -export async function EamScoreConverterWrapper( - eamScore: EamusementScoreData, - song: AnySongDocument, - context: IIDXEamusementCSVContext, - data: IIDXEamusementCSVData, - isLegacyLeggendaria: boolean, - logger: KtLogger -) { - try { - const results = await EamScoreConverter( - eamScore, - song!, - context, - data, - isLegacyLeggendaria, - logger - ); - - if (!results) { - return null; - } - - return { - song, - chart: results.ktchiChart, - dryScore: results.dryScore, - }; - } catch (err) { - if ( - err instanceof KTDataNotFoundFailure || - err instanceof InternalFailure || - err instanceof InvalidScoreFailure - ) { - return err; - } else if (err instanceof ScoreImportFatalError) { - throw err; // throw it all the way up. - } - - logger.error(`Unknown error: `, { err }); - return new InternalFailure("An unknown internal failure has occured."); - } -} - -const ConverterFn: ConverterFunction = async ( - data, - context, - importType, - logger -): Promise => { - let isLegacyLeggendaria = false; - - // if pre-HV, leggendarias were stored in a wacky form. - if (!context.hasBeginnerAndLegg) { - // hack fix for legacy LEGGENDARIA titles - if (data.title.match(/(†|†LEGGENDARIA)$/u)) { - data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd(); - isLegacyLeggendaria = true; - } - } - - const ktchiSong = await FindSongOnTitleInsensitive("iidx", data.title); - - if (!ktchiSong) { - return new KTDataNotFoundFailure( - `Could not find song for ${data.title}.`, - importType, - data, - context - ); - } - // ts thinks ktchiSong might be null. It's not, though! - const results = await Promise.all( - data.scores.map((e) => - EamScoreConverterWrapper(e, ktchiSong!, context, data, isLegacyLeggendaria, logger) - ) - ); - - return results; + return { chart: ktchiChart, dryScore, song: ktchiSong }; }; -export default ConverterFn; +export default ConvertEamIIDXCSV; diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts index 7102a5e0a..241706b6d 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts @@ -18,7 +18,7 @@ t.test("#ParseEamusementCSV", (t) => { logger ); - t.equal(iterableData.length, 152, "Should return exactly 152 datapoints."); + t.equal(iterableData.length, 456, "Should return exactly 456 datapoints."); t.equal(hasBeginnerAndLegg, false, "Should not mark as a HV csv."); t.equal(version, "26", "Should correctly assert that the version of this CSV is ROOTAGE."); @@ -31,7 +31,7 @@ t.test("#ParseEamusementCSV", (t) => { logger ); - t.equal(iterableData.length, 1257, "Should return exactly 1257 datapoints."); + t.equal(iterableData.length, 6285, "Should return exactly 6285 datapoints."); t.equal(hasBeginnerAndLegg, true, "Should mark as a HV csv."); t.equal( version, diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts index 958e2dc32..7115246b9 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts @@ -2,7 +2,7 @@ import { Difficulties } from "kamaitachi-common"; import { KtLogger } from "../../../../logger/logger"; import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; import { ParserFunctionReturnsSync } from "../types"; -import ConverterFn from "./converter"; +import ConvertEamIIDXCSV from "./converter"; import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; enum EAM_VERSION_NAMES { @@ -199,11 +199,13 @@ export function NaiveCSVParse(csvBuffer: Buffer, logger: KtLogger) { }); } - iterableData.push({ - scores, - timestamp, - title, - }); + iterableData.push( + ...scores.map((e) => ({ + score: e, + timestamp, + title, + })) + ); } return { iterableData, version: gameVersion.toString(), hasBeginnerAndLegg }; @@ -270,7 +272,6 @@ function GenericParseEamIIDXCSV( return { iterable: iterableData, context, - ConverterFunction: ConverterFn, game: "iidx", classHandler: null, }; diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts index de8651052..7314e6bed 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts @@ -1,3 +1,5 @@ +import { SongDocument } from "kamaitachi-common"; + export interface EamusementScoreData { difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"; lamp: string; @@ -14,7 +16,7 @@ type BaseProps = { }; export type IIDXEamusementCSVData = { - scores: EamusementScoreData[]; + score: EamusementScoreData; } & BaseProps; type Props = "exscore" | "lamp" | "pgreat" | "great" | "bp" | "level"; diff --git a/server/src/lib/score-import/import-types/common/types.ts b/server/src/lib/score-import/import-types/common/types.ts index 66ae0c34e..17727b396 100644 --- a/server/src/lib/score-import/import-types/common/types.ts +++ b/server/src/lib/score-import/import-types/common/types.ts @@ -8,13 +8,20 @@ import { FervidexContext, FervidexScore } from "../ir/fervidex/types"; import { KaiContext } from "./api-kai/types"; import { BatchManualContext, BatchManualScore } from "./batch-manual/types"; import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./eamusement-iidx-csv/types"; -import { ImportTypes, Game, AnyChartDocument, AnySongDocument } from "kamaitachi-common"; +import { + ImportTypes, + Game, + AnyChartDocument, + AnySongDocument, + integer, + MongoDBDocument, +} from "kamaitachi-common"; import { ConverterFailure } from "../../framework/common/converter-failures"; import { ClassHandler } from "../../framework/user-game-stats/classes"; import { DryScore } from "../../framework/common/types"; -import { BeatorajaIRScoreFormat } from "../../../../server/router/ir/beatoraja/charts/convert-scores"; -import { BeatorajaContext } from "../ir/beatoraja/types"; - +import { BeatorajaContext, BeatorajaScore } from "../ir/beatoraja/types"; +import { USCClientScore } from "../../../../server/router/ir/usc/usc"; +import { IRUSCContext } from "../ir/usc/types"; export interface ImportTypeDataMap { "file/eamusement-iidx-csv": IIDXEamusementCSVData; "file/batch-manual": BatchManualScore; @@ -27,9 +34,11 @@ export interface ImportTypeDataMap { "ir/fervidex": FervidexScore; "ir/fervidex-static": FervidexStaticScore; "ir/chunitachi": BatchManualScore; - "ir/beatoraja": BeatorajaIRScoreFormat; - "ir/usc": EmptyObject; + "ir/beatoraja": BeatorajaScore; + "ir/usc": USCClientScore; + // These aren't placeholder values - the data is yielded in a way that + // the value of these is legitimately unknown at convert time. "api/arc-iidx": unknown; "api/arc-sdvx": unknown; "api/arc-jubeat": unknown; @@ -44,7 +53,7 @@ export interface ImportTypeContextMap { "file/batch-manual": BatchManualContext; "file/solid-state-squad": EmptyObject; "file/mer-iidx": EmptyObject; - "file/pli-iidx-csv": EmptyObject; + "file/pli-iidx-csv": IIDXEamusementCSVContext; "ir/direct-manual": BatchManualContext; "ir/barbatos": EmptyObject; @@ -52,7 +61,7 @@ export interface ImportTypeContextMap { "ir/fervidex-static": FervidexStaticContext; "ir/chunitachi": BatchManualContext; "ir/beatoraja": BeatorajaContext; - "ir/usc": EmptyObject; + "ir/usc": IRUSCContext; "api/arc-iidx": EmptyObject; "api/arc-sdvx": EmptyObject; @@ -62,11 +71,15 @@ export interface ImportTypeContextMap { "api/eag-iidx": KaiContext; "api/eag-sdvx": KaiContext; } -export interface OrphanedScore { + +export interface OrphanScoreDocument extends MongoDBDocument { importType: T; data: ImportTypeDataMap[T]; - converterContext: ImportTypeContextMap[T]; - humanisedIdentifier: string; + context: ImportTypeContextMap[T]; + errMsg: string | null; + orphanID: string; + userID: integer; + timeInserted: number; } export interface ConverterFnSuccessReturn { @@ -75,9 +88,7 @@ export interface ConverterFnSuccessReturn { song: AnySongDocument; } -export type ConverterFnReturn = ConverterFailure | ConverterFnSuccessReturn | null; - -export type ConverterFunctionReturns = ConverterFnReturn | ConverterFnReturn[]; +export type ConverterFnReturnOrFailure = ConverterFailure | ConverterFnSuccessReturn; export interface ConverterFunction { ( @@ -85,7 +96,7 @@ export interface ConverterFunction { processContext: C, importType: ImportTypes, logger: KtLogger - ): Promise; + ): Promise; } export interface ImportInputParser { @@ -100,7 +111,6 @@ export interface ParserFunctionReturnsAsync { iterable: AsyncIterable; context: C; game: Game; - ConverterFunction: ConverterFunction; classHandler: ClassHandler | null; } @@ -108,6 +118,5 @@ export interface ParserFunctionReturnsSync { iterable: Iterable; context: C; game: Game; - ConverterFunction: ConverterFunction; classHandler: ClassHandler | null; } diff --git a/server/src/lib/score-import/import-types/converters.ts b/server/src/lib/score-import/import-types/converters.ts new file mode 100644 index 000000000..e17ab3380 --- /dev/null +++ b/server/src/lib/score-import/import-types/converters.ts @@ -0,0 +1,38 @@ +import { ImportTypes } from "kamaitachi-common"; +import { ConvertAPIKaiIIDX } from "./common/api-kai/iidx/converter"; +import { ConvertAPIKaiSDVX } from "./common/api-kai/sdvx/converter"; +import { ConverterBatchManual } from "./common/batch-manual/converter"; +import ConvertEamIIDXCSV from "./common/eamusement-iidx-csv/converter"; +import { ConverterFunction, ImportTypeContextMap, ImportTypeDataMap } from "./common/types"; +import { ConvertFileMerIIDX } from "./file/mer-iidx/converter"; +import { ConvertFileS3 } from "./file/solid-state-squad/converter"; +import { ConverterIRBarbatos } from "./ir/barbatos/converter"; +import { ConverterIRBeatoraja } from "./ir/beatoraja/converter"; +import { ConverterIRFervidexStatic } from "./ir/fervidex-static/converter"; +import { ConverterIRFervidex } from "./ir/fervidex/converter"; +import { ConverterIRUSC } from "./ir/usc/converter"; + +export type ConverterMap = { + [I in ImportTypes]: ConverterFunction; +}; + +export const Converters: ConverterMap = { + "file/solid-state-squad": ConvertFileS3, + "file/batch-manual": ConverterBatchManual, + "file/mer-iidx": ConvertFileMerIIDX, + "file/pli-iidx-csv": ConvertEamIIDXCSV, + "file/eamusement-iidx-csv": ConvertEamIIDXCSV, + + "api/eag-iidx": ConvertAPIKaiIIDX, + "api/eag-sdvx": ConvertAPIKaiSDVX, + "api/flo-iidx": ConvertAPIKaiIIDX, + "api/flo-sdvx": ConvertAPIKaiSDVX, + + "ir/barbatos": ConverterIRBarbatos, + "ir/beatoraja": ConverterIRBeatoraja, + "ir/fervidex": ConverterIRFervidex, + "ir/fervidex-static": ConverterIRFervidexStatic, + "ir/direct-manual": ConverterBatchManual, + "ir/usc": ConverterIRUSC, + "ir/chunitachi": ConverterBatchManual, +}; diff --git a/server/src/lib/score-import/import-types/file/mer-iidx/converter.test.ts b/server/src/lib/score-import/import-types/file/mer-iidx/converter.test.ts index ad4290bb9..4d814be3c 100644 --- a/server/src/lib/score-import/import-types/file/mer-iidx/converter.test.ts +++ b/server/src/lib/score-import/import-types/file/mer-iidx/converter.test.ts @@ -49,7 +49,7 @@ t.test("#ConvertFileMerIIDX", (t) => { }, }, scoreMeta: {}, - timeAchieved: 1616570122000, + timeAchieved: 1616537722000, }, } as any, "Should return the converted dry score." @@ -83,7 +83,7 @@ t.test("#ConvertFileMerIIDX", (t) => { }, }, scoreMeta: {}, - timeAchieved: 1616570122000, + timeAchieved: 1616537722000, }, } as any, "Should return the converted dry score." diff --git a/server/src/lib/score-import/import-types/file/mer-iidx/converter.ts b/server/src/lib/score-import/import-types/file/mer-iidx/converter.ts index 07be7c467..e7d8435de 100644 --- a/server/src/lib/score-import/import-types/file/mer-iidx/converter.ts +++ b/server/src/lib/score-import/import-types/file/mer-iidx/converter.ts @@ -70,7 +70,8 @@ export const ConvertFileMerIIDX: ConverterFunction = asyn }, }, scoreMeta: {}, - timeAchieved, + // japan is GMT+9 + timeAchieved: timeAchieved ? timeAchieved - NINE_HOURS : null, }; return { @@ -79,3 +80,5 @@ export const ConvertFileMerIIDX: ConverterFunction = asyn dryScore, }; }; + +const NINE_HOURS = 1000 * 60 * 60 * 9; diff --git a/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts b/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts index 20dbc146e..25db56569 100644 --- a/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts +++ b/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts @@ -58,6 +58,5 @@ export function ParseMerIIDX( context: {}, iterable: jsonData as MerScore[], game: "iidx", - ConverterFunction: ConvertFileMerIIDX, }; } diff --git a/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts b/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts index c2a3507fe..10dee58bb 100644 --- a/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts +++ b/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts @@ -87,6 +87,5 @@ export function ParseSolidStateXML( context: {}, iterable: parsedXML.s3data.scoredata.song as S3Score[], game: "iidx", - ConverterFunction: ConvertFileS3, // todo }; } diff --git a/server/src/lib/score-import/import-types/ir/barbatos/parser.ts b/server/src/lib/score-import/import-types/ir/barbatos/parser.ts index a63db8ac1..4266d20e6 100644 --- a/server/src/lib/score-import/import-types/ir/barbatos/parser.ts +++ b/server/src/lib/score-import/import-types/ir/barbatos/parser.ts @@ -40,7 +40,6 @@ export function ParseBarbatosSingle( context: {}, game: "sdvx", iterable: ([body] as unknown) as BarbatosScore[], - ConverterFunction: ConverterIRBarbatos, classHandler: null, }; } diff --git a/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts b/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts index 8f0757328..930ee0137 100644 --- a/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts +++ b/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts @@ -108,7 +108,6 @@ export function ParseBeatorajaSingle( }, game: "bms", iterable: ([body.score] as unknown) as BeatorajaScore[], - ConverterFunction: ConverterIRBeatoraja, classHandler: null, }; } diff --git a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts index 28ee8ecd4..8cf857ce9 100644 --- a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts +++ b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts @@ -79,7 +79,6 @@ export function ParseFervidexStatic( context: { version }, game: "iidx", iterable: scores, - ConverterFunction: ConverterIRFervidexStatic, classHandler: FerStaticClassHandler(body), }; } diff --git a/server/src/lib/score-import/import-types/ir/fervidex/parser.ts b/server/src/lib/score-import/import-types/ir/fervidex/parser.ts index bdda5c846..95b35315b 100644 --- a/server/src/lib/score-import/import-types/ir/fervidex/parser.ts +++ b/server/src/lib/score-import/import-types/ir/fervidex/parser.ts @@ -115,7 +115,6 @@ export function ParseFervidexSingle( context: { version }, game: "iidx", iterable: ([body] as unknown) as FervidexScore[], - ConverterFunction: ConverterIRFervidex, classHandler: null, }; } diff --git a/server/src/lib/score-import/import-types/ir/usc/parser.ts b/server/src/lib/score-import/import-types/ir/usc/parser.ts index b2d9f7534..220833617 100644 --- a/server/src/lib/score-import/import-types/ir/usc/parser.ts +++ b/server/src/lib/score-import/import-types/ir/usc/parser.ts @@ -46,6 +46,5 @@ export function ParseIRUSC( game: "usc", iterable: [body.score] as USCClientScore[], classHandler: null, - ConverterFunction: ConverterIRUSC, }; } diff --git a/server/src/test-utils/mock-db/orphan-scores.json b/server/src/test-utils/mock-db/orphan-scores.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/server/src/test-utils/mock-db/orphan-scores.json @@ -0,0 +1 @@ +[] \ No newline at end of file