From f536485f9e8ce1f706b9e6d9dfbc6be22a1c04af Mon Sep 17 00:00:00 2001
From: zkldi <notzkldi@zkldi.com>
Date: Mon, 31 May 2021 17:30:56 +0100
Subject: [PATCH] Refactor how Converters work to be called statically instead
 of presented dynamically (1)

---
 server/pnpm-lock.yaml                         |  6 +-
 server/src/external/mongo/db.ts               |  3 +
 .../framework/common/converter-failures.ts    |  1 +
 .../score-importing/score-importing.ts        | 15 ++++-
 .../common/api-kai/iidx/parser.ts             |  1 -
 .../common/api-kai/sdvx/parser.ts             |  1 -
 .../common/batch-manual/parser.ts             |  1 -
 .../eamusement-iidx-csv/converter.test.ts     |  2 +-
 .../common/eamusement-iidx-csv/converter.ts   | 59 +++++++------------
 .../common/eamusement-iidx-csv/parser.ts      |  3 +-
 .../score-import/import-types/common/types.ts | 28 +++++----
 .../score-import/import-types/converters.ts   | 38 ++++++++++++
 .../import-types/file/mer-iidx/parser.ts      |  1 -
 .../file/solid-state-squad/parser.ts          |  1 -
 .../import-types/ir/barbatos/parser.ts        |  1 -
 .../import-types/ir/beatoraja/parser.ts       |  1 -
 .../import-types/ir/fervidex-static/parser.ts |  1 -
 .../import-types/ir/fervidex/parser.ts        |  1 -
 .../import-types/ir/usc/parser.ts             |  1 -
 19 files changed, 97 insertions(+), 68 deletions(-)
 create mode 100644 server/src/lib/score-import/import-types/converters.ts

diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml
index 856d5355e..c65e09519 100644
--- a/server/pnpm-lock.yaml
+++ b/server/pnpm-lock.yaml
@@ -61,7 +61,7 @@ dependencies:
   express-session: 1.17.1
   fast-json-stable-hash: 1.0.2
   fast-xml-parser: 3.19.0
-  kamaitachi-common: github.com/zkldi/Kamaitachi-common/309bc3610aca940ceeb5244ff7d337770cf27201
+  kamaitachi-common: github.com/zkldi/Kamaitachi-common/e3a9f8d1f019d2e0d3e26755b0d6c73c28848f2d
   mongodb: 3.6.6
   monk: 7.3.4
   multer: 1.4.2
@@ -4344,8 +4344,8 @@ packages:
     engines: {node: '>=6'}
     dev: true
 
-  github.com/zkldi/Kamaitachi-common/309bc3610aca940ceeb5244ff7d337770cf27201:
-    resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/309bc3610aca940ceeb5244ff7d337770cf27201}
+  github.com/zkldi/Kamaitachi-common/e3a9f8d1f019d2e0d3e26755b0d6c73c28848f2d:
+    resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/e3a9f8d1f019d2e0d3e26755b0d6c73c28848f2d}
     name: kamaitachi-common
     version: 0.1.0
     dependencies:
diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts
index 1eb3ec51d..2d0c35c66 100644
--- a/server/src/external/mongo/db.ts
+++ b/server/src/external/mongo/db.ts
@@ -28,6 +28,7 @@ import {
 import monk from "monk";
 import { MONGO_BASE_URL } from "../../lib/env/env";
 import CreateLogCtx from "../../lib/logger/logger";
+import { OrphanScoreDocument } from "../../lib/score-import/import-types/common/types";
 
 const logger = CreateLogCtx(__filename);
 
@@ -131,6 +132,8 @@ const db = {
     "usc-auth-tokens": monkDB.get<GenericAuthDocument>("usc-auth-tokens"),
     "beatoraja-auth-tokens": monkDB.get<GenericAuthDocument>("beatoraja-auth-tokens"),
     "bms-course-lookup": monkDB.get<BMSCourseDocument>("bms-course-lookup"),
+
+    "orphan-scores": monkDB.get<OrphanScoreDocument>("orphan-scores"),
 };
 
 export default db;
diff --git a/server/src/lib/score-import/framework/common/converter-failures.ts b/server/src/lib/score-import/framework/common/converter-failures.ts
index bc2ee0064..5b5598fab 100644
--- a/server/src/lib/score-import/framework/common/converter-failures.ts
+++ b/server/src/lib/score-import/framework/common/converter-failures.ts
@@ -39,6 +39,7 @@ export class KTDataNotFoundFailure<T extends ImportTypes> extends ConverterFailu
         this.importType = importType;
         this.data = data;
         this.converterContext = context;
+        this.humanisedIdentifier = humanisedIdentifier ?? null;
     }
 }
 
diff --git a/server/src/lib/score-import/framework/score-importing/score-importing.ts b/server/src/lib/score-import/framework/score-importing/score-importing.ts
index 4ec64a50e..29b0466f7 100644
--- a/server/src/lib/score-import/framework/score-importing/score-importing.ts
+++ b/server/src/lib/score-import/framework/score-importing/score-importing.ts
@@ -25,6 +25,7 @@ import {
     ConverterFunction,
 } from "../../import-types/common/types";
 import { DryScore } from "../common/types";
+import { OrphanScore } from "../orphans/orphans";
 
 /**
  * Processes the iterable data into the Kamaitachi database.
@@ -145,10 +146,20 @@ async function ImportFromConverterReturn(
     // if this conversion failed, return it in the proper format
     if (cfnReturn instanceof ConverterFailure) {
         if (cfnReturn instanceof KTDataNotFoundFailure) {
-            logger.warn(`ConverterFailure: ${cfnReturn.message ?? "No message?"}`, {
+            logger.warn(`KTDataNotFoundFailure: ${cfnReturn.message ?? "No message?"}`, {
                 cfnReturn,
                 hideFromConsole: ["cfnReturn"],
             });
+
+            OrphanScore(
+                cfnReturn.importType,
+                userID,
+                cfnReturn.data,
+                cfnReturn.converterContext,
+                cfnReturn.message,
+                logger
+            );
+
             return {
                 success: false,
                 type: "KTDataNotFound",
@@ -159,7 +170,7 @@ async function ImportFromConverterReturn(
                 },
             };
         } else if (cfnReturn instanceof InvalidScoreFailure) {
-            logger.warn(`ConverterFailure: ${cfnReturn.message ?? "No message?"}`, {
+            logger.info(`InvalidScoreFailure: ${cfnReturn.message ?? "No message?"}`, {
                 cfnReturn,
                 hideFromConsole: ["cfnReturn"],
             });
diff --git a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts
index c26fa6472..534f431d9 100644
--- a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts
+++ b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts
@@ -22,6 +22,5 @@ export function ParseKaiIIDX(
         },
         classHandler: null,
         game: "iidx",
-        ConverterFunction: ConvertAPIKaiIIDX,
     };
 }
diff --git a/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts b/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts
index 21b6fa7a0..cfcfb3ec1 100644
--- a/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts
+++ b/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts
@@ -22,6 +22,5 @@ export function ParseKaiSDVX(
         },
         classHandler: null,
         game: "iidx",
-        ConverterFunction: ConvertAPIKaiSDVX,
     };
 }
diff --git a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts
index 080a0a0b0..f801c3019 100644
--- a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts
+++ b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts
@@ -172,7 +172,6 @@ export function ParseBatchManualFromObject(
             version: batchManual.head.version ?? null,
         },
         iterable: batchManual.body,
-        ConverterFunction: ConverterBatchManual,
         classHandler: null,
     };
 }
diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts
index 9817b0e0f..5ce652364 100644
--- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts
+++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.test.ts
@@ -2,7 +2,7 @@ import t from "tap";
 import db from "../../../../../external/mongo/db";
 import CreateLogCtx from "../../../../logger/logger";
 import ResetDBState from "../../../../../test-utils/reset-db-state";
-import ConverterFn, { EamScoreConverter, EamScoreConverterWrapper } from "./converter";
+import ConvertEamIIDXCSV, { EamScoreConverter, EamScoreConverterWrapper } from "./converter";
 import p from "prudence";
 import { prAssert } from "../../../../../test-utils/asserts";
 import deepmerge from "deepmerge";
diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts
index f3620966c..cde8d1197 100644
--- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts
+++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts
@@ -181,47 +181,30 @@ export async function EamScoreConverterWrapper(
     isLegacyLeggendaria: boolean,
     logger: KtLogger
 ) {
-    try {
-        const results = await EamScoreConverter(
-            eamScore,
-            song!,
-            context,
-            data,
-            isLegacyLeggendaria,
-            logger
-        );
+    const results = await EamScoreConverter(
+        eamScore,
+        song!,
+        context,
+        data,
+        isLegacyLeggendaria,
+        logger
+    );
 
-        if (!results) {
-            return null;
-        }
-
-        return {
-            song,
-            chart: results.ktchiChart,
-            dryScore: results.dryScore,
-        };
-    } catch (err) {
-        if (
-            err instanceof KTDataNotFoundFailure ||
-            err instanceof InternalFailure ||
-            err instanceof InvalidScoreFailure
-        ) {
-            return err;
-        } else if (err instanceof ScoreImportFatalError) {
-            throw err; // throw it all the way up.
-        }
-
-        logger.error(`Unknown error: `, { err });
-        return new InternalFailure("An unknown internal failure has occured.");
+    if (!results) {
+        return null;
     }
+
+    return {
+        song,
+        chart: results.ktchiChart,
+        dryScore: results.dryScore,
+    };
 }
 
-const ConverterFn: ConverterFunction<IIDXEamusementCSVData, IIDXEamusementCSVContext> = async (
-    data,
-    context,
-    importType,
-    logger
-): Promise<ConverterFnReturn[] | ConverterFnReturn> => {
+const ConvertEamIIDXCSV: ConverterFunction<
+    IIDXEamusementCSVData,
+    IIDXEamusementCSVContext
+> = async (data, context, importType, logger): Promise<ConverterFnReturn[] | ConverterFnReturn> => {
     let isLegacyLeggendaria = false;
 
     // if pre-HV, leggendarias were stored in a wacky form.
@@ -254,4 +237,4 @@ const ConverterFn: ConverterFunction<IIDXEamusementCSVData, IIDXEamusementCSVCon
     return results;
 };
 
-export default ConverterFn;
+export default ConvertEamIIDXCSV;
diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts
index 958e2dc32..32624ed37 100644
--- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts
+++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts
@@ -2,7 +2,7 @@ import { Difficulties } from "kamaitachi-common";
 import { KtLogger } from "../../../../logger/logger";
 import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
 import { ParserFunctionReturnsSync } from "../types";
-import ConverterFn from "./converter";
+import ConvertEamIIDXCSV from "./converter";
 import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
 
 enum EAM_VERSION_NAMES {
@@ -270,7 +270,6 @@ function GenericParseEamIIDXCSV(
     return {
         iterable: iterableData,
         context,
-        ConverterFunction: ConverterFn,
         game: "iidx",
         classHandler: null,
     };
diff --git a/server/src/lib/score-import/import-types/common/types.ts b/server/src/lib/score-import/import-types/common/types.ts
index 66ae0c34e..34376d889 100644
--- a/server/src/lib/score-import/import-types/common/types.ts
+++ b/server/src/lib/score-import/import-types/common/types.ts
@@ -8,13 +8,13 @@ import { FervidexContext, FervidexScore } from "../ir/fervidex/types";
 import { KaiContext } from "./api-kai/types";
 import { BatchManualContext, BatchManualScore } from "./batch-manual/types";
 import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./eamusement-iidx-csv/types";
-import { ImportTypes, Game, AnyChartDocument, AnySongDocument } from "kamaitachi-common";
+import { ImportTypes, Game, AnyChartDocument, AnySongDocument, integer } from "kamaitachi-common";
 import { ConverterFailure } from "../../framework/common/converter-failures";
 import { ClassHandler } from "../../framework/user-game-stats/classes";
 import { DryScore } from "../../framework/common/types";
-import { BeatorajaIRScoreFormat } from "../../../../server/router/ir/beatoraja/charts/convert-scores";
-import { BeatorajaContext } from "../ir/beatoraja/types";
-
+import { BeatorajaContext, BeatorajaScore } from "../ir/beatoraja/types";
+import { USCClientScore } from "../../../../server/router/ir/usc/usc";
+import { IRUSCContext } from "../ir/usc/types";
 export interface ImportTypeDataMap {
     "file/eamusement-iidx-csv": IIDXEamusementCSVData;
     "file/batch-manual": BatchManualScore;
@@ -27,9 +27,11 @@ export interface ImportTypeDataMap {
     "ir/fervidex": FervidexScore;
     "ir/fervidex-static": FervidexStaticScore;
     "ir/chunitachi": BatchManualScore;
-    "ir/beatoraja": BeatorajaIRScoreFormat;
-    "ir/usc": EmptyObject;
+    "ir/beatoraja": BeatorajaScore;
+    "ir/usc": USCClientScore;
 
+    // These aren't placeholder values - the data is yielded in a way that
+    // the value of these is legitimately unknown at convert time.
     "api/arc-iidx": unknown;
     "api/arc-sdvx": unknown;
     "api/arc-jubeat": unknown;
@@ -44,7 +46,7 @@ export interface ImportTypeContextMap {
     "file/batch-manual": BatchManualContext;
     "file/solid-state-squad": EmptyObject;
     "file/mer-iidx": EmptyObject;
-    "file/pli-iidx-csv": EmptyObject;
+    "file/pli-iidx-csv": IIDXEamusementCSVContext;
 
     "ir/direct-manual": BatchManualContext;
     "ir/barbatos": EmptyObject;
@@ -52,7 +54,7 @@ export interface ImportTypeContextMap {
     "ir/fervidex-static": FervidexStaticContext;
     "ir/chunitachi": BatchManualContext;
     "ir/beatoraja": BeatorajaContext;
-    "ir/usc": EmptyObject;
+    "ir/usc": IRUSCContext;
 
     "api/arc-iidx": EmptyObject;
     "api/arc-sdvx": EmptyObject;
@@ -62,11 +64,15 @@ export interface ImportTypeContextMap {
     "api/eag-iidx": KaiContext;
     "api/eag-sdvx": KaiContext;
 }
-export interface OrphanedScore<T extends ImportTypes> {
+
+export interface OrphanScoreDocument<T extends ImportTypes = ImportTypes> {
     importType: T;
     data: ImportTypeDataMap[T];
     converterContext: ImportTypeContextMap[T];
-    humanisedIdentifier: string;
+    errMsg: string | null;
+    orphanID: string;
+    userID: integer;
+    timeInserted: number;
 }
 
 export interface ConverterFnSuccessReturn {
@@ -100,7 +106,6 @@ export interface ParserFunctionReturnsAsync<D, C> {
     iterable: AsyncIterable<D>;
     context: C;
     game: Game;
-    ConverterFunction: ConverterFunction<D, C>;
     classHandler: ClassHandler | null;
 }
 
@@ -108,6 +113,5 @@ export interface ParserFunctionReturnsSync<D, C> {
     iterable: Iterable<D>;
     context: C;
     game: Game;
-    ConverterFunction: ConverterFunction<D, C>;
     classHandler: ClassHandler | null;
 }
diff --git a/server/src/lib/score-import/import-types/converters.ts b/server/src/lib/score-import/import-types/converters.ts
new file mode 100644
index 000000000..e17ab3380
--- /dev/null
+++ b/server/src/lib/score-import/import-types/converters.ts
@@ -0,0 +1,38 @@
+import { ImportTypes } from "kamaitachi-common";
+import { ConvertAPIKaiIIDX } from "./common/api-kai/iidx/converter";
+import { ConvertAPIKaiSDVX } from "./common/api-kai/sdvx/converter";
+import { ConverterBatchManual } from "./common/batch-manual/converter";
+import ConvertEamIIDXCSV from "./common/eamusement-iidx-csv/converter";
+import { ConverterFunction, ImportTypeContextMap, ImportTypeDataMap } from "./common/types";
+import { ConvertFileMerIIDX } from "./file/mer-iidx/converter";
+import { ConvertFileS3 } from "./file/solid-state-squad/converter";
+import { ConverterIRBarbatos } from "./ir/barbatos/converter";
+import { ConverterIRBeatoraja } from "./ir/beatoraja/converter";
+import { ConverterIRFervidexStatic } from "./ir/fervidex-static/converter";
+import { ConverterIRFervidex } from "./ir/fervidex/converter";
+import { ConverterIRUSC } from "./ir/usc/converter";
+
+export type ConverterMap = {
+    [I in ImportTypes]: ConverterFunction<ImportTypeDataMap[I], ImportTypeContextMap[I]>;
+};
+
+export const Converters: ConverterMap = {
+    "file/solid-state-squad": ConvertFileS3,
+    "file/batch-manual": ConverterBatchManual,
+    "file/mer-iidx": ConvertFileMerIIDX,
+    "file/pli-iidx-csv": ConvertEamIIDXCSV,
+    "file/eamusement-iidx-csv": ConvertEamIIDXCSV,
+
+    "api/eag-iidx": ConvertAPIKaiIIDX,
+    "api/eag-sdvx": ConvertAPIKaiSDVX,
+    "api/flo-iidx": ConvertAPIKaiIIDX,
+    "api/flo-sdvx": ConvertAPIKaiSDVX,
+
+    "ir/barbatos": ConverterIRBarbatos,
+    "ir/beatoraja": ConverterIRBeatoraja,
+    "ir/fervidex": ConverterIRFervidex,
+    "ir/fervidex-static": ConverterIRFervidexStatic,
+    "ir/direct-manual": ConverterBatchManual,
+    "ir/usc": ConverterIRUSC,
+    "ir/chunitachi": ConverterBatchManual,
+};
diff --git a/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts b/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts
index 20dbc146e..25db56569 100644
--- a/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts
+++ b/server/src/lib/score-import/import-types/file/mer-iidx/parser.ts
@@ -58,6 +58,5 @@ export function ParseMerIIDX(
         context: {},
         iterable: jsonData as MerScore[],
         game: "iidx",
-        ConverterFunction: ConvertFileMerIIDX,
     };
 }
diff --git a/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts b/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts
index c2a3507fe..10dee58bb 100644
--- a/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts
+++ b/server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts
@@ -87,6 +87,5 @@ export function ParseSolidStateXML(
         context: {},
         iterable: parsedXML.s3data.scoredata.song as S3Score[],
         game: "iidx",
-        ConverterFunction: ConvertFileS3, // todo
     };
 }
diff --git a/server/src/lib/score-import/import-types/ir/barbatos/parser.ts b/server/src/lib/score-import/import-types/ir/barbatos/parser.ts
index a63db8ac1..4266d20e6 100644
--- a/server/src/lib/score-import/import-types/ir/barbatos/parser.ts
+++ b/server/src/lib/score-import/import-types/ir/barbatos/parser.ts
@@ -40,7 +40,6 @@ export function ParseBarbatosSingle(
         context: {},
         game: "sdvx",
         iterable: ([body] as unknown) as BarbatosScore[],
-        ConverterFunction: ConverterIRBarbatos,
         classHandler: null,
     };
 }
diff --git a/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts b/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts
index 8f0757328..930ee0137 100644
--- a/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts
+++ b/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts
@@ -108,7 +108,6 @@ export function ParseBeatorajaSingle(
         },
         game: "bms",
         iterable: ([body.score] as unknown) as BeatorajaScore[],
-        ConverterFunction: ConverterIRBeatoraja,
         classHandler: null,
     };
 }
diff --git a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts
index 28ee8ecd4..8cf857ce9 100644
--- a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts
+++ b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts
@@ -79,7 +79,6 @@ export function ParseFervidexStatic(
         context: { version },
         game: "iidx",
         iterable: scores,
-        ConverterFunction: ConverterIRFervidexStatic,
         classHandler: FerStaticClassHandler(body),
     };
 }
diff --git a/server/src/lib/score-import/import-types/ir/fervidex/parser.ts b/server/src/lib/score-import/import-types/ir/fervidex/parser.ts
index bdda5c846..95b35315b 100644
--- a/server/src/lib/score-import/import-types/ir/fervidex/parser.ts
+++ b/server/src/lib/score-import/import-types/ir/fervidex/parser.ts
@@ -115,7 +115,6 @@ export function ParseFervidexSingle(
         context: { version },
         game: "iidx",
         iterable: ([body] as unknown) as FervidexScore[],
-        ConverterFunction: ConverterIRFervidex,
         classHandler: null,
     };
 }
diff --git a/server/src/lib/score-import/import-types/ir/usc/parser.ts b/server/src/lib/score-import/import-types/ir/usc/parser.ts
index b2d9f7534..220833617 100644
--- a/server/src/lib/score-import/import-types/ir/usc/parser.ts
+++ b/server/src/lib/score-import/import-types/ir/usc/parser.ts
@@ -46,6 +46,5 @@ export function ParseIRUSC(
         game: "usc",
         iterable: [body.score] as USCClientScore[],
         classHandler: null,
-        ConverterFunction: ConverterIRUSC,
     };
 }