DB validation groundwork

server/src/db/schemas.ts

@@ -266,7 +266,4 @@ export const SCHEMAS: Partial<RevaluedObject<typeof db, PrudenceSchema>> = {
     users: PRUDENCE_PRIVATE_USER,
     "iidx-bpi-data": PRUDENCE_IIDX_BPI_DATA,
     counters: PRUDENCE_COUNTER,
-    scores: {
-        foo: () => "DO NOT USE THIS VALIDATION FUNCTION. USE PRUDENCE_SCORE FORMATS INSTEAD.",
-    },
 };

server/src/internal-api/import/import.test.ts

@@ -1,11 +1,12 @@
 import t from "tap";
 import mockApi from "../../test-utils/mock-api";
-import { TestingIIDXEamusementCSV26 } from "../../test-utils/test-data";
+import { TestingIIDXEamusementCSV26, TestingIIDXEamusementCSV27 } from "../../test-utils/test-data";
 import { CloseAllConnections } from "../../test-utils/close-connections";
 import { RequireNeutralAuthentication } from "../../test-utils/api-common";
 import { CreateFakeAuthCookie } from "../../test-utils/fake-session";
 import ResetDBState from "../../test-utils/reset-db-state";
 import { rootLogger } from "../../logger";
+import { GetUnsuccessfulScores } from "../../test-utils/score-import-utils";
 
 t.test("POST /internal-api/import/file", async (t) => {
     const cookie = await CreateFakeAuthCookie(mockApi);
@@ -22,7 +23,24 @@ t.test("POST /internal-api/import/file", async (t) => {
                 .field("importType", "csv:eamusement-iidx")
                 .field("playtype", "SP");
 
-            t.equal(res.body.success, true);
+            t.equal(res.body.success, true, "Should be successful.");
+
+            t.equal(GetUnsuccessfulScores(res.body.body), 0, "Should have 0 failed scores.");
+
+            t.end();
+        });
+
+        t.test("Valid Heroic Verse CSV import", async (t) => {
+            let res = await mockApi
+                .post("/internal-api/import/file")
+                .set("Cookie", cookie)
+                .attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv")
+                .field("importType", "csv:eamusement-iidx")
+                .field("playtype", "SP");
+
+            t.equal(res.body.success, true, "Should be successful.");
+
+            t.equal(GetUnsuccessfulScores(res.body.body), 0, "Should have 0 failed scores.");
 
             t.end();
         });

server/src/internal-api/import/import.ts

@@ -87,9 +87,7 @@ router.post(
             return res.status(200).json({
                 success: true,
                 description: "Import successful.",
-                body: {
-                    import: importDocument,
-                },
+                body: importDocument,
             });
         } catch (err) {
             if (err instanceof ScoreImportFatalError) {

server/src/score-import/framework/score-import-main.ts

@@ -47,6 +47,12 @@ export default async function ScoreImportMain<D, C>(
         userID: user.id,
     };
 
+    logger.info(
+        `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${
+            importInfo.length
+        } documents. Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`
+    );
+
     // Add this to the imports database
     // @todo
 

server/src/scripts/validate-database-errs/counters.json

@@ -0,0 +1 @@
+[]

server/src/scripts/validate-database-errs/iidx-bpi-data.json

@@ -0,0 +1 @@
+[]

server/src/scripts/validate-database-errs/users.json

@@ -0,0 +1 @@
+[]

server/src/scripts/validate-database.ts

@@ -0,0 +1,55 @@
+import p from "prudence";
+import db, { monkDB } from "../db/db";
+import { SCHEMAS } from "../db/schemas";
+import CreateLogCtx from "../logger";
+import fs from "fs";
+import path from "path";
+import { PrudenceError } from "prudence/js/error";
+
+const BASE_DIR = path.join(__dirname, "./validate-database-errs");
+
+const logger = CreateLogCtx("validate-database.ts");
+
+async function ValidateStaticSchemas(): Promise<void> {
+    for (const c in SCHEMAS) {
+        // @ts-expect-error shut up
+        const schema = SCHEMAS[c];
+
+        logger.info(`=== Validating Collection ${c}... ===`);
+
+        let successCount = 0;
+        let total = 0;
+        let fails: { err: PrudenceError; doc: unknown }[] = [];
+
+        // eslint-disable-next-line no-await-in-loop
+        await monkDB
+            .get(c)
+            .find({}, { projection: { _id: 0 } })
+            // @ts-expect-error monk's types are just so broken, wtf?
+            .each((c: unknown) => {
+                total++;
+                let res = p(c, schema);
+
+                if (res === null) {
+                    successCount++;
+                } else {
+                    logger.error(res);
+                    fails.push({ err: res, doc: c });
+                }
+            });
+
+        logger.info(`Validated ${total} objects.`);
+        logger.info(`Success: ${successCount} (${((successCount * 100) / total).toFixed(2)}%)`);
+        logger.info(`Fail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`);
+
+        if (fails.length !== 0) {
+            logger.severe(`Invalid documents found! Please resolve them.`);
+            fs.writeFileSync(path.join(BASE_DIR, `${c}.json`), JSON.stringify(fails));
+        }
+    }
+
+    logger.info("Done!");
+    process.exit(0);
+}
+
+ValidateStaticSchemas();

server/src/test-utils/fake-session.ts

@@ -5,9 +5,8 @@ import supertest from "supertest";
 const logger = CreateLogCtx("fake-session.ts");
 
 export async function CreateFakeAuthCookie(mockApi: supertest.SuperTest<supertest.Test>) {
-    logger.info("1");
     await ResetDBState();
-    logger.info("2");
+
     // possible security issue, ask hazel
     let res = await mockApi.post("/internal-api/auth/login").send({
         username: "test_zkldi",
@@ -15,13 +14,10 @@ export async function CreateFakeAuthCookie(mockApi: supertest.SuperTest<supertes
         captcha: "asdf",
     });
 
-    logger.info("3");
-
     if (res.status !== 200) {
         logger.crit("Failed to login. Cannot generate auth cookie.");
         throw res.body;
     }
-    logger.info("4");
 
     return res.headers["set-cookie"] as string[];
 }

server/src/test-utils/score-import-utils.ts

@@ -0,0 +1,5 @@
+import { KTBlackImportDocument } from "kamaitachi-common";
+
+export function GetUnsuccessfulScores(importDoc: KTBlackImportDocument) {
+    return importDoc.importInfo.filter((e) => !e.success).length;
+}