From 121a7bb013328643085bde7d9de90793d738db2b Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 20 Apr 2021 20:56:45 +0100 Subject: [PATCH] DB validation groundwork --- server/src/db/schemas.ts | 3 - server/src/internal-api/import/import.test.ts | 22 +++++++- server/src/internal-api/import/import.ts | 4 +- .../framework/score-import-main.ts | 6 ++ .../validate-database-errs/counters.json | 1 + .../validate-database-errs/iidx-bpi-data.json | 1 + .../scripts/validate-database-errs/users.json | 1 + server/src/scripts/validate-database.ts | 55 +++++++++++++++++++ server/src/test-utils/fake-session.ts | 6 +- server/src/test-utils/score-import-utils.ts | 5 ++ 10 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 server/src/scripts/validate-database-errs/counters.json create mode 100644 server/src/scripts/validate-database-errs/iidx-bpi-data.json create mode 100644 server/src/scripts/validate-database-errs/users.json create mode 100644 server/src/scripts/validate-database.ts create mode 100644 server/src/test-utils/score-import-utils.ts diff --git a/server/src/db/schemas.ts b/server/src/db/schemas.ts index a31d30ea0..227af95fb 100644 --- a/server/src/db/schemas.ts +++ b/server/src/db/schemas.ts @@ -266,7 +266,4 @@ export const SCHEMAS: Partial> = { users: PRUDENCE_PRIVATE_USER, "iidx-bpi-data": PRUDENCE_IIDX_BPI_DATA, counters: PRUDENCE_COUNTER, - scores: { - foo: () => "DO NOT USE THIS VALIDATION FUNCTION. USE PRUDENCE_SCORE FORMATS INSTEAD.", - }, }; diff --git a/server/src/internal-api/import/import.test.ts b/server/src/internal-api/import/import.test.ts index cb0354379..5f39de96e 100644 --- a/server/src/internal-api/import/import.test.ts +++ b/server/src/internal-api/import/import.test.ts @@ -1,11 +1,12 @@ import t from "tap"; import mockApi from "../../test-utils/mock-api"; -import { TestingIIDXEamusementCSV26 } from "../../test-utils/test-data"; +import { TestingIIDXEamusementCSV26, TestingIIDXEamusementCSV27 } from "../../test-utils/test-data"; import { CloseAllConnections } from "../../test-utils/close-connections"; import { RequireNeutralAuthentication } from "../../test-utils/api-common"; import { CreateFakeAuthCookie } from "../../test-utils/fake-session"; import ResetDBState from "../../test-utils/reset-db-state"; import { rootLogger } from "../../logger"; +import { GetUnsuccessfulScores } from "../../test-utils/score-import-utils"; t.test("POST /internal-api/import/file", async (t) => { const cookie = await CreateFakeAuthCookie(mockApi); @@ -22,7 +23,24 @@ t.test("POST /internal-api/import/file", async (t) => { .field("importType", "csv:eamusement-iidx") .field("playtype", "SP"); - t.equal(res.body.success, true); + t.equal(res.body.success, true, "Should be successful."); + + t.equal(GetUnsuccessfulScores(res.body.body), 0, "Should have 0 failed scores."); + + t.end(); + }); + + t.test("Valid Heroic Verse CSV import", async (t) => { + let res = await mockApi + .post("/internal-api/import/file") + .set("Cookie", cookie) + .attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv") + .field("importType", "csv:eamusement-iidx") + .field("playtype", "SP"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(GetUnsuccessfulScores(res.body.body), 0, "Should have 0 failed scores."); t.end(); }); diff --git a/server/src/internal-api/import/import.ts b/server/src/internal-api/import/import.ts index 148be5c34..8169fde53 100644 --- a/server/src/internal-api/import/import.ts +++ b/server/src/internal-api/import/import.ts @@ -87,9 +87,7 @@ router.post( return res.status(200).json({ success: true, description: "Import successful.", - body: { - import: importDocument, - }, + body: importDocument, }); } catch (err) { if (err instanceof ScoreImportFatalError) { diff --git a/server/src/score-import/framework/score-import-main.ts b/server/src/score-import/framework/score-import-main.ts index e2fb4e090..db316721d 100644 --- a/server/src/score-import/framework/score-import-main.ts +++ b/server/src/score-import/framework/score-import-main.ts @@ -47,6 +47,12 @@ export default async function ScoreImportMain( userID: user.id, }; + logger.info( + `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${ + importInfo.length + } documents. Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc` + ); + // Add this to the imports database // @todo diff --git a/server/src/scripts/validate-database-errs/counters.json b/server/src/scripts/validate-database-errs/counters.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/server/src/scripts/validate-database-errs/counters.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/server/src/scripts/validate-database-errs/iidx-bpi-data.json b/server/src/scripts/validate-database-errs/iidx-bpi-data.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/server/src/scripts/validate-database-errs/iidx-bpi-data.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/server/src/scripts/validate-database-errs/users.json b/server/src/scripts/validate-database-errs/users.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/server/src/scripts/validate-database-errs/users.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/server/src/scripts/validate-database.ts b/server/src/scripts/validate-database.ts new file mode 100644 index 000000000..74d60d60d --- /dev/null +++ b/server/src/scripts/validate-database.ts @@ -0,0 +1,55 @@ +import p from "prudence"; +import db, { monkDB } from "../db/db"; +import { SCHEMAS } from "../db/schemas"; +import CreateLogCtx from "../logger"; +import fs from "fs"; +import path from "path"; +import { PrudenceError } from "prudence/js/error"; + +const BASE_DIR = path.join(__dirname, "./validate-database-errs"); + +const logger = CreateLogCtx("validate-database.ts"); + +async function ValidateStaticSchemas(): Promise { + for (const c in SCHEMAS) { + // @ts-expect-error shut up + const schema = SCHEMAS[c]; + + logger.info(`=== Validating Collection ${c}... ===`); + + let successCount = 0; + let total = 0; + let fails: { err: PrudenceError; doc: unknown }[] = []; + + // eslint-disable-next-line no-await-in-loop + await monkDB + .get(c) + .find({}, { projection: { _id: 0 } }) + // @ts-expect-error monk's types are just so broken, wtf? + .each((c: unknown) => { + total++; + let res = p(c, schema); + + if (res === null) { + successCount++; + } else { + logger.error(res); + fails.push({ err: res, doc: c }); + } + }); + + logger.info(`Validated ${total} objects.`); + logger.info(`Success: ${successCount} (${((successCount * 100) / total).toFixed(2)}%)`); + logger.info(`Fail: ${fails.length} (${((fails.length * 100) / total).toFixed(2)}%)`); + + if (fails.length !== 0) { + logger.severe(`Invalid documents found! Please resolve them.`); + fs.writeFileSync(path.join(BASE_DIR, `${c}.json`), JSON.stringify(fails)); + } + } + + logger.info("Done!"); + process.exit(0); +} + +ValidateStaticSchemas(); diff --git a/server/src/test-utils/fake-session.ts b/server/src/test-utils/fake-session.ts index d533e8558..2f2bbb82c 100644 --- a/server/src/test-utils/fake-session.ts +++ b/server/src/test-utils/fake-session.ts @@ -5,9 +5,8 @@ import supertest from "supertest"; const logger = CreateLogCtx("fake-session.ts"); export async function CreateFakeAuthCookie(mockApi: supertest.SuperTest) { - logger.info("1"); await ResetDBState(); - logger.info("2"); + // possible security issue, ask hazel let res = await mockApi.post("/internal-api/auth/login").send({ username: "test_zkldi", @@ -15,13 +14,10 @@ export async function CreateFakeAuthCookie(mockApi: supertest.SuperTest !e.success).length; +}