Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-04 04:48:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

Completely refactor how importTypes work

Browse Source
This commit is contained in:
zkldi
2021-05-13 06:46:26 +01:00
parent 51a69321db
commit 2f053ca617
22 changed files with 272 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/pnpm-lock.yaml
Generated
+3 -3
View File
@@ -60,7 +60,7 @@ dependencies:
express-async-errors: 3.1.1_express@4.17.1
express-session: 1.17.1
fast-json-stable-hash: 1.0.2
kamaitachi-common: github.com/zkldi/Kamaitachi-common/07931179339f18fb85bf4a554b81a1fffe010c1f
kamaitachi-common: github.com/zkldi/Kamaitachi-common/3ee7ec47e9f5e76a956f09af4fe851d159e31b25
mongodb: 3.6.6
monk: 7.3.4
multer: 1.4.2
@@ -4637,8 +4637,8 @@ packages:
'@types/yoga-layout': 1.9.2
dev: true
github.com/zkldi/Kamaitachi-common/07931179339f18fb85bf4a554b81a1fffe010c1f:
resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/07931179339f18fb85bf4a554b81a1fffe010c1f}
github.com/zkldi/Kamaitachi-common/3ee7ec47e9f5e76a956f09af4fe851d159e31b25:
resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/3ee7ec47e9f5e76a956f09af4fe851d159e31b25}
name: kamaitachi-common
version: 0.1.0
dependencies:
server/src/api/api.ts
+3 -1
View File
@@ -1,10 +1,12 @@
import { Router } from "express";
import authRouter from "./auth/auth";
import importRouter from "./import/import";
import irRouter from "./ir/ir";
const router = Router({ mergeParams: true });
const router: Router = Router({ mergeParams: true });
router.use("/auth", authRouter);
router.use("/import", importRouter);
router.use("/ir", irRouter);
export default router;
server/src/api/import/import.ts
+2 -2
View File
@@ -97,8 +97,8 @@ router.post(
}
);
import ParseEamusementCSV from "../../score-import/import-types/file/csv_eamusement-iidx/parser";
import ParseBatchManual from "../../score-import/import-types/file/json_batch-manual/parser";
import ParseEamusementCSV from "../../score-import/import-types/file/csv/eamusement-iidx/parser";
import ParseBatchManual from "../../score-import/import-types/file/json/batch-manual/parser";
/**
* Resolves the data from a file upload into an iterable,
server/src/api/ir/direct-manual/direct-manual.ts
+29 -2
View File
@@ -1,12 +1,39 @@
import { Router } from "express";
import { GetUserWithID } from "../../../core/user-core";
import CreateLogCtx from "../../../logger";
import { RequireLoggedIn } from "../../../middleware/require-logged-in";
import { ExpressWrappedScoreImportMain } from "../../../score-import/framework/express-wrapper";
import ParseDirectManual from "../../../score-import/import-types/ir/json/direct-manual/parser";
const router = Router({ mergeParams: true });
const router: Router = Router({ mergeParams: true });
const logger = CreateLogCtx("direct-manual.ts");
/**
* Imports scores in ir/json:direct-manual form.
* @name /api/ir/direct-manual/import
*/
router.post("/import", RequireLoggedIn, async (req, res) => {});
router.post("/import", RequireLoggedIn, async (req, res) => {
const userDoc = await GetUserWithID(req.session.ktchi!.userID);
if (!userDoc) {
logger.severe(
`User ${req.session.ktchi!.userID} does not have an associated user document.`
);
return res.status(500).json({
success: false,
description: "An internal error has occured.",
});
}
let responseData = await ExpressWrappedScoreImportMain(
userDoc,
true,
"ir/json:direct-manual",
(logger) => ParseDirectManual(req.body, logger)
);
return res.status(responseData.statusCode).json(responseData.body);
});
export default router;
server/src/api/ir/ir.ts
+3
View File
@@ -1,5 +1,8 @@
import { Router } from "express";
import directManualIR from "./direct-manual/direct-manual";
const router: Router = Router({ mergeParams: true });
router.use("/direct-manual", directManualIR);
export default router;
server/src/score-import/framework/score-import-main.ts
+1
View File
@@ -59,6 +59,7 @@ export default async function ScoreImportMain<D, C>(
const importTimeStart = process.hrtime.bigint();
let importInfo = await ImportAllIterableData(
user.id,
importType,
iterable,
ConverterFunction,
context,
server/src/score-import/framework/score-importing/converter-failures.ts
+7 -7
View File
@@ -1,6 +1,6 @@
// Various Errors that can occur during processing.
import { Converters } from "../../import-types/import-types";
import { ImportTypes } from "kamaitachi-common";
import { ImportTypeContextMap, ImportTypeDataMap } from "../../../types";
export class ConverterFailure {
message: string | null;
@@ -15,16 +15,16 @@ export class ConverterFailure {
* belongs to. The identifier used to try and match this chart is stored here.
* Alongside any other fields used in the query.
*/
export class KTDataNotFoundFailure<T extends keyof typeof Converters> extends ConverterFailure {
data: Parameters<typeof Converters[T]>[0];
converterContext: Parameters<typeof Converters[T]>[1];
export class KTDataNotFoundFailure<T extends ImportTypes> extends ConverterFailure {
data: ImportTypeDataMap[T];
converterContext: ImportTypeContextMap[T];
importType: T;
constructor(
message: string | null,
importType: T,
data: Parameters<typeof Converters[T]>[0],
context: Parameters<typeof Converters[T]>[1]
data: ImportTypeDataMap[T],
context: ImportTypeContextMap[T]
) {
super(message);
server/src/score-import/framework/score-importing/score-importing.ts
+7 -2
View File
@@ -4,6 +4,7 @@ import {
integer,
ScoreDocument,
AnySongDocument,
ImportTypes,
} from "kamaitachi-common";
import {
DryScore,
@@ -34,6 +35,7 @@ import { AppendLogCtx } from "../../../logger";
*/
export async function ImportAllIterableData<D, C>(
userID: integer,
importType: ImportTypes,
iterableData: Iterable<D> | AsyncIterable<D>,
ConverterFunction: ConverterFunction<D, C>,
context: C,
@@ -47,7 +49,9 @@ export async function ImportAllIterableData<D, C>(
// An example would be making an api request after exhausting
// the first set of data.
for await (const data of iterableData) {
promises.push(ImportIterableDatapoint(userID, data, ConverterFunction, context, logger));
promises.push(
ImportIterableDatapoint(userID, importType, data, ConverterFunction, context, logger)
);
}
// Due to the fact that ProcessIterableDatapoint may return an array instead of a single result
@@ -104,6 +108,7 @@ export async function ImportAllIterableData<D, C>(
*/
export async function ImportIterableDatapoint<D, C>(
userID: integer,
importType: ImportTypes,
data: D,
ConverterFunction: ConverterFunction<D, C>,
context: C,
@@ -112,7 +117,7 @@ export async function ImportIterableDatapoint<D, C>(
let converterReturns: ConverterFunctionReturns;
try {
converterReturns = await ConverterFunction(data, context, logger);
converterReturns = await ConverterFunction(data, context, importType, logger);
} catch (err) {
converterReturns = err;
}
server/src/score-import/import-types/file/json_batch-manual/converter.test.ts → server/src/score-import/import-types/common/batch-manual/converter.test.ts
+35 -12
View File
@@ -2,7 +2,7 @@
import t from "tap";
import CreateLogCtx from "../../../../logger";
import { Testing511Song, Testing511SPA } from "../../../../test-utils/test-data";
import ConverterFn, { ResolveChartFromSong, ResolveMatchTypeToKTData } from "./converter";
import { ResolveChartFromSong, ResolveMatchTypeToKTData, ConverterFn } from "./converter";
import deepmerge from "deepmerge";
import escapeStringRegexp from "../../../../core/escape-string-regexp";
import { CloseMongoConnection } from "../../../../db/db";
@@ -34,11 +34,13 @@ const ktdWrap = (msg: string, game: Game = "iidx", version = null): any => ({
const logger = CreateLogCtx("converter.test.ts");
const importType = "file/json:batch-manual" as const;
t.test("#ResolveMatchTypeToKTData", (t) => {
t.beforeEach(ResetDBState);
t.test("Should resolve for the songID if the matchType is songID", async (t) => {
let res = await ResolveMatchTypeToKTData(baseBatchManualScore, context, logger);
let res = await ResolveMatchTypeToKTData(baseBatchManualScore, context, importType, logger);
t.hasStrict(
res,
@@ -52,6 +54,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
// @ts-expect-error bad
deepmerge(baseBatchManualScore, { identifier: "90000" }),
context,
importType,
logger
),
ktdWrap("Cannot find song with songID 90000")
@@ -64,6 +67,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
let res = await ResolveMatchTypeToKTData(
deepmerge(baseBatchManualScore, { matchType: "songTitle", identifier: "5.1.1." }),
context,
importType,
logger
);
@@ -81,6 +85,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
identifier: "INVALID_TITLE",
}),
context,
importType,
logger
),
ktdWrap("Cannot find song with title INVALID_TITLE")
@@ -96,6 +101,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
let resMD5 = await ResolveMatchTypeToKTData(
deepmerge(baseBatchManualScore, { matchType: "bmsChartHash", identifier: GAZER17MD5 }),
context,
importType,
logger
);
@@ -112,6 +118,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
identifier: GAZER17SHA256,
}),
context,
importType,
logger
);
@@ -130,6 +137,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
identifier: "bad_hash",
}),
context,
importType,
logger
),
ktdWrap("Cannot find chart with hash ", "bms")
@@ -149,6 +157,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
difficulty: "EXPERT",
}),
context,
importType,
logger
);
@@ -169,6 +178,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
difficulty: "EXPERT",
}),
context,
importType,
logger
),
ktdWrap("Cannot find chart with hash", "ddr")
@@ -185,6 +195,7 @@ t.test("#ResolveMatchTypeToKTData", (t) => {
matchType: "BAD_MATCHTYPE",
}),
context,
importType,
logger
),
new InvalidScoreFailure(`Invalid matchType BAD_MATCHTYPE`) as any
@@ -203,7 +214,8 @@ t.test("#ResolveChartFromSong", (t) => {
let res = await ResolveChartFromSong(
Testing511Song,
baseBatchManualScore, // has playtype + diff
{ game: "iidx", service: "foo", version: null }
{ game: "iidx", service: "foo", version: null },
importType
);
t.hasStrict(res, Testing511SPA as any);
@@ -217,7 +229,8 @@ t.test("#ResolveChartFromSong", (t) => {
ResolveChartFromSong(
Testing511Song,
deepmerge(baseBatchManualScore, { playtype: null }),
{ game: "iidx", service: "foo", version: null }
{ game: "iidx", service: "foo", version: null },
importType
),
new InvalidScoreFailure(
`Missing 'playtype' field, but was necessary for this lookup.`
@@ -233,7 +246,8 @@ t.test("#ResolveChartFromSong", (t) => {
ResolveChartFromSong(
Testing511Song,
deepmerge(baseBatchManualScore, { difficulty: null }),
{ game: "iidx", service: "foo", version: null }
{ game: "iidx", service: "foo", version: null },
importType
),
new InvalidScoreFailure(
`Missing 'difficulty' field, but was necessary for this lookup.`
@@ -250,7 +264,8 @@ t.test("#ResolveChartFromSong", (t) => {
Testing511Song,
// @ts-expect-error faulty deepmerge types
deepmerge(baseBatchManualScore, { difficulty: "NOT_VALID_DIFFICULTY" }),
{ game: "iidx", service: "foo", version: null }
{ game: "iidx", service: "foo", version: null },
importType
),
new InvalidScoreFailure(
`Invalid Difficulty for iidx SP - Expected any of BEGINNER, NORMAL, HYPER, ANOTHER, LEGGENDARIA`
@@ -267,7 +282,8 @@ t.test("#ResolveChartFromSong", (t) => {
Testing511Song,
// @ts-expect-error faulty deepmerge types
deepmerge(baseBatchManualScore, { difficulty: "LEGGENDARIA" }), // 511 has no legg (yet, lol)
{ game: "iidx", service: "foo", version: null }
{ game: "iidx", service: "foo", version: null },
importType
),
ktdWrap("Cannot find chart for 5.1.1. (SP LEGGENDARIA)")
);
@@ -276,11 +292,16 @@ t.test("#ResolveChartFromSong", (t) => {
});
t.test("Should successfully lookup if version is provided.", async (t) => {
let res = await ResolveChartFromSong(Testing511Song, baseBatchManualScore, {
game: "iidx",
service: "foo",
version: "27",
});
let res = await ResolveChartFromSong(
Testing511Song,
baseBatchManualScore,
{
game: "iidx",
service: "foo",
version: "27",
},
importType
);
t.hasStrict(res, Testing511SPA as any);
@@ -295,6 +316,7 @@ t.test("#ConverterFn", (t) => {
let res = await ConverterFn(
baseBatchManualScore,
{ game: "iidx", service: "foo", version: null },
importType,
logger
);
@@ -329,6 +351,7 @@ t.test("#ConverterFn", (t) => {
// @ts-expect-error broken deepmerge
deepmerge(baseBatchManualScore, { score: 1000 }),
{ game: "iidx", service: "foo", version: null },
importType,
logger
),
{ message: /Invalid score of 1000, produced percent of 127/u } as any
server/src/score-import/import-types/file/json_batch-manual/converter.ts → server/src/score-import/import-types/common/batch-manual/converter.ts
+23 -16
View File
@@ -1,6 +1,6 @@
import { ConverterFnReturn, ConverterFunction, DryScore, KtLogger } from "../../../../types";
import { BatchManualContext, BatchManualScore } from "./types";
import { AnyChartDocument, AnySongDocument } from "kamaitachi-common";
import { BatchManualContext, BatchManualScore } from "../../common/batch-manual/types";
import { AnyChartDocument, AnySongDocument, ImportTypes } from "kamaitachi-common";
import {
InternalFailure,
InvalidScoreFailure,
@@ -23,14 +23,21 @@ import {
} from "../../../framework/common/score-utils";
import { gamePercentMax } from "kamaitachi-common/js/config";
const ConverterFn: ConverterFunction<BatchManualScore, BatchManualContext> = async (
/**
* Creates a ConverterFn for the BatchManualScore format. This curries
* the importType into the function, so the right failures can be
* returned.
* @returns A BatchManualScore Converter.
*/
export const ConverterFn: ConverterFunction<BatchManualScore, BatchManualContext> = async (
data,
context,
logger: KtLogger
importType,
logger
): Promise<ConverterFnReturn> => {
let game = context.game;
let { song, chart } = await ResolveMatchTypeToKTData(data, context, logger);
let { song, chart } = await ResolveMatchTypeToKTData(data, context, importType, logger);
let percent = GenericCalculatePercent(game, data.score, chart);
@@ -46,7 +53,7 @@ const ConverterFn: ConverterFunction<BatchManualScore, BatchManualContext> = asy
game: game,
service: context.service,
comment: data.comment ?? null,
importType: "file/json:batch-manual",
importType,
timeAchieved: data.timeAchieved ?? null,
scoreData: {
lamp: data.lamp,
@@ -69,6 +76,7 @@ const ConverterFn: ConverterFunction<BatchManualScore, BatchManualContext> = asy
export async function ResolveMatchTypeToKTData(
data: BatchManualScore,
context: BatchManualContext,
importType: ImportTypes,
logger: KtLogger
): Promise<{ song: AnySongDocument; chart: AnyChartDocument }> {
const game = context.game;
@@ -83,7 +91,7 @@ export async function ResolveMatchTypeToKTData(
if (!chart) {
throw new KTDataNotFoundFailure(
`Cannot find chart for hash ${data.identifier}.`,
"file/json:batch-manual",
importType,
data,
context
);
@@ -121,7 +129,7 @@ export async function ResolveMatchTypeToKTData(
if (!chart) {
throw new KTDataNotFoundFailure(
`Cannot find chart for songHash ${data.identifier} (${data.playtype} ${difficulty}).`,
"file/json:batch-manual",
importType,
data,
context
);
@@ -146,13 +154,13 @@ export async function ResolveMatchTypeToKTData(
if (!song) {
throw new KTDataNotFoundFailure(
`Cannot find song with songID ${data.identifier}.`,
"file/json:batch-manual",
importType,
data,
context
);
}
let chart = await ResolveChartFromSong(song, data, context);
let chart = await ResolveChartFromSong(song, data, context, importType);
return { song, chart };
} else if (data.matchType === "songTitle" || data.matchType === "title") {
@@ -161,13 +169,13 @@ export async function ResolveMatchTypeToKTData(
if (!song) {
throw new KTDataNotFoundFailure(
`Cannot find song with title ${data.identifier}.`,
"file/json:batch-manual",
importType,
data,
context
);
}
let chart = await ResolveChartFromSong(song, data, context);
let chart = await ResolveChartFromSong(song, data, context, importType);
return { song, chart };
}
@@ -182,7 +190,8 @@ export async function ResolveMatchTypeToKTData(
export async function ResolveChartFromSong(
song: AnySongDocument,
data: BatchManualScore,
context: BatchManualContext
context: BatchManualContext,
importType: ImportTypes
) {
let game = context.game;
@@ -217,7 +226,7 @@ export async function ResolveChartFromSong(
if (!chart) {
throw new KTDataNotFoundFailure(
`Cannot find chart for ${song.title} (${data.playtype} ${difficulty})`,
"file/json:batch-manual",
importType,
data,
context
);
@@ -225,5 +234,3 @@ export async function ResolveChartFromSong(
return chart;
}
export default ConverterFn;
server/src/score-import/import-types/file/json_batch-manual/parser.test.ts → server/src/score-import/import-types/common/batch-manual/parser.test.ts
+52 -27
View File
@@ -2,7 +2,7 @@ import t from "tap";
import { CloseMongoConnection } from "../../../../db/db";
import CreateLogCtx from "../../../../logger";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import ParserFn from "./parser";
import { ParseBatchManualFromObject as ParserFn } from "./parser";
import { BatchManual } from "./types";
import escapeRegex from "../../../../core/escape-string-regexp";
import deepmerge from "deepmerge";
@@ -47,7 +47,7 @@ function dm(sc: any) {
t.test("#ParserFn", (t) => {
t.test("Non-Object", (t) => {
t.throws(
() => ParserFn(fileify(false), {}, logger),
() => ParserFn(false, "file/json:batch-manual", logger),
new ScoreImportFatalError(
400,
"Invalid BATCH-MANUAL (Not an object, recieved boolean.)"
@@ -60,7 +60,7 @@ t.test("#ParserFn", (t) => {
t.test("No Header", (t) => {
t.throws(
() => ParserFn(fileify({ body: [] }), {}, logger),
() => ParserFn({ body: [] }, "file/json:batch-manual", logger),
new ScoreImportFatalError(
400,
"Could not retrieve head.game - is this valid BATCH-MANUAL?"
@@ -73,7 +73,8 @@ t.test("#ParserFn", (t) => {
t.test("No Game", (t) => {
t.throws(
() => ParserFn(fileify({ body: [], head: { service: "foo" } }), {}, logger),
() =>
ParserFn({ body: [], head: { service: "foo" } }, "file/json:batch-manual", logger),
new ScoreImportFatalError(
400,
"Could not retrieve head.game - is this valid BATCH-MANUAL?"
@@ -88,8 +89,8 @@ t.test("#ParserFn", (t) => {
t.throws(
() =>
ParserFn(
fileify({ body: [], head: { service: "foo", game: "invalid_game" } }),
{},
{ body: [], head: { service: "foo", game: "invalid_game" } },
"file/json:batch-manual",
logger
),
new ScoreImportFatalError(
@@ -100,7 +101,12 @@ t.test("#ParserFn", (t) => {
);
t.throws(
() => ParserFn(fileify({ body: [], head: { service: "foo", game: 123 } }), {}, logger),
() =>
ParserFn(
{ body: [], head: { service: "foo", game: 123 } },
"file/json:batch-manual",
logger
),
new ScoreImportFatalError(
400,
"Invalid game 123 - expected any of iidx, museca, maimai, jubeat, popn, sdvx, ddr, bms, chunithm, gitadora, usc"
@@ -113,7 +119,12 @@ t.test("#ParserFn", (t) => {
t.test("Invalid Service", (t) => {
t.throws(
() => ParserFn(fileify({ body: [], head: { service: "1", game: "iidx" } }), {}, logger),
() =>
ParserFn(
{ body: [], head: { service: "1", game: "iidx" } },
"file/json:batch-manual",
logger
),
new ScoreImportFatalError(
400,
"Invalid BATCH-MANUAL: head.service | Expected a string with length between 3 and 15. | Received 1 [string]."
@@ -122,7 +133,12 @@ t.test("#ParserFn", (t) => {
);
t.throws(
() => ParserFn(fileify({ body: [], head: { service: 1, game: "iidx" } }), {}, logger),
() =>
ParserFn(
{ body: [], head: { service: 1, game: "iidx" } },
"file/json:batch-manual",
logger
),
new ScoreImportFatalError(
400,
"Invalid BATCH-MANUAL: head.service | Expected a string with length between 3 and 15. | Received 1 [number]."
@@ -135,8 +151,8 @@ t.test("#ParserFn", (t) => {
t.test("Valid Empty BATCH-MANUAL", (t) => {
let res = ParserFn(
fileify({ body: [], head: { service: "foo", game: "iidx" } }),
{},
{ body: [], head: { service: "foo", game: "iidx" } },
"file/json:batch-manual",
logger
);
@@ -190,7 +206,7 @@ t.test("#ParserFn", (t) => {
],
head: { service: "foo", game: "iidx" },
} as BatchManual),
{},
"file/json:batch-manual",
logger
);
@@ -239,8 +255,8 @@ t.test("#ParserFn", (t) => {
t.test("Valid HitMeta", (t) => {
let res = ParserFn(
fileify(dm({ hitMeta: { bp: 10, gauge: 100, gaugeHistory: null, comboBreak: 7 } })),
{},
dm({ hitMeta: { bp: 10, gauge: 100, gaugeHistory: null, comboBreak: 7 } }),
"file/json:batch-manual",
logger
);
@@ -275,8 +291,8 @@ t.test("#ParserFn", (t) => {
t.test("Valid HitData", (t) => {
let res = ParserFn(
fileify(dm({ hitData: { pgreat: 1, great: null, bad: 0 } })),
{},
dm({ hitData: { pgreat: 1, great: null, bad: 0 } }),
"file/json:batch-manual",
logger
);
@@ -328,7 +344,7 @@ t.test("#ParserFn", (t) => {
],
head: { service: "foo", game: "iidx" },
}),
{},
"file/json:batch-manual",
logger
);
@@ -344,7 +360,7 @@ t.test("#ParserFn", (t) => {
});
t.test("Non-numeric score", (t) => {
let fn = () => ParserFn(fileify(dm({ score: "123" })), {}, logger);
let fn = () => ParserFn(dm({ score: "123" }), "file/json:batch-manual", logger);
t.throws(
fn,
@@ -358,7 +374,8 @@ t.test("#ParserFn", (t) => {
});
t.test("Invalid timeAchieved", (t) => {
let fn = () => ParserFn(fileify(dm({ timeAchieved: "string" })), {}, logger);
let fn = () =>
ParserFn(dm({ timeAchieved: "string" }), "file/json:batch-manual", logger);
t.throws(
fn,
@@ -369,7 +386,11 @@ t.test("#ParserFn", (t) => {
);
let fn2 = () =>
ParserFn(fileify(dm({ timeAchieved: 1_620_768_609_637 / 1000 })), {}, logger);
ParserFn(
dm({ timeAchieved: 1_620_768_609_637 / 1000 }),
"file/json:batch-manual",
logger
);
t.throws(
fn2,
@@ -385,7 +406,7 @@ t.test("#ParserFn", (t) => {
t.test("Invalid Playtype", (t) => {
// this is not a valid playtype for IIDX
let fn = () => ParserFn(fileify(dm({ playtype: "Single" })), {}, logger);
let fn = () => ParserFn(dm({ playtype: "Single" }), "file/json:batch-manual", logger);
t.throws(
fn,
@@ -400,7 +421,7 @@ t.test("#ParserFn", (t) => {
t.test("Invalid Identifier", (t) => {
// this is not a valid playtype for IIDX
let fn = () => ParserFn(fileify(dm({ identifier: null })), {}, logger);
let fn = () => ParserFn(dm({ identifier: null }), "file/json:batch-manual", logger);
t.throws(fn, mockErr("body[0].identifier | Expected string", "Received null [null]"));
@@ -408,7 +429,8 @@ t.test("#ParserFn", (t) => {
});
t.test("Invalid MatchType", (t) => {
let fn = () => ParserFn(fileify(dm({ matchType: "Invalid_MatchType" })), {}, logger);
let fn = () =>
ParserFn(dm({ matchType: "Invalid_MatchType" }), "file/json:batch-manual", logger);
t.throws(
fn,
@@ -422,11 +444,13 @@ t.test("#ParserFn", (t) => {
});
t.test("Invalid HitData", (t) => {
let fn = () => ParserFn(fileify(dm({ hitData: { not_key: 123 } })), {}, logger);
let fn = () =>
ParserFn(dm({ hitData: { not_key: 123 } }), "file/json:batch-manual", logger);
t.throws(fn, mockErr("body[0].hitData | Invalid Key not_key"));
let fn2 = () => ParserFn(fileify(dm({ hitData: { pgreat: "123" } })), {}, logger);
let fn2 = () =>
ParserFn(dm({ hitData: { pgreat: "123" } }), "file/json:batch-manual", logger);
t.throws(
fn2,
@@ -437,11 +461,12 @@ t.test("#ParserFn", (t) => {
});
t.test("Invalid HitMeta", (t) => {
let fn = () => ParserFn(fileify(dm({ hitMeta: { not_key: 123 } })), {}, logger);
let fn = () =>
ParserFn(dm({ hitMeta: { not_key: 123 } }), "file/json:batch-manual", logger);
t.throws(fn, mockErr("body[0].hitMeta | Unexpected"));
let fn2 = () => ParserFn(fileify(dm({ hitMeta: { bp: -1 } })), {}, logger);
let fn2 = () => ParserFn(dm({ hitMeta: { bp: -1 } }), "file/json:batch-manual", logger);
t.throws(fn2, mockErr("body[0].hitMeta.bp | Expected a positive integer"));
server/src/score-import/import-types/file/json_batch-manual/parser.ts → server/src/score-import/import-types/common/batch-manual/parser.ts
+12 -25
View File
@@ -3,8 +3,8 @@ import ScoreImportFatalError from "../../../framework/score-importing/score-impo
import { BatchManual, BatchManualContext, BatchManualScore } from "./types";
import p, { PrudenceSchema, ValidSchemaValue } from "prudence";
import { lamps, supportedGames, validHitData, validPlaytypes } from "kamaitachi-common/js/config";
import { Game } from "kamaitachi-common";
import ConverterFn from "./converter";
import { Game, ImportTypes } from "kamaitachi-common";
import { ConverterFn } from "./converter";
import deepmerge from "deepmerge";
const optNull = (v: ValidSchemaValue) => p.optional(p.nullable(v));
@@ -108,41 +108,30 @@ const PR_BatchManual = (game: Game): PrudenceSchema => ({
});
/**
* Parses a buffer of BATCH-MANUAL data.
* @param fileData - The buffer to parse.
* Parses an object of BATCH-MANUAL data.
* @param object - The object to parse.
* @param body - The request body that made this file import request.
*/
function ParseBatchManual(
fileData: Express.Multer.File,
body: Record<string, unknown>,
export function ParseBatchManualFromObject(
object: unknown,
importType: ImportTypes,
logger: KtLogger
): ParserFunctionReturnsSync<BatchManualScore, BatchManualContext> {
let jsonData: unknown;
try {
jsonData = JSON.parse(fileData.buffer.toString("utf-8"));
} catch (err) {
throw new ScoreImportFatalError(
400,
`Invalid JSON. (${err?.message ?? "No Error Message Available."})`
);
}
// now to perform some basic validation so we can return
// the iterable
if (typeof jsonData !== "object" || jsonData === null) {
if (typeof object !== "object" || object === null) {
throw new ScoreImportFatalError(
400,
`Invalid BATCH-MANUAL (Not an object, recieved ${
jsonData === null ? "null" : typeof jsonData
object === null ? "null" : typeof object
}.)`
);
}
// attempt to retrieve game
// @ts-expect-error man.
let possiblyGame = jsonData?.head?.game;
let possiblyGame = object?.head?.game;
if (!possiblyGame) {
throw new ScoreImportFatalError(
@@ -165,7 +154,7 @@ function ParseBatchManual(
// This mostly works as a sanity check, and doesn't
// check things like whether a score is > 100%
// or something.
let err = p(jsonData, PR_BatchManual(game));
let err = p(object, PR_BatchManual(game));
if (err) {
throw new ScoreImportFatalError(
@@ -181,7 +170,7 @@ function ParseBatchManual(
);
}
let batchManual = jsonData as BatchManual;
let batchManual = object as BatchManual;
return {
game,
@@ -194,5 +183,3 @@ function ParseBatchManual(
ConverterFunction: ConverterFn,
};
}
export default ParseBatchManual;
server/src/score-import/import-types/file/json_batch-manual/types.ts → server/src/score-import/import-types/common/batch-manual/types.ts
View File
server/src/score-import/import-types/file/csv_eamusement-iidx/converter.test.ts → server/src/score-import/import-types/file/csv/eamusement-iidx/converter.test.ts
+5 -5
View File
@@ -1,17 +1,17 @@
import t from "tap";
import db, { CloseMongoConnection } from "../../../../db/db";
import CreateLogCtx from "../../../../logger";
import ResetDBState from "../../../../test-utils/reset-db-state";
import db, { CloseMongoConnection } from "../../../../../db/db";
import CreateLogCtx from "../../../../../logger";
import ResetDBState from "../../../../../test-utils/reset-db-state";
import ConverterFn, { EamScoreConverter, EamScoreConverterWrapper } from "./converter";
import p from "prudence";
import { prAssert } from "../../../../test-utils/asserts";
import { prAssert } from "../../../../../test-utils/asserts";
import deepmerge from "deepmerge";
import { EamusementScoreData } from "./types";
import {
ConverterFailure,
InvalidScoreFailure,
KTDataNotFoundFailure,
} from "../../../framework/score-importing/converter-failures";
} from "../../../../framework/score-importing/converter-failures";
const logger = CreateLogCtx("converter.test.ts");
server/src/score-import/import-types/file/csv_eamusement-iidx/converter.ts → server/src/score-import/import-types/file/csv/eamusement-iidx/converter.ts
+9 -8
View File
@@ -1,15 +1,15 @@
import { config, ESDCore, Lamps, AnySongDocument, ChartDocument } from "kamaitachi-common";
import { DryScore, ConverterFunction, ConverterFnReturn, KtLogger } from "../../../../types";
import { FindChartWithPTDFVersion } from "../../../database-lookup/chart";
import { FindSongOnTitle } from "../../../database-lookup/song";
import { DryScore, ConverterFunction, ConverterFnReturn, KtLogger } from "../../../../../types";
import { FindChartWithPTDFVersion } from "../../../../database-lookup/chart";
import { FindSongOnTitle } from "../../../../database-lookup/song";
import {
KTDataNotFoundFailure,
InternalFailure,
InvalidScoreFailure,
} from "../../../framework/score-importing/converter-failures";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { GetGradeFromPercent } from "../../../framework/common/score-utils";
import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts";
} from "../../../../framework/score-importing/converter-failures";
import ScoreImportFatalError from "../../../../framework/score-importing/score-import-error";
import { GetGradeFromPercent } from "../../../../framework/common/score-utils";
import { AssertStrAsPositiveInt } from "../../../../framework/common/string-asserts";
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
const EAMUSEMENT_LAMP_RESOLVER: Map<string, Lamps["iidx:SP" | "iidx:DP"]> = new Map([
@@ -217,7 +217,8 @@ export async function EamScoreConverterWrapper(
const ConverterFn: ConverterFunction<IIDXEamusementCSVData, IIDXEamusementCSVContext> = async (
data,
context,
logger: KtLogger
importType,
logger
): Promise<ConverterFnReturn[] | ConverterFnReturn> => {
let isLegacyLeggendaria = false;
server/src/score-import/import-types/file/csv_eamusement-iidx/parser.test.ts → server/src/score-import/import-types/file/csv/eamusement-iidx/parser.test.ts
+4 -4
View File
@@ -1,12 +1,12 @@
import t from "tap";
import CreateLogCtx from "../../../../logger";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import CreateLogCtx from "../../../../../logger";
import ScoreImportFatalError from "../../../../framework/score-importing/score-import-error";
import ParseEamusementCSV, { NaiveCSVParse, ResolveHeaders } from "./parser";
import { CloseMongoConnection } from "../../../../db/db";
import { CloseMongoConnection } from "../../../../../db/db";
import {
TestingIIDXEamusementCSV26,
TestingIIDXEamusementCSV27,
} from "../../../../test-utils/test-data";
} from "../../../../../test-utils/test-data";
const logger = CreateLogCtx("parser.test.ts");
server/src/score-import/import-types/file/csv_eamusement-iidx/parser.ts → server/src/score-import/import-types/file/csv/eamusement-iidx/parser.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { Difficulties } from "kamaitachi-common";
import { KtLogger, ParserFunctionReturnsSync } from "../../../../types";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { KtLogger, ParserFunctionReturnsSync } from "../../../../../types";
import ScoreImportFatalError from "../../../../framework/score-importing/score-import-error";
import ConverterFn from "./converter";
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
server/src/score-import/import-types/file/csv_eamusement-iidx/types.ts → server/src/score-import/import-types/file/csv/eamusement-iidx/types.ts
View File
server/src/score-import/import-types/file/json/batch-manual/parser.ts
+30
View File
@@ -0,0 +1,30 @@
import { KtLogger, ParserFunctionReturnsSync } from "../../../../../types";
import ScoreImportFatalError from "../../../../framework/score-importing/score-import-error";
import { ParseBatchManualFromObject } from "../../../common/batch-manual/parser";
import { BatchManualContext, BatchManualScore } from "../../../common/batch-manual/types";
/**
* Parses a buffer of BATCH-MANUAL data.
* @param fileData - The buffer to parse.
* @param body - The request body that made this file import request.
*/
function ParseBatchManual(
fileData: Express.Multer.File,
body: Record<string, unknown>,
logger: KtLogger
): ParserFunctionReturnsSync<BatchManualScore, BatchManualContext> {
let jsonData: unknown;
try {
jsonData = JSON.parse(fileData.buffer.toString("utf-8"));
} catch (err) {
throw new ScoreImportFatalError(
400,
`Invalid JSON. (${err?.message ?? "No Error Message Available."})`
);
}
return ParseBatchManualFromObject(jsonData, "file/json:batch-manual", logger);
}
export default ParseBatchManual;
server/src/score-import/import-types/import-types.ts
-30
View File
@@ -1,30 +0,0 @@
import iidxEamusementCsv from "./file/csv_eamusement-iidx/converter";
import fileBatchManual from "./file/json_batch-manual/converter";
/**
* Converters are a set of functions designed to convert an element
* from the iterable into the Kamaitachi Score Format.
*
* Due to some oddities with certain formats, one element may contain multiple possible
* score records, so these functions MAY return an array of their traditional values.
*/
export const Converters = {
"file/csv:eamusement-iidx": iidxEamusementCsv,
"file/json:batch-manual": fileBatchManual,
};
/**
* Parsers are a set of functions designed to convert unparsed data
* (files, request bodies, api responses) into an iterable.
*
* These functions should also perform validation on the "data" recieved,
* such as whether they are valid JSON/csv files.
*
* However, they need not validate the actual data recieved - only that it is
* parsable and sensible, but not whether the contents are semantically correct
* (i.e. just parse the JSON into an array, and check if it can be done)
* (don't check the content!)
*
* This is so we don't end up with dual-iteration over the set of data.
*/
export const Parsers = {};
server/src/score-import/import-types/ir/json/direct-manual/parser.ts
+17
View File
@@ -0,0 +1,17 @@
import { KtLogger, ParserFunctionReturnsSync } from "../../../../../types";
import { ParseBatchManualFromObject } from "../../../common/batch-manual/parser";
import { BatchManualContext, BatchManualScore } from "../../../common/batch-manual/types";
/**
* Parses an object of BATCH-MANUAL data.
* @param fileData - The buffer to parse.
* @param body - The request body that made this file import request.
*/
function ParseDirectManual(
body: Record<string, unknown>,
logger: KtLogger
): ParserFunctionReturnsSync<BatchManualScore, BatchManualContext> {
return ParseBatchManualFromObject(body, "ir/json:direct-manual", logger);
}
export default ParseDirectManual;
server/src/types.ts
+28 -4
View File
@@ -11,7 +11,14 @@ import {
} from "kamaitachi-common";
import { Logger, LeveledLogMethod } from "winston";
import { ConverterFailure } from "./score-import/framework/score-importing/converter-failures";
import { Converters } from "./score-import/import-types/import-types";
import {
BatchManualContext,
BatchManualScore,
} from "./score-import/import-types/common/batch-manual/types";
import {
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
} from "./score-import/import-types/file/csv/eamusement-iidx/types";
declare module "express-session" {
// Inject additional properties on express-session
@@ -49,7 +56,12 @@ export type ConverterFnReturn = ConverterFailure | ConverterFnSuccessReturn | nu
export type ConverterFunctionReturns = ConverterFnReturn | ConverterFnReturn[];
export interface ConverterFunction<D, C> {
(data: D, processContext: C, logger: KtLogger): Promise<ConverterFunctionReturns>;
(
data: D,
processContext: C,
importType: ImportTypes,
logger: KtLogger
): Promise<ConverterFunctionReturns>;
}
export interface ImportInputParser<D, C> {
@@ -96,8 +108,8 @@ export interface ScoreConverterInformation<D, C> {
export interface OrphanedScore<T extends ImportTypes> {
importType: T;
data: Parameters<typeof Converters[T]>[0];
converterContext: Parameters<typeof Converters[T]>[1];
data: ImportTypeDataMap[T];
converterContext: ImportTypeContextMap[T];
humanisedIdentifier: string;
}
@@ -108,3 +120,15 @@ export type RevaluedObject<T, U> = {
export interface TextDocument extends MongoDBDocument {
text: string;
}
export interface ImportTypeDataMap {
"file/csv:eamusement-iidx": IIDXEamusementCSVData;
"file/json:batch-manual": BatchManualScore;
"ir/json:direct-manual": BatchManualScore;
}
export interface ImportTypeContextMap {
"file/csv:eamusement-iidx": IIDXEamusementCSVContext;
"file/json:batch-manual": BatchManualContext;
"ir/json:direct-manual": BatchManualContext;
}