mirror of
https://github.com/zkldi/Tachi.git
synced 2026-10-09 15:19:50 +03:00
Merge pull request #153 from zkldi/zkldi/79-Score-Orphaning
zkldi/79-Score-Orphaning
This commit is contained in:
zkldi
Vendored
+10
-2
@@ -15,9 +15,17 @@
|
||||
"${workspaceFolder}/js/**/*.js"
|
||||
],
|
||||
"skipFiles": [
|
||||
"<node_internals>/**",
|
||||
"${workspaceFolder}/node_modules/**/*.js",
|
||||
"<node_internals>",
|
||||
"node_modules",
|
||||
"loader.js",
|
||||
"async_hooks.js",
|
||||
"**/async_hooks.js",
|
||||
"**/internal/**/*",
|
||||
"**/domain.js",
|
||||
"**/events.js",
|
||||
"*"
|
||||
],
|
||||
"smartStep": true,
|
||||
"outputCapture": "std",
|
||||
},
|
||||
]
|
||||
|
||||
Generated
+3
-3
@@ -61,7 +61,7 @@ dependencies:
|
||||
express-session: 1.17.1
|
||||
fast-json-stable-hash: 1.0.2
|
||||
fast-xml-parser: 3.19.0
|
||||
kamaitachi-common: github.com/zkldi/Kamaitachi-common/309bc3610aca940ceeb5244ff7d337770cf27201
|
||||
kamaitachi-common: github.com/zkldi/Kamaitachi-common/63f9cae1e4e2b8a488e134712ed498a88cb4fd4f
|
||||
mongodb: 3.6.6
|
||||
monk: 7.3.4
|
||||
multer: 1.4.2
|
||||
@@ -4344,8 +4344,8 @@ packages:
|
||||
engines: {node: '>=6'}
|
||||
dev: true
|
||||
|
||||
github.com/zkldi/Kamaitachi-common/309bc3610aca940ceeb5244ff7d337770cf27201:
|
||||
resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/309bc3610aca940ceeb5244ff7d337770cf27201}
|
||||
github.com/zkldi/Kamaitachi-common/63f9cae1e4e2b8a488e134712ed498a88cb4fd4f:
|
||||
resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/63f9cae1e4e2b8a488e134712ed498a88cb4fd4f}
|
||||
name: kamaitachi-common
|
||||
version: 0.1.0
|
||||
dependencies:
|
||||
|
||||
Vendored
+3
@@ -28,6 +28,7 @@ import {
|
||||
import monk from "monk";
|
||||
import { MONGO_BASE_URL } from "../../lib/env/env";
|
||||
import CreateLogCtx from "../../lib/logger/logger";
|
||||
import { OrphanScoreDocument } from "../../lib/score-import/import-types/common/types";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
@@ -131,6 +132,8 @@ const db = {
|
||||
"usc-auth-tokens": monkDB.get<GenericAuthDocument>("usc-auth-tokens"),
|
||||
"beatoraja-auth-tokens": monkDB.get<GenericAuthDocument>("beatoraja-auth-tokens"),
|
||||
"bms-course-lookup": monkDB.get<BMSCourseDocument>("bms-course-lookup"),
|
||||
|
||||
"orphan-scores": monkDB.get<OrphanScoreDocument>("orphan-scores"),
|
||||
};
|
||||
|
||||
export default db;
|
||||
|
||||
@@ -139,9 +139,12 @@ export const rootLogger = winston.createLogger({
|
||||
});
|
||||
|
||||
function CreateLogCtx(filename: string, lg = rootLogger): KtLogger {
|
||||
return lg.child({
|
||||
const logger = lg.child({
|
||||
context: filename.replace(new RegExp(`^${process.cwd()}/`, "u"), ""),
|
||||
}) as KtLogger;
|
||||
|
||||
logger.defaultMeta = { context: [filename] };
|
||||
return logger;
|
||||
}
|
||||
|
||||
export function AppendLogCtx(context: string, lg: KtLogger): KtLogger {
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
import t from "tap";
|
||||
import db from "../../../../external/mongo/db";
|
||||
import { CloseAllConnections } from "../../../../test-utils/close-connections";
|
||||
import ResetDBState from "../../../../test-utils/reset-db-state";
|
||||
import CreateLogCtx from "../../../logger/logger";
|
||||
import { BatchManualContext, BatchManualScore } from "../../import-types/common/batch-manual/types";
|
||||
import { OrphanScore, ReprocessOrphan } from "./orphans";
|
||||
import fjsh from "fast-json-stable-hash";
|
||||
import { OrphanScoreDocument } from "../../import-types/common/types";
|
||||
import deepmerge from "deepmerge";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
const batchManualScore: BatchManualScore = {
|
||||
score: 500,
|
||||
lamp: "HARD CLEAR",
|
||||
matchType: "songTitle",
|
||||
identifier: "5.1.1.",
|
||||
playtype: "SP",
|
||||
difficulty: "ANOTHER",
|
||||
};
|
||||
|
||||
const batchManualContext: BatchManualContext = { game: "iidx", service: "foo", version: "27" };
|
||||
|
||||
t.test("#OrphanScore", (t) => {
|
||||
t.beforeEach(ResetDBState);
|
||||
|
||||
t.test("Should orphan data and context information.", async (t) => {
|
||||
const res = await OrphanScore(
|
||||
"ir/direct-manual",
|
||||
1,
|
||||
batchManualScore,
|
||||
batchManualContext,
|
||||
"Example Error Message",
|
||||
logger
|
||||
);
|
||||
|
||||
t.equal(res.success, true);
|
||||
t.equal(
|
||||
res.orphanID,
|
||||
`O${fjsh.hash(
|
||||
{
|
||||
importType: "ir/direct-manual",
|
||||
data: batchManualScore,
|
||||
context: batchManualContext,
|
||||
userID: 1,
|
||||
},
|
||||
"sha256"
|
||||
)}`,
|
||||
"Should be a checksum of the orphan's contents."
|
||||
);
|
||||
|
||||
const dbCheck = await db["orphan-scores"].findOne({
|
||||
orphanID: res.orphanID,
|
||||
});
|
||||
|
||||
t.hasStrict(dbCheck, {
|
||||
orphanID: res.orphanID,
|
||||
userID: 1,
|
||||
data: batchManualScore,
|
||||
context: batchManualContext,
|
||||
importType: "ir/direct-manual",
|
||||
errMsg: "Example Error Message",
|
||||
} as any);
|
||||
|
||||
t.ok(
|
||||
Math.abs(dbCheck!.timeInserted - Date.now()) < 10_000,
|
||||
"timeInserted should be decently close to now."
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("Should skip orphans that already exist.", async (t) => {
|
||||
const res1 = await OrphanScore(
|
||||
"ir/direct-manual",
|
||||
1,
|
||||
batchManualScore,
|
||||
batchManualContext,
|
||||
"Example Error Message",
|
||||
logger
|
||||
);
|
||||
|
||||
const res2 = await OrphanScore(
|
||||
"ir/direct-manual",
|
||||
1,
|
||||
batchManualScore,
|
||||
batchManualContext,
|
||||
"Example Error Message",
|
||||
logger
|
||||
);
|
||||
|
||||
t.equal(res2.success, false);
|
||||
t.equal(res1.orphanID, res2.orphanID);
|
||||
|
||||
const dbCheck = await db["orphan-scores"].count();
|
||||
|
||||
t.equal(dbCheck, 1);
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("#ReprocessOrphan", (t) => {
|
||||
t.beforeEach(ResetDBState);
|
||||
|
||||
const orphanDoc: OrphanScoreDocument = {
|
||||
context: batchManualContext,
|
||||
data: batchManualScore,
|
||||
errMsg: "foo",
|
||||
importType: "ir/direct-manual",
|
||||
orphanID: "foo",
|
||||
timeInserted: 0,
|
||||
userID: 1,
|
||||
};
|
||||
|
||||
t.test("Should convert and insert an orphan if parents are found", async (t) => {
|
||||
// this orphan doc is for 5.1.1. SPA, which definitely exists in
|
||||
// the test DB.
|
||||
const res = await ReprocessOrphan(orphanDoc, logger);
|
||||
|
||||
t.hasStrict(
|
||||
res,
|
||||
{
|
||||
success: true,
|
||||
type: "ScoreImported",
|
||||
message: null,
|
||||
content: {
|
||||
score: {
|
||||
game: "iidx",
|
||||
service: "foo (DIRECT-MANUAL)",
|
||||
comment: null,
|
||||
importType: "ir/direct-manual",
|
||||
timeAchieved: null,
|
||||
scoreMeta: {},
|
||||
scoreData: {
|
||||
lampIndex: 5,
|
||||
gradeIndex: 1,
|
||||
esd: 60.546875,
|
||||
lamp: "HARD CLEAR",
|
||||
score: 500,
|
||||
grade: "E",
|
||||
percent: 31.806615776081426,
|
||||
hitData: {},
|
||||
hitMeta: {},
|
||||
},
|
||||
highlight: false,
|
||||
userID: 1,
|
||||
calculatedData: {
|
||||
rating: 0.6840944877852578,
|
||||
lampRating: 10.6,
|
||||
gameSpecific: {
|
||||
BPI: -15,
|
||||
KESDC: -51.26832199690281,
|
||||
"K%": null,
|
||||
},
|
||||
},
|
||||
songID: 1,
|
||||
chartID: "c2311194e3897ddb5745b1760d2c0141f933e683",
|
||||
scoreID:
|
||||
"R7a3a2b04bd4882ec06c198d78297fe3d56561502c7b134c067214bbfdf4f1602",
|
||||
playtype: "SP",
|
||||
isPrimary: true,
|
||||
},
|
||||
},
|
||||
} as any,
|
||||
"Should successfully import the score."
|
||||
);
|
||||
|
||||
const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
|
||||
|
||||
t.equal(orphan, null, "Should remove the orphan document.");
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("Should delete the orphan doc and return null if the score is invalid.", async (t) => {
|
||||
await db["orphan-scores"].insert(orphanDoc);
|
||||
const res = await ReprocessOrphan(
|
||||
deepmerge(orphanDoc, {
|
||||
data: {
|
||||
score: 99999,
|
||||
},
|
||||
}),
|
||||
logger
|
||||
);
|
||||
|
||||
t.equal(res, null);
|
||||
|
||||
const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
|
||||
|
||||
t.equal(orphan, null, "Should remove the orphan document.");
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test(
|
||||
"Should keep the orphan doc and return false if no parents could be found.",
|
||||
async (t) => {
|
||||
await db["orphan-scores"].insert(orphanDoc);
|
||||
const res = await ReprocessOrphan(
|
||||
deepmerge(orphanDoc, {
|
||||
data: {
|
||||
identifier: "NONSENSE CHART TITLE",
|
||||
},
|
||||
}),
|
||||
logger
|
||||
);
|
||||
|
||||
t.equal(res, false);
|
||||
|
||||
const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
|
||||
|
||||
t.not(orphan, null, "Should not remove the orphan document.");
|
||||
|
||||
t.end();
|
||||
}
|
||||
);
|
||||
|
||||
t.test(
|
||||
"Should keep the orphan doc and return false if an internal failure was triggered.",
|
||||
async (t) => {
|
||||
await db["orphan-scores"].insert(orphanDoc);
|
||||
const GAZER_SHA256 = "195fe1be5c3e74fccd04dc426e05f8a9cfa8a1059c339d0a23e99f63661f0b7d";
|
||||
|
||||
// force an internal failure by removing all bms songs and then
|
||||
// leaving a valid bms chart to match with.
|
||||
await db.songs.bms.remove({});
|
||||
|
||||
const res = await ReprocessOrphan(
|
||||
deepmerge(orphanDoc, {
|
||||
context: {
|
||||
game: "bms",
|
||||
},
|
||||
data: {
|
||||
matchType: "bmsChartHash",
|
||||
identifier: GAZER_SHA256,
|
||||
},
|
||||
}),
|
||||
logger
|
||||
);
|
||||
|
||||
t.equal(res, false);
|
||||
|
||||
const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
|
||||
|
||||
t.not(orphan, null, "Should not remove the orphan document.");
|
||||
|
||||
t.end();
|
||||
}
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.teardown(CloseAllConnections);
|
||||
@@ -0,0 +1,117 @@
|
||||
import db from "../../../../external/mongo/db";
|
||||
import {
|
||||
ConverterFunction,
|
||||
ConverterFnReturnOrFailure,
|
||||
ImportTypeContextMap,
|
||||
ImportTypeDataMap,
|
||||
OrphanScoreDocument,
|
||||
} from "../../import-types/common/types";
|
||||
import { ImportTypes, integer } from "kamaitachi-common";
|
||||
import fjsh from "fast-json-stable-hash";
|
||||
import { KtLogger } from "../../../logger/logger";
|
||||
import { Converters } from "../../import-types/converters";
|
||||
import {
|
||||
ConverterFailure,
|
||||
InternalFailure,
|
||||
KTDataNotFoundFailure,
|
||||
} from "../common/converter-failures";
|
||||
import { ProcessSuccessfulConverterReturn } from "../score-importing/score-importing";
|
||||
|
||||
/**
|
||||
* Creates an OrphanedScore document from the data and context,
|
||||
* and inserts it into the DB if it is not already in there.
|
||||
*
|
||||
* @returns Returns { success: true | false, orphanID }
|
||||
*/
|
||||
export async function OrphanScore<T extends ImportTypes = ImportTypes>(
|
||||
importType: T,
|
||||
userID: integer,
|
||||
data: ImportTypeDataMap[T],
|
||||
context: ImportTypeContextMap[T],
|
||||
errMsg: string | null,
|
||||
logger: KtLogger
|
||||
) {
|
||||
const orphan: Pick<OrphanScoreDocument, "importType" | "data" | "context" | "userID"> = {
|
||||
importType,
|
||||
data,
|
||||
context: context,
|
||||
userID,
|
||||
};
|
||||
|
||||
const orphanID = `O${fjsh.hash(orphan, "sha256")}`;
|
||||
|
||||
const exists = await db["orphan-scores"].findOne({ orphanID });
|
||||
|
||||
if (exists) {
|
||||
logger.debug(`Skipped orphaning score ${orphanID} because it already exists.`);
|
||||
return { success: false, orphanID };
|
||||
}
|
||||
|
||||
const orphanScoreDoc: OrphanScoreDocument = {
|
||||
...orphan,
|
||||
orphanID,
|
||||
errMsg,
|
||||
timeInserted: Date.now(),
|
||||
};
|
||||
|
||||
await db["orphan-scores"].insert(orphanScoreDoc);
|
||||
|
||||
return { success: true, orphanID };
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes an orphan document and re-runs the converter->scoreimport pipeline on its data.
|
||||
*
|
||||
* @returns False if no parent documents could be found for the score again,
|
||||
* Null if the orphan document was removed, but no score was inserted (i.e. score was orphaned AND invalid, so nothing
|
||||
* could be imported when parents were found).
|
||||
* ImportProcessingInfo on success.
|
||||
*/
|
||||
export async function ReprocessOrphan(orphan: OrphanScoreDocument, logger: KtLogger) {
|
||||
const ConverterFunction = Converters[orphan.importType] as ConverterFunction<
|
||||
ImportTypeDataMap[ImportTypes],
|
||||
ImportTypeContextMap[ImportTypes]
|
||||
>;
|
||||
|
||||
let res: ConverterFnReturnOrFailure;
|
||||
|
||||
try {
|
||||
res = await ConverterFunction(orphan.data, orphan.context, orphan.importType, logger);
|
||||
} catch (err) {
|
||||
// this is impossible to test, so we're going to ignore it
|
||||
/* istanbul ignore next */
|
||||
if (!(err instanceof ConverterFailure)) {
|
||||
logger.error(`Converter function ${orphan.importType} returned unexpected error.`, {
|
||||
err,
|
||||
});
|
||||
throw err; // throw this higher up, i guess.
|
||||
}
|
||||
|
||||
res = err;
|
||||
}
|
||||
|
||||
// If the data still can't be found, we do nothing about it.
|
||||
if (res instanceof KTDataNotFoundFailure) {
|
||||
logger.debug(`Unorphaning ${orphan.orphanID} failed. (${res.message})`);
|
||||
return false;
|
||||
} else if (res instanceof InternalFailure) {
|
||||
logger.error(`Orphan Internal Failure - ${res.message}, OrphanID ${orphan.orphanID}`);
|
||||
|
||||
return false;
|
||||
} else if (res instanceof ConverterFailure) {
|
||||
logger.warn(
|
||||
`Recieved ConverterFailure ${res.message} on orphan ${orphan.orphanID}. Removing orphan.`
|
||||
);
|
||||
|
||||
// @danger - This could go terribly, if there's a mistake in the converterFN we might accidentally
|
||||
// remove a users score.
|
||||
await db["orphan-scores"].remove({ orphanID: orphan.orphanID });
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
await db["orphan-scores"].remove({ orphanID: orphan.orphanID });
|
||||
|
||||
// else, import the orphan.
|
||||
return ProcessSuccessfulConverterReturn(orphan.userID, res, logger);
|
||||
}
|
||||
@@ -19,8 +19,9 @@ import db from "../../../external/mongo/db";
|
||||
import { GetAndUpdateUsersGoals } from "./goals/goals";
|
||||
import { UpdateUsersMilestones } from "./milestones/milestones";
|
||||
import { ClassHandler } from "./user-game-stats/classes";
|
||||
import { ImportInputParser } from "../import-types/common/types";
|
||||
import { ConverterFunction, ImportInputParser } from "../import-types/common/types";
|
||||
import { ScorePlaytypeMap } from "./common/types";
|
||||
import { Converters } from "../import-types/converters";
|
||||
|
||||
/**
|
||||
* Performs a Kamaitachi Score Import.
|
||||
@@ -51,12 +52,15 @@ export default async function ScoreImportMain<D, C>(
|
||||
// We get an iterable from the provided parser function, alongside some context and a converter function.
|
||||
// This iterable does not have to be an array - it's anything that's iterable, like a generator.
|
||||
const parseTimeStart = process.hrtime.bigint();
|
||||
const { iterable, ConverterFunction, context, game, classHandler } = await InputParser(logger);
|
||||
const { iterable, context, game, classHandler } = await InputParser(logger);
|
||||
|
||||
const parseTime = GetMilisecondsSince(parseTimeStart);
|
||||
|
||||
logger.debug(`Parsing took ${parseTime} miliseconds.`);
|
||||
|
||||
// We have to cast here due to typescript generic confusions. This is guaranteed to be correct.
|
||||
const ConverterFunction = (Converters[importType] as unknown) as ConverterFunction<D, C>;
|
||||
|
||||
// --- 2. Importing ---
|
||||
// ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data.
|
||||
const importTimeStart = process.hrtime.bigint();
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
ScoreDocument,
|
||||
AnySongDocument,
|
||||
ImportTypes,
|
||||
IDStrings,
|
||||
} from "kamaitachi-common";
|
||||
import { HydrateScore } from "./hydrate-score";
|
||||
import { InsertQueue, QueueScoreInsert, ScoreIDs } from "./insert-score";
|
||||
@@ -20,11 +21,12 @@ import db from "../../../../external/mongo/db";
|
||||
import { AppendLogCtx, KtLogger } from "../../../logger/logger";
|
||||
|
||||
import {
|
||||
ConverterFunctionReturns,
|
||||
ConverterFnReturn,
|
||||
ConverterFnReturnOrFailure,
|
||||
ConverterFunction,
|
||||
ConverterFnSuccessReturn,
|
||||
} from "../../import-types/common/types";
|
||||
import { DryScore } from "../common/types";
|
||||
import { OrphanScore } from "../orphans/orphans";
|
||||
|
||||
/**
|
||||
* Processes the iterable data into the Kamaitachi database.
|
||||
@@ -55,38 +57,19 @@ export async function ImportAllIterableData<D, C>(
|
||||
);
|
||||
}
|
||||
|
||||
// Due to the fact that ProcessIterableDatapoint may return an array instead of a single result
|
||||
// (e-amusement is the only real example of this);
|
||||
// we need to flatten out the datapoints into a single array. We also use this time
|
||||
// to filter out nulls, which we don't care for (these are neither successes or failures)
|
||||
const nonFlatDatapoints = await Promise.all(promises);
|
||||
// We need to filter out nulls, which we don't care for (these are neither successes or failures)
|
||||
const processedResults = await Promise.all(promises);
|
||||
|
||||
logger.verbose(`Finished Importing Data (${promises.length} datapoints).`);
|
||||
logger.debug(`Flattening returns...`);
|
||||
logger.debug(`Removing null returns...`);
|
||||
|
||||
const flatDatapoints = [];
|
||||
const datapoints = processedResults.filter(
|
||||
(e) => e !== null
|
||||
) as ImportProcessingInfo<IDStrings>[];
|
||||
|
||||
for (const dp of nonFlatDatapoints) {
|
||||
if (dp === null) {
|
||||
continue;
|
||||
}
|
||||
logger.debug(`Removed null from results.`);
|
||||
|
||||
if (Array.isArray(dp)) {
|
||||
for (const dpx of dp) {
|
||||
if (dpx === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
flatDatapoints.push(dpx);
|
||||
}
|
||||
} else {
|
||||
flatDatapoints.push(dp);
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(`Flattened returns.`);
|
||||
|
||||
logger.verbose(`Recieved ${flatDatapoints.length} returns, from ${promises.length} data.`);
|
||||
logger.verbose(`Recieved ${datapoints.length} returns, from ${promises.length} data.`);
|
||||
|
||||
// Flush the score queue out after finishing most of the import. This ensures no scores get left in the
|
||||
// queue.
|
||||
@@ -96,7 +79,7 @@ export async function ImportAllIterableData<D, C>(
|
||||
logger.verbose(`Emptied ${emptied} documents from score queue.`);
|
||||
}
|
||||
|
||||
return flatDatapoints;
|
||||
return datapoints;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -114,52 +97,56 @@ export async function ImportIterableDatapoint<D, C>(
|
||||
ConverterFunction: ConverterFunction<D, C>,
|
||||
context: C,
|
||||
logger: KtLogger
|
||||
) {
|
||||
let converterReturns: ConverterFunctionReturns;
|
||||
): Promise<ImportProcessingInfo | null> {
|
||||
// Converter Function Return
|
||||
let cfnReturn: ConverterFnReturnOrFailure;
|
||||
|
||||
try {
|
||||
converterReturns = await ConverterFunction(data, context, importType, logger);
|
||||
cfnReturn = await ConverterFunction(data, context, importType, logger);
|
||||
} catch (err) {
|
||||
converterReturns = err;
|
||||
}
|
||||
|
||||
if (Array.isArray(converterReturns)) {
|
||||
return Promise.all(
|
||||
converterReturns.map((e) => ImportFromConverterReturn(userID, e, logger))
|
||||
);
|
||||
}
|
||||
|
||||
return ImportFromConverterReturn(userID, converterReturns, logger);
|
||||
}
|
||||
|
||||
async function ImportFromConverterReturn(
|
||||
userID: integer,
|
||||
cfnReturn: ConverterFnReturn, // a single return, not an array!
|
||||
logger: KtLogger
|
||||
): Promise<ImportProcessingInfo | null> {
|
||||
// null => processing didnt result in a score document, but not an error, no processing needed!
|
||||
if (cfnReturn === null) {
|
||||
return null;
|
||||
cfnReturn = err;
|
||||
}
|
||||
|
||||
// if this conversion failed, return it in the proper format
|
||||
if (cfnReturn instanceof ConverterFailure) {
|
||||
if (cfnReturn instanceof KTDataNotFoundFailure) {
|
||||
logger.warn(`ConverterFailure: ${cfnReturn.message ?? "No message?"}`, {
|
||||
logger.warn(`KTDataNotFoundFailure: ${cfnReturn.message ?? "No message?"}`, {
|
||||
cfnReturn,
|
||||
hideFromConsole: ["cfnReturn"],
|
||||
});
|
||||
|
||||
const insertOrphan = await OrphanScore(
|
||||
cfnReturn.importType,
|
||||
userID,
|
||||
cfnReturn.data,
|
||||
cfnReturn.converterContext,
|
||||
cfnReturn.message,
|
||||
logger
|
||||
);
|
||||
|
||||
if (insertOrphan.success) {
|
||||
return {
|
||||
success: false,
|
||||
type: "KTDataNotFound",
|
||||
message: cfnReturn.message,
|
||||
content: {
|
||||
context: cfnReturn.converterContext,
|
||||
data: cfnReturn.data,
|
||||
orphanID: insertOrphan.orphanID,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
type: "KTDataNotFound",
|
||||
type: "OrphanExists",
|
||||
message: cfnReturn.message,
|
||||
content: {
|
||||
context: cfnReturn.converterContext,
|
||||
data: cfnReturn.data,
|
||||
orphanID: insertOrphan.orphanID,
|
||||
},
|
||||
};
|
||||
} else if (cfnReturn instanceof InvalidScoreFailure) {
|
||||
logger.warn(`ConverterFailure: ${cfnReturn.message ?? "No message?"}`, {
|
||||
logger.info(`InvalidScoreFailure: ${cfnReturn.message ?? "No message?"}`, {
|
||||
cfnReturn,
|
||||
hideFromConsole: ["cfnReturn"],
|
||||
});
|
||||
@@ -205,6 +192,14 @@ async function ImportFromConverterReturn(
|
||||
};
|
||||
}
|
||||
|
||||
return ProcessSuccessfulConverterReturn(userID, cfnReturn as ConverterFnSuccessReturn, logger);
|
||||
}
|
||||
|
||||
export async function ProcessSuccessfulConverterReturn(
|
||||
userID: integer,
|
||||
cfnReturn: ConverterFnSuccessReturn,
|
||||
logger: KtLogger
|
||||
): Promise<ImportProcessingInfo | null> {
|
||||
const result = await HydrateAndInsertScore(
|
||||
userID,
|
||||
cfnReturn.dryScore,
|
||||
|
||||
@@ -22,6 +22,5 @@ export function ParseKaiIIDX(
|
||||
},
|
||||
classHandler: null,
|
||||
game: "iidx",
|
||||
ConverterFunction: ConvertAPIKaiIIDX,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -22,6 +22,5 @@ export function ParseKaiSDVX(
|
||||
},
|
||||
classHandler: null,
|
||||
game: "iidx",
|
||||
ConverterFunction: ConvertAPIKaiSDVX,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import {
|
||||
import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils";
|
||||
import { KtLogger } from "../../../../logger/logger";
|
||||
import { DryScore } from "../../../framework/common/types";
|
||||
import { ConverterFunction, ConverterFnReturn } from "../types";
|
||||
import { ConverterFunction } from "../types";
|
||||
|
||||
/**
|
||||
* Creates a ConverterFn for the BatchManualScore format. This curries
|
||||
@@ -32,7 +32,7 @@ export const ConverterBatchManual: ConverterFunction<BatchManualScore, BatchManu
|
||||
context,
|
||||
importType,
|
||||
logger
|
||||
): Promise<ConverterFnReturn> => {
|
||||
) => {
|
||||
const game = context.game;
|
||||
|
||||
const { song, chart } = await ResolveMatchTypeToKTData(data, context, importType, logger);
|
||||
|
||||
@@ -172,7 +172,6 @@ export function ParseBatchManualFromObject(
|
||||
version: batchManual.head.version ?? null,
|
||||
},
|
||||
iterable: batchManual.body,
|
||||
ConverterFunction: ConverterBatchManual,
|
||||
classHandler: null,
|
||||
};
|
||||
}
|
||||
|
||||
+1
-325
@@ -2,7 +2,7 @@ import t from "tap";
|
||||
import db from "../../../../../external/mongo/db";
|
||||
import CreateLogCtx from "../../../../logger/logger";
|
||||
import ResetDBState from "../../../../../test-utils/reset-db-state";
|
||||
import ConverterFn, { EamScoreConverter, EamScoreConverterWrapper } from "./converter";
|
||||
import ConvertEamIIDXCSV from "./converter";
|
||||
import p from "prudence";
|
||||
import { prAssert } from "../../../../../test-utils/asserts";
|
||||
import deepmerge from "deepmerge";
|
||||
@@ -63,330 +63,6 @@ const data = {
|
||||
title: "5.1.1",
|
||||
};
|
||||
|
||||
// Testing floats with strictSame under tap is a pain because it uses ===
|
||||
// and there is no way around it. We have isApproximately for this purpose
|
||||
// But it'd be easier if we could just .toFixed() the percent and compare it as
|
||||
// a string.
|
||||
|
||||
t.test("#EamScoreConverter", async (t) => {
|
||||
t.beforeEach(ResetDBState);
|
||||
|
||||
const song = {
|
||||
title: "5.1.1.",
|
||||
artist: "dj nagureo",
|
||||
id: 1,
|
||||
firstVersion: "0",
|
||||
"alt-titles": [],
|
||||
"search-titles": [],
|
||||
data: {
|
||||
genre: "PIANO AMBIENT",
|
||||
},
|
||||
};
|
||||
|
||||
function EamScoreConverterAuto(score: Partial<EamusementScoreData> = {}) {
|
||||
return EamScoreConverter(
|
||||
deepmerge(valid511Score, score) as EamusementScoreData,
|
||||
song,
|
||||
converterContext,
|
||||
data,
|
||||
false,
|
||||
logger
|
||||
);
|
||||
}
|
||||
|
||||
t.test("Valid Scores", (t) => {
|
||||
t.test("Normal 511 SPA Score", async (t) => {
|
||||
const res = await EamScoreConverterAuto();
|
||||
|
||||
t.not(res, null, "Should not be null.");
|
||||
|
||||
t.equal(
|
||||
res!.ktchiChart.chartID,
|
||||
chartID511,
|
||||
"Should have selected 511 SPA as the chart."
|
||||
);
|
||||
|
||||
prAssert(res!.dryScore, DryScorePrudence, "Should return the expected dryscore.");
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("--- BP Score", async (t) => {
|
||||
const res = await EamScoreConverterAuto({ bp: "---" });
|
||||
|
||||
t.not(res, null, "Should not be null.");
|
||||
|
||||
t.equal(
|
||||
res!.ktchiChart.chartID,
|
||||
chartID511,
|
||||
"Should have selected 511 SPA as the chart."
|
||||
);
|
||||
|
||||
prAssert(
|
||||
res!.dryScore,
|
||||
deepmerge(DryScorePrudence, { scoreData: { hitMeta: { bp: "undefined" } } }),
|
||||
"Should return the expected dryscore."
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("Nonsense String BP Score", async (t) => {
|
||||
const res = await EamScoreConverterAuto({ bp: "ASDF" });
|
||||
|
||||
t.not(res, null, "Should not be null.");
|
||||
|
||||
t.equal(
|
||||
res!.ktchiChart.chartID,
|
||||
chartID511,
|
||||
"Should have selected 511 SPA as the chart."
|
||||
);
|
||||
|
||||
prAssert(
|
||||
res!.dryScore,
|
||||
deepmerge(DryScorePrudence, { scoreData: { hitMeta: { bp: "undefined" } } }),
|
||||
"Should return the expected dryscore."
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("0 EX Score", async (t) => {
|
||||
const res = await EamScoreConverterAuto({ exscore: "0", pgreat: "0", great: "0" });
|
||||
|
||||
t.equal(res, null, "Should be null.");
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("1 EX Score", async (t) => {
|
||||
const res = await EamScoreConverterAuto({ exscore: "1", pgreat: "0", great: "1" });
|
||||
|
||||
t.not(res, null, "Should not be null.");
|
||||
|
||||
t.equal(
|
||||
res!.ktchiChart.chartID,
|
||||
chartID511,
|
||||
"Should have selected 511 SPA as the chart."
|
||||
);
|
||||
|
||||
prAssert(
|
||||
res!.dryScore,
|
||||
deepmerge(DryScorePrudence, {
|
||||
scoreData: {
|
||||
score: p.equalTo(1),
|
||||
percent: p.aprx(0.063),
|
||||
hitData: {
|
||||
pgreat: p.equalTo(0),
|
||||
great: p.equalTo(1),
|
||||
},
|
||||
},
|
||||
}),
|
||||
"Should return the expected dryscore."
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("Level 0 chart", async (t) => {
|
||||
const res = await EamScoreConverterAuto({ level: "0" });
|
||||
|
||||
t.equal(res, null, "Should be null.");
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("Invalid Scores", (t) => {
|
||||
t.test("Chart that doesn't exist", (t) => {
|
||||
EamScoreConverterAuto({ difficulty: "LEGGENDARIA" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new KTDataNotFoundFailure(
|
||||
`Could not find chart for 5.1.1. (SP LEGGENDARIA [v27])`,
|
||||
"file/eamusement-iidx-csv",
|
||||
data,
|
||||
converterContext
|
||||
),
|
||||
"Should return a KTDataNotFoundFailure"
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
});
|
||||
|
||||
t.test("Invalid PGreat/Great count", (t) => {
|
||||
EamScoreConverterAuto({ exscore: "100", pgreat: "20", great: "5" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(
|
||||
`5.1.1. (SP ANOTHER [v27]) - PGreats * 2 + Greats did not equal EXScore (20 * 2 + 5 != 100).`
|
||||
),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
});
|
||||
|
||||
t.test(">100%", (t) => {
|
||||
EamScoreConverterAuto({ exscore: "9999" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(
|
||||
`5.1.1. (SP ANOTHER [v27]) - Invalid EX Score of 9999 (Was greater than max chart ex of 1572).`
|
||||
),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
});
|
||||
|
||||
t.test("Nonsense Lamp", (t) => {
|
||||
EamScoreConverterAuto({ lamp: "invalid" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(
|
||||
`5.1.1. (SP ANOTHER [v27]) - Invalid Lamp of invalid.`
|
||||
),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
});
|
||||
|
||||
t.test("Invalid BP", async (t) => {
|
||||
await EamScoreConverterAuto({ bp: "5.73" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of 5.73.`),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
});
|
||||
|
||||
await EamScoreConverterAuto({ bp: "-5.73" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of -5.73.`),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
});
|
||||
|
||||
await EamScoreConverterAuto({ bp: "-1" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of -1.`),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
});
|
||||
|
||||
await EamScoreConverterAuto({ bp: "10000" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(`5.1.1. (SP ANOTHER [v27]) - Invalid BP of 10000.`),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
});
|
||||
|
||||
await EamScoreConverterAuto({ bp: "-10000" })
|
||||
.then(() => t.fail())
|
||||
.catch((err) => {
|
||||
t.strictSame(
|
||||
err,
|
||||
new InvalidScoreFailure(
|
||||
`5.1.1. (SP ANOTHER [v27]) - Invalid BP of -10000.`
|
||||
),
|
||||
"Should return an InvalidScoreFailure"
|
||||
);
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("#EamScoreConverterWrapper", async (t) => {
|
||||
t.beforeEach(ResetDBState);
|
||||
|
||||
const song = {
|
||||
title: "5.1.1.",
|
||||
artist: "dj nagureo",
|
||||
id: 1,
|
||||
firstVersion: "0",
|
||||
"alt-titles": [],
|
||||
"search-titles": [],
|
||||
data: {
|
||||
genre: "PIANO AMBIENT",
|
||||
},
|
||||
};
|
||||
|
||||
// wrapper wrapper, lol
|
||||
function EamScoreConverterWrapperAuto(score: Partial<EamusementScoreData> = {}) {
|
||||
return EamScoreConverterWrapper(
|
||||
deepmerge(valid511Score, score) as EamusementScoreData,
|
||||
song,
|
||||
converterContext,
|
||||
data,
|
||||
false,
|
||||
logger
|
||||
);
|
||||
}
|
||||
|
||||
t.test("Should convert thrown errors into returns", async (t) => {
|
||||
const res = await EamScoreConverterWrapperAuto({ exscore: "-1" });
|
||||
|
||||
t.strictSame(
|
||||
res,
|
||||
new InvalidScoreFailure(
|
||||
"5.1.1. (SP ANOTHER [v27]) - Invalid EX score of -1 (Was negative.)"
|
||||
)
|
||||
);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test("Should return EamScoreConverter on success", async (t) => {
|
||||
const res = await EamScoreConverterWrapperAuto();
|
||||
|
||||
if (res instanceof ConverterFailure || !res) {
|
||||
return t.fail("Unexpected ConverterFailure");
|
||||
}
|
||||
|
||||
t.strictSame(res.song, song, "Should return the right song");
|
||||
t.equal(res.chart.chartID, chartID511, "Should return the right chart");
|
||||
|
||||
prAssert(res.dryScore, DryScorePrudence, "Should return the right DryScore.");
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.todo("#ConverterFn", async (t) => {
|
||||
t.beforeEach(ResetDBState);
|
||||
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
import { Lamps, AnySongDocument, ChartDocument } from "kamaitachi-common";
|
||||
import { Lamps, ChartDocument } from "kamaitachi-common";
|
||||
import { FindChartWithPTDFVersion } from "../../../../../utils/queries/charts";
|
||||
import { FindSongOnTitleInsensitive } from "../../../../../utils/queries/songs";
|
||||
import { FindSongOnTitle } from "../../../../../utils/queries/songs";
|
||||
import {
|
||||
KTDataNotFoundFailure,
|
||||
InternalFailure,
|
||||
InvalidScoreFailure,
|
||||
SkipScoreFailure,
|
||||
} from "../../../framework/common/converter-failures";
|
||||
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
|
||||
import { GetGradeFromPercent } from "../../../framework/common/score-utils";
|
||||
import {
|
||||
GenericGetGradeAndPercent,
|
||||
ParseDateFromString,
|
||||
} from "../../../framework/common/score-utils";
|
||||
import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts";
|
||||
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
|
||||
import { KtLogger } from "../../../../logger/logger";
|
||||
import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
|
||||
import { DryScore } from "../../../framework/common/types";
|
||||
import { ConverterFunction, ConverterFnReturn } from "../types";
|
||||
import { ConverterFunction } from "../types";
|
||||
|
||||
const EAMUSEMENT_LAMP_RESOLVER: Map<string, Lamps["iidx:SP" | "iidx:DP"]> = new Map([
|
||||
["NO PLAY", "NO PLAY"],
|
||||
@@ -25,28 +26,54 @@ const EAMUSEMENT_LAMP_RESOLVER: Map<string, Lamps["iidx:SP" | "iidx:DP"]> = new
|
||||
["ASSIST CLEAR", "ASSIST CLEAR"],
|
||||
]);
|
||||
|
||||
export async function EamScoreConverter(
|
||||
eamScore: EamusementScoreData,
|
||||
ktchiSong: AnySongDocument,
|
||||
context: IIDXEamusementCSVContext,
|
||||
data: IIDXEamusementCSVData,
|
||||
isLegacyLeggendaria: boolean,
|
||||
logger: KtLogger
|
||||
) {
|
||||
const HUMANISED_CHART_TITLE = `${ktchiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`;
|
||||
if (!eamScore.level || eamScore.level === "0") {
|
||||
// charts that dont exist in the format have a level of 0
|
||||
return null;
|
||||
}
|
||||
const NINE_HOURS = 1000 * 60 * 60 * 9;
|
||||
|
||||
if (isLegacyLeggendaria) {
|
||||
eamScore.difficulty = "LEGGENDARIA";
|
||||
}
|
||||
const ConvertEamIIDXCSV: ConverterFunction<
|
||||
IIDXEamusementCSVData,
|
||||
IIDXEamusementCSVContext
|
||||
> = async (data, context, importType, logger) => {
|
||||
const eamScore = data.score;
|
||||
|
||||
if (eamScore.exscore === "0") {
|
||||
// skip scores with an exscore of 0
|
||||
// This also skips things like score resets.
|
||||
return null;
|
||||
throw new SkipScoreFailure("Score has an exscore of 0.");
|
||||
}
|
||||
|
||||
if (!eamScore.level || eamScore.level === "0") {
|
||||
// charts that dont exist in the format have a level of 0
|
||||
throw new SkipScoreFailure("Chart has a level of 0.");
|
||||
}
|
||||
|
||||
let isLegacyLeggendaria = false;
|
||||
|
||||
// if pre-HV, leggendarias were stored in a wacky form.
|
||||
if (!context.hasBeginnerAndLegg) {
|
||||
// hack fix for legacy LEGGENDARIA titles
|
||||
if (data.title.match(/(†|†LEGGENDARIA)$/u)) {
|
||||
data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd();
|
||||
isLegacyLeggendaria = true;
|
||||
}
|
||||
}
|
||||
|
||||
// @optimisable - This is actually a multi-fetch. Since eam-csv scores
|
||||
// are batched up into (song, chart1, chart2, chart3 ...) rows
|
||||
// we actually already have fetched this song a second ago.
|
||||
const ktchiSong = await FindSongOnTitle("iidx", data.title);
|
||||
|
||||
if (!ktchiSong) {
|
||||
throw new KTDataNotFoundFailure(
|
||||
`Could not find song for ${data.title}.`,
|
||||
importType,
|
||||
data,
|
||||
context
|
||||
);
|
||||
}
|
||||
|
||||
const HUMANISED_CHART_TITLE = `${ktchiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`;
|
||||
|
||||
if (isLegacyLeggendaria) {
|
||||
eamScore.difficulty = "LEGGENDARIA";
|
||||
}
|
||||
|
||||
const ktchiChart = (await FindChartWithPTDFVersion(
|
||||
@@ -104,32 +131,15 @@ export async function EamScoreConverter(
|
||||
);
|
||||
}
|
||||
|
||||
const percent = (100 * exscore) / MAX_EX;
|
||||
const grade = GetGradeFromPercent<"iidx:SP" | "iidx:DP">("iidx", percent);
|
||||
const { percent, grade } = GenericGetGradeAndPercent("iidx", exscore, ktchiChart);
|
||||
|
||||
if (!grade) {
|
||||
logger.warn(`${HUMANISED_CHART_TITLE} - Could not resolve ${percent} into a grade?`);
|
||||
throw new InternalFailure(
|
||||
`${HUMANISED_CHART_TITLE} - Could not resolve ${percent} into a grade?`
|
||||
);
|
||||
}
|
||||
|
||||
// Now we need to figure out the timestamp for this score.
|
||||
// Under, well, normal circumstances, we could figure this out quite trivially
|
||||
// But e-amusement provides us the timestamp for the *song*, not the score
|
||||
//
|
||||
// We're going to actually ignore this. Initial drafts of this assumed we could skip the
|
||||
// "epochs" (when a score reset on eamusement happened), but it turns out those are
|
||||
// generated when the user triggers a score migration, NOT (as initially thought) when
|
||||
// KONAMI decides.
|
||||
|
||||
const timestamp = Date.parse(data.timestamp);
|
||||
const timestamp = ParseDateFromString(data.timestamp);
|
||||
|
||||
const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = {
|
||||
service: context.service,
|
||||
comment: null,
|
||||
game: "iidx",
|
||||
importType: "file/eamusement-iidx-csv",
|
||||
importType,
|
||||
scoreData: {
|
||||
score: exscore,
|
||||
lamp,
|
||||
@@ -142,7 +152,8 @@ export async function EamScoreConverter(
|
||||
grade,
|
||||
},
|
||||
scoreMeta: {},
|
||||
timeAchieved: timestamp,
|
||||
// japan is gmt+9
|
||||
timeAchieved: timestamp ? timestamp - NINE_HOURS : null,
|
||||
};
|
||||
|
||||
const numBP = Number(eamScore.bp);
|
||||
@@ -162,96 +173,8 @@ export async function EamScoreConverter(
|
||||
logger.info(`Skipped assigning BP for score. Had unexpected value of ${eamScore.bp}.`);
|
||||
}
|
||||
|
||||
return { ktchiChart, dryScore };
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin wrapper around EamScoreConverter to ensure it returns the expected values
|
||||
* @param eamScore - The eamusement score data to convert.
|
||||
* @param song - The song this score is for.
|
||||
* @param context - Context the converter may need.
|
||||
* @param data - The parent data the eamScore derives from.
|
||||
* @returns ConverterFnReturn
|
||||
*/
|
||||
export async function EamScoreConverterWrapper(
|
||||
eamScore: EamusementScoreData,
|
||||
song: AnySongDocument,
|
||||
context: IIDXEamusementCSVContext,
|
||||
data: IIDXEamusementCSVData,
|
||||
isLegacyLeggendaria: boolean,
|
||||
logger: KtLogger
|
||||
) {
|
||||
try {
|
||||
const results = await EamScoreConverter(
|
||||
eamScore,
|
||||
song!,
|
||||
context,
|
||||
data,
|
||||
isLegacyLeggendaria,
|
||||
logger
|
||||
);
|
||||
|
||||
if (!results) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
song,
|
||||
chart: results.ktchiChart,
|
||||
dryScore: results.dryScore,
|
||||
};
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof KTDataNotFoundFailure ||
|
||||
err instanceof InternalFailure ||
|
||||
err instanceof InvalidScoreFailure
|
||||
) {
|
||||
return err;
|
||||
} else if (err instanceof ScoreImportFatalError) {
|
||||
throw err; // throw it all the way up.
|
||||
}
|
||||
|
||||
logger.error(`Unknown error: `, { err });
|
||||
return new InternalFailure("An unknown internal failure has occured.");
|
||||
}
|
||||
}
|
||||
|
||||
const ConverterFn: ConverterFunction<IIDXEamusementCSVData, IIDXEamusementCSVContext> = async (
|
||||
data,
|
||||
context,
|
||||
importType,
|
||||
logger
|
||||
): Promise<ConverterFnReturn[] | ConverterFnReturn> => {
|
||||
let isLegacyLeggendaria = false;
|
||||
|
||||
// if pre-HV, leggendarias were stored in a wacky form.
|
||||
if (!context.hasBeginnerAndLegg) {
|
||||
// hack fix for legacy LEGGENDARIA titles
|
||||
if (data.title.match(/(†|†LEGGENDARIA)$/u)) {
|
||||
data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd();
|
||||
isLegacyLeggendaria = true;
|
||||
}
|
||||
}
|
||||
|
||||
const ktchiSong = await FindSongOnTitleInsensitive("iidx", data.title);
|
||||
|
||||
if (!ktchiSong) {
|
||||
return new KTDataNotFoundFailure(
|
||||
`Could not find song for ${data.title}.`,
|
||||
importType,
|
||||
data,
|
||||
context
|
||||
);
|
||||
}
|
||||
|
||||
// ts thinks ktchiSong might be null. It's not, though!
|
||||
const results = await Promise.all(
|
||||
data.scores.map((e) =>
|
||||
EamScoreConverterWrapper(e, ktchiSong!, context, data, isLegacyLeggendaria, logger)
|
||||
)
|
||||
);
|
||||
|
||||
return results;
|
||||
return { chart: ktchiChart, dryScore, song: ktchiSong };
|
||||
};
|
||||
|
||||
export default ConverterFn;
|
||||
export default ConvertEamIIDXCSV;
|
||||
|
||||
@@ -18,7 +18,7 @@ t.test("#ParseEamusementCSV", (t) => {
|
||||
logger
|
||||
);
|
||||
|
||||
t.equal(iterableData.length, 152, "Should return exactly 152 datapoints.");
|
||||
t.equal(iterableData.length, 456, "Should return exactly 456 datapoints.");
|
||||
t.equal(hasBeginnerAndLegg, false, "Should not mark as a HV csv.");
|
||||
t.equal(version, "26", "Should correctly assert that the version of this CSV is ROOTAGE.");
|
||||
|
||||
@@ -31,7 +31,7 @@ t.test("#ParseEamusementCSV", (t) => {
|
||||
logger
|
||||
);
|
||||
|
||||
t.equal(iterableData.length, 1257, "Should return exactly 1257 datapoints.");
|
||||
t.equal(iterableData.length, 6285, "Should return exactly 6285 datapoints.");
|
||||
t.equal(hasBeginnerAndLegg, true, "Should mark as a HV csv.");
|
||||
t.equal(
|
||||
version,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Difficulties } from "kamaitachi-common";
|
||||
import { KtLogger } from "../../../../logger/logger";
|
||||
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
|
||||
import { ParserFunctionReturnsSync } from "../types";
|
||||
import ConverterFn from "./converter";
|
||||
import ConvertEamIIDXCSV from "./converter";
|
||||
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
|
||||
|
||||
enum EAM_VERSION_NAMES {
|
||||
@@ -199,11 +199,13 @@ export function NaiveCSVParse(csvBuffer: Buffer, logger: KtLogger) {
|
||||
});
|
||||
}
|
||||
|
||||
iterableData.push({
|
||||
scores,
|
||||
timestamp,
|
||||
title,
|
||||
});
|
||||
iterableData.push(
|
||||
...scores.map((e) => ({
|
||||
score: e,
|
||||
timestamp,
|
||||
title,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
return { iterableData, version: gameVersion.toString(), hasBeginnerAndLegg };
|
||||
@@ -270,7 +272,6 @@ function GenericParseEamIIDXCSV(
|
||||
return {
|
||||
iterable: iterableData,
|
||||
context,
|
||||
ConverterFunction: ConverterFn,
|
||||
game: "iidx",
|
||||
classHandler: null,
|
||||
};
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { SongDocument } from "kamaitachi-common";
|
||||
|
||||
export interface EamusementScoreData {
|
||||
difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
|
||||
lamp: string;
|
||||
@@ -14,7 +16,7 @@ type BaseProps = {
|
||||
};
|
||||
|
||||
export type IIDXEamusementCSVData = {
|
||||
scores: EamusementScoreData[];
|
||||
score: EamusementScoreData;
|
||||
} & BaseProps;
|
||||
|
||||
type Props = "exscore" | "lamp" | "pgreat" | "great" | "bp" | "level";
|
||||
|
||||
@@ -8,13 +8,20 @@ import { FervidexContext, FervidexScore } from "../ir/fervidex/types";
|
||||
import { KaiContext } from "./api-kai/types";
|
||||
import { BatchManualContext, BatchManualScore } from "./batch-manual/types";
|
||||
import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./eamusement-iidx-csv/types";
|
||||
import { ImportTypes, Game, AnyChartDocument, AnySongDocument } from "kamaitachi-common";
|
||||
import {
|
||||
ImportTypes,
|
||||
Game,
|
||||
AnyChartDocument,
|
||||
AnySongDocument,
|
||||
integer,
|
||||
MongoDBDocument,
|
||||
} from "kamaitachi-common";
|
||||
import { ConverterFailure } from "../../framework/common/converter-failures";
|
||||
import { ClassHandler } from "../../framework/user-game-stats/classes";
|
||||
import { DryScore } from "../../framework/common/types";
|
||||
import { BeatorajaIRScoreFormat } from "../../../../server/router/ir/beatoraja/charts/convert-scores";
|
||||
import { BeatorajaContext } from "../ir/beatoraja/types";
|
||||
|
||||
import { BeatorajaContext, BeatorajaScore } from "../ir/beatoraja/types";
|
||||
import { USCClientScore } from "../../../../server/router/ir/usc/usc";
|
||||
import { IRUSCContext } from "../ir/usc/types";
|
||||
export interface ImportTypeDataMap {
|
||||
"file/eamusement-iidx-csv": IIDXEamusementCSVData;
|
||||
"file/batch-manual": BatchManualScore;
|
||||
@@ -27,9 +34,11 @@ export interface ImportTypeDataMap {
|
||||
"ir/fervidex": FervidexScore;
|
||||
"ir/fervidex-static": FervidexStaticScore;
|
||||
"ir/chunitachi": BatchManualScore;
|
||||
"ir/beatoraja": BeatorajaIRScoreFormat;
|
||||
"ir/usc": EmptyObject;
|
||||
"ir/beatoraja": BeatorajaScore;
|
||||
"ir/usc": USCClientScore;
|
||||
|
||||
// These aren't placeholder values - the data is yielded in a way that
|
||||
// the value of these is legitimately unknown at convert time.
|
||||
"api/arc-iidx": unknown;
|
||||
"api/arc-sdvx": unknown;
|
||||
"api/arc-jubeat": unknown;
|
||||
@@ -44,7 +53,7 @@ export interface ImportTypeContextMap {
|
||||
"file/batch-manual": BatchManualContext;
|
||||
"file/solid-state-squad": EmptyObject;
|
||||
"file/mer-iidx": EmptyObject;
|
||||
"file/pli-iidx-csv": EmptyObject;
|
||||
"file/pli-iidx-csv": IIDXEamusementCSVContext;
|
||||
|
||||
"ir/direct-manual": BatchManualContext;
|
||||
"ir/barbatos": EmptyObject;
|
||||
@@ -52,7 +61,7 @@ export interface ImportTypeContextMap {
|
||||
"ir/fervidex-static": FervidexStaticContext;
|
||||
"ir/chunitachi": BatchManualContext;
|
||||
"ir/beatoraja": BeatorajaContext;
|
||||
"ir/usc": EmptyObject;
|
||||
"ir/usc": IRUSCContext;
|
||||
|
||||
"api/arc-iidx": EmptyObject;
|
||||
"api/arc-sdvx": EmptyObject;
|
||||
@@ -62,11 +71,15 @@ export interface ImportTypeContextMap {
|
||||
"api/eag-iidx": KaiContext;
|
||||
"api/eag-sdvx": KaiContext;
|
||||
}
|
||||
export interface OrphanedScore<T extends ImportTypes> {
|
||||
|
||||
export interface OrphanScoreDocument<T extends ImportTypes = ImportTypes> extends MongoDBDocument {
|
||||
importType: T;
|
||||
data: ImportTypeDataMap[T];
|
||||
converterContext: ImportTypeContextMap[T];
|
||||
humanisedIdentifier: string;
|
||||
context: ImportTypeContextMap[T];
|
||||
errMsg: string | null;
|
||||
orphanID: string;
|
||||
userID: integer;
|
||||
timeInserted: number;
|
||||
}
|
||||
|
||||
export interface ConverterFnSuccessReturn {
|
||||
@@ -75,9 +88,7 @@ export interface ConverterFnSuccessReturn {
|
||||
song: AnySongDocument;
|
||||
}
|
||||
|
||||
export type ConverterFnReturn = ConverterFailure | ConverterFnSuccessReturn | null;
|
||||
|
||||
export type ConverterFunctionReturns = ConverterFnReturn | ConverterFnReturn[];
|
||||
export type ConverterFnReturnOrFailure = ConverterFailure | ConverterFnSuccessReturn;
|
||||
|
||||
export interface ConverterFunction<D, C> {
|
||||
(
|
||||
@@ -85,7 +96,7 @@ export interface ConverterFunction<D, C> {
|
||||
processContext: C,
|
||||
importType: ImportTypes,
|
||||
logger: KtLogger
|
||||
): Promise<ConverterFunctionReturns>;
|
||||
): Promise<ConverterFnSuccessReturn>;
|
||||
}
|
||||
|
||||
export interface ImportInputParser<D, C> {
|
||||
@@ -100,7 +111,6 @@ export interface ParserFunctionReturnsAsync<D, C> {
|
||||
iterable: AsyncIterable<D>;
|
||||
context: C;
|
||||
game: Game;
|
||||
ConverterFunction: ConverterFunction<D, C>;
|
||||
classHandler: ClassHandler | null;
|
||||
}
|
||||
|
||||
@@ -108,6 +118,5 @@ export interface ParserFunctionReturnsSync<D, C> {
|
||||
iterable: Iterable<D>;
|
||||
context: C;
|
||||
game: Game;
|
||||
ConverterFunction: ConverterFunction<D, C>;
|
||||
classHandler: ClassHandler | null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { ImportTypes } from "kamaitachi-common";
|
||||
import { ConvertAPIKaiIIDX } from "./common/api-kai/iidx/converter";
|
||||
import { ConvertAPIKaiSDVX } from "./common/api-kai/sdvx/converter";
|
||||
import { ConverterBatchManual } from "./common/batch-manual/converter";
|
||||
import ConvertEamIIDXCSV from "./common/eamusement-iidx-csv/converter";
|
||||
import { ConverterFunction, ImportTypeContextMap, ImportTypeDataMap } from "./common/types";
|
||||
import { ConvertFileMerIIDX } from "./file/mer-iidx/converter";
|
||||
import { ConvertFileS3 } from "./file/solid-state-squad/converter";
|
||||
import { ConverterIRBarbatos } from "./ir/barbatos/converter";
|
||||
import { ConverterIRBeatoraja } from "./ir/beatoraja/converter";
|
||||
import { ConverterIRFervidexStatic } from "./ir/fervidex-static/converter";
|
||||
import { ConverterIRFervidex } from "./ir/fervidex/converter";
|
||||
import { ConverterIRUSC } from "./ir/usc/converter";
|
||||
|
||||
export type ConverterMap = {
|
||||
[I in ImportTypes]: ConverterFunction<ImportTypeDataMap[I], ImportTypeContextMap[I]>;
|
||||
};
|
||||
|
||||
export const Converters: ConverterMap = {
|
||||
"file/solid-state-squad": ConvertFileS3,
|
||||
"file/batch-manual": ConverterBatchManual,
|
||||
"file/mer-iidx": ConvertFileMerIIDX,
|
||||
"file/pli-iidx-csv": ConvertEamIIDXCSV,
|
||||
"file/eamusement-iidx-csv": ConvertEamIIDXCSV,
|
||||
|
||||
"api/eag-iidx": ConvertAPIKaiIIDX,
|
||||
"api/eag-sdvx": ConvertAPIKaiSDVX,
|
||||
"api/flo-iidx": ConvertAPIKaiIIDX,
|
||||
"api/flo-sdvx": ConvertAPIKaiSDVX,
|
||||
|
||||
"ir/barbatos": ConverterIRBarbatos,
|
||||
"ir/beatoraja": ConverterIRBeatoraja,
|
||||
"ir/fervidex": ConverterIRFervidex,
|
||||
"ir/fervidex-static": ConverterIRFervidexStatic,
|
||||
"ir/direct-manual": ConverterBatchManual,
|
||||
"ir/usc": ConverterIRUSC,
|
||||
"ir/chunitachi": ConverterBatchManual,
|
||||
};
|
||||
@@ -49,7 +49,7 @@ t.test("#ConvertFileMerIIDX", (t) => {
|
||||
},
|
||||
},
|
||||
scoreMeta: {},
|
||||
timeAchieved: 1616570122000,
|
||||
timeAchieved: 1616537722000,
|
||||
},
|
||||
} as any,
|
||||
"Should return the converted dry score."
|
||||
@@ -83,7 +83,7 @@ t.test("#ConvertFileMerIIDX", (t) => {
|
||||
},
|
||||
},
|
||||
scoreMeta: {},
|
||||
timeAchieved: 1616570122000,
|
||||
timeAchieved: 1616537722000,
|
||||
},
|
||||
} as any,
|
||||
"Should return the converted dry score."
|
||||
|
||||
@@ -70,7 +70,8 @@ export const ConvertFileMerIIDX: ConverterFunction<MerScore, EmptyObject> = asyn
|
||||
},
|
||||
},
|
||||
scoreMeta: {},
|
||||
timeAchieved,
|
||||
// japan is GMT+9
|
||||
timeAchieved: timeAchieved ? timeAchieved - NINE_HOURS : null,
|
||||
};
|
||||
|
||||
return {
|
||||
@@ -79,3 +80,5 @@ export const ConvertFileMerIIDX: ConverterFunction<MerScore, EmptyObject> = asyn
|
||||
dryScore,
|
||||
};
|
||||
};
|
||||
|
||||
const NINE_HOURS = 1000 * 60 * 60 * 9;
|
||||
|
||||
@@ -58,6 +58,5 @@ export function ParseMerIIDX(
|
||||
context: {},
|
||||
iterable: jsonData as MerScore[],
|
||||
game: "iidx",
|
||||
ConverterFunction: ConvertFileMerIIDX,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -87,6 +87,5 @@ export function ParseSolidStateXML(
|
||||
context: {},
|
||||
iterable: parsedXML.s3data.scoredata.song as S3Score[],
|
||||
game: "iidx",
|
||||
ConverterFunction: ConvertFileS3, // todo
|
||||
};
|
||||
}
|
||||
|
||||
@@ -40,7 +40,6 @@ export function ParseBarbatosSingle(
|
||||
context: {},
|
||||
game: "sdvx",
|
||||
iterable: ([body] as unknown) as BarbatosScore[],
|
||||
ConverterFunction: ConverterIRBarbatos,
|
||||
classHandler: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -108,7 +108,6 @@ export function ParseBeatorajaSingle(
|
||||
},
|
||||
game: "bms",
|
||||
iterable: ([body.score] as unknown) as BeatorajaScore[],
|
||||
ConverterFunction: ConverterIRBeatoraja,
|
||||
classHandler: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -79,7 +79,6 @@ export function ParseFervidexStatic(
|
||||
context: { version },
|
||||
game: "iidx",
|
||||
iterable: scores,
|
||||
ConverterFunction: ConverterIRFervidexStatic,
|
||||
classHandler: FerStaticClassHandler(body),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -115,7 +115,6 @@ export function ParseFervidexSingle(
|
||||
context: { version },
|
||||
game: "iidx",
|
||||
iterable: ([body] as unknown) as FervidexScore[],
|
||||
ConverterFunction: ConverterIRFervidex,
|
||||
classHandler: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -46,6 +46,5 @@ export function ParseIRUSC(
|
||||
game: "usc",
|
||||
iterable: [body.score] as USCClientScore[],
|
||||
classHandler: null,
|
||||
ConverterFunction: ConverterIRUSC,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
Reference in New Issue
Block a user