Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-09-29 10:28:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

eam import refactor

Browse Source
This commit is contained in:
zkldi
2021-04-20 16:29:53 +01:00
parent 4e195634d6
commit edd7aeb73a
3 changed files with 201 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/score-import/import-types/csv-eamusement-iidx/converter.ts
+25 -21
View File
@@ -10,6 +10,7 @@ import {
} from "../../framework/core/converter-errors";
import ScoreImportFatalError from "../../framework/core/score-import-error";
import { GetGradeFromPercent } from "../../framework/core/score-utils";
import { AssertStrAsPositiveInt } from "../../framework/core/string-asserts";
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
export interface DataTest {
@@ -39,7 +40,7 @@ async function EamScoreConverter(
return null;
}
if (eamScore.exscore === 0) {
if (eamScore.exscore === "0") {
// skip scores with an exscore of 0
// This also skips things like score resets.
return null;
@@ -61,24 +62,27 @@ async function EamScoreConverter(
);
}
if (typeof eamScore.exscore !== "number") {
throw new InvalidScoreFailure(
`${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was not a number?).`
);
} else if (!Number.isSafeInteger(eamScore.exscore)) {
throw new InvalidScoreFailure(
`${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Not an integer?).`
);
} else if (eamScore.exscore < 0) {
throw new InvalidScoreFailure(
`${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Not an integer?).`
);
} else if (eamScore.exscore > ktchiChart.notedata.notecount) {
let exscore = AssertStrAsPositiveInt(
eamScore.exscore,
`${HUMANISED_SONG_TITLE} - Invalid EX score of ${eamScore.exscore}`
);
if (exscore > ktchiChart.notedata.notecount) {
throw new InvalidScoreFailure(
`${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was greater than chart notecount of ${ktchiChart.notedata.notecount}).`
);
}
let pgreat = AssertStrAsPositiveInt(
eamScore.pgreat,
`${HUMANISED_SONG_TITLE} - Invalid PGreats of ${eamScore.pgreat}`
);
let great = AssertStrAsPositiveInt(
eamScore.great,
`${HUMANISED_SONG_TITLE} - Invalid Greats of ${eamScore.pgreat}`
);
const lamp = EAMUSEMENT_LAMP_RESOLVER.get(eamScore.lamp);
if (!lamp) {
@@ -88,7 +92,7 @@ async function EamScoreConverter(
);
}
const percent = (100 * eamScore.exscore) / ktchiChart.notedata.notecount;
const percent = (100 * exscore) / ktchiChart.notedata.notecount;
const grade = GetGradeFromPercent<"iidx:SP" | "iidx:DP">("iidx", percent);
if (!grade) {
@@ -121,11 +125,11 @@ async function EamScoreConverter(
game: "iidx",
importType: "csv:eamusement-iidx",
scoreData: {
score: eamScore.exscore,
score: exscore,
lamp,
hitData: {
pgreat: eamScore.pgreat,
great: eamScore.great,
pgreat,
great,
},
hitMeta: {},
percent,
@@ -199,12 +203,12 @@ const ConverterFn: ConverterFunction<IIDXEamusementCSVData, IIDXEamusementCSVCon
data,
context,
logger: Logger
): Promise<ConverterFnReturn[]> => {
): Promise<ConverterFnReturn[] | ConverterFnReturn> => {
let ktchiSong = await FindSongOnTitleVersion("iidx", data.title, context.importVersion);
if (!ktchiSong) {
logger.warn(`Could not find song for ${data.title}.`);
throw new KTDataNotFoundFailure(data.title, "csv:eamusement-iidx", data, context);
// logger.warn(`Could not find song for ${data.title}.`);
return new KTDataNotFoundFailure(data.title, "csv:eamusement-iidx", data, context);
}
// ts thinks ktchiSong might be null. It's not, though!
server/src/score-import/import-types/csv-eamusement-iidx/parser.ts
+171 -175
View File
@@ -1,18 +1,9 @@
import csvParse from "csv-parse/lib/sync";
import { Difficulties, integer } from "kamaitachi-common";
import { Difficulties } from "kamaitachi-common";
import { Logger } from "winston";
import { ParserFunctionReturnsSync } from "../../../types";
import ScoreImportFatalError from "../../framework/core/score-import-error";
import ConverterFn from "./converter";
import {
EamusementScoreData,
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
RawIIDXEamusementCSVData,
} from "./types";
const PRE_HV_HEADER_COUNT = 27;
const HV_HEADER_COUNT = 41;
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
enum EAM_VERSION_NAMES {
"1st&substream" = 1,
@@ -45,6 +36,172 @@ enum EAM_VERSION_NAMES {
"BISTROVER",
}
const PRE_HV_HEADER_COUNT = 27;
const HV_HEADER_COUNT = 41;
// commented out for reference
// [
// "version",
// "title",
// "genre",
// "artist",
// "playcount",
// (?beginnerdata)
// "normal-level",
// "normal-exscore",
// "normal-pgreat",
// "normal-great",
// "normal-bp",
// "normal-lamp",
// "normal-grade",
// "hyper-level",
// "hyper-exscore",
// "hyper-pgreat",
// "hyper-great",
// "hyper-bp",
// "hyper-lamp",
// "hyper-grade",
// "another-level",
// "another-exscore",
// "another-pgreat",
// "another-great",
// "another-bp",
// "another-lamp",
// "another-grade",
// (?leggdata)
// "timestamp",
// ];
function ResolveHeaders(headers: string[], logger: Logger) {
if (headers.length === PRE_HV_HEADER_COUNT) {
logger.verbose("PRE_HV csv recieved.");
return {
hasBeginnerAndLegg: false,
};
} else if (headers.length === HV_HEADER_COUNT) {
logger.verbose("HV+ csv recieved.");
return {
hasBeginnerAndLegg: true,
};
} else {
logger.info(`Invalid CSV header count of ${headers.length} received.`);
throw new ScoreImportFatalError(
400,
"Invalid CSV provided. CSV does not have the correct amount of headers."
);
}
}
function NaiveCSVParse(csvBuffer: Buffer, logger: Logger) {
const csvString = csvBuffer.toString("utf-8");
let csvData = csvString.split("\n");
let rawHeaders = [];
let headerLen = 0;
let curStr = "";
// looks like we're doing it like this.
for (const char of csvData[0]) {
headerLen++;
// safety checks to avoid getting DOS'd
if (headerLen > 1000) {
throw new ScoreImportFatalError(400, "Headers were longer than 1000 characters long.");
} else if (rawHeaders.length > 50) {
throw new ScoreImportFatalError(400, "Invalid CSV Headers.");
}
if (char === ",") {
rawHeaders.push(curStr);
curStr = "";
} else {
curStr += char;
}
}
rawHeaders.push(curStr);
const { hasBeginnerAndLegg } = ResolveHeaders(rawHeaders, logger);
const diffs = hasBeginnerAndLegg
? ["beginner", "normal", "hyper", "another", "leggendaria"]
: ["normal", "hyper", "another"];
let iterableData = [];
let gameVersion = 0;
for (let i = 1; i < csvData.length; i++) {
let data = csvData[i];
let cells = data.split(",");
// weirdly enough, an empty string split on "," is an array with
// one empty value.
// regardless, this line skips empty rows
if (cells.length === 1) {
logger.verbose(`Skipped empty row ${i}.`);
continue;
}
if (cells.length !== rawHeaders.length) {
logger.info(
`eamusement-iidx csv has row (${i}) with invalid cell count of ${cells.length}, rejecting.`,
{
data,
}
);
throw new ScoreImportFatalError(
400,
`Row ${i} has an invalid amount of cells (${cells.length}).`
);
}
let version = cells[0];
let title = cells[1];
let timestamp = cells[rawHeaders.length - 1].replace(/\r/g, "");
// wtf typescript?? what's the point of enums?
const versionNum = EAM_VERSION_NAMES[version as keyof typeof EAM_VERSION_NAMES];
if (!versionNum) {
logger.info(`Invalid/Unknown EAM_VERSION_NAME ${version}.`);
throw new ScoreImportFatalError(400, `Invalid/Unknown EAM_VERSION_NAME ${version}.`);
}
if (versionNum > gameVersion) {
logger.verbose(`Replaced ${version} with ${version} (${versionNum}).`);
gameVersion = versionNum;
}
let scores: EamusementScoreData[] = [];
for (let d = 0; d < diffs.length; d++) {
const diff = diffs[d];
let di = 5 + d * 7;
scores.push({
difficulty: (diff.toUpperCase as unknown) as Difficulties["iidx:SP" | "iidx:DP"],
bp: cells[di + 4],
exscore: cells[di + 1],
pgreat: cells[di + 2],
great: cells[di + 3],
lamp: cells[di + 5],
level: cells[di],
});
}
iterableData.push({
scores,
timestamp,
title,
});
}
return { iterableData, version: gameVersion, hasBeginnerAndLegg };
}
/**
* Parses a buffer of EamusementCSV data.
* @param fileData - The buffer to parse.
@@ -89,171 +246,10 @@ function ParseEamusementCSV(
);
}
let data: IIDXEamusementCSVData[] = [];
let csvData: RawIIDXEamusementCSVData[];
let hasBeginnerAndLegg: boolean | null = null;
try {
csvData = csvParse(fileData.buffer, {
bom: true,
// @ts-expect-error csvParse's types are wrong, see https://github.com/adaltas/node-csv-parse/pull/314
escape: null, // KONMAI do not escape their CSV, disable escaping entirely.
columns: (header) => {
// does not have leggendaria/beginner built in
if (header.length === PRE_HV_HEADER_COUNT) {
logger.verbose("PRE_HV csv recieved.");
hasBeginnerAndLegg = false;
return [
"version",
"title",
"genre",
"artist",
"playcount",
"normal-level",
"normal-exscore",
"normal-pgreat",
"normal-great",
"normal-bp",
"normal-lamp",
"normal-grade",
"hyper-level",
"hyper-exscore",
"hyper-pgreat",
"hyper-great",
"hyper-bp",
"hyper-lamp",
"hyper-grade",
"another-level",
"another-exscore",
"another-pgreat",
"another-great",
"another-bp",
"another-lamp",
"another-grade",
"timestamp",
];
}
// does
else if (header.length === HV_HEADER_COUNT) {
logger.verbose("HV+ csv recieved.");
hasBeginnerAndLegg = true;
return [
"version",
"title",
"genre",
"artist",
"playcount",
"beginner-level",
"beginner-exscore",
"beginner-pgreat",
"beginner-great",
"beginner-bp",
"beginner-lamp",
"beginner-grade",
"normal-level",
"normal-exscore",
"normal-pgreat",
"normal-great",
"normal-bp",
"normal-lamp",
"normal-grade",
"hyper-level",
"hyper-exscore",
"hyper-pgreat",
"hyper-great",
"hyper-bp",
"hyper-lamp",
"hyper-grade",
"another-level",
"another-exscore",
"another-pgreat",
"another-great",
"another-bp",
"another-lamp",
"another-grade",
"leggendaria-level",
"leggendaria-exscore",
"leggendaria-pgreat",
"leggendaria-great",
"leggendaria-bp",
"leggendaria-lamp",
"leggendaria-grade",
"timestamp",
];
} else {
logger.warn(`Invalid CSV header count of ${header.length} received.`);
throw new ScoreImportFatalError(
400,
"Invalid CSV provided. CSV does not have the correct amount of headers."
);
}
},
skipEmptyLines: true,
});
} catch (err) {
logger.warn(`CSV Parser Error`, { err });
throw new ScoreImportFatalError(400, "CSV Could not be parsed.");
}
let { hasBeginnerAndLegg, version, iterableData } = NaiveCSVParse(fileData.buffer, logger);
logger.verbose("Successfully parsed CSV.");
let firstEl = csvData[0];
if (!firstEl) {
throw new ScoreImportFatalError(400, "This CSV has no scores.");
}
let version = 0;
const diffs = hasBeginnerAndLegg
? ["beginner", "normal", "hyper", "another", "leggendaria"]
: ["normal", "hyper", "another"];
// @optimisable - Can hook into the CSV parser perhaps and read this value immediately?
for (const d of csvData) {
// wtf typescript?? what's the point of enums?
const versionNum = EAM_VERSION_NAMES[d.version as keyof typeof EAM_VERSION_NAMES];
if (!versionNum) {
logger.info(`Invalid/Unknown EAM_VERSION_NAME ${d.version}.`);
throw new ScoreImportFatalError(400, `Invalid/Unknown EAM_VERSION_NAME ${d.version}.`);
}
if (versionNum > version) {
logger.verbose(`Replaced ${version} with ${d.version} (${versionNum}).`);
version = versionNum;
}
let scores: EamusementScoreData[] = [];
for (const diff of diffs) {
scores.push({
difficulty: (diff.toUpperCase as unknown) as Difficulties["iidx:SP" | "iidx:DP"],
bp: d[`${diff}-bp`] as integer | "---",
exscore: d[`${diff}-exscore`] as integer,
pgreat: d[`${diff}-pgreat`] as integer,
great: d[`${diff}-great`] as integer,
lamp: d[`${diff}-lamp`] as string,
level: d[`${diff}-level`] as integer,
});
}
data.push({
scores,
artist: d.artist,
genre: d.genre,
playcount: d.playcount,
timestamp: d.timestamp,
title: d.title,
version: d.version,
});
}
if (hasBeginnerAndLegg === null) {
logger.error(`hasBeginnerAndLegg was not set, but the end of parsings was reached?`);
throw new ScoreImportFatalError(500, "An internal service error has occured.");
}
let context: IIDXEamusementCSVContext = {
playtype,
importVersion: version,
@@ -261,10 +257,10 @@ function ParseEamusementCSV(
serviceOrigin: "e-amusement",
};
logger.verbose(`Successfully Parsed with ${data.length} results.`);
logger.verbose(`Successfully Parsed with ${iterableData.length} results.`);
return {
iterable: data,
iterable: iterableData,
context,
ConverterFunction: ConverterFn,
idStrings: [`iidx:${context.playtype}` as "iidx:SP" | "iidx:DP"],
server/src/score-import/import-types/csv-eamusement-iidx/types.ts
+5 -9
View File
@@ -3,19 +3,15 @@ import { integer } from "kamaitachi-common";
export interface EamusementScoreData {
difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
lamp: string;
exscore: integer;
pgreat: integer;
great: integer;
bp: integer | "---"; // lol
level: integer;
exscore: string;
pgreat: string;
great: string;
bp: string;
level: string;
}
type BaseProps = {
version: string;
title: string;
genre: string;
artist: string;
playcount: number;
timestamp: string;
};