From edd7aeb73a262ebfb7b0afaae39b694cea915554 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 20 Apr 2021 16:29:53 +0100 Subject: [PATCH] eam import refactor --- .../csv-eamusement-iidx/converter.ts | 46 +-- .../csv-eamusement-iidx/parser.ts | 346 +++++++++--------- .../import-types/csv-eamusement-iidx/types.ts | 14 +- 3 files changed, 201 insertions(+), 205 deletions(-) diff --git a/server/src/score-import/import-types/csv-eamusement-iidx/converter.ts b/server/src/score-import/import-types/csv-eamusement-iidx/converter.ts index f987ea957..2fb0a3498 100644 --- a/server/src/score-import/import-types/csv-eamusement-iidx/converter.ts +++ b/server/src/score-import/import-types/csv-eamusement-iidx/converter.ts @@ -10,6 +10,7 @@ import { } from "../../framework/core/converter-errors"; import ScoreImportFatalError from "../../framework/core/score-import-error"; import { GetGradeFromPercent } from "../../framework/core/score-utils"; +import { AssertStrAsPositiveInt } from "../../framework/core/string-asserts"; import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; export interface DataTest { @@ -39,7 +40,7 @@ async function EamScoreConverter( return null; } - if (eamScore.exscore === 0) { + if (eamScore.exscore === "0") { // skip scores with an exscore of 0 // This also skips things like score resets. return null; @@ -61,24 +62,27 @@ async function EamScoreConverter( ); } - if (typeof eamScore.exscore !== "number") { - throw new InvalidScoreFailure( - `${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was not a number?).` - ); - } else if (!Number.isSafeInteger(eamScore.exscore)) { - throw new InvalidScoreFailure( - `${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Not an integer?).` - ); - } else if (eamScore.exscore < 0) { - throw new InvalidScoreFailure( - `${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Not an integer?).` - ); - } else if (eamScore.exscore > ktchiChart.notedata.notecount) { + let exscore = AssertStrAsPositiveInt( + eamScore.exscore, + `${HUMANISED_SONG_TITLE} - Invalid EX score of ${eamScore.exscore}` + ); + + if (exscore > ktchiChart.notedata.notecount) { throw new InvalidScoreFailure( `${HUMANISED_SONG_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was greater than chart notecount of ${ktchiChart.notedata.notecount}).` ); } + let pgreat = AssertStrAsPositiveInt( + eamScore.pgreat, + `${HUMANISED_SONG_TITLE} - Invalid PGreats of ${eamScore.pgreat}` + ); + + let great = AssertStrAsPositiveInt( + eamScore.great, + `${HUMANISED_SONG_TITLE} - Invalid Greats of ${eamScore.pgreat}` + ); + const lamp = EAMUSEMENT_LAMP_RESOLVER.get(eamScore.lamp); if (!lamp) { @@ -88,7 +92,7 @@ async function EamScoreConverter( ); } - const percent = (100 * eamScore.exscore) / ktchiChart.notedata.notecount; + const percent = (100 * exscore) / ktchiChart.notedata.notecount; const grade = GetGradeFromPercent<"iidx:SP" | "iidx:DP">("iidx", percent); if (!grade) { @@ -121,11 +125,11 @@ async function EamScoreConverter( game: "iidx", importType: "csv:eamusement-iidx", scoreData: { - score: eamScore.exscore, + score: exscore, lamp, hitData: { - pgreat: eamScore.pgreat, - great: eamScore.great, + pgreat, + great, }, hitMeta: {}, percent, @@ -199,12 +203,12 @@ const ConverterFn: ConverterFunction => { +): Promise => { let ktchiSong = await FindSongOnTitleVersion("iidx", data.title, context.importVersion); if (!ktchiSong) { - logger.warn(`Could not find song for ${data.title}.`); - throw new KTDataNotFoundFailure(data.title, "csv:eamusement-iidx", data, context); + // logger.warn(`Could not find song for ${data.title}.`); + return new KTDataNotFoundFailure(data.title, "csv:eamusement-iidx", data, context); } // ts thinks ktchiSong might be null. It's not, though! diff --git a/server/src/score-import/import-types/csv-eamusement-iidx/parser.ts b/server/src/score-import/import-types/csv-eamusement-iidx/parser.ts index 1b3c87304..e5b27491a 100644 --- a/server/src/score-import/import-types/csv-eamusement-iidx/parser.ts +++ b/server/src/score-import/import-types/csv-eamusement-iidx/parser.ts @@ -1,18 +1,9 @@ -import csvParse from "csv-parse/lib/sync"; -import { Difficulties, integer } from "kamaitachi-common"; +import { Difficulties } from "kamaitachi-common"; import { Logger } from "winston"; import { ParserFunctionReturnsSync } from "../../../types"; import ScoreImportFatalError from "../../framework/core/score-import-error"; import ConverterFn from "./converter"; -import { - EamusementScoreData, - IIDXEamusementCSVContext, - IIDXEamusementCSVData, - RawIIDXEamusementCSVData, -} from "./types"; - -const PRE_HV_HEADER_COUNT = 27; -const HV_HEADER_COUNT = 41; +import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; enum EAM_VERSION_NAMES { "1st&substream" = 1, @@ -45,6 +36,172 @@ enum EAM_VERSION_NAMES { "BISTROVER", } +const PRE_HV_HEADER_COUNT = 27; +const HV_HEADER_COUNT = 41; + +// commented out for reference +// [ +// "version", +// "title", +// "genre", +// "artist", +// "playcount", +// (?beginnerdata) +// "normal-level", +// "normal-exscore", +// "normal-pgreat", +// "normal-great", +// "normal-bp", +// "normal-lamp", +// "normal-grade", +// "hyper-level", +// "hyper-exscore", +// "hyper-pgreat", +// "hyper-great", +// "hyper-bp", +// "hyper-lamp", +// "hyper-grade", +// "another-level", +// "another-exscore", +// "another-pgreat", +// "another-great", +// "another-bp", +// "another-lamp", +// "another-grade", +// (?leggdata) +// "timestamp", +// ]; + +function ResolveHeaders(headers: string[], logger: Logger) { + if (headers.length === PRE_HV_HEADER_COUNT) { + logger.verbose("PRE_HV csv recieved."); + return { + hasBeginnerAndLegg: false, + }; + } else if (headers.length === HV_HEADER_COUNT) { + logger.verbose("HV+ csv recieved."); + return { + hasBeginnerAndLegg: true, + }; + } else { + logger.info(`Invalid CSV header count of ${headers.length} received.`); + throw new ScoreImportFatalError( + 400, + "Invalid CSV provided. CSV does not have the correct amount of headers." + ); + } +} + +function NaiveCSVParse(csvBuffer: Buffer, logger: Logger) { + const csvString = csvBuffer.toString("utf-8"); + + let csvData = csvString.split("\n"); + + let rawHeaders = []; + let headerLen = 0; + let curStr = ""; + + // looks like we're doing it like this. + for (const char of csvData[0]) { + headerLen++; + + // safety checks to avoid getting DOS'd + if (headerLen > 1000) { + throw new ScoreImportFatalError(400, "Headers were longer than 1000 characters long."); + } else if (rawHeaders.length > 50) { + throw new ScoreImportFatalError(400, "Invalid CSV Headers."); + } + + if (char === ",") { + rawHeaders.push(curStr); + curStr = ""; + } else { + curStr += char; + } + } + + rawHeaders.push(curStr); + + const { hasBeginnerAndLegg } = ResolveHeaders(rawHeaders, logger); + + const diffs = hasBeginnerAndLegg + ? ["beginner", "normal", "hyper", "another", "leggendaria"] + : ["normal", "hyper", "another"]; + + let iterableData = []; + + let gameVersion = 0; + + for (let i = 1; i < csvData.length; i++) { + let data = csvData[i]; + + let cells = data.split(","); + + // weirdly enough, an empty string split on "," is an array with + // one empty value. + // regardless, this line skips empty rows + if (cells.length === 1) { + logger.verbose(`Skipped empty row ${i}.`); + continue; + } + + if (cells.length !== rawHeaders.length) { + logger.info( + `eamusement-iidx csv has row (${i}) with invalid cell count of ${cells.length}, rejecting.`, + { + data, + } + ); + throw new ScoreImportFatalError( + 400, + `Row ${i} has an invalid amount of cells (${cells.length}).` + ); + } + + let version = cells[0]; + let title = cells[1]; + let timestamp = cells[rawHeaders.length - 1].replace(/\r/g, ""); + + // wtf typescript?? what's the point of enums? + const versionNum = EAM_VERSION_NAMES[version as keyof typeof EAM_VERSION_NAMES]; + + if (!versionNum) { + logger.info(`Invalid/Unknown EAM_VERSION_NAME ${version}.`); + throw new ScoreImportFatalError(400, `Invalid/Unknown EAM_VERSION_NAME ${version}.`); + } + + if (versionNum > gameVersion) { + logger.verbose(`Replaced ${version} with ${version} (${versionNum}).`); + gameVersion = versionNum; + } + + let scores: EamusementScoreData[] = []; + + for (let d = 0; d < diffs.length; d++) { + const diff = diffs[d]; + let di = 5 + d * 7; + + scores.push({ + difficulty: (diff.toUpperCase as unknown) as Difficulties["iidx:SP" | "iidx:DP"], + bp: cells[di + 4], + exscore: cells[di + 1], + pgreat: cells[di + 2], + great: cells[di + 3], + lamp: cells[di + 5], + level: cells[di], + }); + } + + iterableData.push({ + scores, + timestamp, + title, + }); + } + + return { iterableData, version: gameVersion, hasBeginnerAndLegg }; +} + /** * Parses a buffer of EamusementCSV data. * @param fileData - The buffer to parse. @@ -89,171 +246,10 @@ function ParseEamusementCSV( ); } - let data: IIDXEamusementCSVData[] = []; - let csvData: RawIIDXEamusementCSVData[]; - let hasBeginnerAndLegg: boolean | null = null; - - try { - csvData = csvParse(fileData.buffer, { - bom: true, - // @ts-expect-error csvParse's types are wrong, see https://github.com/adaltas/node-csv-parse/pull/314 - escape: null, // KONMAI do not escape their CSV, disable escaping entirely. - columns: (header) => { - // does not have leggendaria/beginner built in - if (header.length === PRE_HV_HEADER_COUNT) { - logger.verbose("PRE_HV csv recieved."); - hasBeginnerAndLegg = false; - return [ - "version", - "title", - "genre", - "artist", - "playcount", - "normal-level", - "normal-exscore", - "normal-pgreat", - "normal-great", - "normal-bp", - "normal-lamp", - "normal-grade", - "hyper-level", - "hyper-exscore", - "hyper-pgreat", - "hyper-great", - "hyper-bp", - "hyper-lamp", - "hyper-grade", - "another-level", - "another-exscore", - "another-pgreat", - "another-great", - "another-bp", - "another-lamp", - "another-grade", - "timestamp", - ]; - } - // does - else if (header.length === HV_HEADER_COUNT) { - logger.verbose("HV+ csv recieved."); - hasBeginnerAndLegg = true; - return [ - "version", - "title", - "genre", - "artist", - "playcount", - "beginner-level", - "beginner-exscore", - "beginner-pgreat", - "beginner-great", - "beginner-bp", - "beginner-lamp", - "beginner-grade", - "normal-level", - "normal-exscore", - "normal-pgreat", - "normal-great", - "normal-bp", - "normal-lamp", - "normal-grade", - "hyper-level", - "hyper-exscore", - "hyper-pgreat", - "hyper-great", - "hyper-bp", - "hyper-lamp", - "hyper-grade", - "another-level", - "another-exscore", - "another-pgreat", - "another-great", - "another-bp", - "another-lamp", - "another-grade", - "leggendaria-level", - "leggendaria-exscore", - "leggendaria-pgreat", - "leggendaria-great", - "leggendaria-bp", - "leggendaria-lamp", - "leggendaria-grade", - "timestamp", - ]; - } else { - logger.warn(`Invalid CSV header count of ${header.length} received.`); - throw new ScoreImportFatalError( - 400, - "Invalid CSV provided. CSV does not have the correct amount of headers." - ); - } - }, - skipEmptyLines: true, - }); - } catch (err) { - logger.warn(`CSV Parser Error`, { err }); - throw new ScoreImportFatalError(400, "CSV Could not be parsed."); - } + let { hasBeginnerAndLegg, version, iterableData } = NaiveCSVParse(fileData.buffer, logger); logger.verbose("Successfully parsed CSV."); - let firstEl = csvData[0]; - - if (!firstEl) { - throw new ScoreImportFatalError(400, "This CSV has no scores."); - } - - let version = 0; - - const diffs = hasBeginnerAndLegg - ? ["beginner", "normal", "hyper", "another", "leggendaria"] - : ["normal", "hyper", "another"]; - - // @optimisable - Can hook into the CSV parser perhaps and read this value immediately? - for (const d of csvData) { - // wtf typescript?? what's the point of enums? - const versionNum = EAM_VERSION_NAMES[d.version as keyof typeof EAM_VERSION_NAMES]; - - if (!versionNum) { - logger.info(`Invalid/Unknown EAM_VERSION_NAME ${d.version}.`); - throw new ScoreImportFatalError(400, `Invalid/Unknown EAM_VERSION_NAME ${d.version}.`); - } - - if (versionNum > version) { - logger.verbose(`Replaced ${version} with ${d.version} (${versionNum}).`); - version = versionNum; - } - - let scores: EamusementScoreData[] = []; - - for (const diff of diffs) { - scores.push({ - difficulty: (diff.toUpperCase as unknown) as Difficulties["iidx:SP" | "iidx:DP"], - bp: d[`${diff}-bp`] as integer | "---", - exscore: d[`${diff}-exscore`] as integer, - pgreat: d[`${diff}-pgreat`] as integer, - great: d[`${diff}-great`] as integer, - lamp: d[`${diff}-lamp`] as string, - level: d[`${diff}-level`] as integer, - }); - } - - data.push({ - scores, - artist: d.artist, - genre: d.genre, - playcount: d.playcount, - timestamp: d.timestamp, - title: d.title, - version: d.version, - }); - } - - if (hasBeginnerAndLegg === null) { - logger.error(`hasBeginnerAndLegg was not set, but the end of parsings was reached?`); - throw new ScoreImportFatalError(500, "An internal service error has occured."); - } - let context: IIDXEamusementCSVContext = { playtype, importVersion: version, @@ -261,10 +257,10 @@ function ParseEamusementCSV( serviceOrigin: "e-amusement", }; - logger.verbose(`Successfully Parsed with ${data.length} results.`); + logger.verbose(`Successfully Parsed with ${iterableData.length} results.`); return { - iterable: data, + iterable: iterableData, context, ConverterFunction: ConverterFn, idStrings: [`iidx:${context.playtype}` as "iidx:SP" | "iidx:DP"], diff --git a/server/src/score-import/import-types/csv-eamusement-iidx/types.ts b/server/src/score-import/import-types/csv-eamusement-iidx/types.ts index 46e0b6163..b706f41ac 100644 --- a/server/src/score-import/import-types/csv-eamusement-iidx/types.ts +++ b/server/src/score-import/import-types/csv-eamusement-iidx/types.ts @@ -3,19 +3,15 @@ import { integer } from "kamaitachi-common"; export interface EamusementScoreData { difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"; lamp: string; - exscore: integer; - pgreat: integer; - great: integer; - bp: integer | "---"; // lol - level: integer; + exscore: string; + pgreat: string; + great: string; + bp: string; + level: string; } type BaseProps = { - version: string; title: string; - genre: string; - artist: string; - playcount: number; timestamp: string; };