diff --git a/server/package.json b/server/package.json index d1a97be53..1f427064b 100644 --- a/server/package.json +++ b/server/package.json @@ -10,7 +10,8 @@ "build": "tsc --project tsconfig.build.json", "lint": "eslint ./src --ext .ts --fix", "start": "tsc --project tsconfig.build.json && export NODE_PATH=js/ && node js/main.js", - "run": "export NODE_PATH=js/ && node js/main.js" + "run": "export NODE_PATH=js/ && node js/main.js", + "runscoreworker": "export NODE_PATH=js/ && node js/lib/score-import/worker/worker.js" }, "author": "zkldi", "license": "AGPL3", diff --git a/server/src/lib/score-import/framework/common/score-utils.ts b/server/src/lib/score-import/framework/common/score-utils.ts index ef3aa86ec..8876e8438 100644 --- a/server/src/lib/score-import/framework/common/score-utils.ts +++ b/server/src/lib/score-import/framework/common/score-utils.ts @@ -146,7 +146,7 @@ export function CalculateESDForGame( /** * Parses and validates a date from a string. - * @returns Millisecondss from the unix epoch, or null if the initial argument was null or undefined. + * @returns Milliseconds from the unix epoch, or null if the initial argument was null or undefined. */ export function ParseDateFromString(str: string | undefined | null): number | null { if (!str) { diff --git a/server/src/lib/score-import/framework/score-importing/score-import-main.ts b/server/src/lib/score-import/framework/score-importing/score-import-main.ts index a0c211a27..7fbfba28b 100644 --- a/server/src/lib/score-import/framework/score-importing/score-import-main.ts +++ b/server/src/lib/score-import/framework/score-importing/score-import-main.ts @@ -1,5 +1,6 @@ import db from "external/mongo/db"; import { KtLogger } from "lib/logger/logger"; +import { ScoreImportJob, ScoreImportProgress } from "lib/score-import/worker/types"; import { Game, IDStrings, @@ -29,6 +30,8 @@ import { ImportAllIterableData } from "./score-importing"; /** * Performs a Score Import. + * + * If a job is passed, progress will be set throughout the job. */ export default async function ScoreImportMain( userID: integer, @@ -36,7 +39,8 @@ export default async function ScoreImportMain( importType: ImportTypes, InputParser: ImportInputParser, importID: string, - providedLogger?: KtLogger + providedLogger?: KtLogger, + job?: ScoreImportJob ) { const user = await GetUserWithID(userID); @@ -59,7 +63,7 @@ export default async function ScoreImportMain( // // Under normal circumstances, there is no scenario where a user would have two ongoing // imports at the same time - even if they were using single-score imports on a 5 second - // chart, as each score import takes only around ~10-15millisecondss. + // chart, as each score import takes only around ~10-15milliseconds. throw new ScoreImportFatalError(409, "This user already has an ongoing import."); } @@ -77,6 +81,8 @@ export default async function ScoreImportMain( logger = providedLogger; } + SetJobProgress(job, "Parsing score data."); + // --- 1. Parsing --- // We get an iterable from the provided parser function, alongside some context and a converter function. // This iterable does not have to be an array - it's anything that's iterable, like a generator. @@ -85,7 +91,14 @@ export default async function ScoreImportMain( const parseTime = GetMillisecondsSince(parseTimeStart); - logger.debug(`Parsing took ${parseTime} millisecondss.`); + logger.debug(`Parsing took ${parseTime} milliseconds.`); + + SetJobProgress( + job, + `Parsed Score Data. Took ${parseTime}ms. Importing ${ + Array.isArray(iterable) ? iterable.length : "an unknown amount of" + } scores.` + ); // We have to cast here due to typescript generic confusions. This is guaranteed` to be correct. const ConverterFunction = Converters[importType] as unknown as ConverterFunction; @@ -106,7 +119,9 @@ export default async function ScoreImportMain( const importTime = GetMillisecondsSince(importTimeStart); const importTimeRel = importTime / importInfo.length; - logger.debug(`Importing took ${importTime} millisecondss. (${importTimeRel}ms/doc)`); + logger.debug(`Importing took ${importTime} milliseconds. (${importTimeRel}ms/doc)`); + + SetJobProgress(job, `Imported scores, took ${importTime} milliseconds. `); // Steps 3-8 are handled inside here. // This was moved inside here so the score de-orphaning process @@ -127,6 +142,8 @@ export default async function ScoreImportMain( const { importParseTime, sessionTime, pbTime, ugsTime, goalTime, milestoneTime } = absoluteTimes; + SetJobProgress(job, "Finalising Import..."); + // --- 9. Finalise Import Document --- // Create and Save an import document to the database, and finish everything up! const ImportDocument: ImportDocument = { @@ -205,7 +222,8 @@ export async function HandlePostImportSteps( importType: ImportTypes, game: Game, classHandler: ClassHandler | null, - logger: KtLogger + logger: KtLogger, + job: ScoreImportJob | undefined ) { // --- 3. ParseImportInfo --- // ImportInfo is a relatively complex structure. We need some information from it for subsequent steps @@ -217,9 +235,11 @@ export async function HandlePostImportSteps( const importParseTimeRel = importParseTime / importInfo.length; logger.debug( - `Import Parsing took ${importParseTime} millisecondss. (${importParseTimeRel}ms/doc)` + `Import Parsing took ${importParseTime} milliseconds. (${importParseTimeRel}ms/doc)` ); + SetJobProgress(job, "Inserting Sessions."); + // --- 4. Sessions --- // We create (or update existing) sessions here. This uses the aforementioned parsed import info // to determine what goes where. @@ -229,7 +249,9 @@ export async function HandlePostImportSteps( const sessionTime = GetMillisecondsSince(sessionTimeStart); const sessionTimeRel = sessionTime / sessionInfo.length; - logger.debug(`Session Processing took ${sessionTime} millisecondss (${sessionTimeRel}ms/doc).`); + logger.debug(`Session Processing took ${sessionTime} milliseconds (${sessionTimeRel}ms/doc).`); + + SetJobProgress(job, "Processing scores and updating PBs."); // --- 5. PersonalBests --- // We want to keep an updated reference of a users best score on a given chart. @@ -241,10 +263,12 @@ export async function HandlePostImportSteps( const pbTime = GetMillisecondsSince(pbTimeStart); const pbTimeRel = pbTime / chartIDs.size; - logger.debug(`PB Processing took ${pbTime} millisecondss (${pbTimeRel}ms/doc)`); + logger.debug(`PB Processing took ${pbTime} milliseconds (${pbTimeRel}ms/doc)`); const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][]; + SetJobProgress(job, "Updating profile statistics."); + // --- 6. Game Stats --- // This function updates the users "stats" for this game - such as their profile rating or their classes. const ugsTimeStart = process.hrtime.bigint(); @@ -252,7 +276,9 @@ export async function HandlePostImportSteps( const ugsTime = GetMillisecondsSince(ugsTimeStart); - logger.debug(`UGS Processing took ${ugsTime} millisecondss.`); + logger.debug(`UGS Processing took ${ugsTime} milliseconds.`); + + SetJobProgress(job, "Updating Goals."); // --- 7. Goals --- // Evaluate and update the users goals. This returns information about goals that have changed. @@ -261,7 +287,9 @@ export async function HandlePostImportSteps( const goalTime = GetMillisecondsSince(goalTimeStart); - logger.debug(`Goal Processing took ${goalTime} millisecondss.`); + logger.debug(`Goal Processing took ${goalTime} milliseconds.`); + + SetJobProgress(job, "Updating Milestones."); // --- 8. Milestones --- // Evaluate and update the users milestones. This returns... @@ -270,7 +298,7 @@ export async function HandlePostImportSteps( const milestoneTime = GetMillisecondsSince(milestoneTimeStart); - logger.debug(`Milestone Processing took ${milestoneTime} millisecondss.`); + logger.debug(`Milestone Processing took ${milestoneTime} milliseconds.`); return { classDeltas, @@ -348,3 +376,9 @@ function ParseImportInfo(importInfo: ImportProcessingInfo[]) { return { scoreIDs, errors, scorePlaytypeMap, chartIDs }; } + +function SetJobProgress(job: ScoreImportJob | undefined, description: string) { + if (job) { + job.progress({ description }); + } +} diff --git a/server/src/lib/score-import/import-types/common/batch-manual/parser.test.ts b/server/src/lib/score-import/import-types/common/batch-manual/parser.test.ts index c4db2f299..081fd25aa 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/parser.test.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/parser.test.ts @@ -379,7 +379,7 @@ t.test("#ParserFn", (t) => { fn, new ScoreImportFatalError( 400, - "Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of millisecondss? | Received string [string]." + "Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of milliseconds? | Received string [string]." ) ); @@ -394,7 +394,7 @@ t.test("#ParserFn", (t) => { fn2, new ScoreImportFatalError( 400, - "Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of millisecondss? | Received 1620768609.637 [number]." + "Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of milliseconds? | Received 1620768609.637 [number]." ), "Should throw if timeAchieved is less than 10_000_000_000." ); diff --git a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts index 1a5976e9f..834ee485d 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts @@ -138,7 +138,7 @@ const PR_BatchManualScore = (game: Game, playtype: Playtypes[Game]): PrudenceSch timeAchieved: optNull( (self) => (typeof self === "number" && self > 1_000_000_000_000) || - "Expected a number greater than 1 Trillion - did you pass unix seconds instead of millisecondss?" + "Expected a number greater than 1 Trillion - did you pass unix seconds instead of milliseconds?" ), judgements: optNull((self) => { if (typeof self !== "object" || self === null) { diff --git a/server/src/lib/score-import/worker/types.ts b/server/src/lib/score-import/worker/types.ts index f9d5a9bf2..cc0792444 100644 --- a/server/src/lib/score-import/worker/types.ts +++ b/server/src/lib/score-import/worker/types.ts @@ -37,3 +37,7 @@ export interface ScoreImportJobData { } export type ScoreImportJob = Job>; + +export interface ScoreImportProgress { + description: string; +} diff --git a/server/src/lib/score-import/worker/worker.ts b/server/src/lib/score-import/worker/worker.ts index 74c152e1a..005e96bbf 100644 --- a/server/src/lib/score-import/worker/worker.ts +++ b/server/src/lib/score-import/worker/worker.ts @@ -51,10 +51,16 @@ ScoreImportQueue.process(async (job: ScoreImportJob) = // This helps us debug what score import did what! const logger = CreateLogCtx(`Score Import ${job.id} ${FormatUserDoc(user)}`); + logger.debug(`Recieved score import job ${job.id}`); + const InputParser = GetInputParser(job.data); logger.debug(`Starting import.`); + job.progress({ + description: "Importing Scores...", + }); + const importDocument = await ScoreImportMain( user.id, job.data.userIntent, diff --git a/server/src/utils/misc.test.ts b/server/src/utils/misc.test.ts index c6c01109b..2004782d2 100644 --- a/server/src/utils/misc.test.ts +++ b/server/src/utils/misc.test.ts @@ -1,7 +1,7 @@ import t from "tap"; import { GetMillisecondsSince, IsValidURL } from "./misc"; -t.test("#GetMillisecondssSince", (t) => { +t.test("#GetMillisecondsSince", (t) => { const time = GetMillisecondsSince(10n); t.ok(typeof time === "number" && time > 0, "Should return a number greater than 0."); diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 312c14631..b3c6cda53 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -18,7 +18,7 @@ export function EscapeStringRegexp(string: string) { } /** - * Takes a process.hrtime.bigint(), and returns the millisecondss elapsed since it. + * Takes a process.hrtime.bigint(), and returns the milliseconds elapsed since it. * This function will not work if more than 100(ish) days have passed since the first reference. */ export function GetMillisecondsSince(ref: bigint) {