Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-09-30 02:48:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add job progress indicators

Browse Source
This commit is contained in:
zkldi
2021-11-17 15:28:57 +00:00
parent be49202c1a
commit fe8cef47ff
9 changed files with 63 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/package.json
+2 -1
View File
@@ -10,7 +10,8 @@
"build": "tsc --project tsconfig.build.json",
"lint": "eslint ./src --ext .ts --fix",
"start": "tsc --project tsconfig.build.json && export NODE_PATH=js/ && node js/main.js",
"run": "export NODE_PATH=js/ && node js/main.js"
"run": "export NODE_PATH=js/ && node js/main.js",
"runscoreworker": "export NODE_PATH=js/ && node js/lib/score-import/worker/worker.js"
},
"author": "zkldi",
"license": "AGPL3",
server/src/lib/score-import/framework/common/score-utils.ts
+1 -1
View File
@@ -146,7 +146,7 @@ export function CalculateESDForGame(
/**
* Parses and validates a date from a string.
* @returns Millisecondss from the unix epoch, or null if the initial argument was null or undefined.
* @returns Milliseconds from the unix epoch, or null if the initial argument was null or undefined.
*/
export function ParseDateFromString(str: string | undefined | null): number | null {
if (!str) {
server/src/lib/score-import/framework/score-importing/score-import-main.ts
+45 -11
View File
@@ -1,5 +1,6 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { ScoreImportJob, ScoreImportProgress } from "lib/score-import/worker/types";
import {
Game,
IDStrings,
@@ -29,6 +30,8 @@ import { ImportAllIterableData } from "./score-importing";
/**
* Performs a Score Import.
*
* If a job is passed, progress will be set throughout the job.
*/
export default async function ScoreImportMain<D, C>(
userID: integer,
@@ -36,7 +39,8 @@ export default async function ScoreImportMain<D, C>(
importType: ImportTypes,
InputParser: ImportInputParser<D, C>,
importID: string,
providedLogger?: KtLogger
providedLogger?: KtLogger,
job?: ScoreImportJob
) {
const user = await GetUserWithID(userID);
@@ -59,7 +63,7 @@ export default async function ScoreImportMain<D, C>(
//
// Under normal circumstances, there is no scenario where a user would have two ongoing
// imports at the same time - even if they were using single-score imports on a 5 second
// chart, as each score import takes only around ~10-15millisecondss.
// chart, as each score import takes only around ~10-15milliseconds.
throw new ScoreImportFatalError(409, "This user already has an ongoing import.");
}
@@ -77,6 +81,8 @@ export default async function ScoreImportMain<D, C>(
logger = providedLogger;
}
SetJobProgress(job, "Parsing score data.");
// --- 1. Parsing ---
// We get an iterable from the provided parser function, alongside some context and a converter function.
// This iterable does not have to be an array - it's anything that's iterable, like a generator.
@@ -85,7 +91,14 @@ export default async function ScoreImportMain<D, C>(
const parseTime = GetMillisecondsSince(parseTimeStart);
logger.debug(`Parsing took ${parseTime} millisecondss.`);
logger.debug(`Parsing took ${parseTime} milliseconds.`);
SetJobProgress(
job,
`Parsed Score Data. Took ${parseTime}ms. Importing ${
Array.isArray(iterable) ? iterable.length : "an unknown amount of"
} scores.`
);
// We have to cast here due to typescript generic confusions. This is guaranteed` to be correct.
const ConverterFunction = Converters[importType] as unknown as ConverterFunction<D, C>;
@@ -106,7 +119,9 @@ export default async function ScoreImportMain<D, C>(
const importTime = GetMillisecondsSince(importTimeStart);
const importTimeRel = importTime / importInfo.length;
logger.debug(`Importing took ${importTime} millisecondss. (${importTimeRel}ms/doc)`);
logger.debug(`Importing took ${importTime} milliseconds. (${importTimeRel}ms/doc)`);
SetJobProgress(job, `Imported scores, took ${importTime} milliseconds. `);
// Steps 3-8 are handled inside here.
// This was moved inside here so the score de-orphaning process
@@ -127,6 +142,8 @@ export default async function ScoreImportMain<D, C>(
const { importParseTime, sessionTime, pbTime, ugsTime, goalTime, milestoneTime } =
absoluteTimes;
SetJobProgress(job, "Finalising Import...");
// --- 9. Finalise Import Document ---
// Create and Save an import document to the database, and finish everything up!
const ImportDocument: ImportDocument = {
@@ -205,7 +222,8 @@ export async function HandlePostImportSteps(
importType: ImportTypes,
game: Game,
classHandler: ClassHandler | null,
logger: KtLogger
logger: KtLogger,
job: ScoreImportJob | undefined
) {
// --- 3. ParseImportInfo ---
// ImportInfo is a relatively complex structure. We need some information from it for subsequent steps
@@ -217,9 +235,11 @@ export async function HandlePostImportSteps(
const importParseTimeRel = importParseTime / importInfo.length;
logger.debug(
`Import Parsing took ${importParseTime} millisecondss. (${importParseTimeRel}ms/doc)`
`Import Parsing took ${importParseTime} milliseconds. (${importParseTimeRel}ms/doc)`
);
SetJobProgress(job, "Inserting Sessions.");
// --- 4. Sessions ---
// We create (or update existing) sessions here. This uses the aforementioned parsed import info
// to determine what goes where.
@@ -229,7 +249,9 @@ export async function HandlePostImportSteps(
const sessionTime = GetMillisecondsSince(sessionTimeStart);
const sessionTimeRel = sessionTime / sessionInfo.length;
logger.debug(`Session Processing took ${sessionTime} millisecondss (${sessionTimeRel}ms/doc).`);
logger.debug(`Session Processing took ${sessionTime} milliseconds (${sessionTimeRel}ms/doc).`);
SetJobProgress(job, "Processing scores and updating PBs.");
// --- 5. PersonalBests ---
// We want to keep an updated reference of a users best score on a given chart.
@@ -241,10 +263,12 @@ export async function HandlePostImportSteps(
const pbTime = GetMillisecondsSince(pbTimeStart);
const pbTimeRel = pbTime / chartIDs.size;
logger.debug(`PB Processing took ${pbTime} millisecondss (${pbTimeRel}ms/doc)`);
logger.debug(`PB Processing took ${pbTime} milliseconds (${pbTimeRel}ms/doc)`);
const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][];
SetJobProgress(job, "Updating profile statistics.");
// --- 6. Game Stats ---
// This function updates the users "stats" for this game - such as their profile rating or their classes.
const ugsTimeStart = process.hrtime.bigint();
@@ -252,7 +276,9 @@ export async function HandlePostImportSteps(
const ugsTime = GetMillisecondsSince(ugsTimeStart);
logger.debug(`UGS Processing took ${ugsTime} millisecondss.`);
logger.debug(`UGS Processing took ${ugsTime} milliseconds.`);
SetJobProgress(job, "Updating Goals.");
// --- 7. Goals ---
// Evaluate and update the users goals. This returns information about goals that have changed.
@@ -261,7 +287,9 @@ export async function HandlePostImportSteps(
const goalTime = GetMillisecondsSince(goalTimeStart);
logger.debug(`Goal Processing took ${goalTime} millisecondss.`);
logger.debug(`Goal Processing took ${goalTime} milliseconds.`);
SetJobProgress(job, "Updating Milestones.");
// --- 8. Milestones ---
// Evaluate and update the users milestones. This returns...
@@ -270,7 +298,7 @@ export async function HandlePostImportSteps(
const milestoneTime = GetMillisecondsSince(milestoneTimeStart);
logger.debug(`Milestone Processing took ${milestoneTime} millisecondss.`);
logger.debug(`Milestone Processing took ${milestoneTime} milliseconds.`);
return {
classDeltas,
@@ -348,3 +376,9 @@ function ParseImportInfo(importInfo: ImportProcessingInfo[]) {
return { scoreIDs, errors, scorePlaytypeMap, chartIDs };
}
function SetJobProgress(job: ScoreImportJob | undefined, description: string) {
if (job) {
job.progress({ description });
}
}
server/src/lib/score-import/import-types/common/batch-manual/parser.test.ts
+2 -2
View File
@@ -379,7 +379,7 @@ t.test("#ParserFn", (t) => {
fn,
new ScoreImportFatalError(
400,
"Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of millisecondss? | Received string [string]."
"Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of milliseconds? | Received string [string]."
)
);
@@ -394,7 +394,7 @@ t.test("#ParserFn", (t) => {
fn2,
new ScoreImportFatalError(
400,
"Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of millisecondss? | Received 1620768609.637 [number]."
"Invalid BATCH-MANUAL: scores[0].timeAchieved | Expected a number greater than 1 Trillion - did you pass unix seconds instead of milliseconds? | Received 1620768609.637 [number]."
),
"Should throw if timeAchieved is less than 10_000_000_000."
);
server/src/lib/score-import/import-types/common/batch-manual/parser.ts
+1 -1
View File
@@ -138,7 +138,7 @@ const PR_BatchManualScore = (game: Game, playtype: Playtypes[Game]): PrudenceSch
timeAchieved: optNull(
(self) =>
(typeof self === "number" && self > 1_000_000_000_000) ||
"Expected a number greater than 1 Trillion - did you pass unix seconds instead of millisecondss?"
"Expected a number greater than 1 Trillion - did you pass unix seconds instead of milliseconds?"
),
judgements: optNull((self) => {
if (typeof self !== "object" || self === null) {
server/src/lib/score-import/worker/types.ts
+4
View File
@@ -37,3 +37,7 @@ export interface ScoreImportJobData<I extends ImportTypes> {
}
export type ScoreImportJob<I extends ImportTypes = ImportTypes> = Job<ScoreImportJobData<I>>;
export interface ScoreImportProgress {
description: string;
}
server/src/lib/score-import/worker/worker.ts
+6
View File
@@ -51,10 +51,16 @@ ScoreImportQueue.process(async <I extends ImportTypes>(job: ScoreImportJob<I>) =
// This helps us debug what score import did what!
const logger = CreateLogCtx(`Score Import ${job.id} ${FormatUserDoc(user)}`);
logger.debug(`Recieved score import job ${job.id}`);
const InputParser = GetInputParser(job.data);
logger.debug(`Starting import.`);
job.progress({
description: "Importing Scores...",
});
const importDocument = await ScoreImportMain(
user.id,
job.data.userIntent,
server/src/utils/misc.test.ts
+1 -1
View File
@@ -1,7 +1,7 @@
import t from "tap";
import { GetMillisecondsSince, IsValidURL } from "./misc";
t.test("#GetMillisecondssSince", (t) => {
t.test("#GetMillisecondsSince", (t) => {
const time = GetMillisecondsSince(10n);
t.ok(typeof time === "number" && time > 0, "Should return a number greater than 0.");
server/src/utils/misc.ts
+1 -1
View File
@@ -18,7 +18,7 @@ export function EscapeStringRegexp(string: string) {
}
/**
* Takes a process.hrtime.bigint(), and returns the millisecondss elapsed since it.
* Takes a process.hrtime.bigint(), and returns the milliseconds elapsed since it.
* This function will not work if more than 100(ish) days have passed since the first reference.
*/
export function GetMillisecondsSince(ref: bigint) {