From cded8d0daccdcc6721248ad7e40ab08a266f2fc8 Mon Sep 17 00:00:00 2001 From: zkldi Date: Sun, 2 May 2021 22:28:07 +0100 Subject: [PATCH] tidy up and document importing --- server/src/db/db.ts | 4 +- .../framework/score-import-main.ts | 70 +++++++++++-------- .../score-importing/score-importing.ts | 10 ++- 3 files changed, 52 insertions(+), 32 deletions(-) diff --git a/server/src/db/db.ts b/server/src/db/db.ts index 65340d79b..d632a0abe 100644 --- a/server/src/db/db.ts +++ b/server/src/db/db.ts @@ -5,7 +5,7 @@ import { GoalDocument, IIDXBPIData, IIDXEamusementScoreDocument, - ImportDocument, + KTBlackImportDocument, InviteCodeDocument, MilestoneDocument, NotificationDocument, @@ -100,7 +100,7 @@ const db = { milestones: monkDB.get("milestones"), "user-milestones": monkDB.get("user-milestones"), users: monkDB.get("users"), - imports: monkDB.get("imports"), + imports: monkDB.get("imports"), notifications: monkDB.get("notifications"), sessions: monkDB.get("sessions"), "iidx-bpi-data": monkDB.get("iidx-bpi-data"), diff --git a/server/src/score-import/framework/score-import-main.ts b/server/src/score-import/framework/score-import-main.ts index 10d848e86..1870d16e0 100644 --- a/server/src/score-import/framework/score-import-main.ts +++ b/server/src/score-import/framework/score-import-main.ts @@ -8,13 +8,13 @@ import { integer, } from "kamaitachi-common"; import { ImportInputParser, KtLogger, ScorePlaytypeMap } from "../../types"; -import { InsertQueue } from "./score-importing/insert-score"; import { ImportAllIterableData } from "./score-importing/score-importing"; import { CreateImportLoggerAndID } from "./common/import-logger"; import { CreateSessions } from "./sessions/sessions"; import { GetMilisecondsSince } from "../../core/hrtime-core"; import { ProcessPBs } from "./pb/process-pbs"; import { UpdateUsersGamePlaytypeStats } from "./user-game-stats/update-ugs"; +import db from "../../db/db"; export default async function ScoreImportMain( user: PublicUserDocument, @@ -23,9 +23,14 @@ export default async function ScoreImportMain( InputParser: ImportInputParser ) { const timeStarted = Date.now(); + + // We create an "import logger" - this holds a reference to the user's name for any future debugging. const { importID, logger } = CreateImportLoggerAndID(user, importType); logger.verbose("Received import request."); + // --- 1. Parsing --- + // We get an iterable from the provided parser function, alongside some context and a converter function. + // This iterable does not have to be an array - it's anything that's iterable, like a generator or similar. const parseTimeStart = process.hrtime.bigint(); const { iterable, ConverterFunction, context, idStrings, game } = await InputParser(logger); @@ -33,8 +38,9 @@ export default async function ScoreImportMain( logger.verbose(`Parsing took ${parseTime} miliseconds.`); + // --- 2. Importing --- + // ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data. const importTimeStart = process.hrtime.bigint(); - let importInfo = await ImportAllIterableData( user.id, iterable, @@ -43,26 +49,21 @@ export default async function ScoreImportMain( logger ); - // Flush the score queue out after importing. - let emptied = await InsertQueue(); - - if (emptied) { - logger.verbose(`Emptied ${emptied} documents from score queue.`); - } - const importTime = GetMilisecondsSince(importTimeStart); logger.verbose( `Importing took ${importTime} miliseconds. (${importTime / importInfo.length}ms/doc)` ); + // --- 3. ParseImportInfo --- + // ImportInfo is a relatively complex structure. We need some information from it for subsequent steps + // such as the list of chartIDs involved in this import. let { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo); - // Update user's rating information - // @todo - + // --- 4. Sessions --- + // We create (or update existing) sessions here. This uses the aforementioned parsed import info + // to determine what goes where. const sessionTimeStart = process.hrtime.bigint(); - let sessionInfo = await CreateSessions(user.id, importType, game, scorePlaytypeMap, logger); const sessionTime = GetMilisecondsSince(sessionTimeStart); @@ -73,18 +74,23 @@ export default async function ScoreImportMain( }ms/doc).` ); - // Update user's PBs and set flags + // --- 5. PersonalBests --- + // We want to keep an updated reference of a users best score on a given chart. + // This function also handles conjoining different scores together (such as unioning best lamp and + // best score). await ProcessPBs(user.id, chartIDs, logger); let playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][]; - // i have honestly no idea what to do with this information. + // --- 6. Game Stats --- + // This function updates the users "stats" for this game - such as their profile rating, or their classes. let classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, logger); - // Update user's goals + // --- 7. Goals --- // @todo - // Create import document + // --- 8. Finalise Import Document --- + // Create and Save an import document to the database, and finish everything up! const ImportDocument: KTBlackImportDocument = { importType, idStrings: idStrings, @@ -99,21 +105,27 @@ export default async function ScoreImportMain( userIntent, }; - logger.info( - `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${ - importInfo.length - } documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${ - sessionInfo.length - }). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc` - ); + const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${ + importInfo.length + } documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${ + sessionInfo.length + }). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`; - // Add this to the imports database - // @todo + // I only really want to log "big" imports. The others are here for debugging purposes. + if (scoreIDs.length > 500) { + logger.info(logMessage); + } else if (scoreIDs.length > 1) { + logger.verbose(logMessage); + } else { + logger.debug(logMessage); + } + + await db.imports.insert(ImportDocument); return ImportDocument; } -function UpdateUsersGameStats( +async function UpdateUsersGameStats( game: Game, playtypes: Playtypes[Game][], userID: integer, @@ -121,12 +133,12 @@ function UpdateUsersGameStats( ) { let promises = []; - // Update user's classes for (const pt of playtypes) { promises.push(UpdateUsersGamePlaytypeStats(game, pt, userID, null, logger)); } - return Promise.all(promises).then((r) => r.flat(1)); + const r = await Promise.all(promises); + return r.flat(1); } function ParseImportInfo(importInfo: ImportProcessingInfo[]) { diff --git a/server/src/score-import/framework/score-importing/score-importing.ts b/server/src/score-import/framework/score-importing/score-importing.ts index 7c87b8e89..cad3afe76 100644 --- a/server/src/score-import/framework/score-importing/score-importing.ts +++ b/server/src/score-import/framework/score-importing/score-importing.ts @@ -7,7 +7,7 @@ import { } from "kamaitachi-common"; import { DryScore, ConverterFunction, ConverterFnReturn, KtLogger } from "../../../types"; import HydrateScore from "./hydrate-score"; -import { QueueScoreInsert, ScoreIDs } from "./insert-score"; +import { InsertQueue, QueueScoreInsert, ScoreIDs } from "./insert-score"; import { ConverterFailure, InternalFailure, @@ -78,6 +78,14 @@ export async function ImportAllIterableData( logger.verbose(`Recieved ${flatDatapoints.length} returns, from ${promises.length} data.`); + // Flush the score queue out after finishing most of the import. This ensures no scores get left in the + // queue. + let emptied = await InsertQueue(); + + if (emptied) { + logger.verbose(`Emptied ${emptied} documents from score queue.`); + } + return flatDatapoints; }