tidy up and document importing

server/src/db/db.ts

@@ -5,7 +5,7 @@ import {
     GoalDocument,
     IIDXBPIData,
     IIDXEamusementScoreDocument,
-    ImportDocument,
+    KTBlackImportDocument,
     InviteCodeDocument,
     MilestoneDocument,
     NotificationDocument,
@@ -100,7 +100,7 @@ const db = {
     milestones: monkDB.get<MilestoneDocument>("milestones"),
     "user-milestones": monkDB.get<UserMilestoneDocument>("user-milestones"),
     users: monkDB.get<PrivateUserDocument>("users"),
-    imports: monkDB.get<ImportDocument>("imports"),
+    imports: monkDB.get<KTBlackImportDocument>("imports"),
     notifications: monkDB.get<NotificationDocument>("notifications"),
     sessions: monkDB.get<SessionDocument>("sessions"),
     "iidx-bpi-data": monkDB.get<IIDXBPIData>("iidx-bpi-data"),

server/src/score-import/framework/score-import-main.ts

@@ -8,13 +8,13 @@ import {
     integer,
 } from "kamaitachi-common";
 import { ImportInputParser, KtLogger, ScorePlaytypeMap } from "../../types";
-import { InsertQueue } from "./score-importing/insert-score";
 import { ImportAllIterableData } from "./score-importing/score-importing";
 import { CreateImportLoggerAndID } from "./common/import-logger";
 import { CreateSessions } from "./sessions/sessions";
 import { GetMilisecondsSince } from "../../core/hrtime-core";
 import { ProcessPBs } from "./pb/process-pbs";
 import { UpdateUsersGamePlaytypeStats } from "./user-game-stats/update-ugs";
+import db from "../../db/db";
 
 export default async function ScoreImportMain<D, C>(
     user: PublicUserDocument,
@@ -23,9 +23,14 @@ export default async function ScoreImportMain<D, C>(
     InputParser: ImportInputParser<D, C>
 ) {
     const timeStarted = Date.now();
+
+    // We create an "import logger" - this holds a reference to the user's name for any future debugging.
     const { importID, logger } = CreateImportLoggerAndID(user, importType);
     logger.verbose("Received import request.");
 
+    // --- 1. Parsing ---
+    // We get an iterable from the provided parser function, alongside some context and a converter function.
+    // This iterable does not have to be an array - it's anything that's iterable, like a generator or similar.
     const parseTimeStart = process.hrtime.bigint();
     const { iterable, ConverterFunction, context, idStrings, game } = await InputParser(logger);
 
@@ -33,8 +38,9 @@ export default async function ScoreImportMain<D, C>(
 
     logger.verbose(`Parsing took ${parseTime} miliseconds.`);
 
+    // --- 2. Importing ---
+    // ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data.
     const importTimeStart = process.hrtime.bigint();
-
     let importInfo = await ImportAllIterableData(
         user.id,
         iterable,
@@ -43,26 +49,21 @@ export default async function ScoreImportMain<D, C>(
         logger
     );
 
-    // Flush the score queue out after importing.
-    let emptied = await InsertQueue();
-
-    if (emptied) {
-        logger.verbose(`Emptied ${emptied} documents from score queue.`);
-    }
-
     const importTime = GetMilisecondsSince(importTimeStart);
 
     logger.verbose(
         `Importing took ${importTime} miliseconds. (${importTime / importInfo.length}ms/doc)`
     );
 
+    // --- 3. ParseImportInfo ---
+    // ImportInfo is a relatively complex structure. We need some information from it for subsequent steps
+    // such as the list of chartIDs involved in this import.
     let { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo);
 
-    // Update user's rating information
-    // @todo
-
+    // --- 4. Sessions ---
+    // We create (or update existing) sessions here. This uses the aforementioned parsed import info
+    // to determine what goes where.
     const sessionTimeStart = process.hrtime.bigint();
-
     let sessionInfo = await CreateSessions(user.id, importType, game, scorePlaytypeMap, logger);
 
     const sessionTime = GetMilisecondsSince(sessionTimeStart);
@@ -73,18 +74,23 @@ export default async function ScoreImportMain<D, C>(
         }ms/doc).`
     );
 
-    // Update user's PBs and set flags
+    // --- 5. PersonalBests ---
+    // We want to keep an updated reference of a users best score on a given chart.
+    // This function also handles conjoining different scores together (such as unioning best lamp and
+    // best score).
     await ProcessPBs(user.id, chartIDs, logger);
 
     let playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][];
 
-    // i have honestly no idea what to do with this information.
+    // --- 6. Game Stats ---
+    // This function updates the users "stats" for this game - such as their profile rating, or their classes.
     let classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, logger);
 
-    // Update user's goals
+    // --- 7. Goals ---
     // @todo
 
-    // Create import document
+    // --- 8. Finalise Import Document ---
+    // Create and Save an import document to the database, and finish everything up!
     const ImportDocument: KTBlackImportDocument = {
         importType,
         idStrings: idStrings,
@@ -99,21 +105,27 @@ export default async function ScoreImportMain<D, C>(
         userIntent,
     };
 
-    logger.info(
-        `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${
-            importInfo.length
-        } documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${
-            sessionInfo.length
-        }). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`
-    );
+    const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${
+        importInfo.length
+    } documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${
+        sessionInfo.length
+    }). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`;
 
-    // Add this to the imports database
-    // @todo
+    // I only really want to log "big" imports. The others are here for debugging purposes.
+    if (scoreIDs.length > 500) {
+        logger.info(logMessage);
+    } else if (scoreIDs.length > 1) {
+        logger.verbose(logMessage);
+    } else {
+        logger.debug(logMessage);
+    }
+
+    await db.imports.insert(ImportDocument);
 
     return ImportDocument;
 }
 
-function UpdateUsersGameStats(
+async function UpdateUsersGameStats(
     game: Game,
     playtypes: Playtypes[Game][],
     userID: integer,
@@ -121,12 +133,12 @@ function UpdateUsersGameStats(
 ) {
     let promises = [];
 
-    // Update user's classes
     for (const pt of playtypes) {
         promises.push(UpdateUsersGamePlaytypeStats(game, pt, userID, null, logger));
     }
 
-    return Promise.all(promises).then((r) => r.flat(1));
+    const r = await Promise.all(promises);
+    return r.flat(1);
 }
 
 function ParseImportInfo(importInfo: ImportProcessingInfo[]) {

server/src/score-import/framework/score-importing/score-importing.ts

@@ -7,7 +7,7 @@ import {
 } from "kamaitachi-common";
 import { DryScore, ConverterFunction, ConverterFnReturn, KtLogger } from "../../../types";
 import HydrateScore from "./hydrate-score";
-import { QueueScoreInsert, ScoreIDs } from "./insert-score";
+import { InsertQueue, QueueScoreInsert, ScoreIDs } from "./insert-score";
 import {
     ConverterFailure,
     InternalFailure,
@@ -78,6 +78,14 @@ export async function ImportAllIterableData<D, C>(
 
     logger.verbose(`Recieved ${flatDatapoints.length} returns, from ${promises.length} data.`);
 
+    // Flush the score queue out after finishing most of the import. This ensures no scores get left in the
+    // queue.
+    let emptied = await InsertQueue();
+
+    if (emptied) {
+        logger.verbose(`Emptied ${emptied} documents from score queue.`);
+    }
+
     return flatDatapoints;
 }