diff --git a/server/.github/test.conf.json5 b/server/.github/test.conf.json5 index 1e80a3800..276f524d8 100644 --- a/server/.github/test.conf.json5 +++ b/server/.github/test.conf.json5 @@ -52,7 +52,6 @@ "ir/fervidex", "ir/fervidex-static", "ir/beatoraja", - "ir/chunitachi", "ir/usc", "ir/kshook-sv3c", "api/arc-iidx", diff --git a/server/package.json b/server/package.json index 8ff6a08ce..f4a9171f8 100644 --- a/server/package.json +++ b/server/package.json @@ -81,7 +81,7 @@ "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", "seq-logging": "^1.1.1", - "tachi-common": "0.2.42", + "tachi-common": "0.2.46", "ts-node": "10.0.0", "typescript": "4.3.4", "winston": "3.3.3", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 7fbf904a1..fbfee0755 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -62,7 +62,7 @@ specifiers: safe-json-stringify: 1.2.0 seq-logging: ^1.1.1 supertest: 6.1.3 - tachi-common: 0.2.42 + tachi-common: 0.2.46 tap: 15.0.9 ts-node: 10.0.0 tsconfig-paths: 3.10.1 @@ -106,7 +106,7 @@ dependencies: rimraf: 3.0.2 safe-json-stringify: 1.2.0 seq-logging: 1.1.1 - tachi-common: 0.2.42 + tachi-common: 0.2.46 ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b typescript: 4.3.4 winston: 3.3.3 @@ -4176,8 +4176,8 @@ packages: strip-ansi: 6.0.0 dev: true - /tachi-common/0.2.42: - resolution: {integrity: sha512-X+oS08NXgVeiPAFiNUzN7DCmiEJor35NCQkh6ia/kcpTAGkRUwst8NFQnHIca2eXDQmndEJg0KGkpYosMs4KYg==} + /tachi-common/0.2.46: + resolution: {integrity: sha512-BTI3j6joaHLh+T/gapHw5lO9BlB9jSCDtaLIECHkEjY0pUBnXcbcA4GEeMjefXUD65hF04BNkg6J7R6X6Cnkmw==} dependencies: monk: 7.3.4 transitivePeerDependencies: diff --git a/server/src/lib/email/client.ts b/server/src/lib/email/client.ts index 3ad3b8c5c..fb38c777b 100644 --- a/server/src/lib/email/client.ts +++ b/server/src/lib/email/client.ts @@ -1,7 +1,7 @@ +import bunyan from "bunyan"; import CreateLogCtx from "lib/logger/logger"; import { Environment, ServerConfig } from "lib/setup/config"; import nodemailer, { SentMessageInfo, Transporter } from "nodemailer"; -import bunyan from "bunyan"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/email/formats.ts b/server/src/lib/email/formats.ts index 6847a7d80..e25c3806f 100644 --- a/server/src/lib/email/formats.ts +++ b/server/src/lib/email/formats.ts @@ -1,7 +1,7 @@ // Email HTML is a hellish mess of IE5 era nonsense. // Good luck. -import { ServerConfig, TachiConfig, TachiServerConfig } from "lib/setup/config"; +import { ServerConfig, TachiConfig } from "lib/setup/config"; export function EmailFormatResetPassword(username: string, resetCode: string, ipAddr: string) { return MainHTMLWrapper( diff --git a/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts b/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts index fefe874c0..9cdfb4ca0 100644 --- a/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts +++ b/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts @@ -1,6 +1,7 @@ -import p from "prudence"; -import t from "tap"; import CreateLogCtx from "lib/logger/logger"; +import p from "prudence"; +import { ChartDocument, ScoreDocument } from "tachi-common"; +import t from "tap"; import { prAssert } from "test-utils/asserts"; import { Testing511SPA, @@ -9,8 +10,7 @@ import { TestingIIDXSPDryScore, TestingSDVXSingleDryScore, } from "test-utils/test-data"; -import { CreateCalculatedData, CalculateDataForGamePT } from "./calculated-data"; -import { ChartDocument, ScoreDocument } from "tachi-common"; +import { CalculateDataForGamePT, CreateCalculatedData } from "./calculated-data"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/score-import/framework/calculated-data/stats.ts b/server/src/lib/score-import/framework/calculated-data/stats.ts index b24cdf379..985c5186a 100644 --- a/server/src/lib/score-import/framework/calculated-data/stats.ts +++ b/server/src/lib/score-import/framework/calculated-data/stats.ts @@ -1,13 +1,13 @@ +import { KtLogger } from "lib/logger/logger"; import { ChartDocument, - integer, - Grades, - Lamps, Game, - Playtypes, GetGamePTConfig, + Grades, + integer, + Lamps, + Playtypes, } from "tachi-common"; -import { KtLogger } from "lib/logger/logger"; import { DryScore } from "../common/types"; /** diff --git a/server/src/lib/score-import/framework/common/get-input-parser.ts b/server/src/lib/score-import/framework/common/get-input-parser.ts new file mode 100644 index 000000000..d09f3330a --- /dev/null +++ b/server/src/lib/score-import/framework/common/get-input-parser.ts @@ -0,0 +1,26 @@ +import { KtLogger } from "lib/logger/logger"; +import { + ParserFunctionReturns, + ImportTypeDataMap, + ImportTypeContextMap, +} from "lib/score-import/import-types/common/types"; +import { Parsers } from "lib/score-import/import-types/parsers"; +import { ScoreImportJobData } from "lib/score-import/worker/types"; +import { ImportTypes } from "tachi-common"; + +export function GetInputParser(jobData: ScoreImportJobData) { + // Retrieve the set parser function for this import type. + const ParserFunction = Parsers[jobData.importType]; + + const InputParser = (logger: KtLogger) => + // @ts-expect-error TypeScript doesn't like the fact that we + // pass this as a rest parameter, since none of the parsers + // actually take rest args. However, this is the only way to + // achieve the dynamic passing we need to, so lets just override + // it here. + ParserFunction(...jobData.parserArguments, logger) as Promise< + ParserFunctionReturns + >; + + return InputParser; +} diff --git a/server/src/lib/score-import/framework/common/import-logger.test.ts b/server/src/lib/score-import/framework/common/import-logger.test.ts deleted file mode 100644 index 6a2fb2161..000000000 --- a/server/src/lib/score-import/framework/common/import-logger.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { PublicUserDocument } from "tachi-common"; -import t from "tap"; -import { CreateImportLoggerAndID } from "./import-logger"; - -t.test("#CreateImportLoggerAndID", (t) => { - const { importID, logger } = CreateImportLoggerAndID( - { username: "foo", id: 1 } as PublicUserDocument, - "file/eamusement-iidx-csv" - ); - - t.match(importID, /^[a-f0-9]{40}$/u, "Should return a 40 character importID."); - - // not possible to automate this, probably. - logger.info( - "Namespaced log test - if this does not have 'file/eamusement-iidx-csv | foo (#1)' next to it, it's broke." - ); - - t.end(); -}); diff --git a/server/src/lib/score-import/framework/common/import-logger.ts b/server/src/lib/score-import/framework/common/import-logger.ts index 43843a5a0..64a3eb9c1 100644 --- a/server/src/lib/score-import/framework/common/import-logger.ts +++ b/server/src/lib/score-import/framework/common/import-logger.ts @@ -1,13 +1,8 @@ -import crypto from "crypto"; import { KtLogger, rootLogger } from "lib/logger/logger"; import { ImportTypes, PublicUserDocument } from "tachi-common"; +import { Random20Hex } from "utils/misc"; import { FormatUserDoc } from "utils/user"; -export function CreateImportLoggerAndID(userDoc: PublicUserDocument, importType: ImportTypes) { - const importID = crypto.randomBytes(20).toString("hex"); - return { logger: CreateScoreLogger(userDoc, importID, importType), importID }; -} - export function CreateScoreLogger( user: PublicUserDocument, importID: string, diff --git a/server/src/lib/score-import/framework/express-wrapper.ts b/server/src/lib/score-import/framework/express-wrapper.ts index 319821179..a7de7c989 100644 --- a/server/src/lib/score-import/framework/express-wrapper.ts +++ b/server/src/lib/score-import/framework/express-wrapper.ts @@ -1,13 +1,14 @@ +import CreateLogCtx from "lib/logger/logger"; import { ImportDocument, ImportTypes, - PublicUserDocument, + integer, SuccessfulAPIResponse, UnsuccessfulAPIResponse, } from "tachi-common"; -import { ImportInputParser } from "../import-types/common/types"; -import { CreateImportLoggerAndID } from "./common/import-logger"; -import ScoreImportMain from "./score-import-main"; +import { Random20Hex } from "utils/misc"; +import { ParserArguments } from "../worker/types"; +import { MakeScoreImport } from "./score-import"; import ScoreImportFatalError from "./score-importing/score-import-error"; export interface WrappedAPIResponse { @@ -15,24 +16,30 @@ export interface WrappedAPIResponse { body: SuccessfulAPIResponse | UnsuccessfulAPIResponse; } +const logger = CreateLogCtx(__filename); + /** * A thin(ish) wrapper for ScoreImportMain which converts thrown * errors and import documents into a WrappedAPIResponse, which can * be immediately sent with res.json(). */ -export async function ExpressWrappedScoreImportMain( - user: PublicUserDocument, +export async function ExpressWrappedScoreImportMain( + userID: integer, userIntent: boolean, - importType: ImportTypes, - InputParser: ImportInputParser + importType: I, + parserArguments: ParserArguments ): Promise { - const { importID, logger } = CreateImportLoggerAndID(user, importType); + const importID = Random20Hex(); + logger.debug("Received import request."); try { - const res = await ScoreImportMain(user, userIntent, importType, InputParser, { + const res = await MakeScoreImport({ importID, - logger, + importType, + userIntent, + userID, + parserArguments, }); return { @@ -60,8 +67,7 @@ export async function ExpressWrappedScoreImportMain( statusCode: 500, body: { success: false, - description: - "An internal service error has occured. Please do not repeat this request.", + description: "An internal service error has occured. This has been reported!", }, }; } diff --git a/server/src/lib/score-import/framework/orphans/orphans.ts b/server/src/lib/score-import/framework/orphans/orphans.ts index b3a263c82..8aa805f08 100644 --- a/server/src/lib/score-import/framework/orphans/orphans.ts +++ b/server/src/lib/score-import/framework/orphans/orphans.ts @@ -16,7 +16,7 @@ import { KTDataNotFoundFailure, } from "../common/converter-failures"; import { ProcessSuccessfulConverterReturn } from "../score-importing/score-importing"; -import { HandlePostImportSteps } from "../score-import-main"; +import { HandlePostImportSteps } from "../score-importing/score-import-main"; import { GetUserWithID } from "utils/user"; /** diff --git a/server/src/lib/score-import/framework/score-import.ts b/server/src/lib/score-import/framework/score-import.ts new file mode 100644 index 000000000..76f8967ce --- /dev/null +++ b/server/src/lib/score-import/framework/score-import.ts @@ -0,0 +1,39 @@ +import { ServerConfig } from "lib/setup/config"; +import { ScoreImportJobData } from "../worker/types"; +import { GetInputParser } from "./common/get-input-parser"; +import ScoreImportMain from "./score-importing/score-import-main"; +import { ImportTypes, ImportDocument } from "tachi-common"; +import ScoreImportQueue from "../worker/queue"; + +/** + * Makes a score import given ScoreImportJobData. + * If USE_EXTERNAL_SCORE_IMPORT_WORKER is set, then this will instead + * place this on the score import queue, and the worker will process it. + * + * Otherwise, it will just perform score importing on the same process. + * @returns An import document if awaited, however, you should not + * await this if you don't need the import document! Import Documents + * may take multiple minutes to generate for large imports. If you control + * the client, make it poll /api/v1/ongoing-imports/:importID. + */ +export async function MakeScoreImport( + jobData: ScoreImportJobData +): Promise { + if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { + const job = await ScoreImportQueue.add(jobData, { + jobId: jobData.importID, + }); + + return job.finished(); + } else { + const InputParser = GetInputParser(jobData); + + return ScoreImportMain( + jobData.userID, + jobData.userIntent, + jobData.importType, + InputParser, + jobData.importID + ); + } +} diff --git a/server/src/lib/score-import/framework/score-import-main.ts b/server/src/lib/score-import/framework/score-importing/score-import-main.ts similarity index 86% rename from server/src/lib/score-import/framework/score-import-main.ts rename to server/src/lib/score-import/framework/score-importing/score-import-main.ts index 0637a224b..5c7ea9ee2 100644 --- a/server/src/lib/score-import/framework/score-import-main.ts +++ b/server/src/lib/score-import/framework/score-importing/score-import-main.ts @@ -1,40 +1,51 @@ +import db from "external/mongo/db"; +import { KtLogger } from "lib/logger/logger"; import { + Game, + IDStrings, + ImportDocument, ImportProcessingInfo, ImportTypes, - ImportDocument, - PublicUserDocument, - Playtypes, - Game, integer, - IDStrings, + Playtypes, + PublicUserDocument, } from "tachi-common"; -import { KtLogger } from "lib/logger/logger"; -import { ImportAllIterableData } from "./score-importing/score-importing"; -import { CreateImportLoggerAndID } from "./common/import-logger"; -import { CreateSessions } from "./sessions/sessions"; import { GetMillisecondsSince } from "utils/misc"; -import { ProcessPBs } from "./pb/process-pbs"; -import { UpdateUsersGamePlaytypeStats } from "./user-game-stats/update-ugs"; -import db from "external/mongo/db"; -import { GetAndUpdateUsersGoals } from "./goals/goals"; -import { UpdateUsersMilestones } from "./milestones/milestones"; -import { ConverterFunction, ImportInputParser } from "../import-types/common/types"; -import { ScorePlaytypeMap } from "./common/types"; -import { Converters } from "../import-types/converters"; -import { ClassHandler } from "./user-game-stats/types"; -import { GetOrSetUserLock, RemoveUserLock } from "./import-locks/lock"; -import ScoreImportFatalError from "./score-importing/score-import-error"; +import { GetUserWithID } from "utils/user"; +import { ConverterFunction, ImportInputParser } from "../../import-types/common/types"; +import { Converters } from "../../import-types/converters"; +import { InternalFailure } from "../common/converter-failures"; +import { CreateScoreLogger } from "../common/import-logger"; +import { ScorePlaytypeMap } from "../common/types"; +import { GetAndUpdateUsersGoals } from "../goals/goals"; +import { GetOrSetUserLock, RemoveUserLock } from "../import-locks/lock"; +import { UpdateUsersMilestones } from "../milestones/milestones"; +import { ProcessPBs } from "../pb/process-pbs"; +import { CreateSessions } from "../sessions/sessions"; +import { ClassHandler } from "../user-game-stats/types"; +import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs"; +import ScoreImportFatalError from "./score-import-error"; +import { ImportAllIterableData } from "./score-importing"; /** * Performs a Score Import. */ export default async function ScoreImportMain( - user: PublicUserDocument, + userID: integer, userIntent: boolean, importType: ImportTypes, InputParser: ImportInputParser, - providedImportObjects?: { logger: KtLogger; importID: string } + importID: string, + providedLogger?: KtLogger ) { + const user = await GetUserWithID(userID); + + if (!user) { + throw new InternalFailure( + `User with ID ${userID} does not exist, but attempted to make an import?` + ); + } + // in the event of any error, we remove the user lock. try { const lock = await GetOrSetUserLock(user.id); @@ -53,18 +64,17 @@ export default async function ScoreImportMain( } const timeStarted = Date.now(); - let importID; let logger; - if (!providedImportObjects) { + if (!providedLogger) { // If they weren't given to us - // we create an "import logger". // this holds a reference to the user's name, ID, and type // of score import for any future debugging. - ({ importID, logger } = CreateImportLoggerAndID(user, importType)); + logger = CreateScoreLogger(user, importID, importType); logger.debug("Received import request."); } else { - ({ importID, logger } = providedImportObjects); + logger = providedLogger; } // --- 1. Parsing --- diff --git a/server/src/lib/score-import/import-types/api/arc-sdvx/converter.test.ts b/server/src/lib/score-import/import-types/api/arc-sdvx/converter.test.ts index c3c8aa5a1..aa34155e8 100644 --- a/server/src/lib/score-import/import-types/api/arc-sdvx/converter.test.ts +++ b/server/src/lib/score-import/import-types/api/arc-sdvx/converter.test.ts @@ -1,11 +1,10 @@ +import deepmerge from "deepmerge"; +import CreateLogCtx from "lib/logger/logger"; import t from "tap"; - import ResetDBState from "test-utils/resets"; import { GetKTDataJSON, TestingAlbidaADV, TestingSDVXAlbidaSong } from "test-utils/test-data"; -import CreateLogCtx from "lib/logger/logger"; import { ConvertAPIArcSDVX, ResolveARCSDVXLamp } from "./converter"; import { ARCSDVXScore } from "./types"; -import deepmerge from "deepmerge"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/score-import/import-types/api/arc-sdvx/parser.test.ts b/server/src/lib/score-import/import-types/api/arc-sdvx/parser.test.ts index 2367c7e96..5cc04a4ed 100644 --- a/server/src/lib/score-import/import-types/api/arc-sdvx/parser.test.ts +++ b/server/src/lib/score-import/import-types/api/arc-sdvx/parser.test.ts @@ -1,11 +1,10 @@ +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; import t from "tap"; - import { agta } from "test-utils/misc"; import { MockJSONFetch } from "test-utils/mock-fetch"; import ResetDBState from "test-utils/resets"; -import CreateLogCtx from "lib/logger/logger"; import { ParseArcSDVX } from "./parser"; -import db from "external/mongo/db"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts b/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts index ac12a7812..5013adb28 100644 --- a/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts +++ b/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts @@ -1,11 +1,11 @@ import { KtLogger } from "lib/logger/logger"; +import { ServerConfig } from "lib/setup/config"; +import { integer } from "tachi-common"; import nodeFetch from "utils/fetch"; +import { GetArcAuthGuaranteed } from "utils/queries/auth"; +import { EmptyObject } from "utils/types"; import { TraverseKaiAPI } from "../../common/api-kai/traverse-api"; import { ParserFunctionReturns } from "../../common/types"; -import { EmptyObject } from "utils/types"; -import { ServerConfig } from "lib/setup/config"; -import { GetArcAuthGuaranteed } from "utils/queries/auth"; -import { integer } from "tachi-common"; export async function ParseArcSDVX( userID: integer, diff --git a/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts b/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts index 6dd530dcf..1758ff18d 100644 --- a/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts +++ b/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts @@ -1,7 +1,7 @@ import { KtLogger } from "lib/logger/logger"; -import { ParseKaiIIDX } from "../../common/api-kai/iidx/parser"; import { integer } from "tachi-common"; import { GetKaiAuthGuaranteed } from "utils/queries/auth"; +import { ParseKaiIIDX } from "../../common/api-kai/iidx/parser"; export async function ParseEagIIDX(userID: integer, logger: KtLogger) { const authDoc = await GetKaiAuthGuaranteed(userID, "EAG", logger); diff --git a/server/src/lib/score-import/import-types/common/api-kai/iidx/class-handler.test.ts b/server/src/lib/score-import/import-types/common/api-kai/iidx/class-handler.test.ts index d48e74e03..9f3dca831 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/iidx/class-handler.test.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/iidx/class-handler.test.ts @@ -1,7 +1,6 @@ import { IIDXDans } from "lib/constants/classes"; import CreateLogCtx from "lib/logger/logger"; import t from "tap"; - import { MockBasicFetch, MockJSONFetch } from "test-utils/mock-fetch"; import ResetDBState from "test-utils/resets"; import { KaiTypeToBaseURL } from "../utils"; diff --git a/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.ts b/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.ts index 843699140..a30685ac7 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.ts @@ -1,20 +1,20 @@ -import { FindIIDXChartOnInGameIDVersion } from "utils/queries/charts"; -import { KaiContext, KaiIIDXScore } from "../types"; import p from "prudence"; +import { Lamps } from "tachi-common"; +import { FormatPrError } from "utils/prudence"; +import { FindIIDXChartOnInGameIDVersion } from "utils/queries/charts"; +import { FindSongOnID } from "utils/queries/songs"; import { InternalFailure, InvalidScoreFailure, KTDataNotFoundFailure, } from "../../../../framework/common/converter-failures"; -import { FormatPrError } from "utils/prudence"; -import { FindSongOnID } from "utils/queries/songs"; import { GenericGetGradeAndPercent, ParseDateFromString, } from "../../../../framework/common/score-utils"; -import { Lamps } from "tachi-common"; -import { ConverterFunction } from "../../types"; import { DryScore } from "../../../../framework/common/types"; +import { ConverterFunction } from "../../types"; +import { KaiContext, KaiIIDXScore } from "../types"; const PR_KaiIIDXScore = { music_id: p.isPositiveInteger, diff --git a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.test.ts b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.test.ts index 4bc758180..d6c7a59fe 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.test.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.test.ts @@ -1,8 +1,7 @@ +import CreateLogCtx from "lib/logger/logger"; import t from "tap"; - import { MockJSONFetch } from "test-utils/mock-fetch"; import ResetDBState from "test-utils/resets"; -import CreateLogCtx from "lib/logger/logger"; import { ParseKaiIIDX } from "./parser"; const fakeAuth = { diff --git a/server/src/lib/score-import/import-types/common/api-kai/reauth.test.ts b/server/src/lib/score-import/import-types/common/api-kai/reauth.test.ts index 07b6c73bb..8696e180e 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/reauth.test.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/reauth.test.ts @@ -4,7 +4,6 @@ import CreateLogCtx from "lib/logger/logger"; import { ServerConfig } from "lib/setup/config"; import { KaiAuthDocument } from "tachi-common"; import t from "tap"; - import { MockJSONFetch } from "test-utils/mock-fetch"; import ResetDBState from "test-utils/resets"; import { CreateKaiReauthFunction } from "./reauth"; diff --git a/server/src/lib/score-import/import-types/common/api-kai/reauth.ts b/server/src/lib/score-import/import-types/common/api-kai/reauth.ts index 8087dc512..d5de63e6b 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/reauth.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/reauth.ts @@ -1,11 +1,11 @@ import db from "external/mongo/db"; import { KtLogger } from "lib/logger/logger"; import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error"; -import { GetKaiTypeClientCredentials, KaiTypeToBaseURL } from "./utils"; -import nodeFetch from "utils/fetch"; import p from "prudence"; import { KaiAuthDocument } from "tachi-common"; +import nodeFetch from "utils/fetch"; import { CreateURLWithParams } from "utils/url"; +import { GetKaiTypeClientCredentials, KaiTypeToBaseURL } from "./utils"; const REAUTH_SCHEMA = { access_token: "string", diff --git a/server/src/lib/score-import/import-types/common/batch-manual/types.ts b/server/src/lib/score-import/import-types/common/batch-manual/types.ts index 13a571b40..42f10dff8 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/types.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/types.ts @@ -1,4 +1,4 @@ -import { Game, Playtypes, IDStrings, GPTSupportedVersions } from "tachi-common"; +import { Game, GPTSupportedVersions, IDStrings, Playtypes } from "tachi-common"; export interface BatchManualContext { game: Game; diff --git a/server/src/lib/score-import/import-types/common/types.ts b/server/src/lib/score-import/import-types/common/types.ts index 5da94cc34..6cf8b4d87 100644 --- a/server/src/lib/score-import/import-types/common/types.ts +++ b/server/src/lib/score-import/import-types/common/types.ts @@ -36,7 +36,6 @@ export interface ImportTypeDataMap { "ir/barbatos": BarbatosScore; "ir/fervidex": FervidexScore; "ir/fervidex-static": FervidexStaticScore; - "ir/chunitachi": BatchManualScore; "ir/beatoraja": BeatorajaScore; "ir/usc": USCClientScore; "ir/kshook-sv3c": KsHookSV3CScore; @@ -63,7 +62,6 @@ export interface ImportTypeContextMap { "ir/barbatos": EmptyObject; "ir/fervidex": FervidexContext; "ir/fervidex-static": FervidexStaticContext; - "ir/chunitachi": BatchManualContext; "ir/beatoraja": BeatorajaContext; "ir/usc": IRUSCContext; "ir/kshook-sv3c": EmptyObject; @@ -115,3 +113,5 @@ export interface ParserFunctionReturns { game: Game; classHandler: ClassHandler | null; } + +export type ParserFunction = (...args: A) => ParserFunctionReturns; diff --git a/server/src/lib/score-import/import-types/converters.ts b/server/src/lib/score-import/import-types/converters.ts index fc3b375c3..7ef1ed9fb 100644 --- a/server/src/lib/score-import/import-types/converters.ts +++ b/server/src/lib/score-import/import-types/converters.ts @@ -40,6 +40,5 @@ export const Converters: ConverterMap = { "ir/fervidex-static": ConverterIRFervidexStatic, "ir/direct-manual": ConverterBatchManual, "ir/usc": ConverterIRUSC, - "ir/chunitachi": ConverterBatchManual, "ir/kshook-sv3c": ConverterIRKsHookSV3C, }; diff --git a/server/src/lib/score-import/import-types/ir/barbatos/converter.test.ts b/server/src/lib/score-import/import-types/ir/barbatos/converter.test.ts index 2c9dce86e..0a1ac0632 100644 --- a/server/src/lib/score-import/import-types/ir/barbatos/converter.test.ts +++ b/server/src/lib/score-import/import-types/ir/barbatos/converter.test.ts @@ -1,11 +1,10 @@ -import t from "tap"; -import db from "external/mongo/db"; -import ResetDBState from "test-utils/resets"; -import CreateLogCtx from "lib/logger/logger"; -import { ConverterIRBarbatos } from "./converter"; import deepmerge from "deepmerge"; +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import t from "tap"; +import ResetDBState from "test-utils/resets"; import { barbScore } from "test-utils/test-data"; - +import { ConverterIRBarbatos } from "./converter"; import { BarbatosScore } from "./types"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/score-import/import-types/ir/barbatos/parser.ts b/server/src/lib/score-import/import-types/ir/barbatos/parser.ts index 9a7c1895e..8293c360f 100644 --- a/server/src/lib/score-import/import-types/ir/barbatos/parser.ts +++ b/server/src/lib/score-import/import-types/ir/barbatos/parser.ts @@ -1,10 +1,10 @@ -import { EmptyObject } from "utils/types"; import { KtLogger } from "lib/logger/logger"; import p from "prudence"; -import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; import { FormatPrError } from "utils/prudence"; -import { BarbatosScore } from "./types"; +import { EmptyObject } from "utils/types"; +import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; import { ParserFunctionReturns } from "../../common/types"; +import { BarbatosScore } from "./types"; const PR_Barbatos = { difficulty: p.isIn(1, 2, 3, 4), diff --git a/server/src/lib/score-import/import-types/ir/fervidex-static/class-handler.test.ts b/server/src/lib/score-import/import-types/ir/fervidex-static/class-handler.test.ts index 09811b900..00caf6418 100644 --- a/server/src/lib/score-import/import-types/ir/fervidex-static/class-handler.test.ts +++ b/server/src/lib/score-import/import-types/ir/fervidex-static/class-handler.test.ts @@ -1,9 +1,8 @@ -import t from "tap"; - -import ResetDBState from "test-utils/resets"; import CreateLogCtx from "lib/logger/logger"; +import { Game, Playtypes } from "tachi-common"; +import t from "tap"; +import ResetDBState from "test-utils/resets"; import { CreateFerStaticClassHandler } from "./class-handler"; -import { Playtypes, Game } from "tachi-common"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/score-import/import-types/ir/fervidex-static/converter.ts b/server/src/lib/score-import/import-types/ir/fervidex-static/converter.ts index d1bd91ce9..53873e79a 100644 --- a/server/src/lib/score-import/import-types/ir/fervidex-static/converter.ts +++ b/server/src/lib/score-import/import-types/ir/fervidex-static/converter.ts @@ -1,15 +1,15 @@ +import { Lamps } from "tachi-common"; +import { FindChartOnInGameIDVersion } from "utils/queries/charts"; import { FindSongOnID } from "utils/queries/songs"; -import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils"; import { InternalFailure, KTDataNotFoundFailure, } from "../../../framework/common/converter-failures"; -import { Lamps } from "tachi-common"; -import { FindChartOnInGameIDVersion } from "utils/queries/charts"; -import { FervidexStaticContext, FervidexStaticScore } from "./types"; -import { FERVIDEX_LAMP_LOOKUP, SplitFervidexChartRef } from "../fervidex/converter"; +import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils"; import { DryScore } from "../../../framework/common/types"; import { ConverterFunction } from "../../common/types"; +import { FERVIDEX_LAMP_LOOKUP, SplitFervidexChartRef } from "../fervidex/converter"; +import { FervidexStaticContext, FervidexStaticScore } from "./types"; export const ConverterIRFervidexStatic: ConverterFunction< FervidexStaticScore, diff --git a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.test.ts b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.test.ts index 6f2273040..3b1569558 100644 --- a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.test.ts +++ b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.test.ts @@ -1,8 +1,7 @@ +import CreateLogCtx from "lib/logger/logger"; import t from "tap"; - import ResetDBState from "test-utils/resets"; import { GetKTDataJSON } from "test-utils/test-data"; -import CreateLogCtx from "lib/logger/logger"; import { ParseFervidexStatic } from "./parser"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts index 62275e53d..d370cee31 100644 --- a/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts +++ b/server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts @@ -1,12 +1,12 @@ import { KtLogger } from "lib/logger/logger"; import p, { PrudenceSchema } from "prudence"; -import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; import { FormatPrError } from "utils/prudence"; -import { FervidexStaticContext, FervidexStaticScore } from "./types"; -import { FerHeaders, SoftwareIDToVersion } from "../fervidex/parser"; import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts"; -import { CreateFerStaticClassHandler } from "./class-handler"; +import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; import { ParserFunctionReturns } from "../../common/types"; +import { FerHeaders, SoftwareIDToVersion } from "../fervidex/parser"; +import { CreateFerStaticClassHandler } from "./class-handler"; +import { FervidexStaticContext, FervidexStaticScore } from "./types"; const PR_FervidexStatic: PrudenceSchema = { ex_score: p.isPositiveInteger, diff --git a/server/src/lib/score-import/import-types/parsers.ts b/server/src/lib/score-import/import-types/parsers.ts new file mode 100644 index 000000000..bcd3bb61d --- /dev/null +++ b/server/src/lib/score-import/import-types/parsers.ts @@ -0,0 +1,44 @@ +import { ParseArcIIDX } from "./api/arc-iidx/parser"; +import { ParseArcSDVX } from "./api/arc-sdvx/parser"; +import { ParseEagIIDX } from "./api/eag-iidx/parser"; +import { ParseEagSDVX } from "./api/eag-sdvx/parser"; +import { ParseFloIIDX } from "./api/flo-iidx/parser"; +import { ParseFloSDVX } from "./api/flo-sdvx/parser"; +import { ParseMinSDVX } from "./api/min-sdvx/parser"; +import { ParseBatchManualFromObject } from "./common/batch-manual/parser"; +import ParseBatchManual from "./file/batch-manual/parser"; +import ParseEamusementIIDXCSV from "./file/eamusement-iidx-csv/parser"; +import { ParseMerIIDX } from "./file/mer-iidx/parser"; +import ParsePLIIIDXCSV from "./file/pli-iidx-csv/parser"; +import { ParseSolidStateXML } from "./file/solid-state-squad/parser"; +import { ParseBarbatosSingle } from "./ir/barbatos/parser"; +import { ParseBeatorajaSingle } from "./ir/beatoraja/parser"; +import ParseDirectManual from "./ir/direct-manual/parser"; +import { ParseFervidexStatic } from "./ir/fervidex-static/parser"; +import { ParseFervidexSingle } from "./ir/fervidex/parser"; +import { ParseKsHookSV3C } from "./ir/kshook-sv3c/parser"; +import { ParseIRUSC } from "./ir/usc/parser"; + +export const Parsers = { + "file/solid-state-squad": ParseSolidStateXML, + "file/batch-manual": ParseBatchManual, + "file/mer-iidx": ParseMerIIDX, + "file/pli-iidx-csv": ParsePLIIIDXCSV, + "file/eamusement-iidx-csv": ParseEamusementIIDXCSV, + + "api/eag-iidx": ParseEagIIDX, + "api/eag-sdvx": ParseEagSDVX, + "api/flo-iidx": ParseFloIIDX, + "api/flo-sdvx": ParseFloSDVX, + "api/min-sdvx": ParseMinSDVX, + "api/arc-iidx": ParseArcIIDX, + "api/arc-sdvx": ParseArcSDVX, + + "ir/barbatos": ParseBarbatosSingle, + "ir/beatoraja": ParseBeatorajaSingle, + "ir/fervidex": ParseFervidexSingle, + "ir/fervidex-static": ParseFervidexStatic, + "ir/direct-manual": ParseDirectManual, + "ir/usc": ParseIRUSC, + "ir/kshook-sv3c": ParseKsHookSV3C, +}; diff --git a/server/src/lib/score-import/worker/queue.ts b/server/src/lib/score-import/worker/queue.ts new file mode 100644 index 000000000..1c02ea8d7 --- /dev/null +++ b/server/src/lib/score-import/worker/queue.ts @@ -0,0 +1,31 @@ +import Queue from "bull"; +import { RedisClient } from "external/redis/redis"; +import CreateLogCtx from "lib/logger/logger"; +import { Environment, TachiConfig } from "lib/setup/config"; +import ScoreImportFatalError from "../framework/score-importing/score-import-error"; + +const ScoreImportQueue = new Queue(`${TachiConfig.NAME} Score Import Queue`, Environment.redisUrl); + +export default ScoreImportQueue; + +const logger = CreateLogCtx("Score Import Queue"); + +// Log errors if and when they occur. +ScoreImportQueue.on("failed", (job, err) => { + if (err instanceof ScoreImportFatalError) { + logger.info( + `Job ${job.id} hit ScoreImportFatalError (User Fault) with message: ${err.message}`, + err + ); + } else { + logger.error(`Job ${job.id} failed unexpectedly with message: ${err.message}`, err); + } +}); + +ScoreImportQueue.on("completed", (job, result) => { + logger.debug(`Job ${job.id} finished successfully.`, result); +}); + +export function CloseScoreImportQueue() { + return ScoreImportQueue.close(); +} diff --git a/server/src/lib/score-import/worker/types.ts b/server/src/lib/score-import/worker/types.ts new file mode 100644 index 000000000..f9d5a9bf2 --- /dev/null +++ b/server/src/lib/score-import/worker/types.ts @@ -0,0 +1,39 @@ +import { Job } from "bull"; +import { ImportTypes, integer } from "tachi-common"; +import { Parsers } from "../import-types/parsers"; + +// Ok so, please hear me out on this one. +// We need a type that removes the last element from an array. +// This is because parser functions have variable signatures, but all +// end with requiring a logger. We don't want to have to pass the logger, +// we just want to pass those other arguments. +type RemoveLast = T extends [...infer RemoveLast, unknown] + ? RemoveLast + : unknown[]; + +// Which means we can use generic access to make ParserArguments<"ir/usc"> +// A type that returns the paramaters of the parser function bound to +// "ir/usc". +// And then, using RemoveLast, we can remove that logger argument we don't +// want to provide. +export type ParserArguments = RemoveLast>; + +// Depending on how you look at it, this is either beautiful +// TypeScript power, or brutal TypeScript abuse... + +export interface ScoreImportJobData { + importType: I; + userID: integer; + userIntent: boolean; + importID: string; + + // ...and well, I'm personally on the fence. + + // Parsers can't have a consistent signature -- they need info + // from different sources! As such, we need to pass the parser + // arguments in an array, and then it can be respreaded into + // the parser by the import code. + parserArguments: ParserArguments; +} + +export type ScoreImportJob = Job>; diff --git a/server/src/lib/score-import/worker/worker.ts b/server/src/lib/score-import/worker/worker.ts new file mode 100644 index 000000000..acb730512 --- /dev/null +++ b/server/src/lib/score-import/worker/worker.ts @@ -0,0 +1,69 @@ +// For scaling performance, running score-importing in a separate worker is preferable +// as that way, other API calls don't get halted by particularly expensive imports on +// all cores. Parallelism can only get us so far in the same process. + +// You don't have to run this. If it is being ran, you need to set USE_EXTERNAL_SCORE_IMPORT_WORKER +// in conf.json5. That will ensure all score import jobs are thrown at redis and eventually +// end up here. + +// If you don't, score importing will happen on the same thread as your router. That's probably +// fine for lower throughputs, but hey. We're aiming a bit higher. + +// Explicitly set this before importing anything! +process.env.IS_SCORE_WORKER_SERVER = "true"; + +import CreateLogCtx from "lib/logger/logger"; +import { ImportTypes } from "tachi-common"; +import { FormatUserDoc, GetUserWithID } from "utils/user"; +import { GetInputParser } from "../framework/common/get-input-parser"; +import ScoreImportMain from "../framework/score-importing/score-import-main"; +import ScoreImportQueue from "./queue"; +import { ScoreImportJob } from "./types"; + +const workerLogger = CreateLogCtx(`Import Worker`); + +// Exit if we're not called with node. Think of this like if __name__ != "__main__" in python. +if (require.main !== module) { + workerLogger.crit( + "The Score Import Worker was imported, instead of ran directly with node. This is a fatal error. Exiting." + ); + process.exit(1); +} + +/** + * When a job is fired, this code will actually process the given data + * and import it into the codebase. + */ +ScoreImportQueue.process(async (job: ScoreImportJob) => { + const user = await GetUserWithID(job.data.userID); + + if (!user) { + workerLogger.severe( + `Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).` + ); + throw new Error( + `Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).` + ); + } + + // Create a logger that we can pass around for context. + // This helps us debug what score import did what! + const logger = CreateLogCtx(`Score Import ${job.id} ${FormatUserDoc(user)}`); + + const InputParser = GetInputParser(job.data); + + logger.debug(`Starting import.`); + + const importDocument = await ScoreImportMain( + user.id, + job.data.userIntent, + job.data.importType, + InputParser, + job.data.importID, + logger + ); + + logger.debug(`Finished import.`); + + return importDocument; +}); diff --git a/server/src/lib/search/search.test.ts b/server/src/lib/search/search.test.ts index 0dc2cff4f..649ca7c4f 100644 --- a/server/src/lib/search/search.test.ts +++ b/server/src/lib/search/search.test.ts @@ -1,6 +1,5 @@ -import t from "tap"; import db from "external/mongo/db"; - +import t from "tap"; import ResetDBState from "test-utils/resets"; import { LoadTachiIIDXData } from "test-utils/test-data"; import { SearchGameSongs, SearchUsersRegExp } from "./search"; diff --git a/server/src/lib/search/search.ts b/server/src/lib/search/search.ts index 4d4dc0e2a..8f8508622 100644 --- a/server/src/lib/search/search.ts +++ b/server/src/lib/search/search.ts @@ -1,20 +1,20 @@ -import { ICollection } from "monk"; -import { FilterQuery } from "mongodb"; -import CreateLogCtx from "lib/logger/logger"; import deepmerge from "deepmerge"; import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import { TachiConfig } from "lib/setup/config"; +import { FilterQuery } from "mongodb"; +import { ICollection } from "monk"; import { - Game, - SessionDocument, - Playtypes, - SongDocument, ChartDocument, + Game, integer, + Playtypes, PublicUserDocument, + SessionDocument, + SongDocument, } from "tachi-common"; import { EscapeStringRegexp } from "utils/misc"; import { GetOnlineCutoff } from "utils/user"; -import { TachiConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index 911cee54d..d1268e51a 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -64,6 +64,7 @@ export interface TachiServerConfig { RATE_LIMIT: integer; OAUTH_CLIENT_CAP: integer; OPTIONS_ALWAYS_SUCCEEDS?: boolean; + USE_EXTERNAL_SCORE_IMPORT_WORKER?: boolean; EMAIL_CONFIG?: { FROM: string; DKIM?: SendMailOptions["dkim"]; @@ -125,6 +126,7 @@ const err = p(config, { RATE_LIMIT: p.optional(p.isPositiveInteger), OAUTH_CLIENT_CAP: p.optional(p.isPositiveInteger), OPTIONS_ALWAYS_SUCCEEDS: "*boolean", + USE_EXTERNAL_SCORE_IMPORT_WORKER: "*boolean", EMAIL_CONFIG: p.optional({ FROM: "string", DKIM: "*object", diff --git a/server/src/main.ts b/server/src/main.ts index 1e75407c0..898ccb08f 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -14,6 +14,7 @@ import { FormatVersion } from "./lib/constants/version"; import fetch from "utils/fetch"; import http from "http"; import { CloseRedisConnection } from "external/redis/redis"; +import { CloseScoreImportQueue } from "lib/score-import/worker/queue"; const logger = CreateLogCtx(__filename); @@ -79,6 +80,9 @@ process.on("SIGTERM", () => { logger.info("Closing Redis Connection.", { shutdownInfo: true }); CloseRedisConnection(); + logger.info("Closing Score Import Queue.", { shutdownInfo: true }); + CloseScoreImportQueue(); + logger.info("Everything closed. Waiting for process to exit naturally.", { shutdownInfo: true, }); diff --git a/server/src/server/middleware/auth.test.ts b/server/src/server/middleware/auth.test.ts index 32fa2ca6b..30313e60f 100644 --- a/server/src/server/middleware/auth.test.ts +++ b/server/src/server/middleware/auth.test.ts @@ -1,10 +1,9 @@ -import t from "tap"; -import { SetRequestPermissions, AllPermissions } from "./auth"; import expMiddlewareMock from "express-request-mock"; import db from "external/mongo/db"; import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; - +import t from "tap"; import ResetDBState from "test-utils/resets"; +import { AllPermissions, SetRequestPermissions } from "./auth"; t.test("#SetRequestPermissions", (t) => { t.beforeEach(ResetDBState); @@ -17,6 +16,7 @@ t.test("#SetRequestPermissions", (t) => { customise_profile: true, }, token: "mock_token", + fromAPIClient: null, }); const { req } = await expMiddlewareMock(SetRequestPermissions, { @@ -32,6 +32,7 @@ t.test("#SetRequestPermissions", (t) => { customise_profile: true, }, token: "mock_token", + fromAPIClient: null, }); t.end(); @@ -45,6 +46,7 @@ t.test("#SetRequestPermissions", (t) => { identifier: "Guest Token", permissions: {}, token: null, + fromAPIClient: null, }); t.end(); @@ -110,6 +112,7 @@ t.test("#SetRequestPermissions", (t) => { identifier: `Session-Key 1`, token: null, permissions: AllPermissions, + fromAPIClient: null, }); t.end(); }); diff --git a/server/src/server/middleware/auth.ts b/server/src/server/middleware/auth.ts index a21bc2e11..7f82cbb0c 100644 --- a/server/src/server/middleware/auth.ts +++ b/server/src/server/middleware/auth.ts @@ -12,6 +12,7 @@ const GuestToken: APITokenDocument = { userID: null, identifier: "Guest Token", permissions: {}, + fromAPIClient: null, }; export const AllPermissions: Record = { @@ -46,6 +47,7 @@ function CreateSetRequestPermissions(errorKeyName: string): RequestHandler { identifier: `Session-Key ${req.session.tachi.user.id}`, token: null, permissions: AllPermissions, + fromAPIClient: null, }; return next(); } @@ -91,6 +93,7 @@ function CreateSetRequestPermissions(errorKeyName: string): RequestHandler { token, permissions: apiTokenData.permissions, identifier: apiTokenData.identifier, + fromAPIClient: apiTokenData.fromAPIClient, }; return next(); diff --git a/server/src/server/router/api/v1/import/router.ts b/server/src/server/router/api/v1/import/router.ts index f1c0778a6..04ca97032 100644 --- a/server/src/server/router/api/v1/import/router.ts +++ b/server/src/server/router/api/v1/import/router.ts @@ -1,30 +1,20 @@ import { Router } from "express"; -import { APIImportTypes, FileUploadImportTypes, integer } from "tachi-common"; -import Prudence from "prudence"; -import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user"; -import CreateLogCtx, { KtLogger } from "lib/logger/logger"; -import prValidate from "server/middleware/prudence-validate"; -import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error"; -import { SIXTEEN_MEGABTYES } from "lib/constants/filesize"; -import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; -import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload"; - -import ParseEamusementIIDXCSV from "lib/score-import/import-types/file/eamusement-iidx-csv/parser"; -import ParseBatchManual from "lib/score-import/import-types/file/batch-manual/parser"; -import { ParseSolidStateXML } from "lib/score-import/import-types/file/solid-state-squad/parser"; -import { ParseMerIIDX } from "lib/score-import/import-types/file/mer-iidx/parser"; -import ParsePLIIIDXCSV from "lib/score-import/import-types/file/pli-iidx-csv/parser"; -import { TachiConfig } from "lib/setup/config"; -import { RequirePermissions } from "server/middleware/auth"; -import { ParseEagIIDX } from "lib/score-import/import-types/api/eag-iidx/parser"; -import { ParseEagSDVX } from "lib/score-import/import-types/api/eag-sdvx/parser"; -import { ParseFloIIDX } from "lib/score-import/import-types/api/flo-iidx/parser"; -import { ParseFloSDVX } from "lib/score-import/import-types/api/flo-sdvx/parser"; -import { ParseMinSDVX } from "lib/score-import/import-types/api/min-sdvx/parser"; -import { ParseArcSDVX } from "lib/score-import/import-types/api/arc-sdvx/parser"; -import { ParseArcIIDX } from "lib/score-import/import-types/api/arc-iidx/parser"; import db from "external/mongo/db"; +import { SIXTEEN_MEGABTYES } from "lib/constants/filesize"; +import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; +import CreateLogCtx from "lib/logger/logger"; +import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; +import { MakeScoreImport } from "lib/score-import/framework/score-import"; +import { ScoreImportJobData } from "lib/score-import/worker/types"; +import { ServerConfig, TachiConfig } from "lib/setup/config"; +import Prudence from "prudence"; +import { RequirePermissions } from "server/middleware/auth"; +import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload"; +import prValidate from "server/middleware/prudence-validate"; +import { APIImportTypes, FileUploadImportTypes } from "tachi-common"; +import { Random20Hex } from "utils/misc"; +import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user"; const logger = CreateLogCtx(__filename); @@ -37,9 +27,14 @@ const ParseMultipartScoredata = CreateMulterSingleUploadMiddleware( ); const fileImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("file/")); +const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/")); /** * Import scores from a file. Expects the post request to be multipart, and to provide a scoreData file. + * + * @param importType - The import type for this file. + * @param file - The actual file. Should be passed as multipart. + * * @name POST /api/v1/import/file */ router.post( @@ -63,30 +58,45 @@ router.post( const importType = req.body.importType as FileUploadImportTypes; - const inputParser = (logger: KtLogger) => - ResolveFileUploadData(importType, req.file!, req.body, logger); + const userIntent = !!req.header("X-User-Intent"); - const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id); + if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { + const importID = Random20Hex(); - // The here is deliberate - TS picks the IIDX-CSV generic values - // for this function call because it sees them first - // but that is ABSOLUTELY not what is actually occuring. - // We use this as an override because we know better. - // see: https://www.typescriptlang.org/play?ts=4.3.0-beta#code/GYVwdgxgLglg9mABAQQDwBUB8AKYc4BciAYvhpgJSIDeiAsAFCKID0LiAJnAKYDOivKCGDBGAX0aMYYKNwBOwAIYRuJMlhqNmzAEaK5RdOMkNQkWAkQAlbkLlgAynAC23UnGxVqW7W0QAHOVtuMA5EKAALGH5o8IjVIN4QABsoRDhgARdVaX8QNOlBbkUwjMQ5RVCXH2YYTOwAWUVIgDoKqudPRFREAAYWgFYvGu1mILskWj0DRAAiQTlpAHNZxAkmbXXRkfGQexpEaaIARgAmAGY14wZGZNt0vfdEAF5rWz3HbPdPAG4TZGwcEe+AoPyAA - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const responseData = await ExpressWrappedScoreImportMain( - userDoc, - true, - importType, - inputParser - ); + const job: ScoreImportJobData = { + importID, + userID: req[SYMBOL_TachiAPIAuth].userID!, + userIntent, + importType, + parserArguments: [req.file, req.body], + }; - return res.status(responseData.statusCode).json(responseData.body); + // Fire the score import, but make no guarantees about its state. + MakeScoreImport(job); + + return res.status(202).json({ + success: true, + description: + "Import loaded into queue. You can poll the provided URL for information on when its complete.", + body: { + url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`, + importID, + }, + }); + } else { + // Fire the score import and wait for it to finish! + const importResponse = await ExpressWrappedScoreImportMain( + req[SYMBOL_TachiAPIAuth].userID!, + userIntent, + importType, + [req.file, req.body] + ); + + return res.status(importResponse.statusCode).json(importResponse.body); + } } ); -const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/")); - /** * Import scores from another API. This typically will perform a full sync. * @name POST /api/v1/import/from-api @@ -104,21 +114,44 @@ router.post( async (req, res) => { const importType = req.body.importType as APIImportTypes; - const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id); + const importID = Random20Hex(); - const inputParser = (logger: KtLogger) => - ResolveAPIImportParser(userDoc.id, importType, logger); + const userID = req[SYMBOL_TachiAPIAuth].userID!; - // see the argument above about typescript falsely expanding types. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const responseData = await ExpressWrappedScoreImportMain( - userDoc, - true, - importType, - inputParser - ); + const userIntent = !!req.header("X-User-Intent"); - return res.status(responseData.statusCode).json(responseData.body); + if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { + const job: ScoreImportJobData = { + importID, + userID, + userIntent, + importType, + parserArguments: [userID], + }; + + // Fire the score import, but make no guarantees about its state. + MakeScoreImport(job); + + return res.status(202).json({ + success: true, + description: + "Import loaded into queue. You can poll the provided URL for information on when its complete.", + body: { + url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`, + importID, + }, + }); + } else { + // Fire the score import and wait for it to finish! + const importResponse = await ExpressWrappedScoreImportMain( + userID, + userIntent, + importType, + [userID] + ); + + return res.status(importResponse.statusCode).json(importResponse.body); + } } ); @@ -178,65 +211,4 @@ router.post("/orphans", RequirePermissions("submit_score"), async (req, res) => }); }); -/** - * Resolves the data from a file upload into an iterable, - * The appropriate processing function to map that iterable over, - * and and any context the processing may need (such as playtype) - * - * This also performs validation on the type of file uploaded. - * @param importType - The type of import request this was. - * @param fileData - The data sent by the user. - * @param body - Other data passed by the user in the request body. - */ -export function ResolveFileUploadData( - importType: FileUploadImportTypes, - fileData: Express.Multer.File, - body: Record, - logger: KtLogger -) { - switch (importType) { - case "file/eamusement-iidx-csv": - return ParseEamusementIIDXCSV(fileData, body, logger); - case "file/pli-iidx-csv": - return ParsePLIIIDXCSV(fileData, body, logger); - case "file/batch-manual": - return ParseBatchManual(fileData, body, logger); - case "file/solid-state-squad": - return ParseSolidStateXML(fileData, body, logger); - case "file/mer-iidx": - return ParseMerIIDX(fileData, body, logger); - default: - logger.error( - `importType ${importType} made it into ResolveFileUploadData, but should have been rejected by Prudence.` - ); - throw new ScoreImportFatalError(400, `Invalid importType of ${importType}.`); - } -} - -export function ResolveAPIImportParser( - userID: integer, - importType: APIImportTypes, - logger: KtLogger -) { - switch (importType) { - case "api/eag-iidx": - return ParseEagIIDX(userID, logger); - case "api/eag-sdvx": - return ParseEagSDVX(userID, logger); - case "api/flo-iidx": - return ParseFloIIDX(userID, logger); - case "api/flo-sdvx": - return ParseFloSDVX(userID, logger); - case "api/min-sdvx": - return ParseMinSDVX(userID, logger); - case "api/arc-iidx": - return ParseArcIIDX(userID, logger); - case "api/arc-sdvx": - return ParseArcSDVX(userID, logger); - default: - logger.error(`Unknown importType ${importType} has no handler?`); - throw new ScoreImportFatalError(500, `Unknown importType ${importType}.`); - } -} - export default router; diff --git a/server/src/server/router/api/v1/imports/router.ts b/server/src/server/router/api/v1/imports/router.ts index bbdde5bd0..48e48fefc 100644 --- a/server/src/server/router/api/v1/imports/router.ts +++ b/server/src/server/router/api/v1/imports/router.ts @@ -1,6 +1,9 @@ import { Router } from "express"; import db from "external/mongo/db"; import CreateLogCtx from "lib/logger/logger"; +import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error"; +import ScoreImportQueue from "lib/score-import/worker/queue"; +import { ServerConfig, TachiConfig } from "lib/setup/config"; import { GetRelevantSongsAndCharts } from "utils/db"; import { GetUserWithID } from "utils/user"; @@ -8,6 +11,11 @@ const router: Router = Router({ mergeParams: true }); const logger = CreateLogCtx(__filename); +/** + * Retrieve an import with this ID. + * + * @name GET /api/v1/imports/:importID + */ router.get("/:importID", async (req, res) => { const importDoc = await db.imports.findOne({ importID: req.params.importID, @@ -54,4 +62,83 @@ router.get("/:importID", async (req, res) => { }); }); +/** + * Retrieve the status of an ongoing import. + * If the import has been finalised and was successful, return 200. + * + * If the import is ongoing, return its progress. + * + * If the import was never ongoing, return 404. + * + * If the import was finalised and was unsuccessful (i.e. threw a fatal error) + * return its error information in expressified form. + * + * @name GET /api/v1/import/:importID/poll-status + */ +router.get("/:importID/poll-status", async (req, res) => { + if (!ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { + return res.status(501).json({ + success: false, + description: `${TachiConfig.NAME} does not use an external score import worker. Polling imports is not possible. This import may be ongoing, or it may have never occured.`, + }); + } + + const importDoc = await db.imports.findOne({ importID: req.params.importID }); + + if (importDoc) { + return res.status(200).json({ + success: true, + description: `Import was completed!`, + body: { + importStatus: "completed", + }, + }); + } + + const job = await ScoreImportQueue.getJob(req.params.importID); + + if (!job) { + return res.status(404).json({ + success: false, + description: `There is no ongoing import here.`, + }); + } + + if (job.isFailed()) { + const err = await job.finished(); + + if (err instanceof ScoreImportFatalError) { + logger.info(err.message); + return res.status(err.statusCode).json({ + success: false, + description: err.message, + }); + } + + return res.status(500).json({ + success: false, + description: `An internal service error has occured with this import. This has been reported!`, + }); + } else if (job.isCompleted()) { + return res.status(200).json({ + success: true, + description: `Import was completed!`, + body: { + importStatus: "completed", + }, + }); + } + + const progress = await job.progress(); + + return res.status(200).json({ + success: true, + description: `Import is ongoing.`, + body: { + importStatus: "ongoing", + progress, + }, + }); +}); + export default router; diff --git a/server/src/server/router/api/v1/scores/_scoreID/router.test.ts b/server/src/server/router/api/v1/scores/_scoreID/router.test.ts index b412bcf8c..8bfdd5f89 100644 --- a/server/src/server/router/api/v1/scores/_scoreID/router.test.ts +++ b/server/src/server/router/api/v1/scores/_scoreID/router.test.ts @@ -134,6 +134,7 @@ t.test("PATCH /api/v1/scores/:scoreID", (t) => { permissions: { customise_score: true, }, + fromAPIClient: null, }); const res = await mockApi @@ -156,6 +157,7 @@ t.test("PATCH /api/v1/scores/:scoreID", (t) => { userID: 1, identifier: "another fake token", permissions: {}, + fromAPIClient: null, }); const res = await mockApi @@ -186,6 +188,7 @@ t.test("DELETE /api/v1/scores/:scoreID", (t) => { delete_score: true, }, token: "foo", + fromAPIClient: null, }); const res = await mockApi @@ -209,6 +212,7 @@ t.test("DELETE /api/v1/scores/:scoreID", (t) => { permissions: { delete_score: true, }, + fromAPIClient: null, }); const res = await mockApi @@ -231,6 +235,7 @@ t.test("DELETE /api/v1/scores/:scoreID", (t) => { userID: 1, identifier: "another fake token", permissions: {}, + fromAPIClient: null, }); const res = await mockApi diff --git a/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts b/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts index 9284b6804..3c6fedb90 100644 --- a/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts +++ b/server/src/server/router/api/v1/sessions/_sessionID/router.test.ts @@ -161,6 +161,7 @@ t.test("PATCH /api/v1/sessions/:sessionID", (t) => { permissions: { customise_session: true, }, + fromAPIClient: null, }); const res = await mockApi @@ -183,6 +184,7 @@ t.test("PATCH /api/v1/sessions/:sessionID", (t) => { userID: 1, identifier: "another fake token", permissions: {}, + fromAPIClient: null, }); const res = await mockApi diff --git a/server/src/server/router/api/v1/users/_userID/api-tokens/router.test.ts b/server/src/server/router/api/v1/users/_userID/api-tokens/router.test.ts index 1f125ca9f..ace228442 100644 --- a/server/src/server/router/api/v1/users/_userID/api-tokens/router.test.ts +++ b/server/src/server/router/api/v1/users/_userID/api-tokens/router.test.ts @@ -15,18 +15,21 @@ t.test("GET /api/v1/users/:userID/api-tokens", async (t) => { identifier: "foo", permissions: {}, token: "tfoo", + fromAPIClient: null, }, { userID: 1, identifier: "bar", permissions: {}, token: "tbar", + fromAPIClient: null, }, { userID: 2, identifier: "baz", permissions: {}, token: "tbaz", + fromAPIClient: null, }, ]); }); @@ -47,12 +50,14 @@ t.test("GET /api/v1/users/:userID/api-tokens", async (t) => { identifier: "foo", permissions: {}, token: "tfoo", + fromAPIClient: null, }, { userID: 1, identifier: "bar", permissions: {}, token: "tbar", + fromAPIClient: null, }, ] ); @@ -117,6 +122,7 @@ t.test("DELETE /api/v1/users/:userID/api-tokens/:token", async (t) => { permissions: {}, token: "foo", userID: 2, + fromAPIClient: null, }); const res = await mockApi.delete("/api/v1/users/1/api-tokens/foo").set("Cookie", cookie); @@ -164,7 +170,7 @@ t.test("POST /api/v1/users/:userID/api-tokens/create", async (t) => { identifier: "Hello World", permissions: { submit_score: true, customise_profile: true }, userID: 1, - fromAPIClient: undefined, + fromAPIClient: null, }, "Should return a conforming API Token." ); @@ -179,7 +185,7 @@ t.test("POST /api/v1/users/:userID/api-tokens/create", async (t) => { identifier: "Hello World", permissions: { submit_score: true, customise_profile: true }, userID: 1, - fromAPIClient: undefined, + fromAPIClient: null, }, "Should insert a conforming API Token into the database." ); diff --git a/server/src/server/router/api/v1/users/_userID/api-tokens/router.ts b/server/src/server/router/api/v1/users/_userID/api-tokens/router.ts index 93a030efd..5ae174f5f 100644 --- a/server/src/server/router/api/v1/users/_userID/api-tokens/router.ts +++ b/server/src/server/router/api/v1/users/_userID/api-tokens/router.ts @@ -66,7 +66,7 @@ router.post( const user = req[SYMBOL_TachiData]!.requestedUser!; let identifier: string; - let fromAPIClient; + let fromAPIClient = null; if (req.body.clientID) { const client = await db["api-clients"].findOne( @@ -131,10 +131,6 @@ router.post( fromAPIClient, }; - // Inserting { fromAPIClient: undefined } results in bson - // conversion to {fromAPIClient: null}, which is not the same - DeleteUndefinedProps(apiTokenDocument); - await db["api-tokens"].insert(apiTokenDocument); logger.info(`Inserted new API Key for ${FormatUserDoc(user)}.`); diff --git a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts index 13fd80340..98b185227 100644 --- a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts +++ b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts @@ -43,6 +43,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => { customise_profile: true, }, token: "api_token", + fromAPIClient: null, }); const res = await mockApi @@ -90,6 +91,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => { customise_profile: true, }, token: "api_token", + fromAPIClient: null, }); const res = await mockApi @@ -140,6 +142,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => { customise_profile: true, }, token: "api_token", + fromAPIClient: null, }); const res = await mockApi @@ -159,6 +162,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => { customise_profile: false, }, token: "api_token", + fromAPIClient: null, }); const res = await mockApi @@ -178,6 +182,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => { customise_profile: true, }, token: "api_token", + fromAPIClient: null, }); for (const key of [ diff --git a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts index 11f001e6c..426010bb6 100644 --- a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts +++ b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts @@ -260,6 +260,7 @@ t.test("PUT /api/v1/users/:userID/games/:game/:playtype/showcase", (t) => { customise_profile: true, }, token: "alt_token", + fromAPIClient: null, }) ); @@ -271,6 +272,7 @@ t.test("PUT /api/v1/users/:userID/games/:game/:playtype/showcase", (t) => { customise_profile: true, }, token: "altz_token", + fromAPIClient: null, }); const res = await mockApi @@ -290,6 +292,7 @@ t.test("PUT /api/v1/users/:userID/games/:game/:playtype/showcase", (t) => { customise_profile: false, }, token: "unauth_token", + fromAPIClient: null, }); const res = await mockApi diff --git a/server/src/server/router/api/v1/users/_userID/integrations/arc/router.test.ts b/server/src/server/router/api/v1/users/_userID/integrations/arc/router.test.ts index 8452c256b..f5baf348d 100644 --- a/server/src/server/router/api/v1/users/_userID/integrations/arc/router.test.ts +++ b/server/src/server/router/api/v1/users/_userID/integrations/arc/router.test.ts @@ -39,6 +39,7 @@ t.test("GET /api/v1/users/:userID/integrations/arc", async (t) => { permissions: {}, token: "foobar", userID: 1, + fromAPIClient: null, }); const res = await mockApi diff --git a/server/src/server/router/api/v1/users/_userID/router.test.ts b/server/src/server/router/api/v1/users/_userID/router.test.ts index 008460943..5b5c3f960 100644 --- a/server/src/server/router/api/v1/users/_userID/router.test.ts +++ b/server/src/server/router/api/v1/users/_userID/router.test.ts @@ -1,10 +1,9 @@ -import t from "tap"; import db from "external/mongo/db"; - +import { UserGameStats } from "tachi-common"; +import t from "tap"; +import { CreateFakeAuthCookie } from "test-utils/fake-auth"; import mockApi from "test-utils/mock-api"; import ResetDBState from "test-utils/resets"; -import { UserGameStats } from "tachi-common"; -import { CreateFakeAuthCookie } from "test-utils/fake-auth"; t.test("GET /api/v1/users/:userID", (t) => { t.beforeEach(ResetDBState); @@ -86,6 +85,7 @@ t.test("PATCH /api/v1/users/:userID", async (t) => { permissions: {}, identifier: "No permissions token", userID: 1, + fromAPIClient: null, }); const res2 = await mockApi.patch("/api/v1/users/1").set("Authorization", "Bearer noperm"); diff --git a/server/src/server/router/api/v1/users/_userID/settings/router.test.ts b/server/src/server/router/api/v1/users/_userID/settings/router.test.ts index 28504637c..a7981f92b 100644 --- a/server/src/server/router/api/v1/users/_userID/settings/router.test.ts +++ b/server/src/server/router/api/v1/users/_userID/settings/router.test.ts @@ -144,6 +144,7 @@ t.test("PATCH /api/v1/users/:userID/settings", async (t) => { }, token: "no_perm", userID: 1, + fromAPIClient: null, }); const res = await mockApi diff --git a/server/src/server/router/ir/barbatos/router.test.ts b/server/src/server/router/ir/barbatos/router.test.ts index 5cd368639..489e2d126 100644 --- a/server/src/server/router/ir/barbatos/router.test.ts +++ b/server/src/server/router/ir/barbatos/router.test.ts @@ -1,10 +1,9 @@ -import t from "tap"; - -import { CreateFakeAuthCookie, InsertFakeTokenWithAllPerms } from "test-utils/fake-auth"; -import ResetDBState from "test-utils/resets"; -import mockApi from "test-utils/mock-api"; -import { TestingBarbatosScore } from "test-utils/test-data"; import db from "external/mongo/db"; +import t from "tap"; +import { InsertFakeTokenWithAllPerms } from "test-utils/fake-auth"; +import mockApi from "test-utils/mock-api"; +import ResetDBState from "test-utils/resets"; +import { TestingBarbatosScore } from "test-utils/test-data"; t.test("POST /ir/barbatos/score/submit", (t) => { t.beforeEach(ResetDBState); diff --git a/server/src/server/router/ir/barbatos/router.ts b/server/src/server/router/ir/barbatos/router.ts index ee999f59c..4e1f6ccec 100644 --- a/server/src/server/router/ir/barbatos/router.ts +++ b/server/src/server/router/ir/barbatos/router.ts @@ -1,9 +1,7 @@ import { Router } from "express"; import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; -import { ParseBarbatosSingle } from "lib/score-import/import-types/ir/barbatos/parser"; import { RequirePermissions } from "server/middleware/auth"; -import { GetUserWithIDGuaranteed } from "utils/user"; const router: Router = Router({ mergeParams: true }); @@ -14,13 +12,11 @@ router.use(RequirePermissions("submit_score")); * @name POST /ir/barbatos/score/submit */ router.post("/score/submit", async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth]!.userID!); - const responseData = await ExpressWrappedScoreImportMain( - userDoc, + req[SYMBOL_TachiAPIAuth]!.userID!, false, "ir/barbatos", - (logger) => ParseBarbatosSingle(req.body, logger) + [req.body] ); return res.status(responseData.statusCode).json(responseData.body); diff --git a/server/src/server/router/ir/beatoraja/auth.test.ts b/server/src/server/router/ir/beatoraja/auth.test.ts index d0f3825d2..3e63ac63b 100644 --- a/server/src/server/router/ir/beatoraja/auth.test.ts +++ b/server/src/server/router/ir/beatoraja/auth.test.ts @@ -1,6 +1,6 @@ +import expMiddlewareMock from "express-request-mock"; import t from "tap"; import { ValidateIRClientVersion } from "./auth"; -import expMiddlewareMock from "express-request-mock"; t.test("#ValidateIRClientVersion", (t) => { t.test("Should reject clients that are not supported", async (t) => { diff --git a/server/src/server/router/ir/beatoraja/router.test.ts b/server/src/server/router/ir/beatoraja/router.test.ts index dfc5cf75a..79afc06f2 100644 --- a/server/src/server/router/ir/beatoraja/router.test.ts +++ b/server/src/server/router/ir/beatoraja/router.test.ts @@ -1,11 +1,10 @@ -import t from "tap"; +import deepmerge from "deepmerge"; import db from "external/mongo/db"; - +import { PublicUserDocument } from "tachi-common"; +import t from "tap"; import mockApi from "test-utils/mock-api"; import ResetDBState from "test-utils/resets"; import { GetKTDataJSON } from "test-utils/test-data"; -import deepmerge from "deepmerge"; -import { PublicUserDocument } from "tachi-common"; t.test("POST /ir/beatoraja/submit-score", (t) => { t.beforeEach(ResetDBState); @@ -17,6 +16,7 @@ t.test("POST /ir/beatoraja/submit-score", (t) => { submit_score: true, }, token: "mock_token", + fromAPIClient: null, }) ); @@ -142,12 +142,14 @@ t.test("POST /ir/beatoraja/submit-score", (t) => { identifier: "token2", permissions: { submit_score: true }, token: "token2", + fromAPIClient: null, }, { userID: 3, identifier: "token3", permissions: { submit_score: true }, token: "token3", + fromAPIClient: null, }, ]); @@ -320,6 +322,7 @@ t.test("POST /ir/beatoraja/submit-course", (t) => { submit_score: true, }, token: "mock_token", + fromAPIClient: null, }) ); diff --git a/server/src/server/router/ir/beatoraja/router.ts b/server/src/server/router/ir/beatoraja/router.ts index a81cf6f0c..326ed42be 100644 --- a/server/src/server/router/ir/beatoraja/router.ts +++ b/server/src/server/router/ir/beatoraja/router.ts @@ -1,14 +1,11 @@ import { Router } from "express"; import db from "external/mongo/db"; import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; -import CreateLogCtx, { KtLogger } from "lib/logger/logger"; -import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue"; +import CreateLogCtx from "lib/logger/logger"; import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; -import { ParseBeatorajaSingle } from "lib/score-import/import-types/ir/beatoraja/parser"; import { ServerConfig } from "lib/setup/config"; import { RequireNotGuest } from "server/middleware/auth"; import { UpdateClassIfGreater } from "utils/class"; -import { GetUserWithIDGuaranteed } from "utils/user"; import { ValidateIRClientVersion } from "./auth"; import chartsRouter from "./charts/router"; @@ -24,21 +21,19 @@ router.use(ValidateIRClientVersion); * @name POST /ir/beatoraja/submit-score */ router.post("/submit-score", RequireNotGuest, async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth]!.userID!); + const userID = req[SYMBOL_TachiAPIAuth]!.userID!; - const ParserFunction = (logger: KtLogger) => ParseBeatorajaSingle(req.body, userDoc.id, logger); - - const importRes = await ExpressWrappedScoreImportMain( - userDoc, - false, - "ir/beatoraja", - ParserFunction - ); + const importRes = await ExpressWrappedScoreImportMain(userID, false, "ir/beatoraja", [ + req.body, + userID, + ]); if (!importRes.body.success) { return res.status(400).json(importRes.body); } else if (importRes.body.body.errors.length !== 0) { const type = importRes.body.body.errors[0].type; + const errMsg = importRes.body.body.errors[0].message; + if (type === "KTDataNotFound") { return res.status(202).json({ success: true, @@ -47,7 +42,7 @@ router.post("/submit-score", RequireNotGuest, async (req, res) => { } else if (type === "InternalError") { return res.status(500).json({ success: false, - description: `[${importRes.body.body.errors[0].type}] - ${importRes.body.body.errors[0].message}`, + description: `[${type}] - ${errMsg}`, }); } @@ -56,7 +51,7 @@ router.post("/submit-score", RequireNotGuest, async (req, res) => { return res.status(400).json({ success: false, - description: `[${importRes.body.body.errors[0].type}] - ${importRes.body.body.errors[0].message}`, + description: `[${type}] - ${errMsg}`, }); } else if (importRes.body.body.scoreIDs.length === 0) { return res.status(400).json({ diff --git a/server/src/server/router/ir/chunitachi/router.test.ts b/server/src/server/router/ir/chunitachi/router.test.ts deleted file mode 100644 index 8786ab3c4..000000000 --- a/server/src/server/router/ir/chunitachi/router.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import t from "tap"; -import db from "external/mongo/db"; - -import { InsertFakeTokenWithAllPerms } from "test-utils/fake-auth"; -import mockApi from "test-utils/mock-api"; -import ResetDBState from "test-utils/resets"; -import { GetKTDataJSON } from "test-utils/test-data"; -import deepmerge from "deepmerge"; - -t.test("POST /ir/chunitachi/import", (t) => { - t.beforeEach(ResetDBState); - t.beforeEach(InsertFakeTokenWithAllPerms("mock_token")); - - const chunitachiBody = GetKTDataJSON("./batch-manual/chunitachi.json"); - - t.test("Should work for CHUNITACHI requests", async (t) => { - const res = await mockApi - .post("/ir/chunitachi/import") - .set("Authorization", `Bearer mock_token`) - .send(chunitachiBody); - - t.equal(res.body.success, true, "Should be successful"); - - t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); - - const scoreCount = await db.scores.count({ service: "ChunItachi" }); - - t.equal(scoreCount, 1, "Should import one score."); - - t.end(); - }); - - t.test("Should reject invalid batch-manual", async (t) => { - const res = await mockApi - .post("/ir/chunitachi/import") - .set("Authorization", `Bearer mock_token`) - .send({}); - - t.equal(res.body.success, false, "Should not be successful"); - - t.end(); - }); - - t.test("Should reject batch-manual requests if game is not chunithm", async (t) => { - const res = await mockApi - .post("/ir/chunitachi/import") - .set("Authorization", `Bearer mock_token`) - .send(deepmerge(chunitachiBody, { meta: { game: "iidx" } })); - - t.equal(res.body.success, false, "Should not be successful"); - - t.end(); - }); - - t.test("Should reject batch-manual requests if service is not Chunitachi", async (t) => { - const res = await mockApi - .post("/ir/chunitachi/import") - .set("Authorization", `Bearer mock_token`) - .send(deepmerge(chunitachiBody, { meta: { service: "foo bar" } })); - - t.equal(res.body.success, false, "Should not be successful"); - - t.end(); - }); - - t.test("Should require authentication.", async (t) => { - const res = await mockApi.post("/ir/chunitachi/import").send(chunitachiBody); - - t.equal(res.statusCode, 401); - - t.end(); - }); - - t.test("Should require a valid auth token.", async (t) => { - const res = await mockApi - .post("/ir/chunitachi/import") - .set("Authorization", "Bearer invalid_token") - .send(chunitachiBody); - - t.equal(res.statusCode, 401); - - t.end(); - }); - - t.end(); -}); diff --git a/server/src/server/router/ir/chunitachi/router.ts b/server/src/server/router/ir/chunitachi/router.ts deleted file mode 100644 index 81e6b6668..000000000 --- a/server/src/server/router/ir/chunitachi/router.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Router } from "express"; -import { GetUserWithIDGuaranteed } from "utils/user"; -import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; -import ParseDirectManual from "lib/score-import/import-types/ir/direct-manual/parser"; -import { RequirePermissions } from "server/middleware/auth"; -import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; - -const router: Router = Router({ mergeParams: true }); - -/** - * Submits a single score document from Chunitachi clients. - * @name POST /ir/chunitachi/import - */ -router.post("/import", RequirePermissions("submit_score"), async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!); - - if (req.body?.meta?.game !== "chunithm") { - return res.status(400).json({ - success: false, - description: `Invalid Game. Expected 'chunithm', but got ${req.body?.meta?.game}`, - }); - } - - if (req.body.meta.service !== "ChunItachi") { - return res.status(400).json({ - success: false, - description: `Unexpected service ${req.body.head.service} -- expected 'Chunitachi'`, - }); - } - - const responseData = await ExpressWrappedScoreImportMain( - userDoc, - false, - "ir/chunitachi", - (logger) => ParseDirectManual(req.body, logger) - ); - - return res.status(responseData.statusCode).json(responseData.body); -}); - -export default router; diff --git a/server/src/server/router/ir/direct-manual/router.test.ts b/server/src/server/router/ir/direct-manual/router.test.ts index c65598947..b9199abd5 100644 --- a/server/src/server/router/ir/direct-manual/router.test.ts +++ b/server/src/server/router/ir/direct-manual/router.test.ts @@ -1,6 +1,6 @@ -import t from "tap"; import db from "external/mongo/db"; - +import { rootLogger } from "lib/logger/logger"; +import t from "tap"; import { CreateFakeAuthCookie } from "test-utils/fake-auth"; import mockApi from "test-utils/mock-api"; import ResetDBState from "test-utils/resets"; @@ -19,6 +19,7 @@ t.test("POST /ir/direct-manual/import", async (t) => { submit_score: false, }, userID: 1, + fromAPIClient: null, }); const res = await mockApi @@ -78,5 +79,64 @@ t.test("POST /ir/direct-manual/import", async (t) => { t.end(); }); + t.beforeEach(() => + db["api-tokens"].insert({ + token: "mock_token", + fromAPIClient: null, + identifier: "Mock CHUNITACHI Token", + permissions: { submit_score: true }, + userID: 1, + }) + ); + + const chunitachiBody = GetKTDataJSON("./batch-manual/chunitachi.json"); + + t.test("Should work for CHUNITACHI requests", async (t) => { + const res = await mockApi + .post("/ir/direct-manual/import") + .set("Authorization", `Bearer mock_token`) + .send(chunitachiBody); + + t.equal(res.body.success, true, "Should be successful"); + + t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); + + const scoreCount = await db.scores.count({ service: "ChunItachi (DIRECT-MANUAL)" }); + + t.equal(scoreCount, 1, "Should import one score."); + + t.end(); + }); + + t.test("Should reject invalid batch-manual", async (t) => { + const res = await mockApi + .post("/ir/direct-manual/import") + .set("Authorization", `Bearer mock_token`) + .send({}); + + t.equal(res.body.success, false, "Should not be successful"); + + t.end(); + }); + + t.test("Should require authentication.", async (t) => { + const res = await mockApi.post("/ir/direct-manual/import").send(chunitachiBody); + + t.equal(res.statusCode, 401); + + t.end(); + }); + + t.test("Should require a valid auth token.", async (t) => { + const res = await mockApi + .post("/ir/direct-manual/import") + .set("Authorization", "Bearer invalid_token") + .send(chunitachiBody); + + t.equal(res.statusCode, 401); + + t.end(); + }); + t.end(); }); diff --git a/server/src/server/router/ir/direct-manual/router.ts b/server/src/server/router/ir/direct-manual/router.ts index e98c6fef0..1feef43a1 100644 --- a/server/src/server/router/ir/direct-manual/router.ts +++ b/server/src/server/router/ir/direct-manual/router.ts @@ -12,15 +12,13 @@ const router: Router = Router({ mergeParams: true }); * @name POST /ir/direct-manual/import */ router.post("/import", RequirePermissions("submit_score"), async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!); - const intent = req.header("X-User-Intent"); const responseData = await ExpressWrappedScoreImportMain( - userDoc, + req[SYMBOL_TachiAPIAuth].userID!, !!intent, "ir/direct-manual", - (logger) => ParseDirectManual(req.body, logger) + [req.body] ); return res.status(responseData.statusCode).json(responseData.body); diff --git a/server/src/server/router/ir/fervidex/router.test.ts b/server/src/server/router/ir/fervidex/router.test.ts index 5ed1c97a3..c4a916dde 100644 --- a/server/src/server/router/ir/fervidex/router.test.ts +++ b/server/src/server/router/ir/fervidex/router.test.ts @@ -1,6 +1,5 @@ -import t from "tap"; import db from "external/mongo/db"; - +import t from "tap"; import { InsertFakeTokenWithAllPerms } from "test-utils/fake-auth"; import mockApi from "test-utils/mock-api"; import ResetDBState from "test-utils/resets"; diff --git a/server/src/server/router/ir/fervidex/router.ts b/server/src/server/router/ir/fervidex/router.ts index a7001b208..501233f35 100644 --- a/server/src/server/router/ir/fervidex/router.ts +++ b/server/src/server/router/ir/fervidex/router.ts @@ -201,18 +201,16 @@ router.use( * @name POST /ir/fervidex/profile/submit */ router.post("/profile/submit", RequireInf2ModelHeaderOrForceStatic, async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!); - const headers = { // guaranteed to exist because of RequireInf2ModelHeader model: req.header("X-Software-Model")!, }; const responseData = await ExpressWrappedScoreImportMain( - userDoc, + req[SYMBOL_TachiAPIAuth].userID!, false, "ir/fervidex-static", - (logger) => ParseFervidexStatic(req.body, headers, logger) + [req.body, headers] ); if (!responseData.body.success) { @@ -234,8 +232,6 @@ router.post("/profile/submit", RequireInf2ModelHeaderOrForceStatic, async (req, * @name POST /ir/fervidex/score/submit */ router.post("/score/submit", ValidateModelHeader, async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!); - const model = req.header("X-Software-Model"); if (!model) { @@ -250,10 +246,10 @@ router.post("/score/submit", ValidateModelHeader, async (req, res) => { }; const responseData = await ExpressWrappedScoreImportMain( - userDoc, + req[SYMBOL_TachiAPIAuth].userID!, true, "ir/fervidex", - (logger) => ParseFervidexSingle(req.body, headers, logger) + [req.body, headers] ); if (!responseData.body.success) { diff --git a/server/src/server/router/ir/kshook/sv3c/router.ts b/server/src/server/router/ir/kshook/sv3c/router.ts index 0f2843ac4..4e6bfaf98 100644 --- a/server/src/server/router/ir/kshook/sv3c/router.ts +++ b/server/src/server/router/ir/kshook/sv3c/router.ts @@ -81,13 +81,11 @@ router.use(ValidateHeaders); * @name POST /ir/kshook/sv3c/score/save */ router.post("/score/save", async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!); - const responseData = await ExpressWrappedScoreImportMain( - userDoc, + req[SYMBOL_TachiAPIAuth].userID!, true, "ir/kshook-sv3c", - (logger) => ParseKsHookSV3C(req.body, logger) + [req.body] ); if (!responseData.body.success) { diff --git a/server/src/server/router/ir/router.ts b/server/src/server/router/ir/router.ts index a53ac1ab4..ec9e66fa9 100644 --- a/server/src/server/router/ir/router.ts +++ b/server/src/server/router/ir/router.ts @@ -1,6 +1,5 @@ import { Router } from "express"; import barbatosIR from "./barbatos/router"; -import chunitachiIR from "./chunitachi/router"; import directManualIR from "./direct-manual/router"; import fervidexIR from "./fervidex/router"; import uscIR from "./usc/router"; @@ -32,7 +31,6 @@ router.use("/beatoraja", SetRequestPermissions, RequireBokutachi, beatorajaIR); // Kamaitachi IRs router.use("/barbatos", SetRequestPermissions, RequireNotGuest, RequireKamaitachi, barbatosIR); -router.use("/chunitachi", SetRequestPermissions, RequireNotGuest, RequireKamaitachi, chunitachiIR); router.use( "/fervidex", SetFervidexStyleRequestPermissions, diff --git a/server/src/server/router/ir/usc/router.test.ts b/server/src/server/router/ir/usc/router.test.ts index 3c753a9e5..e66cb764b 100644 --- a/server/src/server/router/ir/usc/router.test.ts +++ b/server/src/server/router/ir/usc/router.test.ts @@ -1,12 +1,11 @@ -import t from "tap"; +import deepmerge from "deepmerge"; import db from "external/mongo/db"; - +import { CDNRetrieve } from "lib/cdn/cdn"; +import { PBScoreDocument, PublicUserDocument, ScoreDocument } from "tachi-common"; +import t from "tap"; import mockApi from "test-utils/mock-api"; import ResetDBState, { ResetCDN } from "test-utils/resets"; -import deepmerge from "deepmerge"; -import { PBScoreDocument, ScoreDocument, PublicUserDocument } from "tachi-common"; import { GetKTDataBuffer } from "test-utils/test-data"; -import { CDNRetrieve } from "lib/cdn/cdn"; async function InsertFakeUSCAuth() { await db["api-tokens"].insert({ @@ -16,6 +15,7 @@ async function InsertFakeUSCAuth() { submit_score: true, }, token: "foo", + fromAPIClient: null, }); } @@ -47,6 +47,7 @@ t.test("GET /ir/usc", async (t) => { submit_score: true, }, token: "foo", + fromAPIClient: null, }); t.beforeEach(ResetDBState); @@ -466,6 +467,7 @@ t.test("POST /scores", (t) => { identifier: "token", permissions: { submit_score: true }, token: "token", + fromAPIClient: null, }); }); @@ -587,12 +589,14 @@ t.test("POST /scores", (t) => { identifier: "token2", permissions: { submit_score: true }, token: "token2", + fromAPIClient: null, }, { userID: 3, identifier: "token3", permissions: { submit_score: true }, token: "token3", + fromAPIClient: null, }, ]); diff --git a/server/src/server/router/ir/usc/router.ts b/server/src/server/router/ir/usc/router.ts index 7c03f6229..6a9f004a2 100644 --- a/server/src/server/router/ir/usc/router.ts +++ b/server/src/server/router/ir/usc/router.ts @@ -1,37 +1,35 @@ -import { Router, RequestHandler } from "express"; -import p from "prudence"; -import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi"; +import { RequestHandler, Router } from "express"; import db from "external/mongo/db"; +import { CDNStore } from "lib/cdn/cdn"; +import { GetUSCIRReplayURL } from "lib/cdn/url-format"; +import { ONE_MEGABYTE } from "lib/constants/filesize"; +import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi"; +import { USCIR_MAX_LEADERBOARD_N } from "lib/constants/usc-ir"; +import CreateLogCtx from "lib/logger/logger"; +import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue"; +import { AssertStrAsPositiveNonZeroInt } from "lib/score-import/framework/common/string-asserts"; +import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; +import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; +import { ServerConfig, TachiConfig } from "lib/setup/config"; +import p from "prudence"; +import { RequirePermissions } from "server/middleware/auth"; +import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload"; import { ChartDocument, + ImportDocument, PBScoreDocument, SuccessfulAPIResponse, - ImportDocument, } from "tachi-common"; -import { AssertStrAsPositiveNonZeroInt } from "lib/score-import/framework/common/string-asserts"; -import CreateLogCtx, { KtLogger } from "lib/logger/logger"; +import { FormatPrError } from "utils/prudence"; +import { GetBlacklist } from "utils/queries/blacklist"; +import { AssignToReqTachiData } from "utils/req-tachi-data"; +import { USCClientChart } from "./types"; import { ConvertUSCChart, CreatePOSTScoresResponseBody, TachiScoreToServerScore, USCChartIndexToDiff, } from "./usc"; -import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; -import { GetUserWithID } from "utils/user"; -import { ParseIRUSC } from "lib/score-import/import-types/ir/usc/parser"; -import { USCIR_MAX_LEADERBOARD_N } from "lib/constants/usc-ir"; -import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload"; -import { AssignToReqTachiData } from "utils/req-tachi-data"; -import { CDNStore } from "lib/cdn/cdn"; -import { ONE_MEGABYTE } from "lib/constants/filesize"; -import { RequirePermissions } from "server/middleware/auth"; -import { GetUSCIRReplayURL } from "lib/cdn/url-format"; -import { FormatPrError } from "utils/prudence"; -import { USCClientChart } from "./types"; -import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue"; -import { ServerConfig, TachiConfig } from "lib/setup/config"; -import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; -import { GetBlacklist } from "utils/queries/blacklist"; const logger = CreateLogCtx(__filename); @@ -314,25 +312,18 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => { "context.chartHash": chartDoc.data.hashSHA1, }); - for (const score of scoresToDeorphan) { - await ReprocessOrphan(score, blacklist, logger); - } + await Promise.all( + scoresToDeorphan.map((score) => ReprocessOrphan(score, blacklist, logger)) + ); } } - const userDoc = await GetUserWithID(req[SYMBOL_TachiAPIAuth]!.userID!); + const userID = req[SYMBOL_TachiAPIAuth]!.userID!; - if (!userDoc) { - logger.severe(`User ${req[SYMBOL_TachiAPIAuth]!.userID!} as no parent userDoc?`); - return res.status(200).json({ - statusCode: STATUS_CODES.SERVER_ERROR, - description: "An internal server error has occured.", - }); - } - - const importParser = (logger: KtLogger) => ParseIRUSC(req.body, uscChart.chartHash, logger); - - const importRes = await ExpressWrappedScoreImportMain(userDoc, false, "ir/usc", importParser); + const importRes = await ExpressWrappedScoreImportMain(userID, false, "ir/usc", [ + req.body, + uscChart.chartHash, + ]); // If this was an orphan chart request, return ACCEPTED, // since it may be unorphaned in the future @@ -359,11 +350,7 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => { const importDoc = (importRes.body as SuccessfulAPIResponse).body as ImportDocument; try { - const body = await CreatePOSTScoresResponseBody( - userDoc.id, - chartDoc, - importDoc.scoreIDs[0] - ); + const body = await CreatePOSTScoresResponseBody(userID, chartDoc, importDoc.scoreIDs[0]); return res.status(200).json({ statusCode: STATUS_CODES.SUCCESS, diff --git a/server/src/server/router/ir/usc/usc.test.ts b/server/src/server/router/ir/usc/usc.test.ts index 985f77496..bf89e0e87 100644 --- a/server/src/server/router/ir/usc/usc.test.ts +++ b/server/src/server/router/ir/usc/usc.test.ts @@ -1,9 +1,9 @@ -import t from "tap"; +import deepmerge from "deepmerge"; import db from "external/mongo/db"; +import { ChartDocument, PBScoreDocument, PublicUserDocument, ScoreDocument } from "tachi-common"; +import t from "tap"; import ResetDBState from "test-utils/resets"; import { CreatePOSTScoresResponseBody, TachiScoreToServerScore } from "./usc"; -import { ChartDocument, PBScoreDocument, ScoreDocument, PublicUserDocument } from "tachi-common"; -import deepmerge from "deepmerge"; const mockScorePB: PBScoreDocument<"usc:Single"> = { chartID: "USC_CHART_ID", diff --git a/server/src/server/router/ir/usc/usc.ts b/server/src/server/router/ir/usc/usc.ts index 38e3deb7c..17f34a2ae 100644 --- a/server/src/server/router/ir/usc/usc.ts +++ b/server/src/server/router/ir/usc/usc.ts @@ -1,9 +1,9 @@ -import { integer, PBScoreDocument, ScoreDocument, ChartDocument, SongDocument } from "tachi-common"; -import CreateLogCtx from "lib/logger/logger"; -import { GetPBOnChart, GetServerRecordOnChart } from "utils/scores"; -import { MStoS, Random20Hex } from "utils/misc"; -import { USCIR_ADJACENT_SCORE_N } from "lib/constants/usc-ir"; import db from "external/mongo/db"; +import { USCIR_ADJACENT_SCORE_N } from "lib/constants/usc-ir"; +import CreateLogCtx from "lib/logger/logger"; +import { ChartDocument, integer, PBScoreDocument, ScoreDocument, SongDocument } from "tachi-common"; +import { MStoS, Random20Hex } from "utils/misc"; +import { GetPBOnChart, GetServerRecordOnChart } from "utils/scores"; import { USCClientChart, USCServerScore } from "./types"; const logger = CreateLogCtx(__filename); diff --git a/server/src/test-utils/api-common.ts b/server/src/test-utils/api-common.ts index 93e2cdc4f..26bf48827 100644 --- a/server/src/test-utils/api-common.ts +++ b/server/src/test-utils/api-common.ts @@ -1,7 +1,7 @@ +import db from "external/mongo/db"; +import { APIPermissions } from "tachi-common"; import t from "tap"; import mockApi from "./mock-api"; -import { APIPermissions } from "tachi-common"; -import db from "external/mongo/db"; import ResetDBState from "./resets"; export function RequireAuthPerms( @@ -22,6 +22,7 @@ export function RequireAuthPerms( permissions: {}, token: "temp_auth", userID: 1, + fromAPIClient: null, }); const resAuth = await mockApi[m](url).set("Authorization", "Bearer temp_auth"); @@ -35,6 +36,7 @@ export function RequireAuthPerms( permissions: Object.fromEntries(prm.map((e) => [e, true])), token: "temp_auth2", userID: 1, + fromAPIClient: null, }); const resAuthed = await mockApi[m](url).set("Authorization", "Bearer temp_auth2"); diff --git a/server/src/test-utils/close-connections.ts b/server/src/test-utils/close-connections.ts index 08afb7f88..b6d8c1e52 100644 --- a/server/src/test-utils/close-connections.ts +++ b/server/src/test-utils/close-connections.ts @@ -2,10 +2,12 @@ import { CloseMongoConnection } from "external/mongo/db"; import { CloseRedisPubSub } from "external/redis/redis-IPC"; import { CloseRedisConnection } from "external/redis/redis"; import { CloseServerConnection } from "./mock-api"; +import { CloseScoreImportQueue } from "lib/score-import/worker/queue"; export async function CloseAllConnections() { await CloseMongoConnection(); await CloseServerConnection(); await CloseRedisConnection(); await CloseRedisPubSub(); + await CloseScoreImportQueue(); } diff --git a/server/src/test-utils/fake-auth.ts b/server/src/test-utils/fake-auth.ts index decf9d3b7..7dc3a80ac 100644 --- a/server/src/test-utils/fake-auth.ts +++ b/server/src/test-utils/fake-auth.ts @@ -33,6 +33,7 @@ export function InsertFakeTokenWithAllPerms(token: string): () => any { identifier: "Mock API Token", permissions: AllPermissions, token, + fromAPIClient: null, }); }; } diff --git a/server/src/test-utils/single-process-tap.ts b/server/src/test-utils/single-process-tap.ts index fdf9db550..e8f04fcfb 100644 --- a/server/src/test-utils/single-process-tap.ts +++ b/server/src/test-utils/single-process-tap.ts @@ -1,10 +1,9 @@ +import "external/mongo/db"; import glob from "glob"; import path from "path"; import t from "tap"; import { CloseAllConnections } from "./close-connections"; -import "external/mongo/db"; - const files = glob.sync(path.join(__dirname, "../../", "**/*.test.ts")); process.env.NODE_PATH = path.join(__dirname, "../../");