Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-01 19:38:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #423 from TNG-dev:zkldi/issue-406-Score-Import-Refactor-to-use-separate-process

Browse Source 
Refactor Score Import to use workers
This commit is contained in:
zkldi
2021-11-12 03:14:26 +00:00
committed by GitHub
parent c859903be5 36300ab710
commit 66783efcdb
74 changed files with 741 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/.github/test.conf.json5
Vendored
-1
View File
@@ -52,7 +52,6 @@
"ir/fervidex",
"ir/fervidex-static",
"ir/beatoraja",
"ir/chunitachi",
"ir/usc",
"ir/kshook-sv3c",
"api/arc-iidx",
server/package.json
+1 -1
View File
@@ -81,7 +81,7 @@
"rimraf": "3.0.2",
"safe-json-stringify": "1.2.0",
"seq-logging": "^1.1.1",
"tachi-common": "0.2.42",
"tachi-common": "0.2.46",
"ts-node": "10.0.0",
"typescript": "4.3.4",
"winston": "3.3.3",
server/pnpm-lock.yaml
Generated
+4 -4
View File
@@ -62,7 +62,7 @@ specifiers:
safe-json-stringify: 1.2.0
seq-logging: ^1.1.1
supertest: 6.1.3
tachi-common: 0.2.42
tachi-common: 0.2.46
tap: 15.0.9
ts-node: 10.0.0
tsconfig-paths: 3.10.1
@@ -106,7 +106,7 @@ dependencies:
rimraf: 3.0.2
safe-json-stringify: 1.2.0
seq-logging: 1.1.1
tachi-common: 0.2.42
tachi-common: 0.2.46
ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b
typescript: 4.3.4
winston: 3.3.3
@@ -4176,8 +4176,8 @@ packages:
strip-ansi: 6.0.0
dev: true
/tachi-common/0.2.42:
resolution: {integrity: sha512-X+oS08NXgVeiPAFiNUzN7DCmiEJor35NCQkh6ia/kcpTAGkRUwst8NFQnHIca2eXDQmndEJg0KGkpYosMs4KYg==}
/tachi-common/0.2.46:
resolution: {integrity: sha512-BTI3j6joaHLh+T/gapHw5lO9BlB9jSCDtaLIECHkEjY0pUBnXcbcA4GEeMjefXUD65hF04BNkg6J7R6X6Cnkmw==}
dependencies:
monk: 7.3.4
transitivePeerDependencies:
server/src/lib/email/client.ts
+1 -1
View File
@@ -1,7 +1,7 @@
import bunyan from "bunyan";
import CreateLogCtx from "lib/logger/logger";
import { Environment, ServerConfig } from "lib/setup/config";
import nodemailer, { SentMessageInfo, Transporter } from "nodemailer";
import bunyan from "bunyan";
const logger = CreateLogCtx(__filename);
server/src/lib/email/formats.ts
+1 -1
View File
@@ -1,7 +1,7 @@
// Email HTML is a hellish mess of IE5 era nonsense.
// Good luck.
import { ServerConfig, TachiConfig, TachiServerConfig } from "lib/setup/config";
import { ServerConfig, TachiConfig } from "lib/setup/config";
export function EmailFormatResetPassword(username: string, resetCode: string, ipAddr: string) {
return MainHTMLWrapper(
server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts
+4 -4
View File
@@ -1,6 +1,7 @@
import p from "prudence";
import t from "tap";
import CreateLogCtx from "lib/logger/logger";
import p from "prudence";
import { ChartDocument, ScoreDocument } from "tachi-common";
import t from "tap";
import { prAssert } from "test-utils/asserts";
import {
Testing511SPA,
@@ -9,8 +10,7 @@ import {
TestingIIDXSPDryScore,
TestingSDVXSingleDryScore,
} from "test-utils/test-data";
import { CreateCalculatedData, CalculateDataForGamePT } from "./calculated-data";
import { ChartDocument, ScoreDocument } from "tachi-common";
import { CalculateDataForGamePT, CreateCalculatedData } from "./calculated-data";
const logger = CreateLogCtx(__filename);
server/src/lib/score-import/framework/calculated-data/stats.ts
+5 -5
View File
@@ -1,13 +1,13 @@
import { KtLogger } from "lib/logger/logger";
import {
ChartDocument,
integer,
Grades,
Lamps,
Game,
Playtypes,
GetGamePTConfig,
Grades,
integer,
Lamps,
Playtypes,
} from "tachi-common";
import { KtLogger } from "lib/logger/logger";
import { DryScore } from "../common/types";
/**
server/src/lib/score-import/framework/common/get-input-parser.ts
+26
View File
@@ -0,0 +1,26 @@
import { KtLogger } from "lib/logger/logger";
import {
ParserFunctionReturns,
ImportTypeDataMap,
ImportTypeContextMap,
} from "lib/score-import/import-types/common/types";
import { Parsers } from "lib/score-import/import-types/parsers";
import { ScoreImportJobData } from "lib/score-import/worker/types";
import { ImportTypes } from "tachi-common";
export function GetInputParser<I extends ImportTypes>(jobData: ScoreImportJobData<I>) {
// Retrieve the set parser function for this import type.
const ParserFunction = Parsers[jobData.importType];
const InputParser = (logger: KtLogger) =>
// @ts-expect-error TypeScript doesn't like the fact that we
// pass this as a rest parameter, since none of the parsers
// actually take rest args. However, this is the only way to
// achieve the dynamic passing we need to, so lets just override
// it here.
ParserFunction(...jobData.parserArguments, logger) as Promise<
ParserFunctionReturns<ImportTypeDataMap[I], ImportTypeContextMap[I]>
>;
return InputParser;
}
server/src/lib/score-import/framework/common/import-logger.test.ts
-19
View File
@@ -1,19 +0,0 @@
import { PublicUserDocument } from "tachi-common";
import t from "tap";
import { CreateImportLoggerAndID } from "./import-logger";
t.test("#CreateImportLoggerAndID", (t) => {
const { importID, logger } = CreateImportLoggerAndID(
{ username: "foo", id: 1 } as PublicUserDocument,
"file/eamusement-iidx-csv"
);
t.match(importID, /^[a-f0-9]{40}$/u, "Should return a 40 character importID.");
// not possible to automate this, probably.
logger.info(
"Namespaced log test - if this does not have 'file/eamusement-iidx-csv | foo (#1)' next to it, it's broke."
);
t.end();
});
server/src/lib/score-import/framework/common/import-logger.ts
+1 -6
View File
@@ -1,13 +1,8 @@
import crypto from "crypto";
import { KtLogger, rootLogger } from "lib/logger/logger";
import { ImportTypes, PublicUserDocument } from "tachi-common";
import { Random20Hex } from "utils/misc";
import { FormatUserDoc } from "utils/user";
export function CreateImportLoggerAndID(userDoc: PublicUserDocument, importType: ImportTypes) {
const importID = crypto.randomBytes(20).toString("hex");
return { logger: CreateScoreLogger(userDoc, importID, importType), importID };
}
export function CreateScoreLogger(
user: PublicUserDocument,
importID: string,
server/src/lib/score-import/framework/express-wrapper.ts
+19 -13
View File
@@ -1,13 +1,14 @@
import CreateLogCtx from "lib/logger/logger";
import {
ImportDocument,
ImportTypes,
PublicUserDocument,
integer,
SuccessfulAPIResponse,
UnsuccessfulAPIResponse,
} from "tachi-common";
import { ImportInputParser } from "../import-types/common/types";
import { CreateImportLoggerAndID } from "./common/import-logger";
import ScoreImportMain from "./score-import-main";
import { Random20Hex } from "utils/misc";
import { ParserArguments } from "../worker/types";
import { MakeScoreImport } from "./score-import";
import ScoreImportFatalError from "./score-importing/score-import-error";
export interface WrappedAPIResponse {
@@ -15,24 +16,30 @@ export interface WrappedAPIResponse {
body: SuccessfulAPIResponse<ImportDocument> | UnsuccessfulAPIResponse;
}
const logger = CreateLogCtx(__filename);
/**
* A thin(ish) wrapper for ScoreImportMain which converts thrown
* errors and import documents into a WrappedAPIResponse, which can
* be immediately sent with res.json().
*/
export async function ExpressWrappedScoreImportMain<D, C>(
user: PublicUserDocument,
export async function ExpressWrappedScoreImportMain<I extends ImportTypes>(
userID: integer,
userIntent: boolean,
importType: ImportTypes,
InputParser: ImportInputParser<D, C>
importType: I,
parserArguments: ParserArguments<I>
): Promise<WrappedAPIResponse> {
const { importID, logger } = CreateImportLoggerAndID(user, importType);
const importID = Random20Hex();
logger.debug("Received import request.");
try {
const res = await ScoreImportMain(user, userIntent, importType, InputParser, {
const res = await MakeScoreImport({
importID,
logger,
importType,
userIntent,
userID,
parserArguments,
});
return {
@@ -60,8 +67,7 @@ export async function ExpressWrappedScoreImportMain<D, C>(
statusCode: 500,
body: {
success: false,
description:
"An internal service error has occured. Please do not repeat this request.",
description: "An internal service error has occured. This has been reported!",
},
};
}
server/src/lib/score-import/framework/orphans/orphans.ts
+1 -1
View File
@@ -16,7 +16,7 @@ import {
KTDataNotFoundFailure,
} from "../common/converter-failures";
import { ProcessSuccessfulConverterReturn } from "../score-importing/score-importing";
import { HandlePostImportSteps } from "../score-import-main";
import { HandlePostImportSteps } from "../score-importing/score-import-main";
import { GetUserWithID } from "utils/user";
/**
server/src/lib/score-import/framework/score-import.ts
+39
View File
@@ -0,0 +1,39 @@
import { ServerConfig } from "lib/setup/config";
import { ScoreImportJobData } from "../worker/types";
import { GetInputParser } from "./common/get-input-parser";
import ScoreImportMain from "./score-importing/score-import-main";
import { ImportTypes, ImportDocument } from "tachi-common";
import ScoreImportQueue from "../worker/queue";
/**
* Makes a score import given ScoreImportJobData.
* If USE_EXTERNAL_SCORE_IMPORT_WORKER is set, then this will instead
* place this on the score import queue, and the worker will process it.
*
* Otherwise, it will just perform score importing on the same process.
* @returns An import document if awaited, however, you should not
* await this if you don't need the import document! Import Documents
* may take multiple minutes to generate for large imports. If you control
* the client, make it poll /api/v1/ongoing-imports/:importID.
*/
export async function MakeScoreImport<I extends ImportTypes>(
jobData: ScoreImportJobData<I>
): Promise<ImportDocument> {
if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
const job = await ScoreImportQueue.add(jobData, {
jobId: jobData.importID,
});
return job.finished();
} else {
const InputParser = GetInputParser(jobData);
return ScoreImportMain(
jobData.userID,
jobData.userIntent,
jobData.importType,
InputParser,
jobData.importID
);
}
}
server/src/lib/score-import/framework/score-import-main.ts → server/src/lib/score-import/framework/score-importing/score-import-main.ts
+36 -26
View File
@@ -1,40 +1,51 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import {
Game,
IDStrings,
ImportDocument,
ImportProcessingInfo,
ImportTypes,
ImportDocument,
PublicUserDocument,
Playtypes,
Game,
integer,
IDStrings,
Playtypes,
PublicUserDocument,
} from "tachi-common";
import { KtLogger } from "lib/logger/logger";
import { ImportAllIterableData } from "./score-importing/score-importing";
import { CreateImportLoggerAndID } from "./common/import-logger";
import { CreateSessions } from "./sessions/sessions";
import { GetMillisecondsSince } from "utils/misc";
import { ProcessPBs } from "./pb/process-pbs";
import { UpdateUsersGamePlaytypeStats } from "./user-game-stats/update-ugs";
import db from "external/mongo/db";
import { GetAndUpdateUsersGoals } from "./goals/goals";
import { UpdateUsersMilestones } from "./milestones/milestones";
import { ConverterFunction, ImportInputParser } from "../import-types/common/types";
import { ScorePlaytypeMap } from "./common/types";
import { Converters } from "../import-types/converters";
import { ClassHandler } from "./user-game-stats/types";
import { GetOrSetUserLock, RemoveUserLock } from "./import-locks/lock";
import ScoreImportFatalError from "./score-importing/score-import-error";
import { GetUserWithID } from "utils/user";
import { ConverterFunction, ImportInputParser } from "../../import-types/common/types";
import { Converters } from "../../import-types/converters";
import { InternalFailure } from "../common/converter-failures";
import { CreateScoreLogger } from "../common/import-logger";
import { ScorePlaytypeMap } from "../common/types";
import { GetAndUpdateUsersGoals } from "../goals/goals";
import { GetOrSetUserLock, RemoveUserLock } from "../import-locks/lock";
import { UpdateUsersMilestones } from "../milestones/milestones";
import { ProcessPBs } from "../pb/process-pbs";
import { CreateSessions } from "../sessions/sessions";
import { ClassHandler } from "../user-game-stats/types";
import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs";
import ScoreImportFatalError from "./score-import-error";
import { ImportAllIterableData } from "./score-importing";
/**
* Performs a Score Import.
*/
export default async function ScoreImportMain<D, C>(
user: PublicUserDocument,
userID: integer,
userIntent: boolean,
importType: ImportTypes,
InputParser: ImportInputParser<D, C>,
providedImportObjects?: { logger: KtLogger; importID: string }
importID: string,
providedLogger?: KtLogger
) {
const user = await GetUserWithID(userID);
if (!user) {
throw new InternalFailure(
`User with ID ${userID} does not exist, but attempted to make an import?`
);
}
// in the event of any error, we remove the user lock.
try {
const lock = await GetOrSetUserLock(user.id);
@@ -53,18 +64,17 @@ export default async function ScoreImportMain<D, C>(
}
const timeStarted = Date.now();
let importID;
let logger;
if (!providedImportObjects) {
if (!providedLogger) {
// If they weren't given to us -
// we create an "import logger".
// this holds a reference to the user's name, ID, and type
// of score import for any future debugging.
({ importID, logger } = CreateImportLoggerAndID(user, importType));
logger = CreateScoreLogger(user, importID, importType);
logger.debug("Received import request.");
} else {
({ importID, logger } = providedImportObjects);
logger = providedLogger;
}
// --- 1. Parsing ---
server/src/lib/score-import/import-types/api/arc-sdvx/converter.test.ts
+2 -3
View File
@@ -1,11 +1,10 @@
import deepmerge from "deepmerge";
import CreateLogCtx from "lib/logger/logger";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { GetKTDataJSON, TestingAlbidaADV, TestingSDVXAlbidaSong } from "test-utils/test-data";
import CreateLogCtx from "lib/logger/logger";
import { ConvertAPIArcSDVX, ResolveARCSDVXLamp } from "./converter";
import { ARCSDVXScore } from "./types";
import deepmerge from "deepmerge";
const logger = CreateLogCtx(__filename);
server/src/lib/score-import/import-types/api/arc-sdvx/parser.test.ts
+2 -3
View File
@@ -1,11 +1,10 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import t from "tap";
import { agta } from "test-utils/misc";
import { MockJSONFetch } from "test-utils/mock-fetch";
import ResetDBState from "test-utils/resets";
import CreateLogCtx from "lib/logger/logger";
import { ParseArcSDVX } from "./parser";
import db from "external/mongo/db";
const logger = CreateLogCtx(__filename);
server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts
+4 -4
View File
@@ -1,11 +1,11 @@
import { KtLogger } from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import { integer } from "tachi-common";
import nodeFetch from "utils/fetch";
import { GetArcAuthGuaranteed } from "utils/queries/auth";
import { EmptyObject } from "utils/types";
import { TraverseKaiAPI } from "../../common/api-kai/traverse-api";
import { ParserFunctionReturns } from "../../common/types";
import { EmptyObject } from "utils/types";
import { ServerConfig } from "lib/setup/config";
import { GetArcAuthGuaranteed } from "utils/queries/auth";
import { integer } from "tachi-common";
export async function ParseArcSDVX(
userID: integer,
server/src/lib/score-import/import-types/api/eag-iidx/parser.ts
+1 -1
View File
@@ -1,7 +1,7 @@
import { KtLogger } from "lib/logger/logger";
import { ParseKaiIIDX } from "../../common/api-kai/iidx/parser";
import { integer } from "tachi-common";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import { ParseKaiIIDX } from "../../common/api-kai/iidx/parser";
export async function ParseEagIIDX(userID: integer, logger: KtLogger) {
const authDoc = await GetKaiAuthGuaranteed(userID, "EAG", logger);
server/src/lib/score-import/import-types/common/api-kai/iidx/class-handler.test.ts
-1
View File
@@ -1,7 +1,6 @@
import { IIDXDans } from "lib/constants/classes";
import CreateLogCtx from "lib/logger/logger";
import t from "tap";
import { MockBasicFetch, MockJSONFetch } from "test-utils/mock-fetch";
import ResetDBState from "test-utils/resets";
import { KaiTypeToBaseURL } from "../utils";
server/src/lib/score-import/import-types/common/api-kai/iidx/converter.ts
+6 -6
View File
@@ -1,20 +1,20 @@
import { FindIIDXChartOnInGameIDVersion } from "utils/queries/charts";
import { KaiContext, KaiIIDXScore } from "../types";
import p from "prudence";
import { Lamps } from "tachi-common";
import { FormatPrError } from "utils/prudence";
import { FindIIDXChartOnInGameIDVersion } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import {
InternalFailure,
InvalidScoreFailure,
KTDataNotFoundFailure,
} from "../../../../framework/common/converter-failures";
import { FormatPrError } from "utils/prudence";
import { FindSongOnID } from "utils/queries/songs";
import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../../framework/common/score-utils";
import { Lamps } from "tachi-common";
import { ConverterFunction } from "../../types";
import { DryScore } from "../../../../framework/common/types";
import { ConverterFunction } from "../../types";
import { KaiContext, KaiIIDXScore } from "../types";
const PR_KaiIIDXScore = {
music_id: p.isPositiveInteger,
server/src/lib/score-import/import-types/common/api-kai/iidx/parser.test.ts
+1 -2
View File
@@ -1,8 +1,7 @@
import CreateLogCtx from "lib/logger/logger";
import t from "tap";
import { MockJSONFetch } from "test-utils/mock-fetch";
import ResetDBState from "test-utils/resets";
import CreateLogCtx from "lib/logger/logger";
import { ParseKaiIIDX } from "./parser";
const fakeAuth = {
server/src/lib/score-import/import-types/common/api-kai/reauth.test.ts
-1
View File
@@ -4,7 +4,6 @@ import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import { KaiAuthDocument } from "tachi-common";
import t from "tap";
import { MockJSONFetch } from "test-utils/mock-fetch";
import ResetDBState from "test-utils/resets";
import { CreateKaiReauthFunction } from "./reauth";
server/src/lib/score-import/import-types/common/api-kai/reauth.ts
+2 -2
View File
@@ -1,11 +1,11 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error";
import { GetKaiTypeClientCredentials, KaiTypeToBaseURL } from "./utils";
import nodeFetch from "utils/fetch";
import p from "prudence";
import { KaiAuthDocument } from "tachi-common";
import nodeFetch from "utils/fetch";
import { CreateURLWithParams } from "utils/url";
import { GetKaiTypeClientCredentials, KaiTypeToBaseURL } from "./utils";
const REAUTH_SCHEMA = {
access_token: "string",
server/src/lib/score-import/import-types/common/batch-manual/types.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { Game, Playtypes, IDStrings, GPTSupportedVersions } from "tachi-common";
import { Game, GPTSupportedVersions, IDStrings, Playtypes } from "tachi-common";
export interface BatchManualContext {
game: Game;
server/src/lib/score-import/import-types/common/types.ts
+2 -2
View File
@@ -36,7 +36,6 @@ export interface ImportTypeDataMap {
"ir/barbatos": BarbatosScore;
"ir/fervidex": FervidexScore;
"ir/fervidex-static": FervidexStaticScore;
"ir/chunitachi": BatchManualScore;
"ir/beatoraja": BeatorajaScore;
"ir/usc": USCClientScore;
"ir/kshook-sv3c": KsHookSV3CScore;
@@ -63,7 +62,6 @@ export interface ImportTypeContextMap {
"ir/barbatos": EmptyObject;
"ir/fervidex": FervidexContext;
"ir/fervidex-static": FervidexStaticContext;
"ir/chunitachi": BatchManualContext;
"ir/beatoraja": BeatorajaContext;
"ir/usc": IRUSCContext;
"ir/kshook-sv3c": EmptyObject;
@@ -115,3 +113,5 @@ export interface ParserFunctionReturns<D, C> {
game: Game;
classHandler: ClassHandler | null;
}
export type ParserFunction<D, C, A extends unknown[]> = (...args: A) => ParserFunctionReturns<D, C>;
server/src/lib/score-import/import-types/converters.ts
-1
View File
@@ -40,6 +40,5 @@ export const Converters: ConverterMap = {
"ir/fervidex-static": ConverterIRFervidexStatic,
"ir/direct-manual": ConverterBatchManual,
"ir/usc": ConverterIRUSC,
"ir/chunitachi": ConverterBatchManual,
"ir/kshook-sv3c": ConverterIRKsHookSV3C,
};
server/src/lib/score-import/import-types/ir/barbatos/converter.test.ts
+5 -6
View File
@@ -1,11 +1,10 @@
import t from "tap";
import db from "external/mongo/db";
import ResetDBState from "test-utils/resets";
import CreateLogCtx from "lib/logger/logger";
import { ConverterIRBarbatos } from "./converter";
import deepmerge from "deepmerge";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { barbScore } from "test-utils/test-data";
import { ConverterIRBarbatos } from "./converter";
import { BarbatosScore } from "./types";
const logger = CreateLogCtx(__filename);
server/src/lib/score-import/import-types/ir/barbatos/parser.ts
+3 -3
View File
@@ -1,10 +1,10 @@
import { EmptyObject } from "utils/types";
import { KtLogger } from "lib/logger/logger";
import p from "prudence";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { FormatPrError } from "utils/prudence";
import { BarbatosScore } from "./types";
import { EmptyObject } from "utils/types";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../../common/types";
import { BarbatosScore } from "./types";
const PR_Barbatos = {
difficulty: p.isIn(1, 2, 3, 4),
server/src/lib/score-import/import-types/ir/fervidex-static/class-handler.test.ts
+3 -4
View File
@@ -1,9 +1,8 @@
import t from "tap";
import ResetDBState from "test-utils/resets";
import CreateLogCtx from "lib/logger/logger";
import { Game, Playtypes } from "tachi-common";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { CreateFerStaticClassHandler } from "./class-handler";
import { Playtypes, Game } from "tachi-common";
const logger = CreateLogCtx(__filename);
server/src/lib/score-import/import-types/ir/fervidex-static/converter.ts
+5 -5
View File
@@ -1,15 +1,15 @@
import { Lamps } from "tachi-common";
import { FindChartOnInGameIDVersion } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils";
import {
InternalFailure,
KTDataNotFoundFailure,
} from "../../../framework/common/converter-failures";
import { Lamps } from "tachi-common";
import { FindChartOnInGameIDVersion } from "utils/queries/charts";
import { FervidexStaticContext, FervidexStaticScore } from "./types";
import { FERVIDEX_LAMP_LOOKUP, SplitFervidexChartRef } from "../fervidex/converter";
import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../../common/types";
import { FERVIDEX_LAMP_LOOKUP, SplitFervidexChartRef } from "../fervidex/converter";
import { FervidexStaticContext, FervidexStaticScore } from "./types";
export const ConverterIRFervidexStatic: ConverterFunction<
FervidexStaticScore,
server/src/lib/score-import/import-types/ir/fervidex-static/parser.test.ts
+1 -2
View File
@@ -1,8 +1,7 @@
import CreateLogCtx from "lib/logger/logger";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { GetKTDataJSON } from "test-utils/test-data";
import CreateLogCtx from "lib/logger/logger";
import { ParseFervidexStatic } from "./parser";
const logger = CreateLogCtx(__filename);
server/src/lib/score-import/import-types/ir/fervidex-static/parser.ts
+4 -4
View File
@@ -1,12 +1,12 @@
import { KtLogger } from "lib/logger/logger";
import p, { PrudenceSchema } from "prudence";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { FormatPrError } from "utils/prudence";
import { FervidexStaticContext, FervidexStaticScore } from "./types";
import { FerHeaders, SoftwareIDToVersion } from "../fervidex/parser";
import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts";
import { CreateFerStaticClassHandler } from "./class-handler";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../../common/types";
import { FerHeaders, SoftwareIDToVersion } from "../fervidex/parser";
import { CreateFerStaticClassHandler } from "./class-handler";
import { FervidexStaticContext, FervidexStaticScore } from "./types";
const PR_FervidexStatic: PrudenceSchema = {
ex_score: p.isPositiveInteger,
server/src/lib/score-import/import-types/parsers.ts
+44
View File
@@ -0,0 +1,44 @@
import { ParseArcIIDX } from "./api/arc-iidx/parser";
import { ParseArcSDVX } from "./api/arc-sdvx/parser";
import { ParseEagIIDX } from "./api/eag-iidx/parser";
import { ParseEagSDVX } from "./api/eag-sdvx/parser";
import { ParseFloIIDX } from "./api/flo-iidx/parser";
import { ParseFloSDVX } from "./api/flo-sdvx/parser";
import { ParseMinSDVX } from "./api/min-sdvx/parser";
import { ParseBatchManualFromObject } from "./common/batch-manual/parser";
import ParseBatchManual from "./file/batch-manual/parser";
import ParseEamusementIIDXCSV from "./file/eamusement-iidx-csv/parser";
import { ParseMerIIDX } from "./file/mer-iidx/parser";
import ParsePLIIIDXCSV from "./file/pli-iidx-csv/parser";
import { ParseSolidStateXML } from "./file/solid-state-squad/parser";
import { ParseBarbatosSingle } from "./ir/barbatos/parser";
import { ParseBeatorajaSingle } from "./ir/beatoraja/parser";
import ParseDirectManual from "./ir/direct-manual/parser";
import { ParseFervidexStatic } from "./ir/fervidex-static/parser";
import { ParseFervidexSingle } from "./ir/fervidex/parser";
import { ParseKsHookSV3C } from "./ir/kshook-sv3c/parser";
import { ParseIRUSC } from "./ir/usc/parser";
export const Parsers = {
"file/solid-state-squad": ParseSolidStateXML,
"file/batch-manual": ParseBatchManual,
"file/mer-iidx": ParseMerIIDX,
"file/pli-iidx-csv": ParsePLIIIDXCSV,
"file/eamusement-iidx-csv": ParseEamusementIIDXCSV,
"api/eag-iidx": ParseEagIIDX,
"api/eag-sdvx": ParseEagSDVX,
"api/flo-iidx": ParseFloIIDX,
"api/flo-sdvx": ParseFloSDVX,
"api/min-sdvx": ParseMinSDVX,
"api/arc-iidx": ParseArcIIDX,
"api/arc-sdvx": ParseArcSDVX,
"ir/barbatos": ParseBarbatosSingle,
"ir/beatoraja": ParseBeatorajaSingle,
"ir/fervidex": ParseFervidexSingle,
"ir/fervidex-static": ParseFervidexStatic,
"ir/direct-manual": ParseDirectManual,
"ir/usc": ParseIRUSC,
"ir/kshook-sv3c": ParseKsHookSV3C,
};
server/src/lib/score-import/worker/queue.ts
+31
View File
@@ -0,0 +1,31 @@
import Queue from "bull";
import { RedisClient } from "external/redis/redis";
import CreateLogCtx from "lib/logger/logger";
import { Environment, TachiConfig } from "lib/setup/config";
import ScoreImportFatalError from "../framework/score-importing/score-import-error";
const ScoreImportQueue = new Queue(`${TachiConfig.NAME} Score Import Queue`, Environment.redisUrl);
export default ScoreImportQueue;
const logger = CreateLogCtx("Score Import Queue");
// Log errors if and when they occur.
ScoreImportQueue.on("failed", (job, err) => {
if (err instanceof ScoreImportFatalError) {
logger.info(
`Job ${job.id} hit ScoreImportFatalError (User Fault) with message: ${err.message}`,
err
);
} else {
logger.error(`Job ${job.id} failed unexpectedly with message: ${err.message}`, err);
}
});
ScoreImportQueue.on("completed", (job, result) => {
logger.debug(`Job ${job.id} finished successfully.`, result);
});
export function CloseScoreImportQueue() {
return ScoreImportQueue.close();
}
server/src/lib/score-import/worker/types.ts
+39
View File
@@ -0,0 +1,39 @@
import { Job } from "bull";
import { ImportTypes, integer } from "tachi-common";
import { Parsers } from "../import-types/parsers";
// Ok so, please hear me out on this one.
// We need a type that removes the last element from an array.
// This is because parser functions have variable signatures, but all
// end with requiring a logger. We don't want to have to pass the logger,
// we just want to pass those other arguments.
type RemoveLast<T extends unknown[]> = T extends [...infer RemoveLast, unknown]
? RemoveLast
: unknown[];
// Which means we can use generic access to make ParserArguments<"ir/usc">
// A type that returns the paramaters of the parser function bound to
// "ir/usc".
// And then, using RemoveLast, we can remove that logger argument we don't
// want to provide.
export type ParserArguments<I extends ImportTypes> = RemoveLast<Parameters<typeof Parsers[I]>>;
// Depending on how you look at it, this is either beautiful
// TypeScript power, or brutal TypeScript abuse...
export interface ScoreImportJobData<I extends ImportTypes> {
importType: I;
userID: integer;
userIntent: boolean;
importID: string;
// ...and well, I'm personally on the fence.
// Parsers can't have a consistent signature -- they need info
// from different sources! As such, we need to pass the parser
// arguments in an array, and then it can be respreaded into
// the parser by the import code.
parserArguments: ParserArguments<I>;
}
export type ScoreImportJob<I extends ImportTypes = ImportTypes> = Job<ScoreImportJobData<I>>;
server/src/lib/score-import/worker/worker.ts
+69
View File
@@ -0,0 +1,69 @@
// For scaling performance, running score-importing in a separate worker is preferable
// as that way, other API calls don't get halted by particularly expensive imports on
// all cores. Parallelism can only get us so far in the same process.
// You don't have to run this. If it is being ran, you need to set USE_EXTERNAL_SCORE_IMPORT_WORKER
// in conf.json5. That will ensure all score import jobs are thrown at redis and eventually
// end up here.
// If you don't, score importing will happen on the same thread as your router. That's probably
// fine for lower throughputs, but hey. We're aiming a bit higher.
// Explicitly set this before importing anything!
process.env.IS_SCORE_WORKER_SERVER = "true";
import CreateLogCtx from "lib/logger/logger";
import { ImportTypes } from "tachi-common";
import { FormatUserDoc, GetUserWithID } from "utils/user";
import { GetInputParser } from "../framework/common/get-input-parser";
import ScoreImportMain from "../framework/score-importing/score-import-main";
import ScoreImportQueue from "./queue";
import { ScoreImportJob } from "./types";
const workerLogger = CreateLogCtx(`Import Worker`);
// Exit if we're not called with node. Think of this like if __name__ != "__main__" in python.
if (require.main !== module) {
workerLogger.crit(
"The Score Import Worker was imported, instead of ran directly with node. This is a fatal error. Exiting."
);
process.exit(1);
}
/**
* When a job is fired, this code will actually process the given data
* and import it into the codebase.
*/
ScoreImportQueue.process(async <I extends ImportTypes>(job: ScoreImportJob<I>) => {
const user = await GetUserWithID(job.data.userID);
if (!user) {
workerLogger.severe(
`Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).`
);
throw new Error(
`Couldn't find user with ID ${job.data.userID}. Yet a score import from them was made? (Job ID ${job.id}).`
);
}
// Create a logger that we can pass around for context.
// This helps us debug what score import did what!
const logger = CreateLogCtx(`Score Import ${job.id} ${FormatUserDoc(user)}`);
const InputParser = GetInputParser(job.data);
logger.debug(`Starting import.`);
const importDocument = await ScoreImportMain(
user.id,
job.data.userIntent,
job.data.importType,
InputParser,
job.data.importID,
logger
);
logger.debug(`Finished import.`);
return importDocument;
});
server/src/lib/search/search.test.ts
+1 -2
View File
@@ -1,6 +1,5 @@
import t from "tap";
import db from "external/mongo/db";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { LoadTachiIIDXData } from "test-utils/test-data";
import { SearchGameSongs, SearchUsersRegExp } from "./search";
server/src/lib/search/search.ts
+8 -8
View File
@@ -1,20 +1,20 @@
import { ICollection } from "monk";
import { FilterQuery } from "mongodb";
import CreateLogCtx from "lib/logger/logger";
import deepmerge from "deepmerge";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
import { FilterQuery } from "mongodb";
import { ICollection } from "monk";
import {
Game,
SessionDocument,
Playtypes,
SongDocument,
ChartDocument,
Game,
integer,
Playtypes,
PublicUserDocument,
SessionDocument,
SongDocument,
} from "tachi-common";
import { EscapeStringRegexp } from "utils/misc";
import { GetOnlineCutoff } from "utils/user";
import { TachiConfig } from "lib/setup/config";
const logger = CreateLogCtx(__filename);
server/src/lib/setup/config.ts
+2
View File
@@ -64,6 +64,7 @@ export interface TachiServerConfig {
RATE_LIMIT: integer;
OAUTH_CLIENT_CAP: integer;
OPTIONS_ALWAYS_SUCCEEDS?: boolean;
USE_EXTERNAL_SCORE_IMPORT_WORKER?: boolean;
EMAIL_CONFIG?: {
FROM: string;
DKIM?: SendMailOptions["dkim"];
@@ -125,6 +126,7 @@ const err = p(config, {
RATE_LIMIT: p.optional(p.isPositiveInteger),
OAUTH_CLIENT_CAP: p.optional(p.isPositiveInteger),
OPTIONS_ALWAYS_SUCCEEDS: "*boolean",
USE_EXTERNAL_SCORE_IMPORT_WORKER: "*boolean",
EMAIL_CONFIG: p.optional({
FROM: "string",
DKIM: "*object",
server/src/main.ts
+4
View File
@@ -14,6 +14,7 @@ import { FormatVersion } from "./lib/constants/version";
import fetch from "utils/fetch";
import http from "http";
import { CloseRedisConnection } from "external/redis/redis";
import { CloseScoreImportQueue } from "lib/score-import/worker/queue";
const logger = CreateLogCtx(__filename);
@@ -79,6 +80,9 @@ process.on("SIGTERM", () => {
logger.info("Closing Redis Connection.", { shutdownInfo: true });
CloseRedisConnection();
logger.info("Closing Score Import Queue.", { shutdownInfo: true });
CloseScoreImportQueue();
logger.info("Everything closed. Waiting for process to exit naturally.", {
shutdownInfo: true,
});
server/src/server/middleware/auth.test.ts
+6 -3
View File
@@ -1,10 +1,9 @@
import t from "tap";
import { SetRequestPermissions, AllPermissions } from "./auth";
import expMiddlewareMock from "express-request-mock";
import db from "external/mongo/db";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { AllPermissions, SetRequestPermissions } from "./auth";
t.test("#SetRequestPermissions", (t) => {
t.beforeEach(ResetDBState);
@@ -17,6 +16,7 @@ t.test("#SetRequestPermissions", (t) => {
customise_profile: true,
},
token: "mock_token",
fromAPIClient: null,
});
const { req } = await expMiddlewareMock(SetRequestPermissions, {
@@ -32,6 +32,7 @@ t.test("#SetRequestPermissions", (t) => {
customise_profile: true,
},
token: "mock_token",
fromAPIClient: null,
});
t.end();
@@ -45,6 +46,7 @@ t.test("#SetRequestPermissions", (t) => {
identifier: "Guest Token",
permissions: {},
token: null,
fromAPIClient: null,
});
t.end();
@@ -110,6 +112,7 @@ t.test("#SetRequestPermissions", (t) => {
identifier: `Session-Key 1`,
token: null,
permissions: AllPermissions,
fromAPIClient: null,
});
t.end();
});
server/src/server/middleware/auth.ts
+3
View File
@@ -12,6 +12,7 @@ const GuestToken: APITokenDocument = {
userID: null,
identifier: "Guest Token",
permissions: {},
fromAPIClient: null,
};
export const AllPermissions: Record<APIPermissions, true> = {
@@ -46,6 +47,7 @@ function CreateSetRequestPermissions(errorKeyName: string): RequestHandler {
identifier: `Session-Key ${req.session.tachi.user.id}`,
token: null,
permissions: AllPermissions,
fromAPIClient: null,
};
return next();
}
@@ -91,6 +93,7 @@ function CreateSetRequestPermissions(errorKeyName: string): RequestHandler {
token,
permissions: apiTokenData.permissions,
identifier: apiTokenData.identifier,
fromAPIClient: apiTokenData.fromAPIClient,
};
return next();
server/src/server/router/api/v1/import/router.ts
+87 -115
View File
@@ -1,30 +1,20 @@
import { Router } from "express";
import { APIImportTypes, FileUploadImportTypes, integer } from "tachi-common";
import Prudence from "prudence";
import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user";
import CreateLogCtx, { KtLogger } from "lib/logger/logger";
import prValidate from "server/middleware/prudence-validate";
import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error";
import { SIXTEEN_MEGABTYES } from "lib/constants/filesize";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload";
import ParseEamusementIIDXCSV from "lib/score-import/import-types/file/eamusement-iidx-csv/parser";
import ParseBatchManual from "lib/score-import/import-types/file/batch-manual/parser";
import { ParseSolidStateXML } from "lib/score-import/import-types/file/solid-state-squad/parser";
import { ParseMerIIDX } from "lib/score-import/import-types/file/mer-iidx/parser";
import ParsePLIIIDXCSV from "lib/score-import/import-types/file/pli-iidx-csv/parser";
import { TachiConfig } from "lib/setup/config";
import { RequirePermissions } from "server/middleware/auth";
import { ParseEagIIDX } from "lib/score-import/import-types/api/eag-iidx/parser";
import { ParseEagSDVX } from "lib/score-import/import-types/api/eag-sdvx/parser";
import { ParseFloIIDX } from "lib/score-import/import-types/api/flo-iidx/parser";
import { ParseFloSDVX } from "lib/score-import/import-types/api/flo-sdvx/parser";
import { ParseMinSDVX } from "lib/score-import/import-types/api/min-sdvx/parser";
import { ParseArcSDVX } from "lib/score-import/import-types/api/arc-sdvx/parser";
import { ParseArcIIDX } from "lib/score-import/import-types/api/arc-iidx/parser";
import db from "external/mongo/db";
import { SIXTEEN_MEGABTYES } from "lib/constants/filesize";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import CreateLogCtx from "lib/logger/logger";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
import { MakeScoreImport } from "lib/score-import/framework/score-import";
import { ScoreImportJobData } from "lib/score-import/worker/types";
import { ServerConfig, TachiConfig } from "lib/setup/config";
import Prudence from "prudence";
import { RequirePermissions } from "server/middleware/auth";
import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload";
import prValidate from "server/middleware/prudence-validate";
import { APIImportTypes, FileUploadImportTypes } from "tachi-common";
import { Random20Hex } from "utils/misc";
import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user";
const logger = CreateLogCtx(__filename);
@@ -37,9 +27,14 @@ const ParseMultipartScoredata = CreateMulterSingleUploadMiddleware(
);
const fileImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("file/"));
const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/"));
/**
* Import scores from a file. Expects the post request to be multipart, and to provide a scoreData file.
*
* @param importType - The import type for this file.
* @param file - The actual file. Should be passed as multipart.
*
* @name POST /api/v1/import/file
*/
router.post(
@@ -63,30 +58,45 @@ router.post(
const importType = req.body.importType as FileUploadImportTypes;
const inputParser = (logger: KtLogger) =>
ResolveFileUploadData(importType, req.file!, req.body, logger);
const userIntent = !!req.header("X-User-Intent");
const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id);
if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
const importID = Random20Hex();
// The <any, any> here is deliberate - TS picks the IIDX-CSV generic values
// for this function call because it sees them first
// but that is ABSOLUTELY not what is actually occuring.
// We use this as an override because we know better.
// see: https://www.typescriptlang.org/play?ts=4.3.0-beta#code/GYVwdgxgLglg9mABAQQDwBUB8AKYc4BciAYvhpgJSIDeiAsAFCKID0LiAJnAKYDOivKCGDBGAX0aMYYKNwBOwAIYRuJMlhqNmzAEaK5RdOMkNQkWAkQAlbkLlgAynAC23UnGxVqW7W0QAHOVtuMA5EKAALGH5o8IjVIN4QABsoRDhgARdVaX8QNOlBbkUwjMQ5RVCXH2YYTOwAWUVIgDoKqudPRFREAAYWgFYvGu1mILskWj0DRAAiQTlpAHNZxAkmbXXRkfGQexpEaaIARgAmAGY14wZGZNt0vfdEAF5rWz3HbPdPAG4TZGwcEe+AoPyAA
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const responseData = await ExpressWrappedScoreImportMain<any, any>(
userDoc,
true,
importType,
inputParser
);
const job: ScoreImportJobData<FileUploadImportTypes> = {
importID,
userID: req[SYMBOL_TachiAPIAuth].userID!,
userIntent,
importType,
parserArguments: [req.file, req.body],
};
return res.status(responseData.statusCode).json(responseData.body);
// Fire the score import, but make no guarantees about its state.
MakeScoreImport<FileUploadImportTypes>(job);
return res.status(202).json({
success: true,
description:
"Import loaded into queue. You can poll the provided URL for information on when its complete.",
body: {
url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`,
importID,
},
});
} else {
// Fire the score import and wait for it to finish!
const importResponse = await ExpressWrappedScoreImportMain<FileUploadImportTypes>(
req[SYMBOL_TachiAPIAuth].userID!,
userIntent,
importType,
[req.file, req.body]
);
return res.status(importResponse.statusCode).json(importResponse.body);
}
}
);
const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/"));
/**
* Import scores from another API. This typically will perform a full sync.
* @name POST /api/v1/import/from-api
@@ -104,21 +114,44 @@ router.post(
async (req, res) => {
const importType = req.body.importType as APIImportTypes;
const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id);
const importID = Random20Hex();
const inputParser = (logger: KtLogger) =>
ResolveAPIImportParser(userDoc.id, importType, logger);
const userID = req[SYMBOL_TachiAPIAuth].userID!;
// see the argument above about typescript falsely expanding types.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const responseData = await ExpressWrappedScoreImportMain<any, any>(
userDoc,
true,
importType,
inputParser
);
const userIntent = !!req.header("X-User-Intent");
return res.status(responseData.statusCode).json(responseData.body);
if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
const job: ScoreImportJobData<APIImportTypes> = {
importID,
userID,
userIntent,
importType,
parserArguments: [userID],
};
// Fire the score import, but make no guarantees about its state.
MakeScoreImport<APIImportTypes>(job);
return res.status(202).json({
success: true,
description:
"Import loaded into queue. You can poll the provided URL for information on when its complete.",
body: {
url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`,
importID,
},
});
} else {
// Fire the score import and wait for it to finish!
const importResponse = await ExpressWrappedScoreImportMain<APIImportTypes>(
userID,
userIntent,
importType,
[userID]
);
return res.status(importResponse.statusCode).json(importResponse.body);
}
}
);
@@ -178,65 +211,4 @@ router.post("/orphans", RequirePermissions("submit_score"), async (req, res) =>
});
});
/**
* Resolves the data from a file upload into an iterable,
* The appropriate processing function to map that iterable over,
* and and any context the processing may need (such as playtype)
*
* This also performs validation on the type of file uploaded.
* @param importType - The type of import request this was.
* @param fileData - The data sent by the user.
* @param body - Other data passed by the user in the request body.
*/
export function ResolveFileUploadData(
importType: FileUploadImportTypes,
fileData: Express.Multer.File,
body: Record<string, unknown>,
logger: KtLogger
) {
switch (importType) {
case "file/eamusement-iidx-csv":
return ParseEamusementIIDXCSV(fileData, body, logger);
case "file/pli-iidx-csv":
return ParsePLIIIDXCSV(fileData, body, logger);
case "file/batch-manual":
return ParseBatchManual(fileData, body, logger);
case "file/solid-state-squad":
return ParseSolidStateXML(fileData, body, logger);
case "file/mer-iidx":
return ParseMerIIDX(fileData, body, logger);
default:
logger.error(
`importType ${importType} made it into ResolveFileUploadData, but should have been rejected by Prudence.`
);
throw new ScoreImportFatalError(400, `Invalid importType of ${importType}.`);
}
}
export function ResolveAPIImportParser(
userID: integer,
importType: APIImportTypes,
logger: KtLogger
) {
switch (importType) {
case "api/eag-iidx":
return ParseEagIIDX(userID, logger);
case "api/eag-sdvx":
return ParseEagSDVX(userID, logger);
case "api/flo-iidx":
return ParseFloIIDX(userID, logger);
case "api/flo-sdvx":
return ParseFloSDVX(userID, logger);
case "api/min-sdvx":
return ParseMinSDVX(userID, logger);
case "api/arc-iidx":
return ParseArcIIDX(userID, logger);
case "api/arc-sdvx":
return ParseArcSDVX(userID, logger);
default:
logger.error(`Unknown importType ${importType} has no handler?`);
throw new ScoreImportFatalError(500, `Unknown importType ${importType}.`);
}
}
export default router;
server/src/server/router/api/v1/imports/router.ts
+87
View File
@@ -1,6 +1,9 @@
import { Router } from "express";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error";
import ScoreImportQueue from "lib/score-import/worker/queue";
import { ServerConfig, TachiConfig } from "lib/setup/config";
import { GetRelevantSongsAndCharts } from "utils/db";
import { GetUserWithID } from "utils/user";
@@ -8,6 +11,11 @@ const router: Router = Router({ mergeParams: true });
const logger = CreateLogCtx(__filename);
/**
* Retrieve an import with this ID.
*
* @name GET /api/v1/imports/:importID
*/
router.get("/:importID", async (req, res) => {
const importDoc = await db.imports.findOne({
importID: req.params.importID,
@@ -54,4 +62,83 @@ router.get("/:importID", async (req, res) => {
});
});
/**
* Retrieve the status of an ongoing import.
* If the import has been finalised and was successful, return 200.
*
* If the import is ongoing, return its progress.
*
* If the import was never ongoing, return 404.
*
* If the import was finalised and was unsuccessful (i.e. threw a fatal error)
* return its error information in expressified form.
*
* @name GET /api/v1/import/:importID/poll-status
*/
router.get("/:importID/poll-status", async (req, res) => {
if (!ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
return res.status(501).json({
success: false,
description: `${TachiConfig.NAME} does not use an external score import worker. Polling imports is not possible. This import may be ongoing, or it may have never occured.`,
});
}
const importDoc = await db.imports.findOne({ importID: req.params.importID });
if (importDoc) {
return res.status(200).json({
success: true,
description: `Import was completed!`,
body: {
importStatus: "completed",
},
});
}
const job = await ScoreImportQueue.getJob(req.params.importID);
if (!job) {
return res.status(404).json({
success: false,
description: `There is no ongoing import here.`,
});
}
if (job.isFailed()) {
const err = await job.finished();
if (err instanceof ScoreImportFatalError) {
logger.info(err.message);
return res.status(err.statusCode).json({
success: false,
description: err.message,
});
}
return res.status(500).json({
success: false,
description: `An internal service error has occured with this import. This has been reported!`,
});
} else if (job.isCompleted()) {
return res.status(200).json({
success: true,
description: `Import was completed!`,
body: {
importStatus: "completed",
},
});
}
const progress = await job.progress();
return res.status(200).json({
success: true,
description: `Import is ongoing.`,
body: {
importStatus: "ongoing",
progress,
},
});
});
export default router;
server/src/server/router/api/v1/scores/_scoreID/router.test.ts
+5
View File
@@ -134,6 +134,7 @@ t.test("PATCH /api/v1/scores/:scoreID", (t) => {
permissions: {
customise_score: true,
},
fromAPIClient: null,
});
const res = await mockApi
@@ -156,6 +157,7 @@ t.test("PATCH /api/v1/scores/:scoreID", (t) => {
userID: 1,
identifier: "another fake token",
permissions: {},
fromAPIClient: null,
});
const res = await mockApi
@@ -186,6 +188,7 @@ t.test("DELETE /api/v1/scores/:scoreID", (t) => {
delete_score: true,
},
token: "foo",
fromAPIClient: null,
});
const res = await mockApi
@@ -209,6 +212,7 @@ t.test("DELETE /api/v1/scores/:scoreID", (t) => {
permissions: {
delete_score: true,
},
fromAPIClient: null,
});
const res = await mockApi
@@ -231,6 +235,7 @@ t.test("DELETE /api/v1/scores/:scoreID", (t) => {
userID: 1,
identifier: "another fake token",
permissions: {},
fromAPIClient: null,
});
const res = await mockApi
server/src/server/router/api/v1/sessions/_sessionID/router.test.ts
+2
View File
@@ -161,6 +161,7 @@ t.test("PATCH /api/v1/sessions/:sessionID", (t) => {
permissions: {
customise_session: true,
},
fromAPIClient: null,
});
const res = await mockApi
@@ -183,6 +184,7 @@ t.test("PATCH /api/v1/sessions/:sessionID", (t) => {
userID: 1,
identifier: "another fake token",
permissions: {},
fromAPIClient: null,
});
const res = await mockApi
server/src/server/router/api/v1/users/_userID/api-tokens/router.test.ts
+8 -2
View File
@@ -15,18 +15,21 @@ t.test("GET /api/v1/users/:userID/api-tokens", async (t) => {
identifier: "foo",
permissions: {},
token: "tfoo",
fromAPIClient: null,
},
{
userID: 1,
identifier: "bar",
permissions: {},
token: "tbar",
fromAPIClient: null,
},
{
userID: 2,
identifier: "baz",
permissions: {},
token: "tbaz",
fromAPIClient: null,
},
]);
});
@@ -47,12 +50,14 @@ t.test("GET /api/v1/users/:userID/api-tokens", async (t) => {
identifier: "foo",
permissions: {},
token: "tfoo",
fromAPIClient: null,
},
{
userID: 1,
identifier: "bar",
permissions: {},
token: "tbar",
fromAPIClient: null,
},
]
);
@@ -117,6 +122,7 @@ t.test("DELETE /api/v1/users/:userID/api-tokens/:token", async (t) => {
permissions: {},
token: "foo",
userID: 2,
fromAPIClient: null,
});
const res = await mockApi.delete("/api/v1/users/1/api-tokens/foo").set("Cookie", cookie);
@@ -164,7 +170,7 @@ t.test("POST /api/v1/users/:userID/api-tokens/create", async (t) => {
identifier: "Hello World",
permissions: { submit_score: true, customise_profile: true },
userID: 1,
fromAPIClient: undefined,
fromAPIClient: null,
},
"Should return a conforming API Token."
);
@@ -179,7 +185,7 @@ t.test("POST /api/v1/users/:userID/api-tokens/create", async (t) => {
identifier: "Hello World",
permissions: { submit_score: true, customise_profile: true },
userID: 1,
fromAPIClient: undefined,
fromAPIClient: null,
},
"Should insert a conforming API Token into the database."
);
server/src/server/router/api/v1/users/_userID/api-tokens/router.ts
+1 -5
View File
@@ -66,7 +66,7 @@ router.post(
const user = req[SYMBOL_TachiData]!.requestedUser!;
let identifier: string;
let fromAPIClient;
let fromAPIClient = null;
if (req.body.clientID) {
const client = await db["api-clients"].findOne(
@@ -131,10 +131,6 @@ router.post(
fromAPIClient,
};
// Inserting { fromAPIClient: undefined } results in bson
// conversion to {fromAPIClient: null}, which is not the same
DeleteUndefinedProps(apiTokenDocument);
await db["api-tokens"].insert(apiTokenDocument);
logger.info(`Inserted new API Key for ${FormatUserDoc(user)}.`);
server/src/server/router/api/v1/users/_userID/games/_game/_playtype/settings/router.test.ts
+5
View File
@@ -43,6 +43,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => {
customise_profile: true,
},
token: "api_token",
fromAPIClient: null,
});
const res = await mockApi
@@ -90,6 +91,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => {
customise_profile: true,
},
token: "api_token",
fromAPIClient: null,
});
const res = await mockApi
@@ -140,6 +142,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => {
customise_profile: true,
},
token: "api_token",
fromAPIClient: null,
});
const res = await mockApi
@@ -159,6 +162,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => {
customise_profile: false,
},
token: "api_token",
fromAPIClient: null,
});
const res = await mockApi
@@ -178,6 +182,7 @@ t.test("PATCH /api/v1/users/:userID/games/:game/:playtype/settings", (t) => {
customise_profile: true,
},
token: "api_token",
fromAPIClient: null,
});
for (const key of [
server/src/server/router/api/v1/users/_userID/games/_game/_playtype/showcase/router.test.ts
+3
View File
@@ -260,6 +260,7 @@ t.test("PUT /api/v1/users/:userID/games/:game/:playtype/showcase", (t) => {
customise_profile: true,
},
token: "alt_token",
fromAPIClient: null,
})
);
@@ -271,6 +272,7 @@ t.test("PUT /api/v1/users/:userID/games/:game/:playtype/showcase", (t) => {
customise_profile: true,
},
token: "altz_token",
fromAPIClient: null,
});
const res = await mockApi
@@ -290,6 +292,7 @@ t.test("PUT /api/v1/users/:userID/games/:game/:playtype/showcase", (t) => {
customise_profile: false,
},
token: "unauth_token",
fromAPIClient: null,
});
const res = await mockApi
server/src/server/router/api/v1/users/_userID/integrations/arc/router.test.ts
+1
View File
@@ -39,6 +39,7 @@ t.test("GET /api/v1/users/:userID/integrations/arc", async (t) => {
permissions: {},
token: "foobar",
userID: 1,
fromAPIClient: null,
});
const res = await mockApi
server/src/server/router/api/v1/users/_userID/router.test.ts
+4 -4
View File
@@ -1,10 +1,9 @@
import t from "tap";
import db from "external/mongo/db";
import { UserGameStats } from "tachi-common";
import t from "tap";
import { CreateFakeAuthCookie } from "test-utils/fake-auth";
import mockApi from "test-utils/mock-api";
import ResetDBState from "test-utils/resets";
import { UserGameStats } from "tachi-common";
import { CreateFakeAuthCookie } from "test-utils/fake-auth";
t.test("GET /api/v1/users/:userID", (t) => {
t.beforeEach(ResetDBState);
@@ -86,6 +85,7 @@ t.test("PATCH /api/v1/users/:userID", async (t) => {
permissions: {},
identifier: "No permissions token",
userID: 1,
fromAPIClient: null,
});
const res2 = await mockApi.patch("/api/v1/users/1").set("Authorization", "Bearer noperm");
server/src/server/router/api/v1/users/_userID/settings/router.test.ts
+1
View File
@@ -144,6 +144,7 @@ t.test("PATCH /api/v1/users/:userID/settings", async (t) => {
},
token: "no_perm",
userID: 1,
fromAPIClient: null,
});
const res = await mockApi
server/src/server/router/ir/barbatos/router.test.ts
+5 -6
View File
@@ -1,10 +1,9 @@
import t from "tap";
import { CreateFakeAuthCookie, InsertFakeTokenWithAllPerms } from "test-utils/fake-auth";
import ResetDBState from "test-utils/resets";
import mockApi from "test-utils/mock-api";
import { TestingBarbatosScore } from "test-utils/test-data";
import db from "external/mongo/db";
import t from "tap";
import { InsertFakeTokenWithAllPerms } from "test-utils/fake-auth";
import mockApi from "test-utils/mock-api";
import ResetDBState from "test-utils/resets";
import { TestingBarbatosScore } from "test-utils/test-data";
t.test("POST /ir/barbatos/score/submit", (t) => {
t.beforeEach(ResetDBState);
server/src/server/router/ir/barbatos/router.ts
+2 -6
View File
@@ -1,9 +1,7 @@
import { Router } from "express";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { ParseBarbatosSingle } from "lib/score-import/import-types/ir/barbatos/parser";
import { RequirePermissions } from "server/middleware/auth";
import { GetUserWithIDGuaranteed } from "utils/user";
const router: Router = Router({ mergeParams: true });
@@ -14,13 +12,11 @@ router.use(RequirePermissions("submit_score"));
* @name POST /ir/barbatos/score/submit
*/
router.post("/score/submit", async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth]!.userID!);
const responseData = await ExpressWrappedScoreImportMain(
userDoc,
req[SYMBOL_TachiAPIAuth]!.userID!,
false,
"ir/barbatos",
(logger) => ParseBarbatosSingle(req.body, logger)
[req.body]
);
return res.status(responseData.statusCode).json(responseData.body);
server/src/server/router/ir/beatoraja/auth.test.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import expMiddlewareMock from "express-request-mock";
import t from "tap";
import { ValidateIRClientVersion } from "./auth";
import expMiddlewareMock from "express-request-mock";
t.test("#ValidateIRClientVersion", (t) => {
t.test("Should reject clients that are not supported", async (t) => {
server/src/server/router/ir/beatoraja/router.test.ts
+7 -4
View File
@@ -1,11 +1,10 @@
import t from "tap";
import deepmerge from "deepmerge";
import db from "external/mongo/db";
import { PublicUserDocument } from "tachi-common";
import t from "tap";
import mockApi from "test-utils/mock-api";
import ResetDBState from "test-utils/resets";
import { GetKTDataJSON } from "test-utils/test-data";
import deepmerge from "deepmerge";
import { PublicUserDocument } from "tachi-common";
t.test("POST /ir/beatoraja/submit-score", (t) => {
t.beforeEach(ResetDBState);
@@ -17,6 +16,7 @@ t.test("POST /ir/beatoraja/submit-score", (t) => {
submit_score: true,
},
token: "mock_token",
fromAPIClient: null,
})
);
@@ -142,12 +142,14 @@ t.test("POST /ir/beatoraja/submit-score", (t) => {
identifier: "token2",
permissions: { submit_score: true },
token: "token2",
fromAPIClient: null,
},
{
userID: 3,
identifier: "token3",
permissions: { submit_score: true },
token: "token3",
fromAPIClient: null,
},
]);
@@ -320,6 +322,7 @@ t.test("POST /ir/beatoraja/submit-course", (t) => {
submit_score: true,
},
token: "mock_token",
fromAPIClient: null,
})
);
server/src/server/router/ir/beatoraja/router.ts
+10 -15
View File
@@ -1,14 +1,11 @@
import { Router } from "express";
import db from "external/mongo/db";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import CreateLogCtx, { KtLogger } from "lib/logger/logger";
import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue";
import CreateLogCtx from "lib/logger/logger";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { ParseBeatorajaSingle } from "lib/score-import/import-types/ir/beatoraja/parser";
import { ServerConfig } from "lib/setup/config";
import { RequireNotGuest } from "server/middleware/auth";
import { UpdateClassIfGreater } from "utils/class";
import { GetUserWithIDGuaranteed } from "utils/user";
import { ValidateIRClientVersion } from "./auth";
import chartsRouter from "./charts/router";
@@ -24,21 +21,19 @@ router.use(ValidateIRClientVersion);
* @name POST /ir/beatoraja/submit-score
*/
router.post("/submit-score", RequireNotGuest, async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth]!.userID!);
const userID = req[SYMBOL_TachiAPIAuth]!.userID!;
const ParserFunction = (logger: KtLogger) => ParseBeatorajaSingle(req.body, userDoc.id, logger);
const importRes = await ExpressWrappedScoreImportMain(
userDoc,
false,
"ir/beatoraja",
ParserFunction
);
const importRes = await ExpressWrappedScoreImportMain(userID, false, "ir/beatoraja", [
req.body,
userID,
]);
if (!importRes.body.success) {
return res.status(400).json(importRes.body);
} else if (importRes.body.body.errors.length !== 0) {
const type = importRes.body.body.errors[0].type;
const errMsg = importRes.body.body.errors[0].message;
if (type === "KTDataNotFound") {
return res.status(202).json({
success: true,
@@ -47,7 +42,7 @@ router.post("/submit-score", RequireNotGuest, async (req, res) => {
} else if (type === "InternalError") {
return res.status(500).json({
success: false,
description: `[${importRes.body.body.errors[0].type}] - ${importRes.body.body.errors[0].message}`,
description: `[${type}] - ${errMsg}`,
});
}
@@ -56,7 +51,7 @@ router.post("/submit-score", RequireNotGuest, async (req, res) => {
return res.status(400).json({
success: false,
description: `[${importRes.body.body.errors[0].type}] - ${importRes.body.body.errors[0].message}`,
description: `[${type}] - ${errMsg}`,
});
} else if (importRes.body.body.scoreIDs.length === 0) {
return res.status(400).json({
server/src/server/router/ir/chunitachi/router.test.ts
-86
View File
@@ -1,86 +0,0 @@
import t from "tap";
import db from "external/mongo/db";
import { InsertFakeTokenWithAllPerms } from "test-utils/fake-auth";
import mockApi from "test-utils/mock-api";
import ResetDBState from "test-utils/resets";
import { GetKTDataJSON } from "test-utils/test-data";
import deepmerge from "deepmerge";
t.test("POST /ir/chunitachi/import", (t) => {
t.beforeEach(ResetDBState);
t.beforeEach(InsertFakeTokenWithAllPerms("mock_token"));
const chunitachiBody = GetKTDataJSON("./batch-manual/chunitachi.json");
t.test("Should work for CHUNITACHI requests", async (t) => {
const res = await mockApi
.post("/ir/chunitachi/import")
.set("Authorization", `Bearer mock_token`)
.send(chunitachiBody);
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.count({ service: "ChunItachi" });
t.equal(scoreCount, 1, "Should import one score.");
t.end();
});
t.test("Should reject invalid batch-manual", async (t) => {
const res = await mockApi
.post("/ir/chunitachi/import")
.set("Authorization", `Bearer mock_token`)
.send({});
t.equal(res.body.success, false, "Should not be successful");
t.end();
});
t.test("Should reject batch-manual requests if game is not chunithm", async (t) => {
const res = await mockApi
.post("/ir/chunitachi/import")
.set("Authorization", `Bearer mock_token`)
.send(deepmerge(chunitachiBody, { meta: { game: "iidx" } }));
t.equal(res.body.success, false, "Should not be successful");
t.end();
});
t.test("Should reject batch-manual requests if service is not Chunitachi", async (t) => {
const res = await mockApi
.post("/ir/chunitachi/import")
.set("Authorization", `Bearer mock_token`)
.send(deepmerge(chunitachiBody, { meta: { service: "foo bar" } }));
t.equal(res.body.success, false, "Should not be successful");
t.end();
});
t.test("Should require authentication.", async (t) => {
const res = await mockApi.post("/ir/chunitachi/import").send(chunitachiBody);
t.equal(res.statusCode, 401);
t.end();
});
t.test("Should require a valid auth token.", async (t) => {
const res = await mockApi
.post("/ir/chunitachi/import")
.set("Authorization", "Bearer invalid_token")
.send(chunitachiBody);
t.equal(res.statusCode, 401);
t.end();
});
t.end();
});
server/src/server/router/ir/chunitachi/router.ts
-41
View File
@@ -1,41 +0,0 @@
import { Router } from "express";
import { GetUserWithIDGuaranteed } from "utils/user";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import ParseDirectManual from "lib/score-import/import-types/ir/direct-manual/parser";
import { RequirePermissions } from "server/middleware/auth";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
const router: Router = Router({ mergeParams: true });
/**
* Submits a single score document from Chunitachi clients.
* @name POST /ir/chunitachi/import
*/
router.post("/import", RequirePermissions("submit_score"), async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!);
if (req.body?.meta?.game !== "chunithm") {
return res.status(400).json({
success: false,
description: `Invalid Game. Expected 'chunithm', but got ${req.body?.meta?.game}`,
});
}
if (req.body.meta.service !== "ChunItachi") {
return res.status(400).json({
success: false,
description: `Unexpected service ${req.body.head.service} -- expected 'Chunitachi'`,
});
}
const responseData = await ExpressWrappedScoreImportMain(
userDoc,
false,
"ir/chunitachi",
(logger) => ParseDirectManual(req.body, logger)
);
return res.status(responseData.statusCode).json(responseData.body);
});
export default router;
server/src/server/router/ir/direct-manual/router.test.ts
+62 -2
View File
@@ -1,6 +1,6 @@
import t from "tap";
import db from "external/mongo/db";
import { rootLogger } from "lib/logger/logger";
import t from "tap";
import { CreateFakeAuthCookie } from "test-utils/fake-auth";
import mockApi from "test-utils/mock-api";
import ResetDBState from "test-utils/resets";
@@ -19,6 +19,7 @@ t.test("POST /ir/direct-manual/import", async (t) => {
submit_score: false,
},
userID: 1,
fromAPIClient: null,
});
const res = await mockApi
@@ -78,5 +79,64 @@ t.test("POST /ir/direct-manual/import", async (t) => {
t.end();
});
t.beforeEach(() =>
db["api-tokens"].insert({
token: "mock_token",
fromAPIClient: null,
identifier: "Mock CHUNITACHI Token",
permissions: { submit_score: true },
userID: 1,
})
);
const chunitachiBody = GetKTDataJSON("./batch-manual/chunitachi.json");
t.test("Should work for CHUNITACHI requests", async (t) => {
const res = await mockApi
.post("/ir/direct-manual/import")
.set("Authorization", `Bearer mock_token`)
.send(chunitachiBody);
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.count({ service: "ChunItachi (DIRECT-MANUAL)" });
t.equal(scoreCount, 1, "Should import one score.");
t.end();
});
t.test("Should reject invalid batch-manual", async (t) => {
const res = await mockApi
.post("/ir/direct-manual/import")
.set("Authorization", `Bearer mock_token`)
.send({});
t.equal(res.body.success, false, "Should not be successful");
t.end();
});
t.test("Should require authentication.", async (t) => {
const res = await mockApi.post("/ir/direct-manual/import").send(chunitachiBody);
t.equal(res.statusCode, 401);
t.end();
});
t.test("Should require a valid auth token.", async (t) => {
const res = await mockApi
.post("/ir/direct-manual/import")
.set("Authorization", "Bearer invalid_token")
.send(chunitachiBody);
t.equal(res.statusCode, 401);
t.end();
});
t.end();
});
server/src/server/router/ir/direct-manual/router.ts
+2 -4
View File
@@ -12,15 +12,13 @@ const router: Router = Router({ mergeParams: true });
* @name POST /ir/direct-manual/import
*/
router.post("/import", RequirePermissions("submit_score"), async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!);
const intent = req.header("X-User-Intent");
const responseData = await ExpressWrappedScoreImportMain(
userDoc,
req[SYMBOL_TachiAPIAuth].userID!,
!!intent,
"ir/direct-manual",
(logger) => ParseDirectManual(req.body, logger)
[req.body]
);
return res.status(responseData.statusCode).json(responseData.body);
server/src/server/router/ir/fervidex/router.test.ts
+1 -2
View File
@@ -1,6 +1,5 @@
import t from "tap";
import db from "external/mongo/db";
import t from "tap";
import { InsertFakeTokenWithAllPerms } from "test-utils/fake-auth";
import mockApi from "test-utils/mock-api";
import ResetDBState from "test-utils/resets";
server/src/server/router/ir/fervidex/router.ts
+4 -8
View File
@@ -201,18 +201,16 @@ router.use(
* @name POST /ir/fervidex/profile/submit
*/
router.post("/profile/submit", RequireInf2ModelHeaderOrForceStatic, async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!);
const headers = {
// guaranteed to exist because of RequireInf2ModelHeader
model: req.header("X-Software-Model")!,
};
const responseData = await ExpressWrappedScoreImportMain(
userDoc,
req[SYMBOL_TachiAPIAuth].userID!,
false,
"ir/fervidex-static",
(logger) => ParseFervidexStatic(req.body, headers, logger)
[req.body, headers]
);
if (!responseData.body.success) {
@@ -234,8 +232,6 @@ router.post("/profile/submit", RequireInf2ModelHeaderOrForceStatic, async (req,
* @name POST /ir/fervidex/score/submit
*/
router.post("/score/submit", ValidateModelHeader, async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!);
const model = req.header("X-Software-Model");
if (!model) {
@@ -250,10 +246,10 @@ router.post("/score/submit", ValidateModelHeader, async (req, res) => {
};
const responseData = await ExpressWrappedScoreImportMain(
userDoc,
req[SYMBOL_TachiAPIAuth].userID!,
true,
"ir/fervidex",
(logger) => ParseFervidexSingle(req.body, headers, logger)
[req.body, headers]
);
if (!responseData.body.success) {
server/src/server/router/ir/kshook/sv3c/router.ts
+2 -4
View File
@@ -81,13 +81,11 @@ router.use(ValidateHeaders);
* @name POST /ir/kshook/sv3c/score/save
*/
router.post("/score/save", async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth].userID!);
const responseData = await ExpressWrappedScoreImportMain(
userDoc,
req[SYMBOL_TachiAPIAuth].userID!,
true,
"ir/kshook-sv3c",
(logger) => ParseKsHookSV3C(req.body, logger)
[req.body]
);
if (!responseData.body.success) {
server/src/server/router/ir/router.ts
-2
View File
@@ -1,6 +1,5 @@
import { Router } from "express";
import barbatosIR from "./barbatos/router";
import chunitachiIR from "./chunitachi/router";
import directManualIR from "./direct-manual/router";
import fervidexIR from "./fervidex/router";
import uscIR from "./usc/router";
@@ -32,7 +31,6 @@ router.use("/beatoraja", SetRequestPermissions, RequireBokutachi, beatorajaIR);
// Kamaitachi IRs
router.use("/barbatos", SetRequestPermissions, RequireNotGuest, RequireKamaitachi, barbatosIR);
router.use("/chunitachi", SetRequestPermissions, RequireNotGuest, RequireKamaitachi, chunitachiIR);
router.use(
"/fervidex",
SetFervidexStyleRequestPermissions,
server/src/server/router/ir/usc/router.test.ts
+9 -5
View File
@@ -1,12 +1,11 @@
import t from "tap";
import deepmerge from "deepmerge";
import db from "external/mongo/db";
import { CDNRetrieve } from "lib/cdn/cdn";
import { PBScoreDocument, PublicUserDocument, ScoreDocument } from "tachi-common";
import t from "tap";
import mockApi from "test-utils/mock-api";
import ResetDBState, { ResetCDN } from "test-utils/resets";
import deepmerge from "deepmerge";
import { PBScoreDocument, ScoreDocument, PublicUserDocument } from "tachi-common";
import { GetKTDataBuffer } from "test-utils/test-data";
import { CDNRetrieve } from "lib/cdn/cdn";
async function InsertFakeUSCAuth() {
await db["api-tokens"].insert({
@@ -16,6 +15,7 @@ async function InsertFakeUSCAuth() {
submit_score: true,
},
token: "foo",
fromAPIClient: null,
});
}
@@ -47,6 +47,7 @@ t.test("GET /ir/usc", async (t) => {
submit_score: true,
},
token: "foo",
fromAPIClient: null,
});
t.beforeEach(ResetDBState);
@@ -466,6 +467,7 @@ t.test("POST /scores", (t) => {
identifier: "token",
permissions: { submit_score: true },
token: "token",
fromAPIClient: null,
});
});
@@ -587,12 +589,14 @@ t.test("POST /scores", (t) => {
identifier: "token2",
permissions: { submit_score: true },
token: "token2",
fromAPIClient: null,
},
{
userID: 3,
identifier: "token3",
permissions: { submit_score: true },
token: "token3",
fromAPIClient: null,
},
]);
server/src/server/router/ir/usc/router.ts
+29 -42
View File
@@ -1,37 +1,35 @@
import { Router, RequestHandler } from "express";
import p from "prudence";
import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi";
import { RequestHandler, Router } from "express";
import db from "external/mongo/db";
import { CDNStore } from "lib/cdn/cdn";
import { GetUSCIRReplayURL } from "lib/cdn/url-format";
import { ONE_MEGABYTE } from "lib/constants/filesize";
import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi";
import { USCIR_MAX_LEADERBOARD_N } from "lib/constants/usc-ir";
import CreateLogCtx from "lib/logger/logger";
import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue";
import { AssertStrAsPositiveNonZeroInt } from "lib/score-import/framework/common/string-asserts";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
import { ServerConfig, TachiConfig } from "lib/setup/config";
import p from "prudence";
import { RequirePermissions } from "server/middleware/auth";
import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload";
import {
ChartDocument,
ImportDocument,
PBScoreDocument,
SuccessfulAPIResponse,
ImportDocument,
} from "tachi-common";
import { AssertStrAsPositiveNonZeroInt } from "lib/score-import/framework/common/string-asserts";
import CreateLogCtx, { KtLogger } from "lib/logger/logger";
import { FormatPrError } from "utils/prudence";
import { GetBlacklist } from "utils/queries/blacklist";
import { AssignToReqTachiData } from "utils/req-tachi-data";
import { USCClientChart } from "./types";
import {
ConvertUSCChart,
CreatePOSTScoresResponseBody,
TachiScoreToServerScore,
USCChartIndexToDiff,
} from "./usc";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { GetUserWithID } from "utils/user";
import { ParseIRUSC } from "lib/score-import/import-types/ir/usc/parser";
import { USCIR_MAX_LEADERBOARD_N } from "lib/constants/usc-ir";
import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload";
import { AssignToReqTachiData } from "utils/req-tachi-data";
import { CDNStore } from "lib/cdn/cdn";
import { ONE_MEGABYTE } from "lib/constants/filesize";
import { RequirePermissions } from "server/middleware/auth";
import { GetUSCIRReplayURL } from "lib/cdn/url-format";
import { FormatPrError } from "utils/prudence";
import { USCClientChart } from "./types";
import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue";
import { ServerConfig, TachiConfig } from "lib/setup/config";
import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
import { GetBlacklist } from "utils/queries/blacklist";
const logger = CreateLogCtx(__filename);
@@ -314,25 +312,18 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => {
"context.chartHash": chartDoc.data.hashSHA1,
});
for (const score of scoresToDeorphan) {
await ReprocessOrphan(score, blacklist, logger);
}
await Promise.all(
scoresToDeorphan.map((score) => ReprocessOrphan(score, blacklist, logger))
);
}
}
const userDoc = await GetUserWithID(req[SYMBOL_TachiAPIAuth]!.userID!);
const userID = req[SYMBOL_TachiAPIAuth]!.userID!;
if (!userDoc) {
logger.severe(`User ${req[SYMBOL_TachiAPIAuth]!.userID!} as no parent userDoc?`);
return res.status(200).json({
statusCode: STATUS_CODES.SERVER_ERROR,
description: "An internal server error has occured.",
});
}
const importParser = (logger: KtLogger) => ParseIRUSC(req.body, uscChart.chartHash, logger);
const importRes = await ExpressWrappedScoreImportMain(userDoc, false, "ir/usc", importParser);
const importRes = await ExpressWrappedScoreImportMain(userID, false, "ir/usc", [
req.body,
uscChart.chartHash,
]);
// If this was an orphan chart request, return ACCEPTED,
// since it may be unorphaned in the future
@@ -359,11 +350,7 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => {
const importDoc = (importRes.body as SuccessfulAPIResponse).body as ImportDocument;
try {
const body = await CreatePOSTScoresResponseBody(
userDoc.id,
chartDoc,
importDoc.scoreIDs[0]
);
const body = await CreatePOSTScoresResponseBody(userID, chartDoc, importDoc.scoreIDs[0]);
return res.status(200).json({
statusCode: STATUS_CODES.SUCCESS,
server/src/server/router/ir/usc/usc.test.ts
+3 -3
View File
@@ -1,9 +1,9 @@
import t from "tap";
import deepmerge from "deepmerge";
import db from "external/mongo/db";
import { ChartDocument, PBScoreDocument, PublicUserDocument, ScoreDocument } from "tachi-common";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { CreatePOSTScoresResponseBody, TachiScoreToServerScore } from "./usc";
import { ChartDocument, PBScoreDocument, ScoreDocument, PublicUserDocument } from "tachi-common";
import deepmerge from "deepmerge";
const mockScorePB: PBScoreDocument<"usc:Single"> = {
chartID: "USC_CHART_ID",
server/src/server/router/ir/usc/usc.ts
+5 -5
View File
@@ -1,9 +1,9 @@
import { integer, PBScoreDocument, ScoreDocument, ChartDocument, SongDocument } from "tachi-common";
import CreateLogCtx from "lib/logger/logger";
import { GetPBOnChart, GetServerRecordOnChart } from "utils/scores";
import { MStoS, Random20Hex } from "utils/misc";
import { USCIR_ADJACENT_SCORE_N } from "lib/constants/usc-ir";
import db from "external/mongo/db";
import { USCIR_ADJACENT_SCORE_N } from "lib/constants/usc-ir";
import CreateLogCtx from "lib/logger/logger";
import { ChartDocument, integer, PBScoreDocument, ScoreDocument, SongDocument } from "tachi-common";
import { MStoS, Random20Hex } from "utils/misc";
import { GetPBOnChart, GetServerRecordOnChart } from "utils/scores";
import { USCClientChart, USCServerScore } from "./types";
const logger = CreateLogCtx(__filename);
server/src/test-utils/api-common.ts
+4 -2
View File
@@ -1,7 +1,7 @@
import db from "external/mongo/db";
import { APIPermissions } from "tachi-common";
import t from "tap";
import mockApi from "./mock-api";
import { APIPermissions } from "tachi-common";
import db from "external/mongo/db";
import ResetDBState from "./resets";
export function RequireAuthPerms(
@@ -22,6 +22,7 @@ export function RequireAuthPerms(
permissions: {},
token: "temp_auth",
userID: 1,
fromAPIClient: null,
});
const resAuth = await mockApi[m](url).set("Authorization", "Bearer temp_auth");
@@ -35,6 +36,7 @@ export function RequireAuthPerms(
permissions: Object.fromEntries(prm.map((e) => [e, true])),
token: "temp_auth2",
userID: 1,
fromAPIClient: null,
});
const resAuthed = await mockApi[m](url).set("Authorization", "Bearer temp_auth2");
server/src/test-utils/close-connections.ts
+2
View File
@@ -2,10 +2,12 @@ import { CloseMongoConnection } from "external/mongo/db";
import { CloseRedisPubSub } from "external/redis/redis-IPC";
import { CloseRedisConnection } from "external/redis/redis";
import { CloseServerConnection } from "./mock-api";
import { CloseScoreImportQueue } from "lib/score-import/worker/queue";
export async function CloseAllConnections() {
await CloseMongoConnection();
await CloseServerConnection();
await CloseRedisConnection();
await CloseRedisPubSub();
await CloseScoreImportQueue();
}
server/src/test-utils/fake-auth.ts
+1
View File
@@ -33,6 +33,7 @@ export function InsertFakeTokenWithAllPerms(token: string): () => any {
identifier: "Mock API Token",
permissions: AllPermissions,
token,
fromAPIClient: null,
});
};
}
server/src/test-utils/single-process-tap.ts
+1 -2
View File
@@ -1,10 +1,9 @@
import "external/mongo/db";
import glob from "glob";
import path from "path";
import t from "tap";
import { CloseAllConnections } from "./close-connections";
import "external/mongo/db";
const files = glob.sync(path.join(__dirname, "../../", "**/*.test.ts"));
process.env.NODE_PATH = path.join(__dirname, "../../");