diff --git a/server/.eslintrc b/server/.eslintrc index 50a4f38d1..9cf44c8b1 100644 --- a/server/.eslintrc +++ b/server/.eslintrc @@ -1,94 +1,94 @@ -{ - "root": true, - "env": { - "node": true, - "es6": true - }, - "plugins": [ - "@typescript-eslint", - "import" - ], - "parser": "@typescript-eslint/parser", - "parserOptions": { - "sourceType": "module", - "ecmaVersion": 2020 - }, - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended", - "plugin:prettier/recommended", - "plugin:import/typescript", - "plugin:import/warnings", - "plugin:import/errors" - ], - "rules": { - "prettier/prettier": [ - "error", - { - "printWidth": 100, - "tabWidth": 4, - "semi": true, - "singleQuote": false, - "trailingComma": "es5", - "bracketSpacing": true, - "endOfLine": "auto", - "useTabs": true - } - ], - "@typescript-eslint/explicit-module-boundary-types": "off", - "@typescript-eslint/no-non-null-assertion": "off", - "@typescript-eslint/no-unused-vars": [ - "warn", - { - "argsIgnorePattern": "logger" - } - ], - // breaks with baseUrl - "import/no-unresolved": "off", - "require-await": "warn", - "no-await-in-loop": "warn", - "no-unused-labels": "error", - "prefer-const": "error", - "no-console": "error", - "eqeqeq": "error", - "no-eval": "error", - "no-var": "error", - "no-new-object": "error", - "quote-props": [ - "error", - "as-needed" - ], - "no-array-constructor": "error", - "prefer-template": "error", - "prefer-promise-reject-errors": "error", - "no-useless-escape": "error", - "no-loop-func": "error", - "prefer-rest-params": "error", - "default-param-last": "error", - "no-new-func": "error", - "arrow-body-style": "error", - "no-useless-constructor": "error", - "no-duplicate-imports": "error", - "dot-notation": "error", - "space-before-blocks": "error", - "no-prototype-builtins": "off", - "prefer-arrow-callback": "error", - "curly": "error", - "no-template-curly-in-string": "error", - "no-implied-eval": "error", - "yoda": "error", - "no-labels": "error", - "no-useless-concat": "error", - "radix": "error", - "no-octal-escape": "error", - "no-param-reassign": "error", - "no-proto": "error", - "no-return-await": "error", - "no-throw-literal": "error", - "no-unmodified-loop-condition": "error", - "no-unused-expressions": "error", - "no-useless-call": "error", - "no-useless-return": "error", - "require-unicode-regexp": "error" - } +{ + "root": true, + "env": { + "node": true, + "es6": true + }, + "plugins": [ + "@typescript-eslint", + "import" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "sourceType": "module", + "ecmaVersion": 2020 + }, + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:prettier/recommended", + "plugin:import/typescript", + "plugin:import/warnings", + "plugin:import/errors" + ], + "rules": { + "prettier/prettier": [ + "error", + { + "printWidth": 100, + "tabWidth": 4, + "semi": true, + "singleQuote": false, + "trailingComma": "es5", + "bracketSpacing": true, + "endOfLine": "lf", + "useTabs": true + } + ], + "@typescript-eslint/explicit-module-boundary-types": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/no-unused-vars": [ + "warn", + { + "argsIgnorePattern": "logger" + } + ], + // breaks with baseUrl + "import/no-unresolved": "off", + "require-await": "warn", + "no-await-in-loop": "warn", + "no-unused-labels": "error", + "prefer-const": "error", + "no-console": "error", + "eqeqeq": "error", + "no-eval": "error", + "no-var": "error", + "no-new-object": "error", + "quote-props": [ + "error", + "as-needed" + ], + "no-array-constructor": "error", + "prefer-template": "error", + "prefer-promise-reject-errors": "error", + "no-useless-escape": "error", + "no-loop-func": "error", + "prefer-rest-params": "error", + "default-param-last": "error", + "no-new-func": "error", + "arrow-body-style": "error", + "no-useless-constructor": "error", + "no-duplicate-imports": "error", + "dot-notation": "error", + "space-before-blocks": "error", + "no-prototype-builtins": "off", + "prefer-arrow-callback": "error", + "curly": "error", + "no-template-curly-in-string": "error", + "no-implied-eval": "error", + "yoda": "error", + "no-labels": "error", + "no-useless-concat": "error", + "radix": "error", + "no-octal-escape": "error", + "no-param-reassign": "error", + "no-proto": "error", + "no-return-await": "error", + "no-throw-literal": "error", + "no-unmodified-loop-condition": "error", + "no-unused-expressions": "error", + "no-useless-call": "error", + "no-useless-return": "error", + "require-unicode-regexp": "error" + } } \ No newline at end of file diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index 3d4c7bb6c..38389cee9 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -1,230 +1,230 @@ -import { ONE_MINUTE } from "lib/constants/time"; -import CreateLogCtx from "lib/logger/logger"; -import { OrphanScoreDocument } from "lib/score-import/import-types/common/types"; -import { Environment, ServerConfig } from "lib/setup/config"; -import monk, { TMiddleware } from "monk"; -import { - APITokenDocument, - ARCSavedProfileDocument, - BMSCourseDocument, - ChartDocument, - ClassAchievementDocument, - CounterDocument, - FervidexSettingsDocument, - FolderChartLookup, - FolderDocument, - Game, - GoalDocument, - IIDXBPIData, - ImportDocument, - ImportLockDocument, - ImportTimingsDocument, - integer, - InviteCodeDocument, - KaiAuthDocument, - MilestoneDocument, - OrphanChart, - PBScoreDocument, - PrivateUserInfoDocument, - PublicUserDocument, - ScoreDocument, - SessionDocument, - SessionViewDocument, - SongDocument, - TableDocument, - TachiAPIClientDocument, - UGPTSettings, - UserGameStats, - UserGameStatsSnapshot, - UserGoalDocument, - UserMilestoneDocument, - UserSettings, -} from "tachi-common"; -import { GetMillisecondsSince } from "utils/misc"; - -const logger = CreateLogCtx(__filename); - -let dbName = ServerConfig.MONGO_DATABASE_NAME; - -/* istanbul ignore next */ -if (Environment.nodeEnv === "test") { - dbName = `testingdb`; -} - -logger.info(`Connecting to database ${Environment.mongoUrl}/${dbName}...`, { bootInfo: true }); -const dbtime = process.hrtime.bigint(); - -// By default the connectTimeoutMS is 30 seconds. This has been upped to 2 minutes, due to poor performance -// inside githubs test runners. -export const monkDB = monk(`${Environment.mongoUrl}/${dbName}`, { - serverSelectionTimeoutMS: ONE_MINUTE * 2, -}); - -/* istanbul ignore next */ -monkDB - .then(() => { - logger.info(`Database connection successful: took ${GetMillisecondsSince(dbtime)}ms`, { - bootInfo: true, - }); - }) - .catch((err) => { - logger.crit(err); - process.exit(1); - }); - -const RemoveIDMiddleware: TMiddleware = - () => - (next) => - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (args: any, method) => { - if ((method === "find" || method === "findOne") && !args.options.projectID) { - if (args.options.projection) { - args.options.projection._id = 0; - } else { - args.options.projection = { _id: 0 }; - } - } - - return next(args, method); - }; - -// a bug in monks types means that :any has to be used here. Maybe we'll make a PR for this? -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const StripIDMiddleware: TMiddleware = () => (next) => (args: any, method) => { - if (method === "insert") { - if (Array.isArray(args.data)) { - for (const d of args.data) { - delete d._id; - } - } else { - delete args.data._id; - } - } - - return next(args, method); -}; - -monkDB.addMiddleware(StripIDMiddleware); -monkDB.addMiddleware(RemoveIDMiddleware); - -export async function CloseMongoConnection() { - await monkDB.close(); -} - -const songs = { - bms: monkDB.get(`songs-bms`), - chunithm: monkDB.get(`songs-chunithm`), - ddr: monkDB.get(`songs-ddr`), - gitadora: monkDB.get(`songs-gitadora`), - iidx: monkDB.get(`songs-iidx`), - jubeat: monkDB.get(`songs-jubeat`), - maimai: monkDB.get(`songs-maimai`), - museca: monkDB.get(`songs-museca`), - popn: monkDB.get(`songs-popn`), - sdvx: monkDB.get(`songs-sdvx`), - usc: monkDB.get(`songs-usc`), -}; - -const charts = { - bms: monkDB.get(`charts-bms`), - chunithm: monkDB.get(`charts-chunithm`), - ddr: monkDB.get(`charts-ddr`), - gitadora: monkDB.get(`charts-gitadora`), - iidx: monkDB.get(`charts-iidx`), - jubeat: monkDB.get(`charts-jubeat`), - maimai: monkDB.get(`charts-maimai`), - museca: monkDB.get(`charts-museca`), - popn: monkDB.get(`charts-popn`), - sdvx: monkDB.get(`charts-sdvx`), - usc: monkDB.get(`charts-usc`), -}; - -const db = { - // i have to handwrite this out for TS... :( - // dont worry, it was all macro'd. - songs, - charts, - scores: monkDB.get("scores"), - "personal-bests": monkDB.get("personal-bests"), - folders: monkDB.get("folders"), - "folder-chart-lookup": monkDB.get("folder-chart-lookup"), - goals: monkDB.get("goals"), - "user-goals": monkDB.get("user-goals"), - milestones: monkDB.get("milestones"), - "user-milestones": monkDB.get("user-milestones"), - users: monkDB.get("users"), - imports: monkDB.get("imports"), - "import-timings": monkDB.get("import-timings"), - sessions: monkDB.get("sessions"), - "iidx-bpi-data": monkDB.get("iidx-bpi-data"), - invites: monkDB.get("invites"), - counters: monkDB.get("counters"), - "game-stats": monkDB.get("game-stats"), - "kai-auth-tokens": monkDB.get("kai-auth-tokens"), - "bms-course-lookup": monkDB.get("bms-course-lookup"), - "api-tokens": monkDB.get("api-tokens"), - "orphan-scores": monkDB.get("orphan-scores"), - "import-locks": monkDB.get("import-locks"), - tables: monkDB.get("tables"), - "game-settings": monkDB.get("game-settings"), - "game-stats-snapshots": monkDB.get("game-stats-snapshots"), - "session-view-cache": monkDB.get("session-view-cache"), - "arc-saved-profiles": monkDB.get("arc-saved-profiles"), - "user-settings": monkDB.get("user-settings"), - "user-private-information": monkDB.get("user-private-information"), - "api-clients": monkDB.get("api-clients"), - "oauth2-auth-codes": - // i've inlined this one because i don't see it appearing anywhere else. - monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"), - "fer-settings": monkDB.get("fer-settings"), - "orphan-chart-queue": monkDB.get("orphan-chart-queue"), - "password-reset-codes": monkDB.get<{ - code: string; - userID: integer; - createdOn: number; - }>("password-reset-codes"), - "class-achievements": monkDB.get("class-achievements"), - "score-blacklist": - monkDB.get<{ scoreID: string; userID: integer; score: ScoreDocument }>("score-blacklist"), - "verify-email-codes": - monkDB.get<{ userID: integer; code: string; email: string }>("verify-email-codes"), -}; - -export type StaticDatabases = - | "sessions" - | "session-view-cache" - | "folders" - | "folder-chart-lookup" - | "scores" - | "personal-bests" - | "imports" - | "import-timings" - | "goals" - | "user-goals" - | "user-milestones" - | "milestones" - | "game-stats" - | "game-settings" - | "users" - | "kai-auth-tokens" - | "bms-course-lookup" - | "api-tokens" - | "import-locks" - | "tables" - | "game-stats-snapshots" - | "arc-saved-profiles" - | "user-private-information" - | "api-clients" - | "oauth2-auth-codes" - | "fer-settings" - | "orphan-chart-queue" - | "password-reset-codes" - | "user-settings" - | "counters" - | "score-blacklist" - | "verify-email-codes" - | "class-achievements"; - -export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`; - -export default db; +import { ONE_MINUTE } from "lib/constants/time"; +import CreateLogCtx from "lib/logger/logger"; +import { OrphanScoreDocument } from "lib/score-import/import-types/common/types"; +import { Environment, ServerConfig } from "lib/setup/config"; +import monk, { TMiddleware } from "monk"; +import { + APITokenDocument, + ARCSavedProfileDocument, + BMSCourseDocument, + ChartDocument, + ClassAchievementDocument, + CounterDocument, + FervidexSettingsDocument, + FolderChartLookup, + FolderDocument, + Game, + GoalDocument, + IIDXBPIData, + ImportDocument, + ImportLockDocument, + ImportTimingsDocument, + integer, + InviteCodeDocument, + KaiAuthDocument, + MilestoneDocument, + OrphanChart, + PBScoreDocument, + PrivateUserInfoDocument, + PublicUserDocument, + ScoreDocument, + SessionDocument, + SessionViewDocument, + SongDocument, + TableDocument, + TachiAPIClientDocument, + UGPTSettings, + UserGameStats, + UserGameStatsSnapshot, + UserGoalDocument, + UserMilestoneDocument, + UserSettings, +} from "tachi-common"; +import { GetMillisecondsSince } from "utils/misc"; + +const logger = CreateLogCtx(__filename); + +let dbName = ServerConfig.MONGO_DATABASE_NAME; + +/* istanbul ignore next */ +if (Environment.nodeEnv === "test") { + dbName = `testingdb`; +} + +logger.info(`Connecting to database ${Environment.mongoUrl}/${dbName}...`, { bootInfo: true }); +const dbtime = process.hrtime.bigint(); + +// By default the connectTimeoutMS is 30 seconds. This has been upped to 2 minutes, due to poor performance +// inside githubs test runners. +export const monkDB = monk(`${Environment.mongoUrl}/${dbName}`, { + serverSelectionTimeoutMS: ONE_MINUTE * 2, +}); + +/* istanbul ignore next */ +monkDB + .then(() => { + logger.info(`Database connection successful: took ${GetMillisecondsSince(dbtime)}ms`, { + bootInfo: true, + }); + }) + .catch((err) => { + logger.crit(err); + process.exit(1); + }); + +const RemoveIDMiddleware: TMiddleware = + () => + (next) => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (args: any, method) => { + if ((method === "find" || method === "findOne") && !args.options.projectID) { + if (args.options.projection) { + args.options.projection._id = 0; + } else { + args.options.projection = { _id: 0 }; + } + } + + return next(args, method); + }; + +// a bug in monks types means that :any has to be used here. Maybe we'll make a PR for this? +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const StripIDMiddleware: TMiddleware = () => (next) => (args: any, method) => { + if (method === "insert") { + if (Array.isArray(args.data)) { + for (const d of args.data) { + delete d._id; + } + } else { + delete args.data._id; + } + } + + return next(args, method); +}; + +monkDB.addMiddleware(StripIDMiddleware); +monkDB.addMiddleware(RemoveIDMiddleware); + +export async function CloseMongoConnection() { + await monkDB.close(); +} + +const songs = { + bms: monkDB.get(`songs-bms`), + chunithm: monkDB.get(`songs-chunithm`), + ddr: monkDB.get(`songs-ddr`), + gitadora: monkDB.get(`songs-gitadora`), + iidx: monkDB.get(`songs-iidx`), + jubeat: monkDB.get(`songs-jubeat`), + maimai: monkDB.get(`songs-maimai`), + museca: monkDB.get(`songs-museca`), + popn: monkDB.get(`songs-popn`), + sdvx: monkDB.get(`songs-sdvx`), + usc: monkDB.get(`songs-usc`), +}; + +const charts = { + bms: monkDB.get(`charts-bms`), + chunithm: monkDB.get(`charts-chunithm`), + ddr: monkDB.get(`charts-ddr`), + gitadora: monkDB.get(`charts-gitadora`), + iidx: monkDB.get(`charts-iidx`), + jubeat: monkDB.get(`charts-jubeat`), + maimai: monkDB.get(`charts-maimai`), + museca: monkDB.get(`charts-museca`), + popn: monkDB.get(`charts-popn`), + sdvx: monkDB.get(`charts-sdvx`), + usc: monkDB.get(`charts-usc`), +}; + +const db = { + // i have to handwrite this out for TS... :( + // dont worry, it was all macro'd. + songs, + charts, + scores: monkDB.get("scores"), + "personal-bests": monkDB.get("personal-bests"), + folders: monkDB.get("folders"), + "folder-chart-lookup": monkDB.get("folder-chart-lookup"), + goals: monkDB.get("goals"), + "user-goals": monkDB.get("user-goals"), + milestones: monkDB.get("milestones"), + "user-milestones": monkDB.get("user-milestones"), + users: monkDB.get("users"), + imports: monkDB.get("imports"), + "import-timings": monkDB.get("import-timings"), + sessions: monkDB.get("sessions"), + "iidx-bpi-data": monkDB.get("iidx-bpi-data"), + invites: monkDB.get("invites"), + counters: monkDB.get("counters"), + "game-stats": monkDB.get("game-stats"), + "kai-auth-tokens": monkDB.get("kai-auth-tokens"), + "bms-course-lookup": monkDB.get("bms-course-lookup"), + "api-tokens": monkDB.get("api-tokens"), + "orphan-scores": monkDB.get("orphan-scores"), + "import-locks": monkDB.get("import-locks"), + tables: monkDB.get("tables"), + "game-settings": monkDB.get("game-settings"), + "game-stats-snapshots": monkDB.get("game-stats-snapshots"), + "session-view-cache": monkDB.get("session-view-cache"), + "arc-saved-profiles": monkDB.get("arc-saved-profiles"), + "user-settings": monkDB.get("user-settings"), + "user-private-information": monkDB.get("user-private-information"), + "api-clients": monkDB.get("api-clients"), + "oauth2-auth-codes": + // i've inlined this one because i don't see it appearing anywhere else. + monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"), + "fer-settings": monkDB.get("fer-settings"), + "orphan-chart-queue": monkDB.get("orphan-chart-queue"), + "password-reset-codes": monkDB.get<{ + code: string; + userID: integer; + createdOn: number; + }>("password-reset-codes"), + "class-achievements": monkDB.get("class-achievements"), + "score-blacklist": + monkDB.get<{ scoreID: string; userID: integer; score: ScoreDocument }>("score-blacklist"), + "verify-email-codes": + monkDB.get<{ userID: integer; code: string; email: string }>("verify-email-codes"), +}; + +export type StaticDatabases = + | "sessions" + | "session-view-cache" + | "folders" + | "folder-chart-lookup" + | "scores" + | "personal-bests" + | "imports" + | "import-timings" + | "goals" + | "user-goals" + | "user-milestones" + | "milestones" + | "game-stats" + | "game-settings" + | "users" + | "kai-auth-tokens" + | "bms-course-lookup" + | "api-tokens" + | "import-locks" + | "tables" + | "game-stats-snapshots" + | "arc-saved-profiles" + | "user-private-information" + | "api-clients" + | "oauth2-auth-codes" + | "fer-settings" + | "orphan-chart-queue" + | "password-reset-codes" + | "user-settings" + | "counters" + | "score-blacklist" + | "verify-email-codes" + | "class-achievements"; + +export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`; + +export default db; diff --git a/server/src/lib/logger/logger.ts b/server/src/lib/logger/logger.ts index 4d7b487c1..46b09254e 100644 --- a/server/src/lib/logger/logger.ts +++ b/server/src/lib/logger/logger.ts @@ -1,253 +1,253 @@ -import { Transport as SeqTransport } from "@valuabletouch/winston-seq"; -import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; -import SafeJSONStringify from "safe-json-stringify"; -import { SeqLogLevel } from "seq-logging"; -import { EscapeStringRegexp } from "utils/misc"; -import winston, { format, LeveledLogMethod, Logger, transports } from "winston"; -import "winston-daily-rotate-file"; -import DiscordWinstonTransport from "./discord-transport"; - -export type KtLogger = Logger & { severe: LeveledLogMethod }; - -const level = process.env.LOG_LEVEL ?? ServerConfig.LOGGER_CONFIG.LOG_LEVEL; - -const formatExcessProperties = (meta: Record, limit = false) => { - let i = 0; - for (const key in meta) { - const val = meta[key]; - - if (val instanceof Error) { - meta[key] = { message: val.message, stack: val.stack }; - } - i++; - } - - if (!i) { - return ""; - } - - const content = SafeJSONStringify(meta); - - return ` ${limit ? StrCap(content) : content}`; -}; - -function StrCap(string: string) { - if (string.length > 300) { - return `${string.slice(0, 297)}...`; - } - - return string; -} - -const formatExcessPropertiesNoStack = ( - meta: Record, - omitKeys: string[] = [], - limit = false -) => { - const realMeta: Record = {}; - - for (const key in meta) { - if (omitKeys.includes(key)) { - continue; - } - - const val = meta[key]; - - if (val instanceof Error) { - realMeta[key] = { message: val.message }; - } else if (!key.startsWith("__") && !key.startsWith("!")) { - realMeta[key] = val; - } - } - - if (Object.keys(realMeta).length === 0) { - return ""; - } - - const content = SafeJSONStringify(realMeta); - - return ` ${limit ? StrCap(content) : content}`; -}; - -const replicaInfo = Environment.replicaIdentity ? ` (${Environment.replicaIdentity})` : ""; - -const tachiPrintf = format.printf( - ({ level, message, context = "tachi-root", timestamp, ...meta }) => - `${timestamp}${replicaInfo} [${ - Array.isArray(context) ? context.join(" | ") : context - }] ${level}: ${message}${formatExcessProperties(meta)}` -); - -const tachiConsolePrintf = format.printf( - ({ level, message, context = "tachi-root", timestamp, hideFromConsole, ...meta }) => - `${timestamp}${replicaInfo} [${ - Array.isArray(context) ? context.join(" | ") : context - }] ${level}: ${message}${formatExcessPropertiesNoStack(meta, hideFromConsole, true)}` -); - -winston.addColors({ - crit: ["bgRed", "black"], - severe: ["bgBrightRed", "black"], - error: ["red"], - warn: ["yellow"], - info: ["blue"], - verbose: ["cyan"], - debug: ["white"], -}); - -const baseFormatRoute = format.combine( - format.timestamp({ - format: "YYYY-MM-DD HH:mm:ss", - }) -); - -const defaultFormatRoute = format.combine( - baseFormatRoute, - format.errors({ stack: false }), - tachiPrintf -); - -const consoleFormatRoute = format.combine( - baseFormatRoute, - format.errors({ stack: false }), - tachiConsolePrintf, - format.colorize({ - all: true, - }) -); - -const tports: winston.transport[] = []; - -if (ServerConfig.LOGGER_CONFIG.FILE) { - tports.push( - new transports.DailyRotateFile({ - filename: "logs/tachi-%DATE%.log", - datePattern: "YYYY-MM-DD-HH", - zippedArchive: true, - maxSize: "20m", - maxFiles: "14d", - createSymlink: true, - symlinkName: "tachi.log", - format: defaultFormatRoute, - }) - ); -} - -if (ServerConfig.LOGGER_CONFIG.CONSOLE) { - tports.push( - new transports.Console({ - format: consoleFormatRoute, - }) - ); -} - -if (ServerConfig.LOGGER_CONFIG.DISCORD) { - tports.push( - new DiscordWinstonTransport({ - webhook: ServerConfig.LOGGER_CONFIG.DISCORD.WEBHOOK_URL, - level: "warn", - }) - ); -} - -if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) { - // Turns winston log levels into seq format. - const levelMap: Record = { - crit: "Fatal", - severe: "Error", - error: "Error", - warn: "Warning", - info: "Information", - // Note that Seq interprets these in reverse, - // however, it's easier to read this code if I just - // use the same levels, instead of the right onesQ. - verbose: "Verbose", - debug: "Debug", - }; - - tports.push( - new SeqTransport({ - apiKey: ServerConfig.LOGGER_CONFIG.SEQ_API_KEY, - serverUrl: Environment.seqUrl, - levelMapper(level = "") { - return levelMap[level] ?? "information"; - }, - }) - ); -} - -export const rootLogger = winston.createLogger({ - levels: { - crit: 0, // entire process termination is necessary - severe: 1, // something is wrong, and more than one function is affected (such as a failed assertion that is definitely expected to be true). - error: 2, // function call (or related process) has failed unexpectedly - warn: 3, // function call has hit something it didn't want, but can recover - info: 4, // something has happened that is expected, but worth logging - verbose: 5, // something has happened - debug: 6, // glorified console.log debugging - }, - level, - format: defaultFormatRoute, - transports: tports, - defaultMeta: { - __ServerName: TachiConfig.NAME, - __Worker: !!process.env.IS_WORKER, - __ReplicaID: Environment.replicaIdentity, - }, -}); - -if (!!ServerConfig.LOGGER_CONFIG.SEQ_API_KEY !== !!Environment.seqUrl) { - rootLogger.warn( - `Only one of SEQ_API_KEY (conf.json5) and SEQ_URL (Environment) were set. Not sending logs to Seq, as both must be provided.` - ); -} - -if (tports.length === 0) { - // eslint-disable-next-line no-console - console.warn( - "You have no transports set. Absolutely no logs will be saved. This is a terrible idea!" - ); -} - -function CreateLogCtx(filename: string, lg = rootLogger): KtLogger { - const replacedFilename = filename.replace( - new RegExp(`^${EscapeStringRegexp(process.cwd())}/((js|src)/)?`, "u"), - "" - ); - - const logger = lg.child({ - context: [replacedFilename], - }) as KtLogger; - - logger.defaultMeta = Object.assign({}, logger.defaultMeta ?? {}, { - context: [replacedFilename], - }); - return logger; -} - -export function AppendLogCtx(context: string, lg: KtLogger): KtLogger { - const newContext = [...lg.defaultMeta.context, context]; - - return lg.child({ context: newContext }) as KtLogger; -} - -export function ChangeRootLogLevel( - level: "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug" -) { - rootLogger.info(`Changing log level to ${level}.`); - - for (const tp of tports) { - tp.level = level; - } -} - -export function GetLogLevel() { - return ( - tports.map((e) => e.level).find((e) => typeof e === "string") ?? - ServerConfig.LOGGER_CONFIG.LOG_LEVEL - ); -} - -export const Transports = tports; - -export default CreateLogCtx; +import { Transport as SeqTransport } from "@valuabletouch/winston-seq"; +import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; +import SafeJSONStringify from "safe-json-stringify"; +import { SeqLogLevel } from "seq-logging"; +import { EscapeStringRegexp } from "utils/misc"; +import winston, { format, LeveledLogMethod, Logger, transports } from "winston"; +import "winston-daily-rotate-file"; +import DiscordWinstonTransport from "./discord-transport"; + +export type KtLogger = Logger & { severe: LeveledLogMethod }; + +const level = process.env.LOG_LEVEL ?? ServerConfig.LOGGER_CONFIG.LOG_LEVEL; + +const formatExcessProperties = (meta: Record, limit = false) => { + let i = 0; + for (const key in meta) { + const val = meta[key]; + + if (val instanceof Error) { + meta[key] = { message: val.message, stack: val.stack }; + } + i++; + } + + if (!i) { + return ""; + } + + const content = SafeJSONStringify(meta); + + return ` ${limit ? StrCap(content) : content}`; +}; + +function StrCap(string: string) { + if (string.length > 300) { + return `${string.slice(0, 297)}...`; + } + + return string; +} + +const formatExcessPropertiesNoStack = ( + meta: Record, + omitKeys: string[] = [], + limit = false +) => { + const realMeta: Record = {}; + + for (const key in meta) { + if (omitKeys.includes(key)) { + continue; + } + + const val = meta[key]; + + if (val instanceof Error) { + realMeta[key] = { message: val.message }; + } else if (!key.startsWith("__") && !key.startsWith("!")) { + realMeta[key] = val; + } + } + + if (Object.keys(realMeta).length === 0) { + return ""; + } + + const content = SafeJSONStringify(realMeta); + + return ` ${limit ? StrCap(content) : content}`; +}; + +const replicaInfo = Environment.replicaIdentity ? ` (${Environment.replicaIdentity})` : ""; + +const tachiPrintf = format.printf( + ({ level, message, context = "tachi-root", timestamp, ...meta }) => + `${timestamp}${replicaInfo} [${ + Array.isArray(context) ? context.join(" | ") : context + }] ${level}: ${message}${formatExcessProperties(meta)}` +); + +const tachiConsolePrintf = format.printf( + ({ level, message, context = "tachi-root", timestamp, hideFromConsole, ...meta }) => + `${timestamp}${replicaInfo} [${ + Array.isArray(context) ? context.join(" | ") : context + }] ${level}: ${message}${formatExcessPropertiesNoStack(meta, hideFromConsole, true)}` +); + +winston.addColors({ + crit: ["bgRed", "black"], + severe: ["bgBrightRed", "black"], + error: ["red"], + warn: ["yellow"], + info: ["blue"], + verbose: ["cyan"], + debug: ["white"], +}); + +const baseFormatRoute = format.combine( + format.timestamp({ + format: "YYYY-MM-DD HH:mm:ss", + }) +); + +const defaultFormatRoute = format.combine( + baseFormatRoute, + format.errors({ stack: false }), + tachiPrintf +); + +const consoleFormatRoute = format.combine( + baseFormatRoute, + format.errors({ stack: false }), + tachiConsolePrintf, + format.colorize({ + all: true, + }) +); + +const tports: winston.transport[] = []; + +if (ServerConfig.LOGGER_CONFIG.FILE) { + tports.push( + new transports.DailyRotateFile({ + filename: "logs/tachi-%DATE%.log", + datePattern: "YYYY-MM-DD-HH", + zippedArchive: true, + maxSize: "20m", + maxFiles: "14d", + createSymlink: true, + symlinkName: "tachi.log", + format: defaultFormatRoute, + }) + ); +} + +if (ServerConfig.LOGGER_CONFIG.CONSOLE) { + tports.push( + new transports.Console({ + format: consoleFormatRoute, + }) + ); +} + +if (ServerConfig.LOGGER_CONFIG.DISCORD) { + tports.push( + new DiscordWinstonTransport({ + webhook: ServerConfig.LOGGER_CONFIG.DISCORD.WEBHOOK_URL, + level: "warn", + }) + ); +} + +if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) { + // Turns winston log levels into seq format. + const levelMap: Record = { + crit: "Fatal", + severe: "Error", + error: "Error", + warn: "Warning", + info: "Information", + // Note that Seq interprets these in reverse, + // however, it's easier to read this code if I just + // use the same levels, instead of the right onesQ. + verbose: "Verbose", + debug: "Debug", + }; + + tports.push( + new SeqTransport({ + apiKey: ServerConfig.LOGGER_CONFIG.SEQ_API_KEY, + serverUrl: Environment.seqUrl, + levelMapper(level = "") { + return levelMap[level] ?? "information"; + }, + }) + ); +} + +export const rootLogger = winston.createLogger({ + levels: { + crit: 0, // entire process termination is necessary + severe: 1, // something is wrong, and more than one function is affected (such as a failed assertion that is definitely expected to be true). + error: 2, // function call (or related process) has failed unexpectedly + warn: 3, // function call has hit something it didn't want, but can recover + info: 4, // something has happened that is expected, but worth logging + verbose: 5, // something has happened + debug: 6, // glorified console.log debugging + }, + level, + format: defaultFormatRoute, + transports: tports, + defaultMeta: { + __ServerName: TachiConfig.NAME, + __Worker: !!process.env.IS_WORKER, + __ReplicaID: Environment.replicaIdentity, + }, +}); + +if (!!ServerConfig.LOGGER_CONFIG.SEQ_API_KEY !== !!Environment.seqUrl) { + rootLogger.warn( + `Only one of SEQ_API_KEY (conf.json5) and SEQ_URL (Environment) were set. Not sending logs to Seq, as both must be provided.` + ); +} + +if (tports.length === 0) { + // eslint-disable-next-line no-console + console.warn( + "You have no transports set. Absolutely no logs will be saved. This is a terrible idea!" + ); +} + +function CreateLogCtx(filename: string, lg = rootLogger): KtLogger { + const replacedFilename = filename.replace( + new RegExp(`^${EscapeStringRegexp(process.cwd())}/((js|src)/)?`, "u"), + "" + ); + + const logger = lg.child({ + context: [replacedFilename], + }) as KtLogger; + + logger.defaultMeta = Object.assign({}, logger.defaultMeta ?? {}, { + context: [replacedFilename], + }); + return logger; +} + +export function AppendLogCtx(context: string, lg: KtLogger): KtLogger { + const newContext = [...lg.defaultMeta.context, context]; + + return lg.child({ context: newContext }) as KtLogger; +} + +export function ChangeRootLogLevel( + level: "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug" +) { + rootLogger.info(`Changing log level to ${level}.`); + + for (const tp of tports) { + tp.level = level; + } +} + +export function GetLogLevel() { + return ( + tports.map((e) => e.level).find((e) => typeof e === "string") ?? + ServerConfig.LOGGER_CONFIG.LOG_LEVEL + ); +} + +export const Transports = tports; + +export default CreateLogCtx; diff --git a/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts b/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts index 63ab1c96f..ebdaf6ab3 100644 --- a/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts +++ b/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts @@ -1,414 +1,414 @@ -import CreateLogCtx from "lib/logger/logger"; -import p from "prudence"; -import { ChartDocument, ScoreDocument } from "tachi-common"; -import t from "tap"; -import { prAssert } from "test-utils/asserts"; -import { - Testing511SPA, - TestingDoraChart, - TestingGITADORADoraDryScore, - TestingIIDXSPDryScore, - TestingSDVXSingleDryScore, -} from "test-utils/test-data"; -import { CalculateDataForGamePT, CreateCalculatedData } from "./calculated-data"; - -const logger = CreateLogCtx(__filename); - -t.test("#CreateCalculatedData", async (t) => { - const res = await CreateCalculatedData(TestingIIDXSPDryScore, Testing511SPA, 30, logger); - - prAssert( - res, - { - ktRating: p.aprx(2.65), - ktLampRating: p.equalTo(10), - BPI: "?number", - }, - "Should correctly produce calculatedData" - ); - - const gitadoraRes = await CreateCalculatedData( - TestingGITADORADoraDryScore, - TestingDoraChart, - 30, - logger - ); - - prAssert( - gitadoraRes, - { - skill: p.isPositiveNonZero, - }, - "Should correctly call rating function overrides for different games" - ); - - const uscRes = await CreateCalculatedData( - { game: "usc", playtype: "Controller" } as ScoreDocument, - { data: { isOfficial: false }, playtype: "Controller" } as ChartDocument, - null, - logger - ); - - t.strictSame(uscRes, { VF6: null }, "Should return null if chart was not an official."); - - const uscKbRes = await CreateCalculatedData( - { game: "usc", playtype: "Keyboard" } as ScoreDocument, - { data: { isOfficial: false }, playtype: "Keyboard" } as ChartDocument, - null, - logger - ); - - t.strictSame( - uscKbRes, - { VF6: null }, - "Should return null if chart was not an official (Keyboard)." - ); - - t.end(); -}); - -/** - * These tests only check that the right properties are assigned. - */ -t.test("#CalculateDataForGamePT", (t) => { - t.test("IIDX:SP", async (t) => { - const res = await CalculateDataForGamePT( - "iidx", - "SP", - Testing511SPA, - TestingIIDXSPDryScore, - 30, - - logger - ); - - prAssert( - res, - { - ktRating: "?number", - ktLampRating: "?number", - BPI: "?number", - }, - "Response should contain keys for IIDX:SP" - ); - - t.end(); - }); - - t.test("IIDX:DP", async (t) => { - const res = await CalculateDataForGamePT( - "iidx", - "DP", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an SP score. but we're testing - 30, - - logger - ); - - prAssert( - res, - { - ktRating: "?number", - ktLampRating: "?number", - BPI: "?number", - }, - "Response should contain keys for IIDX:DP" - ); - - t.end(); - }); - - t.test("SDVX:Single", async (t) => { - const res = await CalculateDataForGamePT( - "sdvx", - "Single", - Testing511SPA, - TestingSDVXSingleDryScore, - null, - logger - ); - - prAssert( - res, - { - VF6: p.nullable(p.isPositive), - }, - "Response should contain keys for SDVX:Single" - ); - - t.end(); - }); - - t.test("DDR:SP", async (t) => { - const res = await CalculateDataForGamePT( - "ddr", - "SP", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - MFCP: p.nullable(p.isPositiveInteger), - ktRating: "?number", - }, - "Response should contain keys for DDR:SP" - ); - - t.end(); - }); - - t.test("DDR:DP", async (t) => { - const res = await CalculateDataForGamePT( - "ddr", - "DP", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - MFCP: "null", - ktRating: "?number", - }, - "Response should contain nulled keys for DDR:DP" - ); - - t.end(); - }); - - t.test("chunithm:Single", async (t) => { - const res = await CalculateDataForGamePT( - "chunithm", - "Single", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - rating: "?number", - }, - "Response should contain nulled keys for chunithm:Single" - ); - - t.end(); - }); - - t.test("maimai:Single", async (t) => { - const res = await CalculateDataForGamePT( - "maimai", - "Single", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - ktRating: "?number", - }, - "Response should contain nulled keys for maimai:Single" - ); - - t.end(); - }); - - t.test("museca:Single", async (t) => { - const res = await CalculateDataForGamePT( - "museca", - "Single", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - ktRating: "?number", - }, - "Response should contain nulled keys for museca:Single" - ); - - t.end(); - }); - - t.test("bms:7K", async (t) => { - const res = await CalculateDataForGamePT( - "bms", - "7K", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - sieglinde: "?number", - }, - "Response should contain nulled keys for bms:7K" - ); - - t.end(); - }); - - t.test("bms:14K", async (t) => { - const res = await CalculateDataForGamePT( - "bms", - "14K", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - sieglinde: "?number", - }, - "Response should contain nulled keys for bms:14K" - ); - - t.end(); - }); - - t.test("gitadora:Gita", async (t) => { - const res = await CalculateDataForGamePT( - "gitadora", - "Gita", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - skill: "?number", - }, - "Response should contain nulled keys for gitadora:Gita" - ); - - t.end(); - }); - - t.test("gitadora:Dora", async (t) => { - const res = await CalculateDataForGamePT( - "gitadora", - "Dora", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - skill: "?number", - }, - "Response should contain nulled keys for gitadora:Dora" - ); - - t.end(); - }); - - t.test("usc:Controller", async (t) => { - const res = await CalculateDataForGamePT( - "usc", - "Controller", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - VF6: "?number", - }, - "Response should contain nulled keys for usc:Controller" - ); - - t.end(); - }); - - t.test("usc:Keyboard", async (t) => { - const res = await CalculateDataForGamePT( - "usc", - "Keyboard", - Testing511SPA, - TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - null, - logger - ); - - prAssert( - res, - { - VF6: "?number", - }, - "Response should contain nulled keys for usc:Keyboard" - ); - - t.end(); - }); - // t.test("jubeat:Single", async (t) => { - // const res = await CalculateDataForGamePT( - // "jubeat", - // "Single", - // Testing511SPA, - // TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - // null, - // - // logger - // ); - - // prAssert( - // res, - // { - // jubility: "?number", - // }, - // "Response should contain nulled keys for jubeat:Single" - // ); - - // t.end(); - // }); - - // t.test("popn:9B", async (t) => { - // const res = await CalculateDataForGamePT( - // "popn", - // "9B", - // Testing511SPA, - // TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing - // null, - // - // logger - // ); - - // prAssert(res, {}, "Response should contain nulled keys for popn:9B"); - - // t.end(); - // }); - - t.end(); -}); +import CreateLogCtx from "lib/logger/logger"; +import p from "prudence"; +import { ChartDocument, ScoreDocument } from "tachi-common"; +import t from "tap"; +import { prAssert } from "test-utils/asserts"; +import { + Testing511SPA, + TestingDoraChart, + TestingGITADORADoraDryScore, + TestingIIDXSPDryScore, + TestingSDVXSingleDryScore, +} from "test-utils/test-data"; +import { CalculateDataForGamePT, CreateCalculatedData } from "./calculated-data"; + +const logger = CreateLogCtx(__filename); + +t.test("#CreateCalculatedData", async (t) => { + const res = await CreateCalculatedData(TestingIIDXSPDryScore, Testing511SPA, 30, logger); + + prAssert( + res, + { + ktRating: p.aprx(2.65), + ktLampRating: p.equalTo(10), + BPI: "?number", + }, + "Should correctly produce calculatedData" + ); + + const gitadoraRes = await CreateCalculatedData( + TestingGITADORADoraDryScore, + TestingDoraChart, + 30, + logger + ); + + prAssert( + gitadoraRes, + { + skill: p.isPositiveNonZero, + }, + "Should correctly call rating function overrides for different games" + ); + + const uscRes = await CreateCalculatedData( + { game: "usc", playtype: "Controller" } as ScoreDocument, + { data: { isOfficial: false }, playtype: "Controller" } as ChartDocument, + null, + logger + ); + + t.strictSame(uscRes, { VF6: null }, "Should return null if chart was not an official."); + + const uscKbRes = await CreateCalculatedData( + { game: "usc", playtype: "Keyboard" } as ScoreDocument, + { data: { isOfficial: false }, playtype: "Keyboard" } as ChartDocument, + null, + logger + ); + + t.strictSame( + uscKbRes, + { VF6: null }, + "Should return null if chart was not an official (Keyboard)." + ); + + t.end(); +}); + +/** + * These tests only check that the right properties are assigned. + */ +t.test("#CalculateDataForGamePT", (t) => { + t.test("IIDX:SP", async (t) => { + const res = await CalculateDataForGamePT( + "iidx", + "SP", + Testing511SPA, + TestingIIDXSPDryScore, + 30, + + logger + ); + + prAssert( + res, + { + ktRating: "?number", + ktLampRating: "?number", + BPI: "?number", + }, + "Response should contain keys for IIDX:SP" + ); + + t.end(); + }); + + t.test("IIDX:DP", async (t) => { + const res = await CalculateDataForGamePT( + "iidx", + "DP", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an SP score. but we're testing + 30, + + logger + ); + + prAssert( + res, + { + ktRating: "?number", + ktLampRating: "?number", + BPI: "?number", + }, + "Response should contain keys for IIDX:DP" + ); + + t.end(); + }); + + t.test("SDVX:Single", async (t) => { + const res = await CalculateDataForGamePT( + "sdvx", + "Single", + Testing511SPA, + TestingSDVXSingleDryScore, + null, + logger + ); + + prAssert( + res, + { + VF6: p.nullable(p.isPositive), + }, + "Response should contain keys for SDVX:Single" + ); + + t.end(); + }); + + t.test("DDR:SP", async (t) => { + const res = await CalculateDataForGamePT( + "ddr", + "SP", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + MFCP: p.nullable(p.isPositiveInteger), + ktRating: "?number", + }, + "Response should contain keys for DDR:SP" + ); + + t.end(); + }); + + t.test("DDR:DP", async (t) => { + const res = await CalculateDataForGamePT( + "ddr", + "DP", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + MFCP: "null", + ktRating: "?number", + }, + "Response should contain nulled keys for DDR:DP" + ); + + t.end(); + }); + + t.test("chunithm:Single", async (t) => { + const res = await CalculateDataForGamePT( + "chunithm", + "Single", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + rating: "?number", + }, + "Response should contain nulled keys for chunithm:Single" + ); + + t.end(); + }); + + t.test("maimai:Single", async (t) => { + const res = await CalculateDataForGamePT( + "maimai", + "Single", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + ktRating: "?number", + }, + "Response should contain nulled keys for maimai:Single" + ); + + t.end(); + }); + + t.test("museca:Single", async (t) => { + const res = await CalculateDataForGamePT( + "museca", + "Single", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + ktRating: "?number", + }, + "Response should contain nulled keys for museca:Single" + ); + + t.end(); + }); + + t.test("bms:7K", async (t) => { + const res = await CalculateDataForGamePT( + "bms", + "7K", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + sieglinde: "?number", + }, + "Response should contain nulled keys for bms:7K" + ); + + t.end(); + }); + + t.test("bms:14K", async (t) => { + const res = await CalculateDataForGamePT( + "bms", + "14K", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + sieglinde: "?number", + }, + "Response should contain nulled keys for bms:14K" + ); + + t.end(); + }); + + t.test("gitadora:Gita", async (t) => { + const res = await CalculateDataForGamePT( + "gitadora", + "Gita", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + skill: "?number", + }, + "Response should contain nulled keys for gitadora:Gita" + ); + + t.end(); + }); + + t.test("gitadora:Dora", async (t) => { + const res = await CalculateDataForGamePT( + "gitadora", + "Dora", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + skill: "?number", + }, + "Response should contain nulled keys for gitadora:Dora" + ); + + t.end(); + }); + + t.test("usc:Controller", async (t) => { + const res = await CalculateDataForGamePT( + "usc", + "Controller", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + VF6: "?number", + }, + "Response should contain nulled keys for usc:Controller" + ); + + t.end(); + }); + + t.test("usc:Keyboard", async (t) => { + const res = await CalculateDataForGamePT( + "usc", + "Keyboard", + Testing511SPA, + TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + null, + logger + ); + + prAssert( + res, + { + VF6: "?number", + }, + "Response should contain nulled keys for usc:Keyboard" + ); + + t.end(); + }); + // t.test("jubeat:Single", async (t) => { + // const res = await CalculateDataForGamePT( + // "jubeat", + // "Single", + // Testing511SPA, + // TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + // null, + // + // logger + // ); + + // prAssert( + // res, + // { + // jubility: "?number", + // }, + // "Response should contain nulled keys for jubeat:Single" + // ); + + // t.end(); + // }); + + // t.test("popn:9B", async (t) => { + // const res = await CalculateDataForGamePT( + // "popn", + // "9B", + // Testing511SPA, + // TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing + // null, + // + // logger + // ); + + // prAssert(res, {}, "Response should contain nulled keys for popn:9B"); + + // t.end(); + // }); + + t.end(); +}); diff --git a/server/src/lib/score-import/framework/calculated-data/calculated-data.ts b/server/src/lib/score-import/framework/calculated-data/calculated-data.ts index 6e1e24072..8bc2e0340 100644 --- a/server/src/lib/score-import/framework/calculated-data/calculated-data.ts +++ b/server/src/lib/score-import/framework/calculated-data/calculated-data.ts @@ -1,291 +1,291 @@ -import db from "external/mongo/db"; -import { KtLogger } from "lib/logger/logger"; -import { - ChartDocument, - Game, - Grades, - IDStrings, - Lamps, - Playtypes, - ScoreDocument, -} from "tachi-common"; -import { HasOwnProperty } from "utils/misc"; -import { DryScore } from "../common/types"; -import { - CalculateBPI, - CalculateCHUNITHMRating, - CalculateGITADORASkill, - CalculateKTLampRating, - CalculateKTRating, - CalculateMFCP, - CalculateVF6, -} from "./stats"; - -export async function CreateCalculatedData( - dryScore: DryScore, - chart: ChartDocument, - esd: number | null, - logger: KtLogger -): Promise { - const game = dryScore.game; - const playtype = chart.playtype; - - const calculatedData = await CalculateDataForGamePT( - game, - playtype, - chart, - dryScore, - esd, - logger - ); - - return calculatedData; -} - -type CalculatedDataFunctions = { - [G in Game]: { - [P in Playtypes[G]]: ( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger - ) => Promise | ScoreDocument["calculatedData"]; - }; -}; - -const CalculatedDataFunctions: CalculatedDataFunctions = { - iidx: { - SP: CalculateDataIIDXSP, - DP: CalculateDataIIDXDP, - }, - sdvx: { - Single: CalculateDataSDVXorUSC, - }, - // popn: { - // "9B": () => ({}), - // }, - museca: { - Single: CalculateDataMuseca, - }, - chunithm: { - Single: CalculateDataCHUNITHM, - }, - maimai: { - Single: CalculateDataMaimai, - }, - gitadora: { - Gita: CalculateDataGitadora, - Dora: CalculateDataGitadora, - }, - bms: { - "7K": CalculateDataBMS7K, - "14K": CalculateDataBMS14K, - }, - ddr: { - SP: CalculateDataDDR, - DP: CalculateDataDDR, - }, - // jubeat: { - // Single: CalculateDataJubeat, - // }, - usc: { - Controller: CalculateDataSDVXorUSC, - Keyboard: CalculateDataSDVXorUSC, - }, -}; - -// Creates Game-Specific calculatedData for the provided game & playtype. -// eslint-disable-next-line require-await -export async function CalculateDataForGamePT( - game: G, - playtype: Playtypes[G], - chart: ChartDocument, - dryScore: DryScore, - // ESD gets specially passed through because it's not part of the DryScore, but - // can be used for statistics anyway. - esd: number | null, - logger: KtLogger -): Promise { - const GameRatingFns = CalculatedDataFunctions[game]; - - if (!HasOwnProperty(GameRatingFns, playtype)) { - logger.error( - `Invalid playtype of ${playtype} given for game ${game} in CalculateDataForGamePT, returning an empty object.` - ); - return {}; - } - - // @ts-expect-error standard game->pt stuff. - return GameRatingFns[playtype](dryScore, chart, logger); -} - -type CalculatedData = Required["calculatedData"]>; - -async function CalculateDataIIDXSP( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): Promise> { - const BPIData = await db["iidx-bpi-data"].findOne({ - chartID: chart.chartID, - }); - - let bpi; - - if (BPIData) { - bpi = CalculateBPI( - BPIData.kavg, - BPIData.wr, - dryScore.scoreData.score, - (chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2, - BPIData.coef - ); - - // kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled - } else { - bpi = null; - } - - return { - BPI: bpi, - ktRating: await CalculateKTRating(dryScore, "iidx", "SP", chart, logger), - ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "SP", chart), - }; -} - -async function CalculateDataIIDXDP( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): Promise> { - const BPIData = await db["iidx-bpi-data"].findOne({ - chartID: chart.chartID, - }); - - let bpi; - - if (BPIData) { - bpi = CalculateBPI( - BPIData.kavg, - BPIData.wr, - dryScore.scoreData.score, - (chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2, - BPIData.coef - ); - - // kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled - } else { - bpi = null; - } - - return { - BPI: bpi, - ktRating: await CalculateKTRating(dryScore, "iidx", "DP", chart, logger), - ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "DP", chart), - }; -} - -function CalculateDataSDVXorUSC( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): CalculatedData<"sdvx:Single" | "usc:Keyboard" | "usc:Controller"> { - // for usc, unofficial charts currently have no VF6 value. - if ( - dryScore.game === "usc" && - !(chart as ChartDocument<"usc:Controller" | "usc:Keyboard">).data.isOfficial - ) { - return { VF6: null }; - } - - const VF6 = CalculateVF6( - dryScore.scoreData.grade as Grades["sdvx:Single"], - dryScore.scoreData.lamp as Lamps["sdvx:Single"], - dryScore.scoreData.percent, - chart.levelNum, - logger - ); - - return { - VF6, - }; -} - -async function CalculateDataMuseca( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): Promise> { - return { - ktRating: await CalculateKTRating(dryScore, "museca", "Single", chart, logger), - }; -} - -function CalculateDataCHUNITHM( - dryScore: DryScore, - chart: ChartDocument -): CalculatedData<"chunithm:Single"> { - return { - rating: CalculateCHUNITHMRating(dryScore, chart), - }; -} - -async function CalculateDataMaimai( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): Promise> { - // @todo #373 Add maimai rating algorithms. - return { - ktRating: 0, - }; -} - -function CalculateDataGitadora( - dryScore: DryScore, - chart: ChartDocument -): CalculatedData<"gitadora:Gita" | "gitadora:Dora"> { - return { - skill: CalculateGITADORASkill(dryScore, chart), - }; -} - -async function CalculateDataDDR( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): Promise> { - return { - MFCP: CalculateMFCP(dryScore, chart, logger), - ktRating: await CalculateKTRating(dryScore, "ddr", chart.playtype, chart, logger), - }; -} - -export async function CalculateDataBMS14K( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): Promise> { - return { - sieglinde: 0, // @todo #33 - }; -} - -export async function CalculateDataBMS7K( - dryScore: DryScore, - chart: ChartDocument, - logger: KtLogger -): Promise> { - return { - sieglinde: 0, // @todo #33 - }; -} - -// async function CalculateDataJubeat( -// dryScore: DryScore, -// chart: ChartDocument, -// logger: KtLogger -// ): Promise> { -// return { -// jubility: 0, // @todo #163 Jubeat Jubility -// }; -// } +import db from "external/mongo/db"; +import { KtLogger } from "lib/logger/logger"; +import { + ChartDocument, + Game, + Grades, + IDStrings, + Lamps, + Playtypes, + ScoreDocument, +} from "tachi-common"; +import { HasOwnProperty } from "utils/misc"; +import { DryScore } from "../common/types"; +import { + CalculateBPI, + CalculateCHUNITHMRating, + CalculateGITADORASkill, + CalculateKTLampRating, + CalculateKTRating, + CalculateMFCP, + CalculateVF6, +} from "./stats"; + +export async function CreateCalculatedData( + dryScore: DryScore, + chart: ChartDocument, + esd: number | null, + logger: KtLogger +): Promise { + const game = dryScore.game; + const playtype = chart.playtype; + + const calculatedData = await CalculateDataForGamePT( + game, + playtype, + chart, + dryScore, + esd, + logger + ); + + return calculatedData; +} + +type CalculatedDataFunctions = { + [G in Game]: { + [P in Playtypes[G]]: ( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger + ) => Promise | ScoreDocument["calculatedData"]; + }; +}; + +const CalculatedDataFunctions: CalculatedDataFunctions = { + iidx: { + SP: CalculateDataIIDXSP, + DP: CalculateDataIIDXDP, + }, + sdvx: { + Single: CalculateDataSDVXorUSC, + }, + // popn: { + // "9B": () => ({}), + // }, + museca: { + Single: CalculateDataMuseca, + }, + chunithm: { + Single: CalculateDataCHUNITHM, + }, + maimai: { + Single: CalculateDataMaimai, + }, + gitadora: { + Gita: CalculateDataGitadora, + Dora: CalculateDataGitadora, + }, + bms: { + "7K": CalculateDataBMS7K, + "14K": CalculateDataBMS14K, + }, + ddr: { + SP: CalculateDataDDR, + DP: CalculateDataDDR, + }, + // jubeat: { + // Single: CalculateDataJubeat, + // }, + usc: { + Controller: CalculateDataSDVXorUSC, + Keyboard: CalculateDataSDVXorUSC, + }, +}; + +// Creates Game-Specific calculatedData for the provided game & playtype. +// eslint-disable-next-line require-await +export async function CalculateDataForGamePT( + game: G, + playtype: Playtypes[G], + chart: ChartDocument, + dryScore: DryScore, + // ESD gets specially passed through because it's not part of the DryScore, but + // can be used for statistics anyway. + esd: number | null, + logger: KtLogger +): Promise { + const GameRatingFns = CalculatedDataFunctions[game]; + + if (!HasOwnProperty(GameRatingFns, playtype)) { + logger.error( + `Invalid playtype of ${playtype} given for game ${game} in CalculateDataForGamePT, returning an empty object.` + ); + return {}; + } + + // @ts-expect-error standard game->pt stuff. + return GameRatingFns[playtype](dryScore, chart, logger); +} + +type CalculatedData = Required["calculatedData"]>; + +async function CalculateDataIIDXSP( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): Promise> { + const BPIData = await db["iidx-bpi-data"].findOne({ + chartID: chart.chartID, + }); + + let bpi; + + if (BPIData) { + bpi = CalculateBPI( + BPIData.kavg, + BPIData.wr, + dryScore.scoreData.score, + (chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2, + BPIData.coef + ); + + // kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled + } else { + bpi = null; + } + + return { + BPI: bpi, + ktRating: await CalculateKTRating(dryScore, "iidx", "SP", chart, logger), + ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "SP", chart), + }; +} + +async function CalculateDataIIDXDP( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): Promise> { + const BPIData = await db["iidx-bpi-data"].findOne({ + chartID: chart.chartID, + }); + + let bpi; + + if (BPIData) { + bpi = CalculateBPI( + BPIData.kavg, + BPIData.wr, + dryScore.scoreData.score, + (chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2, + BPIData.coef + ); + + // kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled + } else { + bpi = null; + } + + return { + BPI: bpi, + ktRating: await CalculateKTRating(dryScore, "iidx", "DP", chart, logger), + ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "DP", chart), + }; +} + +function CalculateDataSDVXorUSC( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): CalculatedData<"sdvx:Single" | "usc:Keyboard" | "usc:Controller"> { + // for usc, unofficial charts currently have no VF6 value. + if ( + dryScore.game === "usc" && + !(chart as ChartDocument<"usc:Controller" | "usc:Keyboard">).data.isOfficial + ) { + return { VF6: null }; + } + + const VF6 = CalculateVF6( + dryScore.scoreData.grade as Grades["sdvx:Single"], + dryScore.scoreData.lamp as Lamps["sdvx:Single"], + dryScore.scoreData.percent, + chart.levelNum, + logger + ); + + return { + VF6, + }; +} + +async function CalculateDataMuseca( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): Promise> { + return { + ktRating: await CalculateKTRating(dryScore, "museca", "Single", chart, logger), + }; +} + +function CalculateDataCHUNITHM( + dryScore: DryScore, + chart: ChartDocument +): CalculatedData<"chunithm:Single"> { + return { + rating: CalculateCHUNITHMRating(dryScore, chart), + }; +} + +async function CalculateDataMaimai( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): Promise> { + // @todo #373 Add maimai rating algorithms. + return { + ktRating: 0, + }; +} + +function CalculateDataGitadora( + dryScore: DryScore, + chart: ChartDocument +): CalculatedData<"gitadora:Gita" | "gitadora:Dora"> { + return { + skill: CalculateGITADORASkill(dryScore, chart), + }; +} + +async function CalculateDataDDR( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): Promise> { + return { + MFCP: CalculateMFCP(dryScore, chart, logger), + ktRating: await CalculateKTRating(dryScore, "ddr", chart.playtype, chart, logger), + }; +} + +export async function CalculateDataBMS14K( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): Promise> { + return { + sieglinde: 0, // @todo #33 + }; +} + +export async function CalculateDataBMS7K( + dryScore: DryScore, + chart: ChartDocument, + logger: KtLogger +): Promise> { + return { + sieglinde: 0, // @todo #33 + }; +} + +// async function CalculateDataJubeat( +// dryScore: DryScore, +// chart: ChartDocument, +// logger: KtLogger +// ): Promise> { +// return { +// jubility: 0, // @todo #163 Jubeat Jubility +// }; +// } diff --git a/server/src/lib/score-import/framework/calculated-data/stats.test.ts b/server/src/lib/score-import/framework/calculated-data/stats.test.ts index 4f23e9dae..6121ae303 100644 --- a/server/src/lib/score-import/framework/calculated-data/stats.test.ts +++ b/server/src/lib/score-import/framework/calculated-data/stats.test.ts @@ -1,361 +1,361 @@ -import deepmerge from "deepmerge"; -import CreateLogCtx from "lib/logger/logger"; -import { ChartDocument, Difficulties, Lamps, ScoreDocument } from "tachi-common"; -import t from "tap"; -import { isApproximately } from "test-utils/asserts"; -import { GetKTDataJSON, Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data"; -import { DryScore } from "../common/types"; -import { - CalculateBPI, - CalculateCHUNITHMRating, - CalculateGITADORASkill, - CalculateKTRating, - CalculateMFCP, - CalculateVF6, -} from "./stats"; - -t.test("#CalculateBPI", (t) => { - t.test("AA BPI tests", (t) => { - // data accurate as of 14/04/2021 - const WR_AA = 3650; - const KAVG_AA = 3204; - const MAX_AA = 1834 * 2; - const COEF_AA = 1.25945; - - // AA, the song. - function AA_BPI(yourEx: number) { - return CalculateBPI(KAVG_AA, WR_AA, yourEx, MAX_AA, COEF_AA); - } - - t.equal(AA_BPI(KAVG_AA), 0, "A score of KAVG should be exactly 0BPI"); - isApproximately(AA_BPI(3393), 10.02, "A score with 3393 should be approximately 10.02BPI"); - isApproximately(AA_BPI(3481), 20.09, "A score with 3481 should be approximately 20.09BPI"); - isApproximately(AA_BPI(3535), 30.01, "A score with 3535 should be approximately 30.01BPI"); - isApproximately(AA_BPI(3572), 40.18, "A score with 3572 should be approximately 40.18BPI"); - isApproximately(AA_BPI(3597), 50.1, "A score with 3597 should be approximately 50.1BPI"); - isApproximately(AA_BPI(3615), 60.12, "A score with 3615 should be approximately 60.12BPI"); - isApproximately(AA_BPI(3628), 70.11, "A score with 3628 should be approximately 70.11BPI"); - isApproximately(AA_BPI(3638), 80.62, "A score with 3638 should be approximately 80.62BPI"); - isApproximately(AA_BPI(3645), 90.59, "A score with 3645 should be approximately 90.59BPI"); - t.equal(AA_BPI(WR_AA), 100, "A score of WR should be exactly 100BPI"); - - isApproximately(AA_BPI(3041), -5, "A score with 3628 should be approximately -5BPI"); - isApproximately(AA_BPI(2886), -9.99, "A score with 3638 should be approximately -9.99BPI"); - - t.equal(AA_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI"); - - isApproximately(AA_BPI(MAX_AA), 244.56, "A score of MAX should be approximately 244.56BPI"); - - t.end(); - }); - - t.test("COLOSSEUM BPI Tests", (t) => { - const WR_AFT = 2312; - const KAVG_CL = 2131; - const MAX_CL = 2318; - const COEF_CL = -1; - - function CL_BPI(yourEx: number) { - return CalculateBPI(KAVG_CL, WR_AFT, yourEx, MAX_CL, COEF_CL); - } - - isApproximately(CL_BPI(2307), 79.63); - - t.end(); - }); - - // This song has both no co-efficient and is an sp11, so it serves - // as another interesting test. - t.test("Afterimage d'automne BPI tests", (t) => { - // data accurate as of 14/04/2021 - const WR_AFT = 2891; - const KAVG_AFT = 2497; - const MAX_AFT = 1480 * 2; - const COEF_AFT = null; - - function AFT_BPI(yourEx: number) { - return CalculateBPI(KAVG_AFT, WR_AFT, yourEx, MAX_AFT, COEF_AFT); - } - - t.equal(AFT_BPI(KAVG_AFT), 0, "A score of KAVG should be exactly 0BPI"); - isApproximately(AFT_BPI(2606), 10, "A score with 2606 should be approximately 10BPI"); - isApproximately(AFT_BPI(2675), 20.07, "A score with 2675 should be approximately 20.07BPI"); - isApproximately(AFT_BPI(2727), 30.18, "A score with 2727 should be approximately 30.18BPI"); - isApproximately(AFT_BPI(2767), 40.12, "A score with 2767 should be approximately 40.12BPI"); - isApproximately(AFT_BPI(2799), 50.06, "A score with 2799 should be approximately 50.06BPI"); - isApproximately(AFT_BPI(2826), 60.43, "A score with 2826 should be approximately 60.43BPI"); - isApproximately(AFT_BPI(2847), 70.3, "A score with 2847 should be approximately 70.3BPI"); - isApproximately(AFT_BPI(2865), 80.57, "A score with 2865 should be approximately 80.57BPI"); - isApproximately(AFT_BPI(2879), 90.18, "A score with 2879 should be approximately 90.18BPI"); - t.equal(AFT_BPI(WR_AFT), 100, "A score of WR should be exactly 100BPI"); - - isApproximately(AFT_BPI(2423), -4.99, "A score with 2423 should be approximately -4.99BPI"); - isApproximately( - AFT_BPI(2354), - -10.04, - "A score with 2354 should be approximately -10.04BPI" - ); - - t.equal(AFT_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI"); - - isApproximately( - AFT_BPI(MAX_AFT), - 431.57, - "A score of MAX should be approximately 431.57BPI" - ); - - t.end(); - }); - - t.end(); -}); - -t.test("#CalculateGITADORARating", (t) => { - function TestGitadoraRating(percent: number, levelNum: number) { - return CalculateGITADORASkill( - { scoreData: { percent } } as DryScore, - { levelNum } as ChartDocument - ); - } - - t.equal( - TestGitadoraRating(89.48, 3.4), - 60.84, - "Test GITADORA Rating function aligns with game (1)" - ); - - t.equal( - TestGitadoraRating(70.76, 5.8), - 82.08, - "Test GITADORA Rating function aligns with game (2)" - ); - - t.end(); -}); - -const logger = CreateLogCtx(__filename); - -t.test("#CalculateMFCP", (t) => { - function TestMFCP( - lamp: Lamps["ddr:SP" | "ddr:DP"], - levelNum: number, - difficulty: Difficulties["ddr:DP" | "ddr:SP"] - ) { - return CalculateMFCP( - { - scoreData: { - lamp, - }, - } as ScoreDocument, - { - difficulty, - levelNum, - } as ChartDocument, - logger - ); - } - - t.equal(TestMFCP("FAILED", 10, "EXPERT"), null, "Should return null for non-mfcs"); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 10, "BASIC"), - null, - "Should reject charts on BASIC difficulty." - ); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 10, "BEGINNER"), - null, - "Should reject charts on BEGINNER difficulty." - ); - - t.test("Should return null for charts with level less than 8", (t) => { - for (let i = 1; i <= 7; i++) { - t.equal( - TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"), - null, - `Should return null for charts with level ${i}` - ); - } - - t.end(); - }); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 8, "EXPERT"), - 1, - "Should return 1 for charts with level 8" - ); - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 9, "EXPERT"), - 1, - "Should return 1 for charts with level 9" - ); - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 10, "EXPERT"), - 1, - "Should return 1 for charts with level 10" - ); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 11, "EXPERT"), - 2, - "Should return 2 for charts with level 11" - ); - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 12, "EXPERT"), - 2, - "Should return 2 for charts with level 12" - ); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 13, "EXPERT"), - 4, - "Should return 4 for charts with level 13" - ); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 14, "EXPERT"), - 8, - "Should return 8 for charts with level 14" - ); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", 15, "EXPERT"), - 15, - "Should return 15 for charts with level 15" - ); - - t.test("Should return 25 for charts with level 16-20", (t) => { - for (let i = 16; i <= 20; i++) { - t.equal( - TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"), - 25, - `Should return 25 for charts with level ${i}` - ); - } - t.end(); - }); - - t.equal( - TestMFCP("MARVELOUS FULL COMBO", NaN, "EXPERT"), - null, - "Invalid level triggers failsafe." - ); - - t.end(); -}); - -const bbkk = GetKTDataJSON("./tachi/chunithm-bbkk-chart.json"); - -// unit testing a mathematical function is a square-round-hole problem. -t.test("#CalculateCHUNITHMRating", (t) => { - // cutoffs - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_010_000 } } as DryScore, bbkk), 5.0); - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_005_000 } } as DryScore, bbkk), 4.5); - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_000_000 } } as DryScore, bbkk), 4); - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 975_000 } } as DryScore, bbkk), 3); - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 925_000 } } as DryScore, bbkk), 0); - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 900_000 } } as DryScore, bbkk), 0); - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 800_000 } } as DryScore, bbkk), 0); - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 0 } } as DryScore, bbkk), 0); - - // inbetweens - t.equal(CalculateCHUNITHMRating({ scoreData: { score: 987_000 } } as DryScore, bbkk), 3.48); - - t.end(); -}); - -// Random assertions plucked from bemaniwiki. -t.test("#CalculateVF6", (t) => { - t.equal(CalculateVF6("S", "CLEAR", 99, 16, logger), 0.332); - - t.equal(CalculateVF6("AAA+", "ULTIMATE CHAIN", 98, 17, logger), 0.356); - - t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 16, logger), 0.369); - t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 17, logger), 0.392); - t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 18, logger), 0.415); - t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 19, logger), 0.438); - t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 20, logger), 0.462); - - t.end(); -}); - -t.test("#CalculateRating", (t) => { - t.test("Should call the success calculator if percent > pivotPercent", async (t) => { - const r = await CalculateKTRating( - deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 80 } }), - "iidx", - "SP", - Testing511SPA, - logger - ); - - t.ok(r > 10, "Should return rating greater than the levelNum of the chart."); - - t.end(); - }); - - t.test("Should call the fail calculator if percent > pivotPercent", async (t) => { - const r = await CalculateKTRating( - TestingIIDXSPDryScore, - "iidx", - "SP", - Testing511SPA, - logger - ); - - t.ok(r < 10, "Should return rating less than the levelNum of the chart."); - - t.end(); - }); - - t.test("Should call levelNum if percent === pivotPercent", async (t) => { - const r = await CalculateKTRating( - deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 77.7777 } }), - "iidx", - "SP", - Testing511SPA, - logger - ); - - t.equal( - // hack for approximate tests - parseFloat(r.toFixed(2)), - 10, - "Should return rating exactly that of the levelNum of the chart." - ); - - t.end(); - }); - - t.test( - "Should trigger safety if completely invalid percent somehow gets through", - async (t) => { - let r = await CalculateKTRating( - deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 1000000000 } }), - "iidx", - "SP", - Testing511SPA, - logger - ); - - t.equal(r, 0, "Should safely return 0 and log a warning."); - - r = await CalculateKTRating( - // not high enough to be non-finite but high enough to be > 1000 - deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 200 } }), - "iidx", - "SP", - Testing511SPA, - logger - ); - - t.equal(r, 0, "Should safely return 0 and log a warning."); - - t.end(); - } - ); - - t.end(); -}); +import deepmerge from "deepmerge"; +import CreateLogCtx from "lib/logger/logger"; +import { ChartDocument, Difficulties, Lamps, ScoreDocument } from "tachi-common"; +import t from "tap"; +import { isApproximately } from "test-utils/asserts"; +import { GetKTDataJSON, Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data"; +import { DryScore } from "../common/types"; +import { + CalculateBPI, + CalculateCHUNITHMRating, + CalculateGITADORASkill, + CalculateKTRating, + CalculateMFCP, + CalculateVF6, +} from "./stats"; + +t.test("#CalculateBPI", (t) => { + t.test("AA BPI tests", (t) => { + // data accurate as of 14/04/2021 + const WR_AA = 3650; + const KAVG_AA = 3204; + const MAX_AA = 1834 * 2; + const COEF_AA = 1.25945; + + // AA, the song. + function AA_BPI(yourEx: number) { + return CalculateBPI(KAVG_AA, WR_AA, yourEx, MAX_AA, COEF_AA); + } + + t.equal(AA_BPI(KAVG_AA), 0, "A score of KAVG should be exactly 0BPI"); + isApproximately(AA_BPI(3393), 10.02, "A score with 3393 should be approximately 10.02BPI"); + isApproximately(AA_BPI(3481), 20.09, "A score with 3481 should be approximately 20.09BPI"); + isApproximately(AA_BPI(3535), 30.01, "A score with 3535 should be approximately 30.01BPI"); + isApproximately(AA_BPI(3572), 40.18, "A score with 3572 should be approximately 40.18BPI"); + isApproximately(AA_BPI(3597), 50.1, "A score with 3597 should be approximately 50.1BPI"); + isApproximately(AA_BPI(3615), 60.12, "A score with 3615 should be approximately 60.12BPI"); + isApproximately(AA_BPI(3628), 70.11, "A score with 3628 should be approximately 70.11BPI"); + isApproximately(AA_BPI(3638), 80.62, "A score with 3638 should be approximately 80.62BPI"); + isApproximately(AA_BPI(3645), 90.59, "A score with 3645 should be approximately 90.59BPI"); + t.equal(AA_BPI(WR_AA), 100, "A score of WR should be exactly 100BPI"); + + isApproximately(AA_BPI(3041), -5, "A score with 3628 should be approximately -5BPI"); + isApproximately(AA_BPI(2886), -9.99, "A score with 3638 should be approximately -9.99BPI"); + + t.equal(AA_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI"); + + isApproximately(AA_BPI(MAX_AA), 244.56, "A score of MAX should be approximately 244.56BPI"); + + t.end(); + }); + + t.test("COLOSSEUM BPI Tests", (t) => { + const WR_AFT = 2312; + const KAVG_CL = 2131; + const MAX_CL = 2318; + const COEF_CL = -1; + + function CL_BPI(yourEx: number) { + return CalculateBPI(KAVG_CL, WR_AFT, yourEx, MAX_CL, COEF_CL); + } + + isApproximately(CL_BPI(2307), 79.63); + + t.end(); + }); + + // This song has both no co-efficient and is an sp11, so it serves + // as another interesting test. + t.test("Afterimage d'automne BPI tests", (t) => { + // data accurate as of 14/04/2021 + const WR_AFT = 2891; + const KAVG_AFT = 2497; + const MAX_AFT = 1480 * 2; + const COEF_AFT = null; + + function AFT_BPI(yourEx: number) { + return CalculateBPI(KAVG_AFT, WR_AFT, yourEx, MAX_AFT, COEF_AFT); + } + + t.equal(AFT_BPI(KAVG_AFT), 0, "A score of KAVG should be exactly 0BPI"); + isApproximately(AFT_BPI(2606), 10, "A score with 2606 should be approximately 10BPI"); + isApproximately(AFT_BPI(2675), 20.07, "A score with 2675 should be approximately 20.07BPI"); + isApproximately(AFT_BPI(2727), 30.18, "A score with 2727 should be approximately 30.18BPI"); + isApproximately(AFT_BPI(2767), 40.12, "A score with 2767 should be approximately 40.12BPI"); + isApproximately(AFT_BPI(2799), 50.06, "A score with 2799 should be approximately 50.06BPI"); + isApproximately(AFT_BPI(2826), 60.43, "A score with 2826 should be approximately 60.43BPI"); + isApproximately(AFT_BPI(2847), 70.3, "A score with 2847 should be approximately 70.3BPI"); + isApproximately(AFT_BPI(2865), 80.57, "A score with 2865 should be approximately 80.57BPI"); + isApproximately(AFT_BPI(2879), 90.18, "A score with 2879 should be approximately 90.18BPI"); + t.equal(AFT_BPI(WR_AFT), 100, "A score of WR should be exactly 100BPI"); + + isApproximately(AFT_BPI(2423), -4.99, "A score with 2423 should be approximately -4.99BPI"); + isApproximately( + AFT_BPI(2354), + -10.04, + "A score with 2354 should be approximately -10.04BPI" + ); + + t.equal(AFT_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI"); + + isApproximately( + AFT_BPI(MAX_AFT), + 431.57, + "A score of MAX should be approximately 431.57BPI" + ); + + t.end(); + }); + + t.end(); +}); + +t.test("#CalculateGITADORARating", (t) => { + function TestGitadoraRating(percent: number, levelNum: number) { + return CalculateGITADORASkill( + { scoreData: { percent } } as DryScore, + { levelNum } as ChartDocument + ); + } + + t.equal( + TestGitadoraRating(89.48, 3.4), + 60.84, + "Test GITADORA Rating function aligns with game (1)" + ); + + t.equal( + TestGitadoraRating(70.76, 5.8), + 82.08, + "Test GITADORA Rating function aligns with game (2)" + ); + + t.end(); +}); + +const logger = CreateLogCtx(__filename); + +t.test("#CalculateMFCP", (t) => { + function TestMFCP( + lamp: Lamps["ddr:SP" | "ddr:DP"], + levelNum: number, + difficulty: Difficulties["ddr:DP" | "ddr:SP"] + ) { + return CalculateMFCP( + { + scoreData: { + lamp, + }, + } as ScoreDocument, + { + difficulty, + levelNum, + } as ChartDocument, + logger + ); + } + + t.equal(TestMFCP("FAILED", 10, "EXPERT"), null, "Should return null for non-mfcs"); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 10, "BASIC"), + null, + "Should reject charts on BASIC difficulty." + ); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 10, "BEGINNER"), + null, + "Should reject charts on BEGINNER difficulty." + ); + + t.test("Should return null for charts with level less than 8", (t) => { + for (let i = 1; i <= 7; i++) { + t.equal( + TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"), + null, + `Should return null for charts with level ${i}` + ); + } + + t.end(); + }); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 8, "EXPERT"), + 1, + "Should return 1 for charts with level 8" + ); + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 9, "EXPERT"), + 1, + "Should return 1 for charts with level 9" + ); + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 10, "EXPERT"), + 1, + "Should return 1 for charts with level 10" + ); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 11, "EXPERT"), + 2, + "Should return 2 for charts with level 11" + ); + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 12, "EXPERT"), + 2, + "Should return 2 for charts with level 12" + ); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 13, "EXPERT"), + 4, + "Should return 4 for charts with level 13" + ); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 14, "EXPERT"), + 8, + "Should return 8 for charts with level 14" + ); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", 15, "EXPERT"), + 15, + "Should return 15 for charts with level 15" + ); + + t.test("Should return 25 for charts with level 16-20", (t) => { + for (let i = 16; i <= 20; i++) { + t.equal( + TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"), + 25, + `Should return 25 for charts with level ${i}` + ); + } + t.end(); + }); + + t.equal( + TestMFCP("MARVELOUS FULL COMBO", NaN, "EXPERT"), + null, + "Invalid level triggers failsafe." + ); + + t.end(); +}); + +const bbkk = GetKTDataJSON("./tachi/chunithm-bbkk-chart.json"); + +// unit testing a mathematical function is a square-round-hole problem. +t.test("#CalculateCHUNITHMRating", (t) => { + // cutoffs + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_010_000 } } as DryScore, bbkk), 5.0); + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_005_000 } } as DryScore, bbkk), 4.5); + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_000_000 } } as DryScore, bbkk), 4); + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 975_000 } } as DryScore, bbkk), 3); + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 925_000 } } as DryScore, bbkk), 0); + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 900_000 } } as DryScore, bbkk), 0); + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 800_000 } } as DryScore, bbkk), 0); + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 0 } } as DryScore, bbkk), 0); + + // inbetweens + t.equal(CalculateCHUNITHMRating({ scoreData: { score: 987_000 } } as DryScore, bbkk), 3.48); + + t.end(); +}); + +// Random assertions plucked from bemaniwiki. +t.test("#CalculateVF6", (t) => { + t.equal(CalculateVF6("S", "CLEAR", 99, 16, logger), 0.332); + + t.equal(CalculateVF6("AAA+", "ULTIMATE CHAIN", 98, 17, logger), 0.356); + + t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 16, logger), 0.369); + t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 17, logger), 0.392); + t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 18, logger), 0.415); + t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 19, logger), 0.438); + t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 20, logger), 0.462); + + t.end(); +}); + +t.test("#CalculateRating", (t) => { + t.test("Should call the success calculator if percent > pivotPercent", async (t) => { + const r = await CalculateKTRating( + deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 80 } }), + "iidx", + "SP", + Testing511SPA, + logger + ); + + t.ok(r > 10, "Should return rating greater than the levelNum of the chart."); + + t.end(); + }); + + t.test("Should call the fail calculator if percent > pivotPercent", async (t) => { + const r = await CalculateKTRating( + TestingIIDXSPDryScore, + "iidx", + "SP", + Testing511SPA, + logger + ); + + t.ok(r < 10, "Should return rating less than the levelNum of the chart."); + + t.end(); + }); + + t.test("Should call levelNum if percent === pivotPercent", async (t) => { + const r = await CalculateKTRating( + deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 77.7777 } }), + "iidx", + "SP", + Testing511SPA, + logger + ); + + t.equal( + // hack for approximate tests + parseFloat(r.toFixed(2)), + 10, + "Should return rating exactly that of the levelNum of the chart." + ); + + t.end(); + }); + + t.test( + "Should trigger safety if completely invalid percent somehow gets through", + async (t) => { + let r = await CalculateKTRating( + deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 1000000000 } }), + "iidx", + "SP", + Testing511SPA, + logger + ); + + t.equal(r, 0, "Should safely return 0 and log a warning."); + + r = await CalculateKTRating( + // not high enough to be non-finite but high enough to be > 1000 + deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 200 } }), + "iidx", + "SP", + Testing511SPA, + logger + ); + + t.equal(r, 0, "Should safely return 0 and log a warning."); + + t.end(); + } + ); + + t.end(); +}); diff --git a/server/src/lib/score-import/framework/calculated-data/stats.ts b/server/src/lib/score-import/framework/calculated-data/stats.ts index 985c5186a..67714211a 100644 --- a/server/src/lib/score-import/framework/calculated-data/stats.ts +++ b/server/src/lib/score-import/framework/calculated-data/stats.ts @@ -1,404 +1,404 @@ -import { KtLogger } from "lib/logger/logger"; -import { - ChartDocument, - Game, - GetGamePTConfig, - Grades, - integer, - Lamps, - Playtypes, -} from "tachi-common"; -import { DryScore } from "../common/types"; - -/** - * Calculates the in-game CHUNITHM rating for a score. - */ -export function CalculateCHUNITHMRating(dryScore: DryScore, chartData: ChartDocument) { - const score = dryScore.scoreData.score; - const levelBase = chartData.levelNum * 100; - - let val = 0; - - if (score >= 1_007_500) { - val = levelBase + 200; - } else if (score >= 1_005_000) { - val = levelBase + 150 + ((score - 1_005_000) * 10) / 500; - } else if (score >= 1_000_000) { - val = levelBase + 100 + ((score - 1_000_000) * 5) / 100; - } else if (score >= 975_000) { - val = levelBase + ((score - 975_000) * 2) / 500; - } else if (score >= 925_000) { - val = levelBase - 300 + ((score - 925_000) * 3) / 500; - } else if (score >= 900_000) { - val = levelBase - 500 + ((score - 900_000) * 4) / 500; - } else if (score >= 800_000) { - val = (levelBase - 500) / 2 + ((score - 800_000) * ((levelBase - 500) / 2)) / 100_000; - } - - return Math.max(Math.floor(val) / 100, 0); -} - -/** - * Calculates the in-game GITADORA rating for a score. - */ -export function CalculateGITADORASkill(dryScore: DryScore, chartData: ChartDocument) { - const trueRating = (dryScore.scoreData.percent / 100) * chartData.levelNum * 20; - const flooredRating = Math.floor(trueRating * 100) / 100; - return flooredRating; -} - -/** - * Calculates the PikaGreatFunction, used in BPI. I have no idea what this does. - * @returns - */ -function BPIPikaGreatFn(score: integer, max: integer) { - return score === max ? max * 0.8 : 1 + (score / max - 0.5) / (1 - score / max); -} - -/** - * Oh boy. - * - * Calculates the "Beat Performance Index" of an IIDX score. This algorithm has many issues, - * but is a direct port of Poyashi's implementation for consistencies sake. - * https://github.com/potakusan/iidx_score_manager/blob/f21ba6b85fcc0bf8b7ca888fa2239a3951a9c9c2/src/components/bpi/index.tsx#L120 - * - * @param kaidenEx The kaiden average EX score. - * @param wrEx The world record's EX score. - * @param yourEx Your EX score. - * @param max The maximum amount of EX achievable on this chart. - * @param powCoef What power the BPI should be raised to. This is arbitrary, and assigned on a per-song basis. Defaults to 1.175. - * @returns A number between -15 and 100. Unless your score is better than the world record, in which case - * returns can be above 100. - */ -export function CalculateBPI( - kaidenEx: integer, - wrEx: integer, - yourEx: integer, - max: integer, - pc: number | null -) { - let powCoef = pc ?? 1.175; - if (powCoef === -1) { - powCoef = 1.175; - } - - const yourPGF = BPIPikaGreatFn(yourEx, max); - const kaidenPGF = BPIPikaGreatFn(kaidenEx, max); - const wrPGF = BPIPikaGreatFn(wrEx, max); - - // no idea what these var names are - const _s_ = yourPGF / kaidenPGF; - const _z_ = wrPGF / kaidenPGF; - - const isBetterThanKavg = yourEx >= kaidenEx; - - // this line of code isn't mine, and that's why it's *really* bad here. - const bpi = - Math.round( - (isBetterThanKavg ? 100 : -100) * - Math.pow( - (isBetterThanKavg ? Math.log(_s_) : -Math.log(_s_)) / Math.log(_z_), - powCoef - ) * - 100 - ) / 100; - - if (bpi < -15) { - return -15; - } - - return bpi; -} - -/** - * Calculate Marvelous Full Combo Points. This algorithm - * is used in LIFE4, and described here: - * https://life4ddr.com/requirements/#mfcpoints - * @returns Null if this score was not eligible, a number otherwise. - */ -export function CalculateMFCP(dryScore: DryScore, chartData: ChartDocument, logger: KtLogger) { - if (dryScore.scoreData.lamp !== "MARVELOUS FULL COMBO") { - return null; - } - - // Beginner and BASIC scores are explicitly excluded. - if (chartData.difficulty === "BEGINNER" || chartData.difficulty === "BASIC") { - return null; - } - - if (chartData.levelNum < 8) { - return null; - } else if (chartData.levelNum <= 10) { - return 1; - } else if (chartData.levelNum <= 12) { - return 2; - } else if (chartData.levelNum === 13) { - return 4; - } else if (chartData.levelNum === 14) { - return 8; - } else if (chartData.levelNum === 15) { - return 15; - } else if (chartData.levelNum >= 16) { - return 25; - } - - logger.warn( - `Invalid levelNum passed to MFCP ${chartData.levelNum}. ChartID ${chartData.chartID}.` - ); - - // failsafe - return null; -} - -const VF5GradeCoefficients = { - PUC: 1.05, - S: 1.05, - "AAA+": 1.02, - AAA: 1.0, - "AA+": 0.97, - AA: 0.94, - "A+": 0.91, // everything below this point (incl. this) is marked with a (?) in bemaniwiki. - A: 0.88, - B: 0.85, - C: 0.82, - D: 0.8, -}; - -const VF5LampCoefficients = { - "PERFECT ULTIMATE CHAIN": 1.1, - "ULTIMATE CHAIN": 1.05, - "EXCESSIVE CLEAR": 1.02, - CLEAR: 1.0, - FAILED: 0.5, -}; - -function FloorToNDP(number: number, dp: integer) { - const mul = 10 ** dp; - return Math.floor(number * mul) / mul; -} - -export function CalculateVF6( - grade: Grades["sdvx:Single"], - lamp: Lamps["sdvx:Single"], - per: number, - levelNum: number, - logger: KtLogger -) { - const gradeCoefficient = VF5GradeCoefficients[grade]; - const lampCoefficient = VF5LampCoefficients[lamp]; - - if (!lampCoefficient) { - logger.warn(`Invalid lamp of ${lamp} passed to CalculateVF5. Returning null.`); - return null; - } - - if (!gradeCoefficient) { - logger.warn(`Invalid grade of ${grade} passed to CalculateVF5. Returning null.`); - return null; - } - - const percent = per / 100; - if (!levelNum || !percent) { - return 0; - } - - const realVF6 = (levelNum * 2 * percent * gradeCoefficient * lampCoefficient) / 100; - - return FloorToNDP(realVF6, 3); -} - -// function CalculateJubility( -// dryScore: DryScore<"jubeat:Single">, -// chartData: ChartDocument, -// logger: KtLogger -// ) { -// let rate = dryScore.calculatedData.musicRate; eurgh, this is hard. -// } - -interface RatingParameters { - failHarshnessMultiplier: number; - pivotPercent: number; - clearExpMultiplier: number; -} - -// Generic Rating Calc that is guaranteed to work for everything. This is unspecialised, and not great. -function KTRatingCalcV1( - percent: number, - levelNum: number, - parameters: RatingParameters, - logger: KtLogger -) { - const percentDiv100 = percent / 100; - - if (percentDiv100 < parameters.pivotPercent) { - return RatingCalcV0Fail(percentDiv100, levelNum, parameters); - } - - return RatingCalcV1Clear(percentDiv100, levelNum, parameters, logger); -} - -function RatingCalcV1Clear( - percentDiv100: number, - levelNum: number, - parameters: RatingParameters, - logger: KtLogger -) { - // https://www.desmos.com/calculator/hn7uxjmjkc - - const rating = - Math.cosh( - parameters.clearExpMultiplier * levelNum * (percentDiv100 - parameters.pivotPercent) - ) + - (levelNum - 1); - - // checks for Infinity or NaN. I'm not sure how this would happen, but it's a failsafe. - if (!Number.isFinite(rating)) { - logger.warn( - `Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid. Defaulting to 0.` - ); - return 0; - } else if (rating > 1000) { - logger.warn( - `Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid (> 1000). Defaulting to 0.` - ); - return 0; - } - - return rating; -} - -function RatingCalcV0Fail(percentDiv100: number, levelNum: number, parameters: RatingParameters) { - // https://www.desmos.com/calculator/hn7uxjmjkc - return ( - percentDiv100 ** (parameters.failHarshnessMultiplier * levelNum) * - (levelNum / parameters.pivotPercent ** (parameters.failHarshnessMultiplier * levelNum)) - ); -} - -const ratingParameters = { - iidx: { - failHarshnessMultiplier: 0.3, - pivotPercent: 0.7777, // Grade: AA - clearExpMultiplier: 1, - }, - bms: { - failHarshnessMultiplier: 0.5, - pivotPercent: 0.7777, // Grade: AA - clearExpMultiplier: 0.75, - }, - museca: { - failHarshnessMultiplier: 1, - pivotPercent: 0.8, // grade: not fail - clearExpMultiplier: 1, // no real reason - }, - maimai: { - failHarshnessMultiplier: 1, - pivotPercent: 0.8, - clearExpMultiplier: 1, - }, - ddr: { - failHarshnessMultiplier: 0.9, - pivotPercent: 0.9, - clearExpMultiplier: 1, - }, -}; - -export function CalculateKTRating( - dryScore: DryScore, - game: "iidx" | "ddr" | "museca", - playtype: Playtypes[Game], - chart: ChartDocument, - logger: KtLogger -) { - const parameters = ratingParameters[game]; - - const levelNum = chart.levelNum; - - return KTRatingCalcV1(dryScore.scoreData.percent, levelNum, parameters, logger); -} - -export function CalculateKTLampRating( - dryScore: DryScore, - game: Game, - playtype: Playtypes[Game], - chart: ChartDocument -) { - return LampRatingNoTierlistInfo(dryScore, game, playtype, chart); -} - -function LampRatingNoTierlistInfo( - dryScore: DryScore, - game: Game, - playtype: Playtypes[Game], - chart: ChartDocument -) { - const gptConfig = GetGamePTConfig(game, playtype); - - const lamps = gptConfig.lamps; - - const CLEAR_LAMP_INDEX = lamps.indexOf(gptConfig.clearLamp); - - // if this is a clear - if (lamps.indexOf(dryScore.scoreData.lamp) >= CLEAR_LAMP_INDEX) { - // return this chart's numeric level as the lamp rating - return chart.levelNum; - } - - // else, this score is worth 0. - return 0; -} - -// deprecated calcs - -// const VF4GradeCoefficients = { -// S: 1.0, -// "AAA+": 0.99, -// AAA: 0.98, -// "AA+": 0.97, -// AA: 0.96, -// "A+": 0.95, -// A: 0.94, -// B: 0.93, -// C: 0.92, -// D: 0.91, -// }; - -// export function CalculateVF4( -// grade: Grades["sdvx:Single"], -// per: number, -// levelNum: number, -// logger: KtLogger -// ) { -// const multiplier = 25; - -// const gradeCoefficient = VF4GradeCoefficients[grade]; - -// if (!gradeCoefficient) { -// logger.warn(`Invalid grade of ${grade} passed to CalculateVF4. Returning null.`); -// return null; -// } - -// const percent = per / 100; -// if (!levelNum || !percent) { -// return 0; -// } - -// return Math.floor(multiplier * (levelNum + 1) * percent * gradeCoefficient); -// } - -// // VF5 is just VF6 but to three decimal places instead of four. -// export function CalculateVF5( -// grade: Grades["sdvx:Single"], -// lamp: Lamps["sdvx:Single"], -// per: number, -// levelNum: number, -// logger: KtLogger -// ) { -// const vf6 = CalculateVF6(grade, lamp, per, levelNum, logger); - -// if (vf6 === null) { -// return null; -// } - -// return FloorToNDP(vf6, 2); -// } +import { KtLogger } from "lib/logger/logger"; +import { + ChartDocument, + Game, + GetGamePTConfig, + Grades, + integer, + Lamps, + Playtypes, +} from "tachi-common"; +import { DryScore } from "../common/types"; + +/** + * Calculates the in-game CHUNITHM rating for a score. + */ +export function CalculateCHUNITHMRating(dryScore: DryScore, chartData: ChartDocument) { + const score = dryScore.scoreData.score; + const levelBase = chartData.levelNum * 100; + + let val = 0; + + if (score >= 1_007_500) { + val = levelBase + 200; + } else if (score >= 1_005_000) { + val = levelBase + 150 + ((score - 1_005_000) * 10) / 500; + } else if (score >= 1_000_000) { + val = levelBase + 100 + ((score - 1_000_000) * 5) / 100; + } else if (score >= 975_000) { + val = levelBase + ((score - 975_000) * 2) / 500; + } else if (score >= 925_000) { + val = levelBase - 300 + ((score - 925_000) * 3) / 500; + } else if (score >= 900_000) { + val = levelBase - 500 + ((score - 900_000) * 4) / 500; + } else if (score >= 800_000) { + val = (levelBase - 500) / 2 + ((score - 800_000) * ((levelBase - 500) / 2)) / 100_000; + } + + return Math.max(Math.floor(val) / 100, 0); +} + +/** + * Calculates the in-game GITADORA rating for a score. + */ +export function CalculateGITADORASkill(dryScore: DryScore, chartData: ChartDocument) { + const trueRating = (dryScore.scoreData.percent / 100) * chartData.levelNum * 20; + const flooredRating = Math.floor(trueRating * 100) / 100; + return flooredRating; +} + +/** + * Calculates the PikaGreatFunction, used in BPI. I have no idea what this does. + * @returns + */ +function BPIPikaGreatFn(score: integer, max: integer) { + return score === max ? max * 0.8 : 1 + (score / max - 0.5) / (1 - score / max); +} + +/** + * Oh boy. + * + * Calculates the "Beat Performance Index" of an IIDX score. This algorithm has many issues, + * but is a direct port of Poyashi's implementation for consistencies sake. + * https://github.com/potakusan/iidx_score_manager/blob/f21ba6b85fcc0bf8b7ca888fa2239a3951a9c9c2/src/components/bpi/index.tsx#L120 + * + * @param kaidenEx The kaiden average EX score. + * @param wrEx The world record's EX score. + * @param yourEx Your EX score. + * @param max The maximum amount of EX achievable on this chart. + * @param powCoef What power the BPI should be raised to. This is arbitrary, and assigned on a per-song basis. Defaults to 1.175. + * @returns A number between -15 and 100. Unless your score is better than the world record, in which case + * returns can be above 100. + */ +export function CalculateBPI( + kaidenEx: integer, + wrEx: integer, + yourEx: integer, + max: integer, + pc: number | null +) { + let powCoef = pc ?? 1.175; + if (powCoef === -1) { + powCoef = 1.175; + } + + const yourPGF = BPIPikaGreatFn(yourEx, max); + const kaidenPGF = BPIPikaGreatFn(kaidenEx, max); + const wrPGF = BPIPikaGreatFn(wrEx, max); + + // no idea what these var names are + const _s_ = yourPGF / kaidenPGF; + const _z_ = wrPGF / kaidenPGF; + + const isBetterThanKavg = yourEx >= kaidenEx; + + // this line of code isn't mine, and that's why it's *really* bad here. + const bpi = + Math.round( + (isBetterThanKavg ? 100 : -100) * + Math.pow( + (isBetterThanKavg ? Math.log(_s_) : -Math.log(_s_)) / Math.log(_z_), + powCoef + ) * + 100 + ) / 100; + + if (bpi < -15) { + return -15; + } + + return bpi; +} + +/** + * Calculate Marvelous Full Combo Points. This algorithm + * is used in LIFE4, and described here: + * https://life4ddr.com/requirements/#mfcpoints + * @returns Null if this score was not eligible, a number otherwise. + */ +export function CalculateMFCP(dryScore: DryScore, chartData: ChartDocument, logger: KtLogger) { + if (dryScore.scoreData.lamp !== "MARVELOUS FULL COMBO") { + return null; + } + + // Beginner and BASIC scores are explicitly excluded. + if (chartData.difficulty === "BEGINNER" || chartData.difficulty === "BASIC") { + return null; + } + + if (chartData.levelNum < 8) { + return null; + } else if (chartData.levelNum <= 10) { + return 1; + } else if (chartData.levelNum <= 12) { + return 2; + } else if (chartData.levelNum === 13) { + return 4; + } else if (chartData.levelNum === 14) { + return 8; + } else if (chartData.levelNum === 15) { + return 15; + } else if (chartData.levelNum >= 16) { + return 25; + } + + logger.warn( + `Invalid levelNum passed to MFCP ${chartData.levelNum}. ChartID ${chartData.chartID}.` + ); + + // failsafe + return null; +} + +const VF5GradeCoefficients = { + PUC: 1.05, + S: 1.05, + "AAA+": 1.02, + AAA: 1.0, + "AA+": 0.97, + AA: 0.94, + "A+": 0.91, // everything below this point (incl. this) is marked with a (?) in bemaniwiki. + A: 0.88, + B: 0.85, + C: 0.82, + D: 0.8, +}; + +const VF5LampCoefficients = { + "PERFECT ULTIMATE CHAIN": 1.1, + "ULTIMATE CHAIN": 1.05, + "EXCESSIVE CLEAR": 1.02, + CLEAR: 1.0, + FAILED: 0.5, +}; + +function FloorToNDP(number: number, dp: integer) { + const mul = 10 ** dp; + return Math.floor(number * mul) / mul; +} + +export function CalculateVF6( + grade: Grades["sdvx:Single"], + lamp: Lamps["sdvx:Single"], + per: number, + levelNum: number, + logger: KtLogger +) { + const gradeCoefficient = VF5GradeCoefficients[grade]; + const lampCoefficient = VF5LampCoefficients[lamp]; + + if (!lampCoefficient) { + logger.warn(`Invalid lamp of ${lamp} passed to CalculateVF5. Returning null.`); + return null; + } + + if (!gradeCoefficient) { + logger.warn(`Invalid grade of ${grade} passed to CalculateVF5. Returning null.`); + return null; + } + + const percent = per / 100; + if (!levelNum || !percent) { + return 0; + } + + const realVF6 = (levelNum * 2 * percent * gradeCoefficient * lampCoefficient) / 100; + + return FloorToNDP(realVF6, 3); +} + +// function CalculateJubility( +// dryScore: DryScore<"jubeat:Single">, +// chartData: ChartDocument, +// logger: KtLogger +// ) { +// let rate = dryScore.calculatedData.musicRate; eurgh, this is hard. +// } + +interface RatingParameters { + failHarshnessMultiplier: number; + pivotPercent: number; + clearExpMultiplier: number; +} + +// Generic Rating Calc that is guaranteed to work for everything. This is unspecialised, and not great. +function KTRatingCalcV1( + percent: number, + levelNum: number, + parameters: RatingParameters, + logger: KtLogger +) { + const percentDiv100 = percent / 100; + + if (percentDiv100 < parameters.pivotPercent) { + return RatingCalcV0Fail(percentDiv100, levelNum, parameters); + } + + return RatingCalcV1Clear(percentDiv100, levelNum, parameters, logger); +} + +function RatingCalcV1Clear( + percentDiv100: number, + levelNum: number, + parameters: RatingParameters, + logger: KtLogger +) { + // https://www.desmos.com/calculator/hn7uxjmjkc + + const rating = + Math.cosh( + parameters.clearExpMultiplier * levelNum * (percentDiv100 - parameters.pivotPercent) + ) + + (levelNum - 1); + + // checks for Infinity or NaN. I'm not sure how this would happen, but it's a failsafe. + if (!Number.isFinite(rating)) { + logger.warn( + `Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid. Defaulting to 0.` + ); + return 0; + } else if (rating > 1000) { + logger.warn( + `Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid (> 1000). Defaulting to 0.` + ); + return 0; + } + + return rating; +} + +function RatingCalcV0Fail(percentDiv100: number, levelNum: number, parameters: RatingParameters) { + // https://www.desmos.com/calculator/hn7uxjmjkc + return ( + percentDiv100 ** (parameters.failHarshnessMultiplier * levelNum) * + (levelNum / parameters.pivotPercent ** (parameters.failHarshnessMultiplier * levelNum)) + ); +} + +const ratingParameters = { + iidx: { + failHarshnessMultiplier: 0.3, + pivotPercent: 0.7777, // Grade: AA + clearExpMultiplier: 1, + }, + bms: { + failHarshnessMultiplier: 0.5, + pivotPercent: 0.7777, // Grade: AA + clearExpMultiplier: 0.75, + }, + museca: { + failHarshnessMultiplier: 1, + pivotPercent: 0.8, // grade: not fail + clearExpMultiplier: 1, // no real reason + }, + maimai: { + failHarshnessMultiplier: 1, + pivotPercent: 0.8, + clearExpMultiplier: 1, + }, + ddr: { + failHarshnessMultiplier: 0.9, + pivotPercent: 0.9, + clearExpMultiplier: 1, + }, +}; + +export function CalculateKTRating( + dryScore: DryScore, + game: "iidx" | "ddr" | "museca", + playtype: Playtypes[Game], + chart: ChartDocument, + logger: KtLogger +) { + const parameters = ratingParameters[game]; + + const levelNum = chart.levelNum; + + return KTRatingCalcV1(dryScore.scoreData.percent, levelNum, parameters, logger); +} + +export function CalculateKTLampRating( + dryScore: DryScore, + game: Game, + playtype: Playtypes[Game], + chart: ChartDocument +) { + return LampRatingNoTierlistInfo(dryScore, game, playtype, chart); +} + +function LampRatingNoTierlistInfo( + dryScore: DryScore, + game: Game, + playtype: Playtypes[Game], + chart: ChartDocument +) { + const gptConfig = GetGamePTConfig(game, playtype); + + const lamps = gptConfig.lamps; + + const CLEAR_LAMP_INDEX = lamps.indexOf(gptConfig.clearLamp); + + // if this is a clear + if (lamps.indexOf(dryScore.scoreData.lamp) >= CLEAR_LAMP_INDEX) { + // return this chart's numeric level as the lamp rating + return chart.levelNum; + } + + // else, this score is worth 0. + return 0; +} + +// deprecated calcs + +// const VF4GradeCoefficients = { +// S: 1.0, +// "AAA+": 0.99, +// AAA: 0.98, +// "AA+": 0.97, +// AA: 0.96, +// "A+": 0.95, +// A: 0.94, +// B: 0.93, +// C: 0.92, +// D: 0.91, +// }; + +// export function CalculateVF4( +// grade: Grades["sdvx:Single"], +// per: number, +// levelNum: number, +// logger: KtLogger +// ) { +// const multiplier = 25; + +// const gradeCoefficient = VF4GradeCoefficients[grade]; + +// if (!gradeCoefficient) { +// logger.warn(`Invalid grade of ${grade} passed to CalculateVF4. Returning null.`); +// return null; +// } + +// const percent = per / 100; +// if (!levelNum || !percent) { +// return 0; +// } + +// return Math.floor(multiplier * (levelNum + 1) * percent * gradeCoefficient); +// } + +// // VF5 is just VF6 but to three decimal places instead of four. +// export function CalculateVF5( +// grade: Grades["sdvx:Single"], +// lamp: Lamps["sdvx:Single"], +// per: number, +// levelNum: number, +// logger: KtLogger +// ) { +// const vf6 = CalculateVF6(grade, lamp, per, levelNum, logger); + +// if (vf6 === null) { +// return null; +// } + +// return FloorToNDP(vf6, 2); +// } diff --git a/server/src/lib/score-import/framework/common/converter-failures.ts b/server/src/lib/score-import/framework/common/converter-failures.ts index 163e2b922..7130ecb0d 100644 --- a/server/src/lib/score-import/framework/common/converter-failures.ts +++ b/server/src/lib/score-import/framework/common/converter-failures.ts @@ -1,55 +1,55 @@ -// Various Errors that can occur during processing. - -import { ImportTypes } from "tachi-common"; -import { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types"; - -export class ConverterFailure { - message: string | null; - - constructor(message: string | null) { - this.message = message; - } -} - -/** - * SkipScoreFailure - This score could not be processed because we do not - * support it. This is not an error, but is not a success either. - * An example scenario would be something like a 5KEY score being imported from SSS. - */ -export class SkipScoreFailure extends ConverterFailure {} - -/** - * KTDataNotFoundError - We could not find the song or chart this score - * belongs to. The identifier used to try and match this chart is stored here. - * Alongside any other fields used in the query. - */ -export class KTDataNotFoundFailure extends ConverterFailure { - data: ImportTypeDataMap[T]; - converterContext: ImportTypeContextMap[T]; - importType: T; - - constructor( - message: string | null, - importType: T, - data: ImportTypeDataMap[T], - context: ImportTypeContextMap[T] - ) { - super(message); - - this.importType = importType; - this.data = data; - this.converterContext = context; - } -} - -/** - * InvalidScoreError - This score provided invalid data that we - * can not accept. - */ -export class InvalidScoreFailure extends ConverterFailure {} - -/** - * KTInternalFailure - An unexpected, internal error has occured, - * and the score could not be processed. - */ -export class InternalFailure extends ConverterFailure {} +// Various Errors that can occur during processing. + +import { ImportTypes } from "tachi-common"; +import { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types"; + +export class ConverterFailure { + message: string | null; + + constructor(message: string | null) { + this.message = message; + } +} + +/** + * SkipScoreFailure - This score could not be processed because we do not + * support it. This is not an error, but is not a success either. + * An example scenario would be something like a 5KEY score being imported from SSS. + */ +export class SkipScoreFailure extends ConverterFailure {} + +/** + * KTDataNotFoundError - We could not find the song or chart this score + * belongs to. The identifier used to try and match this chart is stored here. + * Alongside any other fields used in the query. + */ +export class KTDataNotFoundFailure extends ConverterFailure { + data: ImportTypeDataMap[T]; + converterContext: ImportTypeContextMap[T]; + importType: T; + + constructor( + message: string | null, + importType: T, + data: ImportTypeDataMap[T], + context: ImportTypeContextMap[T] + ) { + super(message); + + this.importType = importType; + this.data = data; + this.converterContext = context; + } +} + +/** + * InvalidScoreError - This score provided invalid data that we + * can not accept. + */ +export class InvalidScoreFailure extends ConverterFailure {} + +/** + * KTInternalFailure - An unexpected, internal error has occured, + * and the score could not be processed. + */ +export class InternalFailure extends ConverterFailure {} diff --git a/server/src/lib/score-import/framework/common/import-logger.ts b/server/src/lib/score-import/framework/common/import-logger.ts index 64a3eb9c1..df91b1cea 100644 --- a/server/src/lib/score-import/framework/common/import-logger.ts +++ b/server/src/lib/score-import/framework/common/import-logger.ts @@ -1,22 +1,22 @@ -import { KtLogger, rootLogger } from "lib/logger/logger"; -import { ImportTypes, PublicUserDocument } from "tachi-common"; -import { Random20Hex } from "utils/misc"; -import { FormatUserDoc } from "utils/user"; - -export function CreateScoreLogger( - user: PublicUserDocument, - importID: string, - importType: ImportTypes -): KtLogger { - const meta = { - context: ["Score Import", importType, FormatUserDoc(user)], - importID, - }; - - // used so appendLogCtx works - const childLogger = rootLogger.child(meta); - - childLogger.defaultMeta = meta; - - return childLogger as KtLogger; -} +import { KtLogger, rootLogger } from "lib/logger/logger"; +import { ImportTypes, PublicUserDocument } from "tachi-common"; +import { Random20Hex } from "utils/misc"; +import { FormatUserDoc } from "utils/user"; + +export function CreateScoreLogger( + user: PublicUserDocument, + importID: string, + importType: ImportTypes +): KtLogger { + const meta = { + context: ["Score Import", importType, FormatUserDoc(user)], + importID, + }; + + // used so appendLogCtx works + const childLogger = rootLogger.child(meta); + + childLogger.defaultMeta = meta; + + return childLogger as KtLogger; +} diff --git a/server/src/lib/score-import/framework/common/score-utils.ts b/server/src/lib/score-import/framework/common/score-utils.ts index 847550fa0..ef3aa86ec 100644 --- a/server/src/lib/score-import/framework/common/score-utils.ts +++ b/server/src/lib/score-import/framework/common/score-utils.ts @@ -1,163 +1,163 @@ -import CreateLogCtx from "lib/logger/logger"; -import { - ChartDocument, - ESDCore, - Game, - GameToIDStrings, - GetGamePTConfig, - Grades, - IDStrings, - Playtypes, -} from "tachi-common"; -import { InternalFailure, InvalidScoreFailure } from "./converter-failures"; - -const logger = CreateLogCtx(__filename); - -/** - * Util for getting a games' grade for a given percent. - */ -export function GetGradeFromPercent( - game: Game, - playtype: Playtypes[Game], - percent: number -): Grades[I] { - const gptConfig = GetGamePTConfig(game, playtype); - const boundaries = gptConfig.gradeBoundaries; - const grades = gptConfig.grades; - - // (hey, this for loop is backwards!) - for (let i = boundaries.length; i >= 0; i--) { - if (percent >= boundaries[i]) { - return grades[i] as Grades[I]; - } - } - - logger.error(`Could not resolve grade for percent ${percent} on game ${game}`); - throw new InternalFailure(`Could not resolve grade for percent ${percent} on game ${game}`); -} - -/** - * A Generic function for calculating a percent from a given score on - * a given game. - */ -export function GenericCalculatePercent(game: Game, score: number, chart?: ChartDocument): number { - switch (game) { - case "ddr": - case "museca": - case "chunithm": - return (score / 1_000_000) * 100; - case "sdvx": - case "usc": - return (score / 10_000_000) * 100; - // case "popn": - // return (score / 100_000) * 100; - case "gitadora": - case "maimai": - return score; - case "bms": - case "iidx": - if (!chart) { - logger.severe("No Chart passed to GenericCalcPercent but game was iidx/bms."); - throw new InternalFailure( - "No Chart passed to GenericCalcPercent but game was iidx/bms." - ); - } - - // Yeah, we declare it like this so the below return is actually clear. - // eslint-disable-next-line no-case-declarations - const MAX = - (chart as ChartDocument<"iidx:SP" | "bms:7K" | "bms:14K" | "iidx:DP">).data - .notecount * 2; - - return (100 * score) / MAX; - default: - logger.severe(`Invalid game passed of ${game} to GenericCalcPercent.`); - throw new InternalFailure(`Invalid game passed of ${game} to GenericCalcPercent.`); - } -} - -/** - * Helper utility for validating percents on a game. This throws an InvalidScoreFailure if the percent is - * invalid, and returns void on success. - * - * This exists to support maimai, as it has a dynamic "max percent". - */ -export function ValidatePercent( - game: Game, - playtype: Playtypes[Game], - percent: number, - chart: ChartDocument -) { - // i love needing a helper function for *ONE* game. - if (game === "maimai") { - const mmChart = chart as ChartDocument<"maimai:Single">; - if (percent > mmChart.data.maxPercent) { - throw new InvalidScoreFailure( - `Invalid percent - expected less than ${mmChart.data.maxPercent}.` - ); - } - } - - const gptConfig = GetGamePTConfig(game, playtype); - - if (percent > gptConfig.percentMax) { - throw new InvalidScoreFailure( - `Invalid percent of ${percent} - expected a value less than ${gptConfig.percentMax}% (${chart.songID} ${chart.playtype} ${chart.difficulty}).` - ); - } -} - -/** - * Generically gets the grade and percent for a given score on a given game. This only works for games where - * grades are just percent boundaries. This will throw an InvalidScoreFailure if the percent is invalid, - * or if the grade is invalid. - */ -export function GenericGetGradeAndPercent( - game: G, - score: number, - chart: ChartDocument -) { - const percent = GenericCalculatePercent(game, score, chart); - - ValidatePercent(game, chart.playtype, percent, chart); - - const grade = GetGradeFromPercent(game, chart.playtype, percent) as Grades[GameToIDStrings[G]]; - - return { percent, grade }; -} - -/** - * Calculates the ESD for a given game + percent combo. This function returns - * null if the game does not support support ESD. - */ -export function CalculateESDForGame( - game: Game, - playtype: Playtypes[Game], - percent: number -): number | null { - const gptConfig = GetGamePTConfig(game, playtype); - - if (!gptConfig.supportsESD) { - return null; - } - - return ESDCore.CalculateESD(gptConfig.judgementWindows, percent); -} - -/** - * Parses and validates a date from a string. - * @returns Millisecondss from the unix epoch, or null if the initial argument was null or undefined. - */ -export function ParseDateFromString(str: string | undefined | null): number | null { - if (!str) { - return null; - } - - const date = Date.parse(str); - - if (Number.isNaN(date)) { - throw new InvalidScoreFailure(`Invalid/Unparsable score timestamp of ${str}.`); - } - - return date; -} +import CreateLogCtx from "lib/logger/logger"; +import { + ChartDocument, + ESDCore, + Game, + GameToIDStrings, + GetGamePTConfig, + Grades, + IDStrings, + Playtypes, +} from "tachi-common"; +import { InternalFailure, InvalidScoreFailure } from "./converter-failures"; + +const logger = CreateLogCtx(__filename); + +/** + * Util for getting a games' grade for a given percent. + */ +export function GetGradeFromPercent( + game: Game, + playtype: Playtypes[Game], + percent: number +): Grades[I] { + const gptConfig = GetGamePTConfig(game, playtype); + const boundaries = gptConfig.gradeBoundaries; + const grades = gptConfig.grades; + + // (hey, this for loop is backwards!) + for (let i = boundaries.length; i >= 0; i--) { + if (percent >= boundaries[i]) { + return grades[i] as Grades[I]; + } + } + + logger.error(`Could not resolve grade for percent ${percent} on game ${game}`); + throw new InternalFailure(`Could not resolve grade for percent ${percent} on game ${game}`); +} + +/** + * A Generic function for calculating a percent from a given score on + * a given game. + */ +export function GenericCalculatePercent(game: Game, score: number, chart?: ChartDocument): number { + switch (game) { + case "ddr": + case "museca": + case "chunithm": + return (score / 1_000_000) * 100; + case "sdvx": + case "usc": + return (score / 10_000_000) * 100; + // case "popn": + // return (score / 100_000) * 100; + case "gitadora": + case "maimai": + return score; + case "bms": + case "iidx": + if (!chart) { + logger.severe("No Chart passed to GenericCalcPercent but game was iidx/bms."); + throw new InternalFailure( + "No Chart passed to GenericCalcPercent but game was iidx/bms." + ); + } + + // Yeah, we declare it like this so the below return is actually clear. + // eslint-disable-next-line no-case-declarations + const MAX = + (chart as ChartDocument<"iidx:SP" | "bms:7K" | "bms:14K" | "iidx:DP">).data + .notecount * 2; + + return (100 * score) / MAX; + default: + logger.severe(`Invalid game passed of ${game} to GenericCalcPercent.`); + throw new InternalFailure(`Invalid game passed of ${game} to GenericCalcPercent.`); + } +} + +/** + * Helper utility for validating percents on a game. This throws an InvalidScoreFailure if the percent is + * invalid, and returns void on success. + * + * This exists to support maimai, as it has a dynamic "max percent". + */ +export function ValidatePercent( + game: Game, + playtype: Playtypes[Game], + percent: number, + chart: ChartDocument +) { + // i love needing a helper function for *ONE* game. + if (game === "maimai") { + const mmChart = chart as ChartDocument<"maimai:Single">; + if (percent > mmChart.data.maxPercent) { + throw new InvalidScoreFailure( + `Invalid percent - expected less than ${mmChart.data.maxPercent}.` + ); + } + } + + const gptConfig = GetGamePTConfig(game, playtype); + + if (percent > gptConfig.percentMax) { + throw new InvalidScoreFailure( + `Invalid percent of ${percent} - expected a value less than ${gptConfig.percentMax}% (${chart.songID} ${chart.playtype} ${chart.difficulty}).` + ); + } +} + +/** + * Generically gets the grade and percent for a given score on a given game. This only works for games where + * grades are just percent boundaries. This will throw an InvalidScoreFailure if the percent is invalid, + * or if the grade is invalid. + */ +export function GenericGetGradeAndPercent( + game: G, + score: number, + chart: ChartDocument +) { + const percent = GenericCalculatePercent(game, score, chart); + + ValidatePercent(game, chart.playtype, percent, chart); + + const grade = GetGradeFromPercent(game, chart.playtype, percent) as Grades[GameToIDStrings[G]]; + + return { percent, grade }; +} + +/** + * Calculates the ESD for a given game + percent combo. This function returns + * null if the game does not support support ESD. + */ +export function CalculateESDForGame( + game: Game, + playtype: Playtypes[Game], + percent: number +): number | null { + const gptConfig = GetGamePTConfig(game, playtype); + + if (!gptConfig.supportsESD) { + return null; + } + + return ESDCore.CalculateESD(gptConfig.judgementWindows, percent); +} + +/** + * Parses and validates a date from a string. + * @returns Millisecondss from the unix epoch, or null if the initial argument was null or undefined. + */ +export function ParseDateFromString(str: string | undefined | null): number | null { + if (!str) { + return null; + } + + const date = Date.parse(str); + + if (Number.isNaN(date)) { + throw new InvalidScoreFailure(`Invalid/Unparsable score timestamp of ${str}.`); + } + + return date; +} diff --git a/server/src/lib/score-import/framework/score-importing/hydrate-score.ts b/server/src/lib/score-import/framework/score-importing/hydrate-score.ts index 5e7f8237c..6f8aad22c 100644 --- a/server/src/lib/score-import/framework/score-importing/hydrate-score.ts +++ b/server/src/lib/score-import/framework/score-importing/hydrate-score.ts @@ -1,60 +1,60 @@ -import { ChartDocument, integer, ScoreDocument, SongDocument, GetGamePTConfig } from "tachi-common"; -import { KtLogger } from "lib/logger/logger"; -import { CreateCalculatedData } from "../calculated-data/calculated-data"; -import { CalculateESDForGame } from "../common/score-utils"; -import { DryScore } from "../common/types"; - -/** - * Takes an "intermediate" score and appends the rest of the data it needs. - * @param dryScore The intermediate score to make into a real score. - * @param userID The userID this score is for. - */ -export async function HydrateScore( - userID: integer, - dryScore: DryScore, - chart: ChartDocument, - song: SongDocument, - scoreID: string, - logger: KtLogger -): Promise { - const esd = CalculateESDForGame( - dryScore.game, - chart.playtype, - dryScore.scoreData.percent / 100 - ); - - const calculatedData = await CreateCalculatedData(dryScore, chart, esd, logger); - - const { scoreData: dryScoreData, ...rest } = dryScore; - - const gptConfig = GetGamePTConfig(dryScore.game, chart.playtype); - - // Fill out the rest of the fields we want for scoreData - const scoreData = Object.assign( - { - lampIndex: gptConfig.lamps.indexOf(dryScore.scoreData.lamp), - gradeIndex: gptConfig.grades.indexOf(dryScore.scoreData.grade), - esd, - }, - dryScoreData - ); - - const score: ScoreDocument = { - // extract all of the non-scoreData props from a dry score and push them here - ...rest, - // then push our score data. - scoreData, - // everything below this point is sane - highlight: false, - timeAdded: Date.now(), - userID, - calculatedData, - songID: song.id, - chartID: chart.chartID, - scoreID, - playtype: chart.playtype, - isPrimary: chart.isPrimary, - }; - - return score; -} +import { ChartDocument, integer, ScoreDocument, SongDocument, GetGamePTConfig } from "tachi-common"; +import { KtLogger } from "lib/logger/logger"; +import { CreateCalculatedData } from "../calculated-data/calculated-data"; +import { CalculateESDForGame } from "../common/score-utils"; +import { DryScore } from "../common/types"; + +/** + * Takes an "intermediate" score and appends the rest of the data it needs. + * @param dryScore The intermediate score to make into a real score. + * @param userID The userID this score is for. + */ +export async function HydrateScore( + userID: integer, + dryScore: DryScore, + chart: ChartDocument, + song: SongDocument, + scoreID: string, + logger: KtLogger +): Promise { + const esd = CalculateESDForGame( + dryScore.game, + chart.playtype, + dryScore.scoreData.percent / 100 + ); + + const calculatedData = await CreateCalculatedData(dryScore, chart, esd, logger); + + const { scoreData: dryScoreData, ...rest } = dryScore; + + const gptConfig = GetGamePTConfig(dryScore.game, chart.playtype); + + // Fill out the rest of the fields we want for scoreData + const scoreData = Object.assign( + { + lampIndex: gptConfig.lamps.indexOf(dryScore.scoreData.lamp), + gradeIndex: gptConfig.grades.indexOf(dryScore.scoreData.grade), + esd, + }, + dryScoreData + ); + + const score: ScoreDocument = { + // extract all of the non-scoreData props from a dry score and push them here + ...rest, + // then push our score data. + scoreData, + // everything below this point is sane + highlight: false, + timeAdded: Date.now(), + userID, + calculatedData, + songID: song.id, + chartID: chart.chartID, + scoreID, + playtype: chart.playtype, + isPrimary: chart.isPrimary, + }; + + return score; +} diff --git a/server/src/lib/score-import/framework/score-importing/insert-score.test.ts b/server/src/lib/score-import/framework/score-importing/insert-score.test.ts index 0c9feaebb..cebee5def 100644 --- a/server/src/lib/score-import/framework/score-importing/insert-score.test.ts +++ b/server/src/lib/score-import/framework/score-importing/insert-score.test.ts @@ -1,152 +1,152 @@ -import { ScoreDocument } from "tachi-common"; -import t from "tap"; -import db from "external/mongo/db"; - -import ResetDBState from "test-utils/resets"; -import { InsertQueue, QueueScoreInsert } from "./insert-score"; - -// these two get the same tests, because they're too closely linked -t.test("#QueueScoreInsert, #InsertQueue", async (t) => { - t.beforeEach(ResetDBState); - - // empty scoreDB after - t.afterEach(async () => { - await db.scores.remove({}); - }); - - t.test("Single Queue Test", async (t) => { - // fake score doc - const res = await QueueScoreInsert({ - scoreID: "foo", - userID: 1, - } as unknown as ScoreDocument); - - t.equal( - res, - true, - "QueueScoreInsert should not insert a score when the queue is not full." - ); - - // this is the best way to get the size of the queue - const flushSize = await InsertQueue(1); - - t.equal(flushSize, 1, "QueueScoreInsert should append the score to the queue."); - - const dbRes = await db.scores.find({ - scoreID: "foo", - }); - - t.equal( - dbRes.length, - 1, - "InsertQueue should insert all 1 members of the queue into the database." - ); - - t.end(); - }); - - let r = await InsertQueue(1); // flush queue just incase former test fails. - - t.test("Queue Overflow Test", async (t) => { - for (let i = 0; i < 499; i++) { - // eslint-disable-next-line no-await-in-loop - await QueueScoreInsert({ - scoreID: i, - chartID: "test", - userID: 1, - } as unknown as ScoreDocument); - } - - const overflowRes = await QueueScoreInsert({ - scoreID: "foo", - chartID: "test", - userID: 1, - } as unknown as ScoreDocument); - - t.equal( - overflowRes, - 500, - "Appending 500 items to the queue should result in them being inserted." - ); - - const flushRes = await InsertQueue(1); - - t.equal(flushRes, 0, "The queue should now be empty."); - - const dbRes = await db.scores.find({ - scoreID: { $exists: true }, - chartID: "test", - }); - - t.equal( - dbRes.length, - 500, - "InsertQueue should insert all 500 members of the queue into the database." - ); - - t.end(); - }); - - r = await InsertQueue(1); // flush queue just incase former test fails. - - t.equal(r, 0, "Queue should be empty after test."); - - t.test("Queue Dedupe Test", async (t) => { - await QueueScoreInsert({ - scoreID: 1, - chartID: "foo", - userID: 1, - } as unknown as ScoreDocument); - const r2 = await QueueScoreInsert({ - scoreID: 1, - chartID: "foo", - userID: 1, - } as unknown as ScoreDocument); - - t.equal(r2, null, "Should return null when a duplicate scoreID is submitted"); - - const flushRes = await InsertQueue(1); - - t.equal(flushRes, 1, "Should flush 1 score document"); - - const dbRes = await db.scores.find({ - chartID: "foo", - }); - - t.equal(dbRes.length, 1, "Should only insert one document"); - }); - - t.test("Should give separate users separate queues.", async (t) => { - await QueueScoreInsert({ - scoreID: "1", - chartID: "foo", - userID: 1, - } as unknown as ScoreDocument); - await QueueScoreInsert({ - scoreID: "2", - chartID: "foo", - userID: 2, - } as unknown as ScoreDocument); - - const r1 = await InsertQueue(1); - t.equal(r1, 1, "Queue for userID 1 should have length 1."); - - const r2 = await InsertQueue(2); - t.equal(r2, 1, "Queue for userID 2 should also have length 1."); - - t.end(); - }); - - t.test("Should not throw if InsertQueue is called on an empty queue.", async (t) => { - try { - await InsertQueue(1); - t.pass("Did not throw when inserting an empty queue."); - } catch (err) { - t.fail(err); - } - - t.end(); - }); - - t.end(); -}); +import { ScoreDocument } from "tachi-common"; +import t from "tap"; +import db from "external/mongo/db"; + +import ResetDBState from "test-utils/resets"; +import { InsertQueue, QueueScoreInsert } from "./insert-score"; + +// these two get the same tests, because they're too closely linked +t.test("#QueueScoreInsert, #InsertQueue", async (t) => { + t.beforeEach(ResetDBState); + + // empty scoreDB after + t.afterEach(async () => { + await db.scores.remove({}); + }); + + t.test("Single Queue Test", async (t) => { + // fake score doc + const res = await QueueScoreInsert({ + scoreID: "foo", + userID: 1, + } as unknown as ScoreDocument); + + t.equal( + res, + true, + "QueueScoreInsert should not insert a score when the queue is not full." + ); + + // this is the best way to get the size of the queue + const flushSize = await InsertQueue(1); + + t.equal(flushSize, 1, "QueueScoreInsert should append the score to the queue."); + + const dbRes = await db.scores.find({ + scoreID: "foo", + }); + + t.equal( + dbRes.length, + 1, + "InsertQueue should insert all 1 members of the queue into the database." + ); + + t.end(); + }); + + let r = await InsertQueue(1); // flush queue just incase former test fails. + + t.test("Queue Overflow Test", async (t) => { + for (let i = 0; i < 499; i++) { + // eslint-disable-next-line no-await-in-loop + await QueueScoreInsert({ + scoreID: i, + chartID: "test", + userID: 1, + } as unknown as ScoreDocument); + } + + const overflowRes = await QueueScoreInsert({ + scoreID: "foo", + chartID: "test", + userID: 1, + } as unknown as ScoreDocument); + + t.equal( + overflowRes, + 500, + "Appending 500 items to the queue should result in them being inserted." + ); + + const flushRes = await InsertQueue(1); + + t.equal(flushRes, 0, "The queue should now be empty."); + + const dbRes = await db.scores.find({ + scoreID: { $exists: true }, + chartID: "test", + }); + + t.equal( + dbRes.length, + 500, + "InsertQueue should insert all 500 members of the queue into the database." + ); + + t.end(); + }); + + r = await InsertQueue(1); // flush queue just incase former test fails. + + t.equal(r, 0, "Queue should be empty after test."); + + t.test("Queue Dedupe Test", async (t) => { + await QueueScoreInsert({ + scoreID: 1, + chartID: "foo", + userID: 1, + } as unknown as ScoreDocument); + const r2 = await QueueScoreInsert({ + scoreID: 1, + chartID: "foo", + userID: 1, + } as unknown as ScoreDocument); + + t.equal(r2, null, "Should return null when a duplicate scoreID is submitted"); + + const flushRes = await InsertQueue(1); + + t.equal(flushRes, 1, "Should flush 1 score document"); + + const dbRes = await db.scores.find({ + chartID: "foo", + }); + + t.equal(dbRes.length, 1, "Should only insert one document"); + }); + + t.test("Should give separate users separate queues.", async (t) => { + await QueueScoreInsert({ + scoreID: "1", + chartID: "foo", + userID: 1, + } as unknown as ScoreDocument); + await QueueScoreInsert({ + scoreID: "2", + chartID: "foo", + userID: 2, + } as unknown as ScoreDocument); + + const r1 = await InsertQueue(1); + t.equal(r1, 1, "Queue for userID 1 should have length 1."); + + const r2 = await InsertQueue(2); + t.equal(r2, 1, "Queue for userID 2 should also have length 1."); + + t.end(); + }); + + t.test("Should not throw if InsertQueue is called on an empty queue.", async (t) => { + try { + await InsertQueue(1); + t.pass("Did not throw when inserting an empty queue."); + } catch (err) { + t.fail(err); + } + + t.end(); + }); + + t.end(); +}); diff --git a/server/src/lib/score-import/framework/score-importing/insert-score.ts b/server/src/lib/score-import/framework/score-importing/insert-score.ts index 37738f5ad..5e83d341e 100644 --- a/server/src/lib/score-import/framework/score-importing/insert-score.ts +++ b/server/src/lib/score-import/framework/score-importing/insert-score.ts @@ -1,102 +1,102 @@ -import { ScoreDocument, integer } from "tachi-common"; -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; - -const logger = CreateLogCtx(__filename); -const MAX_PIPELINE_LENGTH = 500; - -interface ScoreQueue { - queue: ScoreDocument[]; - scoreIDSet: Set; -} - -const ScoreQueues: Record = {}; - -/** - * Returns this user's score queue. A score queue is a temporary place scores are saved - * so that they can be inserted into the database in bulk. - * - * This massively improves performance on large imports instead of constantly running single imports. - * - * If a score queue does not exist for the user, one is created. - */ -function GetOrSetScoreQueue(userID: integer) { - const queue = ScoreQueues[userID]; - - if (!queue) { - return SetScoreQueue(userID); - } - - return queue; -} - -export function GetScoreQueueMaybe(userID: integer): ScoreQueue | undefined { - return ScoreQueues[userID]; -} - -function SetScoreQueue(userID: integer) { - const queue: ScoreQueue = { - queue: [], - scoreIDSet: new Set(), - }; - - ScoreQueues[userID] = queue; - - return queue; -} - -/** - * Adds a new score to the given queue. - */ -function AddToScoreQueue(scoreQueue: ScoreQueue, score: ScoreDocument) { - scoreQueue.queue.push(score); - scoreQueue.scoreIDSet.add(score.scoreID); -} - -export async function InsertQueue(userID: integer) { - const scoreQueue = GetOrSetScoreQueue(userID); - - const queuedScores = scoreQueue.queue.splice(0); - - if (queuedScores.length !== 0) { - delete ScoreQueues[userID]; - - try { - await db.scores.insert(queuedScores); - } catch (err) { - logger.warn( - `Triggered duplicate key protection. Race condition protected against, but this is not good.` - ); - return null; - } - } - - return queuedScores.length; -} - -/** - * Adds a score to a queue to be inserted in batch to the database. - * @param score - The score document to queue. - * @returns True on success, The amount of scores inserted on auto-pipeline-flush, and null if - * the score provided is already loaded. - */ -export function QueueScoreInsert(score: ScoreDocument) { - const scoreQueue = GetOrSetScoreQueue(score.userID); - - if (scoreQueue.scoreIDSet.has(score.scoreID)) { - // skip - logger.verbose(`Score ID ${score.scoreID} was already queued to be imported.`); - return null; - } - - AddToScoreQueue(scoreQueue, score); - - logger.debug(`ScoreQueue for ${score.userID} is now at ${scoreQueue.queue.length}.`); - - if (scoreQueue.queue.length >= MAX_PIPELINE_LENGTH) { - logger.verbose(`Triggered pipeline flush with len ${scoreQueue.queue.length}.`); - return InsertQueue(score.userID); - } - - return true; -} +import { ScoreDocument, integer } from "tachi-common"; +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; + +const logger = CreateLogCtx(__filename); +const MAX_PIPELINE_LENGTH = 500; + +interface ScoreQueue { + queue: ScoreDocument[]; + scoreIDSet: Set; +} + +const ScoreQueues: Record = {}; + +/** + * Returns this user's score queue. A score queue is a temporary place scores are saved + * so that they can be inserted into the database in bulk. + * + * This massively improves performance on large imports instead of constantly running single imports. + * + * If a score queue does not exist for the user, one is created. + */ +function GetOrSetScoreQueue(userID: integer) { + const queue = ScoreQueues[userID]; + + if (!queue) { + return SetScoreQueue(userID); + } + + return queue; +} + +export function GetScoreQueueMaybe(userID: integer): ScoreQueue | undefined { + return ScoreQueues[userID]; +} + +function SetScoreQueue(userID: integer) { + const queue: ScoreQueue = { + queue: [], + scoreIDSet: new Set(), + }; + + ScoreQueues[userID] = queue; + + return queue; +} + +/** + * Adds a new score to the given queue. + */ +function AddToScoreQueue(scoreQueue: ScoreQueue, score: ScoreDocument) { + scoreQueue.queue.push(score); + scoreQueue.scoreIDSet.add(score.scoreID); +} + +export async function InsertQueue(userID: integer) { + const scoreQueue = GetOrSetScoreQueue(userID); + + const queuedScores = scoreQueue.queue.splice(0); + + if (queuedScores.length !== 0) { + delete ScoreQueues[userID]; + + try { + await db.scores.insert(queuedScores); + } catch (err) { + logger.warn( + `Triggered duplicate key protection. Race condition protected against, but this is not good.` + ); + return null; + } + } + + return queuedScores.length; +} + +/** + * Adds a score to a queue to be inserted in batch to the database. + * @param score - The score document to queue. + * @returns True on success, The amount of scores inserted on auto-pipeline-flush, and null if + * the score provided is already loaded. + */ +export function QueueScoreInsert(score: ScoreDocument) { + const scoreQueue = GetOrSetScoreQueue(score.userID); + + if (scoreQueue.scoreIDSet.has(score.scoreID)) { + // skip + logger.verbose(`Score ID ${score.scoreID} was already queued to be imported.`); + return null; + } + + AddToScoreQueue(scoreQueue, score); + + logger.debug(`ScoreQueue for ${score.userID} is now at ${scoreQueue.queue.length}.`); + + if (scoreQueue.queue.length >= MAX_PIPELINE_LENGTH) { + logger.verbose(`Triggered pipeline flush with len ${scoreQueue.queue.length}.`); + return InsertQueue(score.userID); + } + + return true; +} diff --git a/server/src/lib/score-import/framework/score-importing/score-id.test.ts b/server/src/lib/score-import/framework/score-importing/score-id.test.ts index 4cfbe5c26..fc0fb57ad 100644 --- a/server/src/lib/score-import/framework/score-importing/score-id.test.ts +++ b/server/src/lib/score-import/framework/score-importing/score-id.test.ts @@ -1,95 +1,95 @@ -import { ScoreDocument } from "tachi-common"; -import t from "tap"; - -import ResetDBState from "test-utils/resets"; -import { Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data"; -import { CreateScoreID, GetWithScoreID } from "./score-id"; - -t.test("#GetWithScoreID", async (t) => { - await ResetDBState(); - - const sc = await GetWithScoreID("TESTING_SCORE_ID"); - - t.not(sc, null, "Return a score for a valid score id"); - - const nl = await GetWithScoreID("INVALID SCORE ID"); - - t.equal(nl, null, "Should return null for score IDs it cannot resolve"); - - t.end(); -}); - -t.test("#CreateScoreID", (t) => { - const scoreID = CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID); - - t.match( - scoreID, - /^R[0-9a-f]{40}/u, - "Should return an R followed by 40 characters of lowercase hex as scoreID." - ); - - t.not( - scoreID, - CreateScoreID(2, TestingIIDXSPDryScore, Testing511SPA.chartID), - "Hash should be affected by userID." - ); - - t.equal( - scoreID, - CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID), - "ScoreIDs should be consistently generatable" - ); - - // duplicate score - const minimalScore = { - scoreData: { - percent: TestingIIDXSPDryScore.scoreData.percent, - score: TestingIIDXSPDryScore.scoreData.score, - grade: TestingIIDXSPDryScore.scoreData.grade, - lamp: TestingIIDXSPDryScore.scoreData.lamp, - }, - }; - - t.equal( - scoreID, - CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), - "ScoreIDs should only be affected by score, percent, grade and lamp." - ); - - minimalScore.scoreData.percent = 0; - - t.not( - scoreID, - CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), - "ScoreIDs should not produce the same value if percent is different." - ); - - minimalScore.scoreData.percent = TestingIIDXSPDryScore.scoreData.percent; - minimalScore.scoreData.score = 0; - - t.not( - scoreID, - CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), - "ScoreIDs should not produce the same value if score is different." - ); - - minimalScore.scoreData.score = TestingIIDXSPDryScore.scoreData.score; - minimalScore.scoreData.lamp = "ASSIST CLEAR"; - - t.not( - scoreID, - CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), - "ScoreIDs should not produce the same value if lamp is different." - ); - - minimalScore.scoreData.lamp = TestingIIDXSPDryScore.scoreData.lamp; - minimalScore.scoreData.grade = "F"; - - t.not( - scoreID, - CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), - "ScoreIDs should not produce the same value if grade is different." - ); - - t.end(); -}); +import { ScoreDocument } from "tachi-common"; +import t from "tap"; + +import ResetDBState from "test-utils/resets"; +import { Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data"; +import { CreateScoreID, GetWithScoreID } from "./score-id"; + +t.test("#GetWithScoreID", async (t) => { + await ResetDBState(); + + const sc = await GetWithScoreID("TESTING_SCORE_ID"); + + t.not(sc, null, "Return a score for a valid score id"); + + const nl = await GetWithScoreID("INVALID SCORE ID"); + + t.equal(nl, null, "Should return null for score IDs it cannot resolve"); + + t.end(); +}); + +t.test("#CreateScoreID", (t) => { + const scoreID = CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID); + + t.match( + scoreID, + /^R[0-9a-f]{40}/u, + "Should return an R followed by 40 characters of lowercase hex as scoreID." + ); + + t.not( + scoreID, + CreateScoreID(2, TestingIIDXSPDryScore, Testing511SPA.chartID), + "Hash should be affected by userID." + ); + + t.equal( + scoreID, + CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID), + "ScoreIDs should be consistently generatable" + ); + + // duplicate score + const minimalScore = { + scoreData: { + percent: TestingIIDXSPDryScore.scoreData.percent, + score: TestingIIDXSPDryScore.scoreData.score, + grade: TestingIIDXSPDryScore.scoreData.grade, + lamp: TestingIIDXSPDryScore.scoreData.lamp, + }, + }; + + t.equal( + scoreID, + CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), + "ScoreIDs should only be affected by score, percent, grade and lamp." + ); + + minimalScore.scoreData.percent = 0; + + t.not( + scoreID, + CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), + "ScoreIDs should not produce the same value if percent is different." + ); + + minimalScore.scoreData.percent = TestingIIDXSPDryScore.scoreData.percent; + minimalScore.scoreData.score = 0; + + t.not( + scoreID, + CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), + "ScoreIDs should not produce the same value if score is different." + ); + + minimalScore.scoreData.score = TestingIIDXSPDryScore.scoreData.score; + minimalScore.scoreData.lamp = "ASSIST CLEAR"; + + t.not( + scoreID, + CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), + "ScoreIDs should not produce the same value if lamp is different." + ); + + minimalScore.scoreData.lamp = TestingIIDXSPDryScore.scoreData.lamp; + minimalScore.scoreData.grade = "F"; + + t.not( + scoreID, + CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID), + "ScoreIDs should not produce the same value if grade is different." + ); + + t.end(); +}); diff --git a/server/src/lib/score-import/framework/score-importing/score-id.ts b/server/src/lib/score-import/framework/score-importing/score-id.ts index 14f32bfe6..67c780d67 100644 --- a/server/src/lib/score-import/framework/score-importing/score-id.ts +++ b/server/src/lib/score-import/framework/score-importing/score-id.ts @@ -1,32 +1,32 @@ -import crypto from "crypto"; -import db from "external/mongo/db"; -import { integer } from "tachi-common"; -import { DryScore } from "../common/types"; - -/** - * Performs sha256 hashing on the input data. - * @param scoreIDString - The string to sha256 hash. - * @returns A sha256 checksum in lowercase hex. - */ -function HashScoreIDString(scoreIDString: string) { - return crypto.createHash("sha256").update(scoreIDString).digest("hex"); -} - -/** - * Creates an identifier for this score. - * This is used to deduplicate repeated scores. - * @returns @see HashScoreIDString - prefixed with R. - */ -export function CreateScoreID(userID: integer, dryScore: DryScore, chartID: string) { - const hash = HashScoreIDString( - `${userID}|${chartID}|${dryScore.scoreData.lamp}|${dryScore.scoreData.grade}|${dryScore.scoreData.score}|${dryScore.scoreData.percent}` - ); - - return `R${hash}`; -} - -export function GetWithScoreID(scoreID: string) { - return db.scores.findOne({ - scoreID, - }); -} +import crypto from "crypto"; +import db from "external/mongo/db"; +import { integer } from "tachi-common"; +import { DryScore } from "../common/types"; + +/** + * Performs sha256 hashing on the input data. + * @param scoreIDString - The string to sha256 hash. + * @returns A sha256 checksum in lowercase hex. + */ +function HashScoreIDString(scoreIDString: string) { + return crypto.createHash("sha256").update(scoreIDString).digest("hex"); +} + +/** + * Creates an identifier for this score. + * This is used to deduplicate repeated scores. + * @returns @see HashScoreIDString - prefixed with R. + */ +export function CreateScoreID(userID: integer, dryScore: DryScore, chartID: string) { + const hash = HashScoreIDString( + `${userID}|${chartID}|${dryScore.scoreData.lamp}|${dryScore.scoreData.grade}|${dryScore.scoreData.score}|${dryScore.scoreData.percent}` + ); + + return `R${hash}`; +} + +export function GetWithScoreID(scoreID: string) { + return db.scores.findOne({ + scoreID, + }); +} diff --git a/server/src/lib/score-import/framework/score-importing/score-import-error.test.ts b/server/src/lib/score-import/framework/score-importing/score-import-error.test.ts index bb5cc9c85..9f5123004 100644 --- a/server/src/lib/score-import/framework/score-importing/score-import-error.test.ts +++ b/server/src/lib/score-import/framework/score-importing/score-import-error.test.ts @@ -1,11 +1,11 @@ -import t from "tap"; -import ScoreImportFatalError from "./score-import-error"; - -t.test("#new ScoreImportFatalError()", (t) => { - const err = new ScoreImportFatalError(500, "error message"); - - t.equal(err.statusCode, 500, "Should store the status code passed to it."); - t.equal(err.message, "error message", "Should store description as the error message."); - - t.end(); -}); +import t from "tap"; +import ScoreImportFatalError from "./score-import-error"; + +t.test("#new ScoreImportFatalError()", (t) => { + const err = new ScoreImportFatalError(500, "error message"); + + t.equal(err.statusCode, 500, "Should store the status code passed to it."); + t.equal(err.message, "error message", "Should store description as the error message."); + + t.end(); +}); diff --git a/server/src/lib/score-import/framework/score-importing/score-import-error.ts b/server/src/lib/score-import/framework/score-importing/score-import-error.ts index 1091eb050..38206fd20 100644 --- a/server/src/lib/score-import/framework/score-importing/score-import-error.ts +++ b/server/src/lib/score-import/framework/score-importing/score-import-error.ts @@ -1,16 +1,16 @@ -/** - * Throw this if the ScoreImport needs to exit and return - * something to the user. This is used for expected errors, - * such as the user passing unparsable data. Unlike Failures, - * throwing this at any time will exit the *entire* score import - * process. - */ -export default class ScoreImportFatalError extends Error { - statusCode: number; - - constructor(statusCode: number, description: string) { - super(description); - - this.statusCode = statusCode; - } -} +/** + * Throw this if the ScoreImport needs to exit and return + * something to the user. This is used for expected errors, + * such as the user passing unparsable data. Unlike Failures, + * throwing this at any time will exit the *entire* score import + * process. + */ +export default class ScoreImportFatalError extends Error { + statusCode: number; + + constructor(statusCode: number, description: string) { + super(description); + + this.statusCode = statusCode; + } +} diff --git a/server/src/lib/score-import/framework/score-importing/score-import-main.ts b/server/src/lib/score-import/framework/score-importing/score-import-main.ts index 5c7ea9ee2..a0c211a27 100644 --- a/server/src/lib/score-import/framework/score-importing/score-import-main.ts +++ b/server/src/lib/score-import/framework/score-importing/score-import-main.ts @@ -1,350 +1,350 @@ -import db from "external/mongo/db"; -import { KtLogger } from "lib/logger/logger"; -import { - Game, - IDStrings, - ImportDocument, - ImportProcessingInfo, - ImportTypes, - integer, - Playtypes, - PublicUserDocument, -} from "tachi-common"; -import { GetMillisecondsSince } from "utils/misc"; -import { GetUserWithID } from "utils/user"; -import { ConverterFunction, ImportInputParser } from "../../import-types/common/types"; -import { Converters } from "../../import-types/converters"; -import { InternalFailure } from "../common/converter-failures"; -import { CreateScoreLogger } from "../common/import-logger"; -import { ScorePlaytypeMap } from "../common/types"; -import { GetAndUpdateUsersGoals } from "../goals/goals"; -import { GetOrSetUserLock, RemoveUserLock } from "../import-locks/lock"; -import { UpdateUsersMilestones } from "../milestones/milestones"; -import { ProcessPBs } from "../pb/process-pbs"; -import { CreateSessions } from "../sessions/sessions"; -import { ClassHandler } from "../user-game-stats/types"; -import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs"; -import ScoreImportFatalError from "./score-import-error"; -import { ImportAllIterableData } from "./score-importing"; - -/** - * Performs a Score Import. - */ -export default async function ScoreImportMain( - userID: integer, - userIntent: boolean, - importType: ImportTypes, - InputParser: ImportInputParser, - importID: string, - providedLogger?: KtLogger -) { - const user = await GetUserWithID(userID); - - if (!user) { - throw new InternalFailure( - `User with ID ${userID} does not exist, but attempted to make an import?` - ); - } - - // in the event of any error, we remove the user lock. - try { - const lock = await GetOrSetUserLock(user.id); - - if (lock) { - // @danger - // Throwing away an import if the user already has one outgoing is *bad*, as in the case - // of degraded performance we might just start throwing scores away. This is obviously - // not great, but any other solution involves making a queue, which can't be done because - // InputParser is a very dynamic function that cannot be stored in redis or something. - // - // Under normal circumstances, there is no scenario where a user would have two ongoing - // imports at the same time - even if they were using single-score imports on a 5 second - // chart, as each score import takes only around ~10-15millisecondss. - throw new ScoreImportFatalError(409, "This user already has an ongoing import."); - } - - const timeStarted = Date.now(); - let logger; - - if (!providedLogger) { - // If they weren't given to us - - // we create an "import logger". - // this holds a reference to the user's name, ID, and type - // of score import for any future debugging. - logger = CreateScoreLogger(user, importID, importType); - logger.debug("Received import request."); - } else { - logger = providedLogger; - } - - // --- 1. Parsing --- - // We get an iterable from the provided parser function, alongside some context and a converter function. - // This iterable does not have to be an array - it's anything that's iterable, like a generator. - const parseTimeStart = process.hrtime.bigint(); - const { iterable, context, game, classHandler } = await InputParser(logger); - - const parseTime = GetMillisecondsSince(parseTimeStart); - - logger.debug(`Parsing took ${parseTime} millisecondss.`); - - // We have to cast here due to typescript generic confusions. This is guaranteed` to be correct. - const ConverterFunction = Converters[importType] as unknown as ConverterFunction; - - // --- 2. Importing --- - // ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data. - const importTimeStart = process.hrtime.bigint(); - const importInfo = await ImportAllIterableData( - user.id, - importType, - iterable, - ConverterFunction, - context, - game, - logger - ); - - const importTime = GetMillisecondsSince(importTimeStart); - const importTimeRel = importTime / importInfo.length; - - logger.debug(`Importing took ${importTime} millisecondss. (${importTimeRel}ms/doc)`); - - // Steps 3-8 are handled inside here. - // This was moved inside here so the score de-orphaning process - // could hook into importing better - const { - playtypes, - scoreIDs, - errors, - sessionInfo, - classDeltas, - goalInfo, - milestoneInfo, - relativeTimes, - absoluteTimes, - } = await HandlePostImportSteps(importInfo, user, importType, game, classHandler, logger); - - const { importParseTimeRel, pbTimeRel, sessionTimeRel } = relativeTimes; - const { importParseTime, sessionTime, pbTime, ugsTime, goalTime, milestoneTime } = - absoluteTimes; - - // --- 9. Finalise Import Document --- - // Create and Save an import document to the database, and finish everything up! - const ImportDocument: ImportDocument = { - importType, - idStrings: playtypes.map((e) => `${game}:${e}`) as IDStrings[], - scoreIDs, - playtypes, - game, - errors, - importID, - timeFinished: Date.now(), - timeStarted, - createdSessions: sessionInfo, - userID: user.id, - classDeltas, - goalInfo, - milestoneInfo, - userIntent, - }; - - const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${ - importInfo.length - } documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${ - sessionInfo.length - }). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`; - - // I only really want to log "big" imports. The others are here for debugging purposes. - if (scoreIDs.length > 500) { - logger.info(logMessage); - } else if (scoreIDs.length > 1) { - logger.verbose(logMessage); - } else { - logger.debug(logMessage); - } - - await db.imports.insert(ImportDocument); - - // we don't await this because we don't - // particularly care about waiting for it. - db["import-timings"].insert({ - importID, - timestamp: Date.now(), - total: ImportDocument.timeFinished - timeStarted, - rel: { - import: importTimeRel, - importParse: importParseTimeRel, - pb: pbTimeRel, - session: sessionTimeRel, - }, - abs: { - parse: parseTime, - import: importTime, - importParse: importParseTime, - session: sessionTime, - pb: pbTime, - ugs: ugsTime, - goal: goalTime, - milestone: milestoneTime, - }, - }); - - return ImportDocument; - } finally { - await RemoveUserLock(user.id); - } -} - -/** - * Handles every single processing step after actually loading scores - * into the database, such as updating goals, reprocessing sessions, - * and updating a users game stats. - */ -export async function HandlePostImportSteps( - importInfo: ImportProcessingInfo[], - user: PublicUserDocument, - importType: ImportTypes, - game: Game, - classHandler: ClassHandler | null, - logger: KtLogger -) { - // --- 3. ParseImportInfo --- - // ImportInfo is a relatively complex structure. We need some information from it for subsequent steps - // such as the list of chartIDs involved in this import. - const importParseTimeStart = process.hrtime.bigint(); - const { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo); - - const importParseTime = GetMillisecondsSince(importParseTimeStart); - const importParseTimeRel = importParseTime / importInfo.length; - - logger.debug( - `Import Parsing took ${importParseTime} millisecondss. (${importParseTimeRel}ms/doc)` - ); - - // --- 4. Sessions --- - // We create (or update existing) sessions here. This uses the aforementioned parsed import info - // to determine what goes where. - const sessionTimeStart = process.hrtime.bigint(); - const sessionInfo = await CreateSessions(user.id, importType, game, scorePlaytypeMap, logger); - - const sessionTime = GetMillisecondsSince(sessionTimeStart); - const sessionTimeRel = sessionTime / sessionInfo.length; - - logger.debug(`Session Processing took ${sessionTime} millisecondss (${sessionTimeRel}ms/doc).`); - - // --- 5. PersonalBests --- - // We want to keep an updated reference of a users best score on a given chart. - // This function also handles conjoining different scores together (such as unioning best lamp and - // best score). - const pbTimeStart = process.hrtime.bigint(); - await ProcessPBs(user.id, chartIDs, logger); - - const pbTime = GetMillisecondsSince(pbTimeStart); - const pbTimeRel = pbTime / chartIDs.size; - - logger.debug(`PB Processing took ${pbTime} millisecondss (${pbTimeRel}ms/doc)`); - - const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][]; - - // --- 6. Game Stats --- - // This function updates the users "stats" for this game - such as their profile rating or their classes. - const ugsTimeStart = process.hrtime.bigint(); - const classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, classHandler, logger); - - const ugsTime = GetMillisecondsSince(ugsTimeStart); - - logger.debug(`UGS Processing took ${ugsTime} millisecondss.`); - - // --- 7. Goals --- - // Evaluate and update the users goals. This returns information about goals that have changed. - const goalTimeStart = process.hrtime.bigint(); - const goalInfo = await GetAndUpdateUsersGoals(game, user.id, chartIDs, logger); - - const goalTime = GetMillisecondsSince(goalTimeStart); - - logger.debug(`Goal Processing took ${goalTime} millisecondss.`); - - // --- 8. Milestones --- - // Evaluate and update the users milestones. This returns... - const milestoneTimeStart = process.hrtime.bigint(); - const milestoneInfo = await UpdateUsersMilestones(goalInfo, game, playtypes, user.id, logger); - - const milestoneTime = GetMillisecondsSince(milestoneTimeStart); - - logger.debug(`Milestone Processing took ${milestoneTime} millisecondss.`); - - return { - classDeltas, - milestoneInfo, - goalInfo, - playtypes, - scoreIDs, - errors, - sessionInfo, - relativeTimes: { - importParseTimeRel, - pbTimeRel, - sessionTimeRel, - }, - absoluteTimes: { - importParseTime, - sessionTime, - pbTime, - ugsTime, - goalTime, - milestoneTime, - }, - }; -} - -/** - * Calls UpdateUsersGamePlaytypeStats for every playtype in the import. - * @returns A flattened array of ClassDeltas - */ -async function UpdateUsersGameStats( - game: Game, - playtypes: Playtypes[Game][], - userID: integer, - classHandler: ClassHandler | null, - logger: KtLogger -) { - const promises = []; - - for (const pt of playtypes) { - promises.push(UpdateUsersGamePlaytypeStats(game, pt, userID, classHandler, logger)); - } - - const r = await Promise.all(promises); - return r.flat(1); -} - -/** - * Parses the return of ImportProcessingInfo into relevant information - * for the rest the import. - * @returns The list of scoreIDs used in the import, the list of errors - * A set of unique chartIDs involved in the import and the scores mapped - * on their playtype. - */ -function ParseImportInfo(importInfo: ImportProcessingInfo[]) { - const scorePlaytypeMap: ScorePlaytypeMap = Object.create(null); - - const scoreIDs = []; - const errors = []; - const chartIDs: Set = new Set(); - - for (const info of importInfo) { - if (info.success) { - scoreIDs.push(info.content.score.scoreID); - chartIDs.add(info.content.score.chartID); - - if (scorePlaytypeMap[info.content.score.playtype]) { - scorePlaytypeMap[info.content.score.playtype]!.push(info.content.score); - } else { - scorePlaytypeMap[info.content.score.playtype] = [info.content.score]; - } - } else { - errors.push({ type: info.type, message: info.message }); - } - } - - return { scoreIDs, errors, scorePlaytypeMap, chartIDs }; -} +import db from "external/mongo/db"; +import { KtLogger } from "lib/logger/logger"; +import { + Game, + IDStrings, + ImportDocument, + ImportProcessingInfo, + ImportTypes, + integer, + Playtypes, + PublicUserDocument, +} from "tachi-common"; +import { GetMillisecondsSince } from "utils/misc"; +import { GetUserWithID } from "utils/user"; +import { ConverterFunction, ImportInputParser } from "../../import-types/common/types"; +import { Converters } from "../../import-types/converters"; +import { InternalFailure } from "../common/converter-failures"; +import { CreateScoreLogger } from "../common/import-logger"; +import { ScorePlaytypeMap } from "../common/types"; +import { GetAndUpdateUsersGoals } from "../goals/goals"; +import { GetOrSetUserLock, RemoveUserLock } from "../import-locks/lock"; +import { UpdateUsersMilestones } from "../milestones/milestones"; +import { ProcessPBs } from "../pb/process-pbs"; +import { CreateSessions } from "../sessions/sessions"; +import { ClassHandler } from "../user-game-stats/types"; +import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs"; +import ScoreImportFatalError from "./score-import-error"; +import { ImportAllIterableData } from "./score-importing"; + +/** + * Performs a Score Import. + */ +export default async function ScoreImportMain( + userID: integer, + userIntent: boolean, + importType: ImportTypes, + InputParser: ImportInputParser, + importID: string, + providedLogger?: KtLogger +) { + const user = await GetUserWithID(userID); + + if (!user) { + throw new InternalFailure( + `User with ID ${userID} does not exist, but attempted to make an import?` + ); + } + + // in the event of any error, we remove the user lock. + try { + const lock = await GetOrSetUserLock(user.id); + + if (lock) { + // @danger + // Throwing away an import if the user already has one outgoing is *bad*, as in the case + // of degraded performance we might just start throwing scores away. This is obviously + // not great, but any other solution involves making a queue, which can't be done because + // InputParser is a very dynamic function that cannot be stored in redis or something. + // + // Under normal circumstances, there is no scenario where a user would have two ongoing + // imports at the same time - even if they were using single-score imports on a 5 second + // chart, as each score import takes only around ~10-15millisecondss. + throw new ScoreImportFatalError(409, "This user already has an ongoing import."); + } + + const timeStarted = Date.now(); + let logger; + + if (!providedLogger) { + // If they weren't given to us - + // we create an "import logger". + // this holds a reference to the user's name, ID, and type + // of score import for any future debugging. + logger = CreateScoreLogger(user, importID, importType); + logger.debug("Received import request."); + } else { + logger = providedLogger; + } + + // --- 1. Parsing --- + // We get an iterable from the provided parser function, alongside some context and a converter function. + // This iterable does not have to be an array - it's anything that's iterable, like a generator. + const parseTimeStart = process.hrtime.bigint(); + const { iterable, context, game, classHandler } = await InputParser(logger); + + const parseTime = GetMillisecondsSince(parseTimeStart); + + logger.debug(`Parsing took ${parseTime} millisecondss.`); + + // We have to cast here due to typescript generic confusions. This is guaranteed` to be correct. + const ConverterFunction = Converters[importType] as unknown as ConverterFunction; + + // --- 2. Importing --- + // ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data. + const importTimeStart = process.hrtime.bigint(); + const importInfo = await ImportAllIterableData( + user.id, + importType, + iterable, + ConverterFunction, + context, + game, + logger + ); + + const importTime = GetMillisecondsSince(importTimeStart); + const importTimeRel = importTime / importInfo.length; + + logger.debug(`Importing took ${importTime} millisecondss. (${importTimeRel}ms/doc)`); + + // Steps 3-8 are handled inside here. + // This was moved inside here so the score de-orphaning process + // could hook into importing better + const { + playtypes, + scoreIDs, + errors, + sessionInfo, + classDeltas, + goalInfo, + milestoneInfo, + relativeTimes, + absoluteTimes, + } = await HandlePostImportSteps(importInfo, user, importType, game, classHandler, logger); + + const { importParseTimeRel, pbTimeRel, sessionTimeRel } = relativeTimes; + const { importParseTime, sessionTime, pbTime, ugsTime, goalTime, milestoneTime } = + absoluteTimes; + + // --- 9. Finalise Import Document --- + // Create and Save an import document to the database, and finish everything up! + const ImportDocument: ImportDocument = { + importType, + idStrings: playtypes.map((e) => `${game}:${e}`) as IDStrings[], + scoreIDs, + playtypes, + game, + errors, + importID, + timeFinished: Date.now(), + timeStarted, + createdSessions: sessionInfo, + userID: user.id, + classDeltas, + goalInfo, + milestoneInfo, + userIntent, + }; + + const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${ + importInfo.length + } documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${ + sessionInfo.length + }). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`; + + // I only really want to log "big" imports. The others are here for debugging purposes. + if (scoreIDs.length > 500) { + logger.info(logMessage); + } else if (scoreIDs.length > 1) { + logger.verbose(logMessage); + } else { + logger.debug(logMessage); + } + + await db.imports.insert(ImportDocument); + + // we don't await this because we don't + // particularly care about waiting for it. + db["import-timings"].insert({ + importID, + timestamp: Date.now(), + total: ImportDocument.timeFinished - timeStarted, + rel: { + import: importTimeRel, + importParse: importParseTimeRel, + pb: pbTimeRel, + session: sessionTimeRel, + }, + abs: { + parse: parseTime, + import: importTime, + importParse: importParseTime, + session: sessionTime, + pb: pbTime, + ugs: ugsTime, + goal: goalTime, + milestone: milestoneTime, + }, + }); + + return ImportDocument; + } finally { + await RemoveUserLock(user.id); + } +} + +/** + * Handles every single processing step after actually loading scores + * into the database, such as updating goals, reprocessing sessions, + * and updating a users game stats. + */ +export async function HandlePostImportSteps( + importInfo: ImportProcessingInfo[], + user: PublicUserDocument, + importType: ImportTypes, + game: Game, + classHandler: ClassHandler | null, + logger: KtLogger +) { + // --- 3. ParseImportInfo --- + // ImportInfo is a relatively complex structure. We need some information from it for subsequent steps + // such as the list of chartIDs involved in this import. + const importParseTimeStart = process.hrtime.bigint(); + const { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo); + + const importParseTime = GetMillisecondsSince(importParseTimeStart); + const importParseTimeRel = importParseTime / importInfo.length; + + logger.debug( + `Import Parsing took ${importParseTime} millisecondss. (${importParseTimeRel}ms/doc)` + ); + + // --- 4. Sessions --- + // We create (or update existing) sessions here. This uses the aforementioned parsed import info + // to determine what goes where. + const sessionTimeStart = process.hrtime.bigint(); + const sessionInfo = await CreateSessions(user.id, importType, game, scorePlaytypeMap, logger); + + const sessionTime = GetMillisecondsSince(sessionTimeStart); + const sessionTimeRel = sessionTime / sessionInfo.length; + + logger.debug(`Session Processing took ${sessionTime} millisecondss (${sessionTimeRel}ms/doc).`); + + // --- 5. PersonalBests --- + // We want to keep an updated reference of a users best score on a given chart. + // This function also handles conjoining different scores together (such as unioning best lamp and + // best score). + const pbTimeStart = process.hrtime.bigint(); + await ProcessPBs(user.id, chartIDs, logger); + + const pbTime = GetMillisecondsSince(pbTimeStart); + const pbTimeRel = pbTime / chartIDs.size; + + logger.debug(`PB Processing took ${pbTime} millisecondss (${pbTimeRel}ms/doc)`); + + const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][]; + + // --- 6. Game Stats --- + // This function updates the users "stats" for this game - such as their profile rating or their classes. + const ugsTimeStart = process.hrtime.bigint(); + const classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, classHandler, logger); + + const ugsTime = GetMillisecondsSince(ugsTimeStart); + + logger.debug(`UGS Processing took ${ugsTime} millisecondss.`); + + // --- 7. Goals --- + // Evaluate and update the users goals. This returns information about goals that have changed. + const goalTimeStart = process.hrtime.bigint(); + const goalInfo = await GetAndUpdateUsersGoals(game, user.id, chartIDs, logger); + + const goalTime = GetMillisecondsSince(goalTimeStart); + + logger.debug(`Goal Processing took ${goalTime} millisecondss.`); + + // --- 8. Milestones --- + // Evaluate and update the users milestones. This returns... + const milestoneTimeStart = process.hrtime.bigint(); + const milestoneInfo = await UpdateUsersMilestones(goalInfo, game, playtypes, user.id, logger); + + const milestoneTime = GetMillisecondsSince(milestoneTimeStart); + + logger.debug(`Milestone Processing took ${milestoneTime} millisecondss.`); + + return { + classDeltas, + milestoneInfo, + goalInfo, + playtypes, + scoreIDs, + errors, + sessionInfo, + relativeTimes: { + importParseTimeRel, + pbTimeRel, + sessionTimeRel, + }, + absoluteTimes: { + importParseTime, + sessionTime, + pbTime, + ugsTime, + goalTime, + milestoneTime, + }, + }; +} + +/** + * Calls UpdateUsersGamePlaytypeStats for every playtype in the import. + * @returns A flattened array of ClassDeltas + */ +async function UpdateUsersGameStats( + game: Game, + playtypes: Playtypes[Game][], + userID: integer, + classHandler: ClassHandler | null, + logger: KtLogger +) { + const promises = []; + + for (const pt of playtypes) { + promises.push(UpdateUsersGamePlaytypeStats(game, pt, userID, classHandler, logger)); + } + + const r = await Promise.all(promises); + return r.flat(1); +} + +/** + * Parses the return of ImportProcessingInfo into relevant information + * for the rest the import. + * @returns The list of scoreIDs used in the import, the list of errors + * A set of unique chartIDs involved in the import and the scores mapped + * on their playtype. + */ +function ParseImportInfo(importInfo: ImportProcessingInfo[]) { + const scorePlaytypeMap: ScorePlaytypeMap = Object.create(null); + + const scoreIDs = []; + const errors = []; + const chartIDs: Set = new Set(); + + for (const info of importInfo) { + if (info.success) { + scoreIDs.push(info.content.score.scoreID); + chartIDs.add(info.content.score.chartID); + + if (scorePlaytypeMap[info.content.score.playtype]) { + scorePlaytypeMap[info.content.score.playtype]!.push(info.content.score); + } else { + scorePlaytypeMap[info.content.score.playtype] = [info.content.score]; + } + } else { + errors.push({ type: info.type, message: info.message }); + } + } + + return { scoreIDs, errors, scorePlaytypeMap, chartIDs }; +} diff --git a/server/src/lib/score-import/framework/score-importing/score-importing.ts b/server/src/lib/score-import/framework/score-importing/score-importing.ts index b394ca251..a98195735 100644 --- a/server/src/lib/score-import/framework/score-importing/score-importing.ts +++ b/server/src/lib/score-import/framework/score-importing/score-importing.ts @@ -1,339 +1,339 @@ -import db from "external/mongo/db"; -import { AppendLogCtx, KtLogger } from "lib/logger/logger"; -import { - ChartDocument, - Game, - IDStrings, - ImportProcessingInfo, - ImportTypes, - integer, - ScoreDocument, - SongDocument, -} from "tachi-common"; -import { - ConverterFnReturnOrFailure, - ConverterFnSuccessReturn, - ConverterFunction, -} from "../../import-types/common/types"; -import { - ConverterFailure, - InternalFailure, - InvalidScoreFailure, - KTDataNotFoundFailure, - SkipScoreFailure, -} from "../common/converter-failures"; -import { DryScore } from "../common/types"; -import { OrphanScore } from "../orphans/orphans"; -import { HydrateScore } from "./hydrate-score"; -import { GetScoreQueueMaybe, InsertQueue, QueueScoreInsert } from "./insert-score"; -import { CreateScoreID } from "./score-id"; - -/** - * Processes the iterable data into the Tachi database. - * @param userID - The user this score import was for. - * @param iterableData - The data to iterate upon. - * @param ConverterFunction - The function needed to convert the data into an IntermediateScore - * @param context - Any context the Converter may need in order to make decisions. - * @returns An array of ImportProcessInfo objects. - */ -export async function ImportAllIterableData( - userID: integer, - importType: ImportTypes, - iterableData: Iterable | AsyncIterable, - ConverterFunction: ConverterFunction, - context: C, - game: Game, - logger: KtLogger -): Promise { - logger.verbose("Getting Blacklist..."); - - // @optimisable: could filter harder with score.game and score.playtype - // stuff. - const blacklist = ( - await db["score-blacklist"].find({ - userID, - }) - ).map((e) => e.scoreID); - - logger.verbose(`Starting Data Processing...`); - - const processedResults = []; - - // for await is used here as iterableData may be an async iterable - // An example would be making an api request after exhausting - // the first set of data. - for await (const data of iterableData) { - processedResults.push( - await ImportIterableDatapoint( - userID, - importType, - data, - ConverterFunction, - context, - game, - blacklist, - logger - ) - ); - } - - // We need to filter out nulls, which we don't care for (these are neither successes or failures) - - logger.verbose(`Finished Importing Data (${processedResults.length} datapoints).`); - logger.debug(`Removing null returns...`); - - const datapoints = processedResults.filter( - (e) => e !== null - ) as ImportProcessingInfo[]; - - logger.debug(`Removed null from results.`); - - logger.verbose(`Recieved ${datapoints.length} returns, from ${processedResults.length} data.`); - - // Flush the score queue out after finishing most of the import. This ensures no scores get left in the - // queue. - const emptied = await InsertQueue(userID); - - if (emptied) { - logger.verbose(`Emptied ${emptied} documents from score queue.`); - } - - return datapoints; -} - -/** - * Processes a single data object into one or many ImportProcessingInfo objects. - * @param userID - The user this score is from. - * @param data - The data to process. - * @param ConverterFunction - The processor function that takes the data and returns the partialScore(s) - * @param context - Any context the processor might need that it can not infer from the data object. - * @returns An array of ImportProcessingInfo objects, or a single ImportProcessingInfo object - */ -export async function ImportIterableDatapoint( - userID: integer, - importType: ImportTypes, - data: D, - ConverterFunction: ConverterFunction, - context: C, - game: Game, - blacklist: string[], - logger: KtLogger -): Promise { - // Converter Function Return - let cfnReturn: ConverterFnReturnOrFailure; - - try { - cfnReturn = await ConverterFunction(data, context, importType, logger); - } catch (err) { - cfnReturn = err as ConverterFailure | Error; - } - - // if this conversion failed, return it in the proper format - if (cfnReturn instanceof ConverterFailure) { - if (cfnReturn instanceof KTDataNotFoundFailure) { - logger.info(`KTDataNotFoundFailure: ${cfnReturn.message ?? "No message?"}`, { - cfnReturn, - hideFromConsole: ["cfnReturn"], - }); - - logger.debug("Inserting orphan...", { cfnReturn }); - - const insertOrphan = await OrphanScore( - cfnReturn.importType, - userID, - cfnReturn.data, - cfnReturn.converterContext, - cfnReturn.message, - game, - logger - ); - - if (insertOrphan.success) { - logger.debug("Orphan inserted successfully.", { orphanID: insertOrphan.orphanID }); - return { - success: false, - type: "KTDataNotFound", - message: cfnReturn.message, - content: { - context: cfnReturn.converterContext, - data: cfnReturn.data, - orphanID: insertOrphan.orphanID, - }, - }; - } - - logger.debug(`Orphan already exists.`, { orphanID: insertOrphan.orphanID }); - - return { - success: false, - type: "OrphanExists", - message: cfnReturn.message, - content: { - orphanID: insertOrphan.orphanID, - }, - }; - } else if (cfnReturn instanceof InvalidScoreFailure) { - logger.info(`InvalidScoreFailure: ${cfnReturn.message ?? "No message?"}`, { - cfnReturn, - hideFromConsole: ["cfnReturn"], - }); - return { - success: false, - type: "InvalidDatapoint", - message: cfnReturn.message, - content: {}, - }; - } else if (cfnReturn instanceof InternalFailure) { - logger.error(`Internal error occured.`, { cfnReturn }); - return { - success: false, - type: "InternalError", - // could return cfnReturn.message here, but we might want to hide the details of the crash. - message: "An internal error has occured.", - content: {}, - }; - } else if (cfnReturn instanceof SkipScoreFailure) { - return null; - } else { - logger.warn(`Unknown error returned as ConverterFailure, Ignoring.`, { - err: cfnReturn, - }); - return { - success: false, - type: "InternalError", - message: "An internal service error has occured.", - content: {}, - }; - } - } - - if (cfnReturn instanceof Error) { - logger.error(`Unknown error thrown from converter, Ignoring.`, { - err: cfnReturn, - }); - return { - success: false, - type: "InternalError", - message: "An internal service error has occured.", - content: {}, - }; - } - - return ProcessSuccessfulConverterReturn( - userID, - cfnReturn as ConverterFnSuccessReturn, - blacklist, - logger - ); -} - -export async function ProcessSuccessfulConverterReturn( - userID: integer, - cfnReturn: ConverterFnSuccessReturn, - blacklist: string[], - logger: KtLogger, - forceImmediateImport = false -): Promise { - const result = await HydrateAndInsertScore( - userID, - cfnReturn.dryScore, - cfnReturn.chart, - cfnReturn.song, - blacklist, - logger, - forceImmediateImport - ); - - // This used to be a ScoreExists error. However, we never actually care about - // handling ScoreExists errors (they're nobodies issue) - // so instead, the function will just return null, and we pass that on here. - if (result === null) { - return null; - } - - logger.debug(`Successfully imported score: ${result.scoreID}`); - - return { - success: true, - type: "ScoreImported", - message: null, - content: { - score: result, - }, - }; -} - -/** - * Hydrates and inserts a score to the Tachi database. - * @param userID - The user this score is from. - * @param dryScore - The score that is to be hydrated and inserted. - * @param chart - The chart this score is on. - * @param song - The song this score is on. - * @param blacklist - A list of ScoreIDs to never write to the database. - * - * @param force - Whether to immediately insert the score into the database - * or not. - */ -async function HydrateAndInsertScore( - userID: integer, - dryScore: DryScore, - chart: ChartDocument, - song: SongDocument, - blacklist: string[], - importLogger: KtLogger, - force = false -): Promise { - const scoreID = CreateScoreID(userID, dryScore, chart.chartID); - - // sub-context the logger so the below logs are more accurate - const logger = AppendLogCtx(scoreID, importLogger); - - if (blacklist.length && blacklist.includes(scoreID)) { - logger.verbose("Skipped score, as it was on the blacklist."); - return null; - } - - const existingScore = await db.scores.findOne( - { - scoreID, - }, - { - // micro-optimisation - mongoDB is significantly faster when returning less fields - // since we only care about whether we have a score or not here, we can minimise returned - // fields. - // - // feel free to test this yourself! - zkldi (09/04/2021) - projection: { - _id: 1, - }, - } - ); - - if (existingScore) { - logger.verbose(`Skipped score.`); - return null; - } - - // If this users score queue - if (GetScoreQueueMaybe(userID)?.scoreIDSet.has(scoreID)) { - logger.verbose(`Skipped score.`); - return null; - } - - const score = await HydrateScore(userID, dryScore, chart, song, scoreID, logger); - - let res; - if (force) { - res = await db.scores.insert(score); - } else { - res = await QueueScoreInsert(score); - } - - // this is a last resort for avoiding doubled imports - if (res === null) { - logger.verbose(`Skipped score - Race Condition protection triggered.`); - return null; - } - - return score; -} +import db from "external/mongo/db"; +import { AppendLogCtx, KtLogger } from "lib/logger/logger"; +import { + ChartDocument, + Game, + IDStrings, + ImportProcessingInfo, + ImportTypes, + integer, + ScoreDocument, + SongDocument, +} from "tachi-common"; +import { + ConverterFnReturnOrFailure, + ConverterFnSuccessReturn, + ConverterFunction, +} from "../../import-types/common/types"; +import { + ConverterFailure, + InternalFailure, + InvalidScoreFailure, + KTDataNotFoundFailure, + SkipScoreFailure, +} from "../common/converter-failures"; +import { DryScore } from "../common/types"; +import { OrphanScore } from "../orphans/orphans"; +import { HydrateScore } from "./hydrate-score"; +import { GetScoreQueueMaybe, InsertQueue, QueueScoreInsert } from "./insert-score"; +import { CreateScoreID } from "./score-id"; + +/** + * Processes the iterable data into the Tachi database. + * @param userID - The user this score import was for. + * @param iterableData - The data to iterate upon. + * @param ConverterFunction - The function needed to convert the data into an IntermediateScore + * @param context - Any context the Converter may need in order to make decisions. + * @returns An array of ImportProcessInfo objects. + */ +export async function ImportAllIterableData( + userID: integer, + importType: ImportTypes, + iterableData: Iterable | AsyncIterable, + ConverterFunction: ConverterFunction, + context: C, + game: Game, + logger: KtLogger +): Promise { + logger.verbose("Getting Blacklist..."); + + // @optimisable: could filter harder with score.game and score.playtype + // stuff. + const blacklist = ( + await db["score-blacklist"].find({ + userID, + }) + ).map((e) => e.scoreID); + + logger.verbose(`Starting Data Processing...`); + + const processedResults = []; + + // for await is used here as iterableData may be an async iterable + // An example would be making an api request after exhausting + // the first set of data. + for await (const data of iterableData) { + processedResults.push( + await ImportIterableDatapoint( + userID, + importType, + data, + ConverterFunction, + context, + game, + blacklist, + logger + ) + ); + } + + // We need to filter out nulls, which we don't care for (these are neither successes or failures) + + logger.verbose(`Finished Importing Data (${processedResults.length} datapoints).`); + logger.debug(`Removing null returns...`); + + const datapoints = processedResults.filter( + (e) => e !== null + ) as ImportProcessingInfo[]; + + logger.debug(`Removed null from results.`); + + logger.verbose(`Recieved ${datapoints.length} returns, from ${processedResults.length} data.`); + + // Flush the score queue out after finishing most of the import. This ensures no scores get left in the + // queue. + const emptied = await InsertQueue(userID); + + if (emptied) { + logger.verbose(`Emptied ${emptied} documents from score queue.`); + } + + return datapoints; +} + +/** + * Processes a single data object into one or many ImportProcessingInfo objects. + * @param userID - The user this score is from. + * @param data - The data to process. + * @param ConverterFunction - The processor function that takes the data and returns the partialScore(s) + * @param context - Any context the processor might need that it can not infer from the data object. + * @returns An array of ImportProcessingInfo objects, or a single ImportProcessingInfo object + */ +export async function ImportIterableDatapoint( + userID: integer, + importType: ImportTypes, + data: D, + ConverterFunction: ConverterFunction, + context: C, + game: Game, + blacklist: string[], + logger: KtLogger +): Promise { + // Converter Function Return + let cfnReturn: ConverterFnReturnOrFailure; + + try { + cfnReturn = await ConverterFunction(data, context, importType, logger); + } catch (err) { + cfnReturn = err as ConverterFailure | Error; + } + + // if this conversion failed, return it in the proper format + if (cfnReturn instanceof ConverterFailure) { + if (cfnReturn instanceof KTDataNotFoundFailure) { + logger.info(`KTDataNotFoundFailure: ${cfnReturn.message ?? "No message?"}`, { + cfnReturn, + hideFromConsole: ["cfnReturn"], + }); + + logger.debug("Inserting orphan...", { cfnReturn }); + + const insertOrphan = await OrphanScore( + cfnReturn.importType, + userID, + cfnReturn.data, + cfnReturn.converterContext, + cfnReturn.message, + game, + logger + ); + + if (insertOrphan.success) { + logger.debug("Orphan inserted successfully.", { orphanID: insertOrphan.orphanID }); + return { + success: false, + type: "KTDataNotFound", + message: cfnReturn.message, + content: { + context: cfnReturn.converterContext, + data: cfnReturn.data, + orphanID: insertOrphan.orphanID, + }, + }; + } + + logger.debug(`Orphan already exists.`, { orphanID: insertOrphan.orphanID }); + + return { + success: false, + type: "OrphanExists", + message: cfnReturn.message, + content: { + orphanID: insertOrphan.orphanID, + }, + }; + } else if (cfnReturn instanceof InvalidScoreFailure) { + logger.info(`InvalidScoreFailure: ${cfnReturn.message ?? "No message?"}`, { + cfnReturn, + hideFromConsole: ["cfnReturn"], + }); + return { + success: false, + type: "InvalidDatapoint", + message: cfnReturn.message, + content: {}, + }; + } else if (cfnReturn instanceof InternalFailure) { + logger.error(`Internal error occured.`, { cfnReturn }); + return { + success: false, + type: "InternalError", + // could return cfnReturn.message here, but we might want to hide the details of the crash. + message: "An internal error has occured.", + content: {}, + }; + } else if (cfnReturn instanceof SkipScoreFailure) { + return null; + } else { + logger.warn(`Unknown error returned as ConverterFailure, Ignoring.`, { + err: cfnReturn, + }); + return { + success: false, + type: "InternalError", + message: "An internal service error has occured.", + content: {}, + }; + } + } + + if (cfnReturn instanceof Error) { + logger.error(`Unknown error thrown from converter, Ignoring.`, { + err: cfnReturn, + }); + return { + success: false, + type: "InternalError", + message: "An internal service error has occured.", + content: {}, + }; + } + + return ProcessSuccessfulConverterReturn( + userID, + cfnReturn as ConverterFnSuccessReturn, + blacklist, + logger + ); +} + +export async function ProcessSuccessfulConverterReturn( + userID: integer, + cfnReturn: ConverterFnSuccessReturn, + blacklist: string[], + logger: KtLogger, + forceImmediateImport = false +): Promise { + const result = await HydrateAndInsertScore( + userID, + cfnReturn.dryScore, + cfnReturn.chart, + cfnReturn.song, + blacklist, + logger, + forceImmediateImport + ); + + // This used to be a ScoreExists error. However, we never actually care about + // handling ScoreExists errors (they're nobodies issue) + // so instead, the function will just return null, and we pass that on here. + if (result === null) { + return null; + } + + logger.debug(`Successfully imported score: ${result.scoreID}`); + + return { + success: true, + type: "ScoreImported", + message: null, + content: { + score: result, + }, + }; +} + +/** + * Hydrates and inserts a score to the Tachi database. + * @param userID - The user this score is from. + * @param dryScore - The score that is to be hydrated and inserted. + * @param chart - The chart this score is on. + * @param song - The song this score is on. + * @param blacklist - A list of ScoreIDs to never write to the database. + * + * @param force - Whether to immediately insert the score into the database + * or not. + */ +async function HydrateAndInsertScore( + userID: integer, + dryScore: DryScore, + chart: ChartDocument, + song: SongDocument, + blacklist: string[], + importLogger: KtLogger, + force = false +): Promise { + const scoreID = CreateScoreID(userID, dryScore, chart.chartID); + + // sub-context the logger so the below logs are more accurate + const logger = AppendLogCtx(scoreID, importLogger); + + if (blacklist.length && blacklist.includes(scoreID)) { + logger.verbose("Skipped score, as it was on the blacklist."); + return null; + } + + const existingScore = await db.scores.findOne( + { + scoreID, + }, + { + // micro-optimisation - mongoDB is significantly faster when returning less fields + // since we only care about whether we have a score or not here, we can minimise returned + // fields. + // + // feel free to test this yourself! - zkldi (09/04/2021) + projection: { + _id: 1, + }, + } + ); + + if (existingScore) { + logger.verbose(`Skipped score.`); + return null; + } + + // If this users score queue + if (GetScoreQueueMaybe(userID)?.scoreIDSet.has(scoreID)) { + logger.verbose(`Skipped score.`); + return null; + } + + const score = await HydrateScore(userID, dryScore, chart, song, scoreID, logger); + + let res; + if (force) { + res = await db.scores.insert(score); + } else { + res = await QueueScoreInsert(score); + } + + // this is a last resort for avoiding doubled imports + if (res === null) { + logger.verbose(`Skipped score - Race Condition protection triggered.`); + return null; + } + + return score; +} diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts index 83121c657..204cc77a3 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts @@ -1,178 +1,178 @@ -import { Lamps, ChartDocument } from "tachi-common"; -import { FindChartWithPTDFVersion } from "utils/queries/charts"; -import { FindSongOnTitle } from "utils/queries/songs"; -import { - KTDataNotFoundFailure, - InvalidScoreFailure, - SkipScoreFailure, -} from "../../../framework/common/converter-failures"; -import { - GenericGetGradeAndPercent, - ParseDateFromString, -} from "../../../framework/common/score-utils"; -import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts"; -import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; -import { DryScore } from "../../../framework/common/types"; -import { ConverterFunction } from "../types"; - -const EAMUSEMENT_LAMP_RESOLVER: Map = new Map([ - ["NO PLAY", "NO PLAY"], - ["FAILED", "FAILED"], - ["FULLCOMBO CLEAR", "FULL COMBO"], - ["EX HARD CLEAR", "EX HARD CLEAR"], - ["HARD CLEAR", "HARD CLEAR"], - ["CLEAR", "CLEAR"], - ["EASY CLEAR", "EASY CLEAR"], - ["ASSIST CLEAR", "ASSIST CLEAR"], -]); - -const NINE_HOURS = 1000 * 60 * 60 * 9; - -const ConvertEamIIDXCSV: ConverterFunction = - async (data, context, importType, logger) => { - const eamScore = data.score; - - if (eamScore.exscore === "0") { - // skip scores with an exscore of 0 - // This also skips things like score resets. - throw new SkipScoreFailure("Score has an exscore of 0."); - } - - if (!eamScore.level || eamScore.level === "0") { - // charts that dont exist in the format have a level of 0 - throw new SkipScoreFailure("Chart has a level of 0."); - } - - let isLegacyLeggendaria = false; - - // if pre-HV, leggendarias were stored in a wacky form. - if (!context.hasBeginnerAndLegg) { - // hack fix for legacy LEGGENDARIA titles - if (data.title.match(/(†|†LEGGENDARIA)$/u)) { - data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd(); - isLegacyLeggendaria = true; - } - } - - // @optimisable - This is actually a multi-fetch. Since eam-csv scores - // are batched up into (song, chart1, chart2, chart3 ...) rows - // we actually already have fetched this song a second ago. - const tachiSong = await FindSongOnTitle("iidx", data.title); - - if (!tachiSong) { - throw new KTDataNotFoundFailure( - `Could not find song for ${data.title}.`, - importType, - data, - context - ); - } - - const HUMANISED_CHART_TITLE = `${tachiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`; - - if (isLegacyLeggendaria) { - eamScore.difficulty = "LEGGENDARIA"; - } - - const tachiChart = (await FindChartWithPTDFVersion( - "iidx", - tachiSong.id, - context.playtype, - eamScore.difficulty, - context.importVersion - )) as ChartDocument<"iidx:SP" | "iidx:DP">; - - if (!tachiChart) { - throw new KTDataNotFoundFailure( - `Could not find chart for ${HUMANISED_CHART_TITLE}`, - "file/eamusement-iidx-csv", - data, - context - ); - } - - const exscore = AssertStrAsPositiveInt( - eamScore.exscore, - `${HUMANISED_CHART_TITLE} - Invalid EX score of ${eamScore.exscore}` - ); - - const MAX_EX = tachiChart.data.notecount * 2; - - if (exscore > MAX_EX) { - throw new InvalidScoreFailure( - `${HUMANISED_CHART_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was greater than max chart ex of ${MAX_EX}).` - ); - } - - const pgreat = AssertStrAsPositiveInt( - eamScore.pgreat, - `${HUMANISED_CHART_TITLE} - Invalid PGreats of ${eamScore.pgreat}` - ); - - const great = AssertStrAsPositiveInt( - eamScore.great, - `${HUMANISED_CHART_TITLE} - Invalid Greats of ${eamScore.pgreat}` - ); - - if (pgreat * 2 + great !== exscore) { - throw new InvalidScoreFailure( - `${HUMANISED_CHART_TITLE} - PGreats * 2 + Greats did not equal EXScore (${pgreat} * 2 + ${great} != ${exscore}).` - ); - } - - const lamp = EAMUSEMENT_LAMP_RESOLVER.get(eamScore.lamp); - - if (!lamp) { - logger.info(`Invalid lamp of ${eamScore.lamp} provided.`); - throw new InvalidScoreFailure( - `${HUMANISED_CHART_TITLE} - Invalid Lamp of ${eamScore.lamp}.` - ); - } - - const { percent, grade } = GenericGetGradeAndPercent("iidx", exscore, tachiChart); - - const timestamp = ParseDateFromString(data.timestamp); - - const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = { - service: context.service, - comment: null, - game: "iidx", - importType, - scoreData: { - score: exscore, - lamp, - judgements: { - pgreat, - great, - }, - hitMeta: {}, - percent, - grade, - }, - scoreMeta: {}, - // japan is gmt+9 - timeAchieved: timestamp ? timestamp - NINE_HOURS : null, - }; - - const numBP = Number(eamScore.bp); - - if (!Number.isNaN(numBP)) { - if (!Number.isInteger(numBP) || numBP < 0 || numBP > 9999) { - throw new InvalidScoreFailure( - `${HUMANISED_CHART_TITLE} - Invalid BP of ${eamScore.bp}.` - ); - } - dryScore.scoreData.hitMeta.bp = numBP; - } else if (eamScore.bp === "---") { - logger.debug( - `Skipped assigning BP for score as it had expected null value of ${eamScore.bp}.` - ); - } else { - logger.info(`Skipped assigning BP for score. Had unexpected value of ${eamScore.bp}.`); - } - - // ts thinks tachiSong might be null. It's not, though! - return { chart: tachiChart, dryScore, song: tachiSong }; - }; - -export default ConvertEamIIDXCSV; +import { Lamps, ChartDocument } from "tachi-common"; +import { FindChartWithPTDFVersion } from "utils/queries/charts"; +import { FindSongOnTitle } from "utils/queries/songs"; +import { + KTDataNotFoundFailure, + InvalidScoreFailure, + SkipScoreFailure, +} from "../../../framework/common/converter-failures"; +import { + GenericGetGradeAndPercent, + ParseDateFromString, +} from "../../../framework/common/score-utils"; +import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts"; +import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; +import { DryScore } from "../../../framework/common/types"; +import { ConverterFunction } from "../types"; + +const EAMUSEMENT_LAMP_RESOLVER: Map = new Map([ + ["NO PLAY", "NO PLAY"], + ["FAILED", "FAILED"], + ["FULLCOMBO CLEAR", "FULL COMBO"], + ["EX HARD CLEAR", "EX HARD CLEAR"], + ["HARD CLEAR", "HARD CLEAR"], + ["CLEAR", "CLEAR"], + ["EASY CLEAR", "EASY CLEAR"], + ["ASSIST CLEAR", "ASSIST CLEAR"], +]); + +const NINE_HOURS = 1000 * 60 * 60 * 9; + +const ConvertEamIIDXCSV: ConverterFunction = + async (data, context, importType, logger) => { + const eamScore = data.score; + + if (eamScore.exscore === "0") { + // skip scores with an exscore of 0 + // This also skips things like score resets. + throw new SkipScoreFailure("Score has an exscore of 0."); + } + + if (!eamScore.level || eamScore.level === "0") { + // charts that dont exist in the format have a level of 0 + throw new SkipScoreFailure("Chart has a level of 0."); + } + + let isLegacyLeggendaria = false; + + // if pre-HV, leggendarias were stored in a wacky form. + if (!context.hasBeginnerAndLegg) { + // hack fix for legacy LEGGENDARIA titles + if (data.title.match(/(†|†LEGGENDARIA)$/u)) { + data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd(); + isLegacyLeggendaria = true; + } + } + + // @optimisable - This is actually a multi-fetch. Since eam-csv scores + // are batched up into (song, chart1, chart2, chart3 ...) rows + // we actually already have fetched this song a second ago. + const tachiSong = await FindSongOnTitle("iidx", data.title); + + if (!tachiSong) { + throw new KTDataNotFoundFailure( + `Could not find song for ${data.title}.`, + importType, + data, + context + ); + } + + const HUMANISED_CHART_TITLE = `${tachiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`; + + if (isLegacyLeggendaria) { + eamScore.difficulty = "LEGGENDARIA"; + } + + const tachiChart = (await FindChartWithPTDFVersion( + "iidx", + tachiSong.id, + context.playtype, + eamScore.difficulty, + context.importVersion + )) as ChartDocument<"iidx:SP" | "iidx:DP">; + + if (!tachiChart) { + throw new KTDataNotFoundFailure( + `Could not find chart for ${HUMANISED_CHART_TITLE}`, + "file/eamusement-iidx-csv", + data, + context + ); + } + + const exscore = AssertStrAsPositiveInt( + eamScore.exscore, + `${HUMANISED_CHART_TITLE} - Invalid EX score of ${eamScore.exscore}` + ); + + const MAX_EX = tachiChart.data.notecount * 2; + + if (exscore > MAX_EX) { + throw new InvalidScoreFailure( + `${HUMANISED_CHART_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was greater than max chart ex of ${MAX_EX}).` + ); + } + + const pgreat = AssertStrAsPositiveInt( + eamScore.pgreat, + `${HUMANISED_CHART_TITLE} - Invalid PGreats of ${eamScore.pgreat}` + ); + + const great = AssertStrAsPositiveInt( + eamScore.great, + `${HUMANISED_CHART_TITLE} - Invalid Greats of ${eamScore.pgreat}` + ); + + if (pgreat * 2 + great !== exscore) { + throw new InvalidScoreFailure( + `${HUMANISED_CHART_TITLE} - PGreats * 2 + Greats did not equal EXScore (${pgreat} * 2 + ${great} != ${exscore}).` + ); + } + + const lamp = EAMUSEMENT_LAMP_RESOLVER.get(eamScore.lamp); + + if (!lamp) { + logger.info(`Invalid lamp of ${eamScore.lamp} provided.`); + throw new InvalidScoreFailure( + `${HUMANISED_CHART_TITLE} - Invalid Lamp of ${eamScore.lamp}.` + ); + } + + const { percent, grade } = GenericGetGradeAndPercent("iidx", exscore, tachiChart); + + const timestamp = ParseDateFromString(data.timestamp); + + const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = { + service: context.service, + comment: null, + game: "iidx", + importType, + scoreData: { + score: exscore, + lamp, + judgements: { + pgreat, + great, + }, + hitMeta: {}, + percent, + grade, + }, + scoreMeta: {}, + // japan is gmt+9 + timeAchieved: timestamp ? timestamp - NINE_HOURS : null, + }; + + const numBP = Number(eamScore.bp); + + if (!Number.isNaN(numBP)) { + if (!Number.isInteger(numBP) || numBP < 0 || numBP > 9999) { + throw new InvalidScoreFailure( + `${HUMANISED_CHART_TITLE} - Invalid BP of ${eamScore.bp}.` + ); + } + dryScore.scoreData.hitMeta.bp = numBP; + } else if (eamScore.bp === "---") { + logger.debug( + `Skipped assigning BP for score as it had expected null value of ${eamScore.bp}.` + ); + } else { + logger.info(`Skipped assigning BP for score. Had unexpected value of ${eamScore.bp}.`); + } + + // ts thinks tachiSong might be null. It's not, though! + return { chart: tachiChart, dryScore, song: tachiSong }; + }; + +export default ConvertEamIIDXCSV; diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts index c93f96e58..a7a9fa0b7 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts @@ -1,286 +1,286 @@ -import { KtLogger } from "lib/logger/logger"; -import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; -import { ParserFunctionReturns } from "../types"; -import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; - -enum EAM_VERSION_NAMES { - "1st&substream" = 1, - "2nd style", - "3rd style", - "4th style", - "5th style", - "6th style", - "7th style", - "8th style", - "9th style", - "10th style", - "IIDX RED", - "HAPPY SKY", - "DistorteD", - "GOLD", - "DJ TROOPERS", - "EMPRESS", - "SIRIUS", - "Resort Anthem", - "Lincle", - "tricoro", - "SPADA", - "PENDUAL", - "copula", - "SINOBUZ", - "CANNON BALLERS", - "Rootage", - "HEROIC VERSE", - "BISTROVER", -} - -const PRE_HV_HEADER_COUNT = 27; -const HV_HEADER_COUNT = 41; - -// commented out for reference -// [ -// "version", -// "title", -// "genre", -// "artist", -// "playcount", -// (?beginnerdata) -// "normal-level", -// "normal-exscore", -// "normal-pgreat", -// "normal-great", -// "normal-bp", -// "normal-lamp", -// "normal-grade", -// "hyper-level", -// "hyper-exscore", -// "hyper-pgreat", -// "hyper-great", -// "hyper-bp", -// "hyper-lamp", -// "hyper-grade", -// "another-level", -// "another-exscore", -// "another-pgreat", -// "another-great", -// "another-bp", -// "another-lamp", -// "another-grade", -// (?leggdata) -// "timestamp", -// ]; - -export function ResolveHeaders(headers: string[], logger: KtLogger) { - if (headers.length === PRE_HV_HEADER_COUNT) { - logger.verbose("PRE_HV csv recieved."); - return { - hasBeginnerAndLegg: false, - }; - } else if (headers.length === HV_HEADER_COUNT) { - logger.verbose("HV+ csv recieved."); - return { - hasBeginnerAndLegg: true, - }; - } - - logger.info(`Invalid CSV header count of ${headers.length} received.`); - throw new ScoreImportFatalError( - 400, - "Invalid CSV provided. CSV does not have the correct amount of headers." - ); -} - -export function NaiveCSVParse(csvBuffer: Buffer, logger: KtLogger) { - const csvString = csvBuffer.toString("utf-8"); - - const csvData = csvString.split("\n"); - - const rawHeaders = []; - let headerLen = 0; - let curStr = ""; - - // looks like we're doing it like this. - for (const char of csvData[0]) { - headerLen++; - - // safety checks to avoid getting DOS'd - if (headerLen > 1000) { - throw new ScoreImportFatalError(400, "Headers were longer than 1000 characters long."); - } else if (rawHeaders.length >= 50) { - // this does not *really* do what it seems. - // because there's inevitably something left in curStr in this fn - // this means that the above check is actually > 50 headers. Not - // >= 50. - throw new ScoreImportFatalError(400, "Too many CSV headers."); - } - - if (char === ",") { - rawHeaders.push(curStr); - curStr = ""; - } else { - curStr += char; - } - } - - rawHeaders.push(curStr); - - const { hasBeginnerAndLegg } = ResolveHeaders(rawHeaders, logger); - - const diffs = hasBeginnerAndLegg - ? (["beginner", "normal", "hyper", "another", "leggendaria"] as const) - : (["normal", "hyper", "another"] as const); - - const iterableData = []; - - let gameVersion = 0; - - for (let i = 1; i < csvData.length; i++) { - const data = csvData[i]; - - // @security: This should probably be safetied from DOSing - const cells = data.split(","); - - // weirdly enough, an empty string split on "," is an array with - // one empty value. - // regardless, this line skips empty rows - if (cells.length === 1) { - logger.verbose(`Skipped empty row ${i}.`); - continue; - } - - if (cells.length !== rawHeaders.length) { - logger.info( - `eamusement-iidx csv has row (${i}) with invalid cell count of ${cells.length}, rejecting.`, - { - data, - } - ); - throw new ScoreImportFatalError( - 400, - `Row ${i} has an invalid amount of cells (${cells.length}, expected ${rawHeaders.length}).` - ); - } - - const version = cells[0]; - const title = cells[1].trim(); // konmai quality - const timestamp = cells[rawHeaders.length - 1].trim(); - - // wtf typescript?? what's the point of enums? - const versionNum = EAM_VERSION_NAMES[version as keyof typeof EAM_VERSION_NAMES]; - - if (!versionNum) { - logger.info(`Invalid/Unsupported EAM_VERSION_NAME ${version}.`); - throw new ScoreImportFatalError( - 400, - `Invalid/Unsupported Eamusement Version Name ${version}.` - ); - } - - if (versionNum > gameVersion) { - gameVersion = versionNum; - } - - const scores: EamusementScoreData[] = []; - - for (let d = 0; d < diffs.length; d++) { - const diff = diffs[d]; - const di = 5 + d * 7; - - scores.push({ - difficulty: diff.toUpperCase() as Uppercase, - bp: cells[di + 4], - exscore: cells[di + 1], - pgreat: cells[di + 2], - great: cells[di + 3], - lamp: cells[di + 5], - level: cells[di], - }); - } - - iterableData.push( - ...scores.map((e) => ({ - score: e, - timestamp, - title, - })) - ); - } - - if (!["26", "27", "28"].includes(gameVersion.toString())) { - throw new ScoreImportFatalError(400, `Only versions 26, 27 and 28 are supported.`); - } - - return { - iterableData, - version: gameVersion.toString() as "26" | "27" | "28", - hasBeginnerAndLegg, - }; -} - -/** - * Parses a buffer of EamusementCSV data. - * @param fileData - The buffer to parse. - * @param body - The request body that made this file import request. Used to infer playtype. - */ -function GenericParseEamIIDXCSV( - fileData: Express.Multer.File, - body: Record, - service: string, - logger: KtLogger -): ParserFunctionReturns { - let playtype: "SP" | "DP"; - - if (body.playtype === "SP") { - playtype = "SP"; - } else if (body.playtype === "DP") { - playtype = "DP"; - } else { - logger.info(`Invalid playtype of ${body.playtype} passed to ParseEamusementCSV.`); - throw new ScoreImportFatalError( - 400, - `Invalid playtype of ${body.playtype ?? "Nothing"} given.` - ); - } - - const lowercaseFilename = fileData.originalname.toLowerCase(); - - // prettier pls - if ( - !body.assertPlaytypeCorrect && - ((lowercaseFilename.includes("sp") && playtype === "DP") || - (lowercaseFilename.includes("dp") && playtype === "SP")) - ) { - logger.info( - `File was uploaded with filename ${fileData.originalname}, but this was set as a ${playtype} import. Sanity check refusing.` - ); - - throw new ScoreImportFatalError( - 400, - `Safety Triggered: Filename contained '${ - playtype === "SP" ? "DP" : "SP" - }', but was marked as a ${playtype} import. Are you *absolutely* sure this is right?` - ); - } - - const { hasBeginnerAndLegg, version, iterableData } = NaiveCSVParse(fileData.buffer, logger); - - logger.verbose("Successfully parsed CSV."); - - const context: IIDXEamusementCSVContext = { - playtype, - importVersion: version, - hasBeginnerAndLegg, - service, - }; - - logger.verbose(`Successfully Parsed with ${iterableData.length} results.`); - - return { - iterable: iterableData, - context, - game: "iidx", - classHandler: null, - }; -} - -export default GenericParseEamIIDXCSV; +import { KtLogger } from "lib/logger/logger"; +import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; +import { ParserFunctionReturns } from "../types"; +import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types"; + +enum EAM_VERSION_NAMES { + "1st&substream" = 1, + "2nd style", + "3rd style", + "4th style", + "5th style", + "6th style", + "7th style", + "8th style", + "9th style", + "10th style", + "IIDX RED", + "HAPPY SKY", + "DistorteD", + "GOLD", + "DJ TROOPERS", + "EMPRESS", + "SIRIUS", + "Resort Anthem", + "Lincle", + "tricoro", + "SPADA", + "PENDUAL", + "copula", + "SINOBUZ", + "CANNON BALLERS", + "Rootage", + "HEROIC VERSE", + "BISTROVER", +} + +const PRE_HV_HEADER_COUNT = 27; +const HV_HEADER_COUNT = 41; + +// commented out for reference +// [ +// "version", +// "title", +// "genre", +// "artist", +// "playcount", +// (?beginnerdata) +// "normal-level", +// "normal-exscore", +// "normal-pgreat", +// "normal-great", +// "normal-bp", +// "normal-lamp", +// "normal-grade", +// "hyper-level", +// "hyper-exscore", +// "hyper-pgreat", +// "hyper-great", +// "hyper-bp", +// "hyper-lamp", +// "hyper-grade", +// "another-level", +// "another-exscore", +// "another-pgreat", +// "another-great", +// "another-bp", +// "another-lamp", +// "another-grade", +// (?leggdata) +// "timestamp", +// ]; + +export function ResolveHeaders(headers: string[], logger: KtLogger) { + if (headers.length === PRE_HV_HEADER_COUNT) { + logger.verbose("PRE_HV csv recieved."); + return { + hasBeginnerAndLegg: false, + }; + } else if (headers.length === HV_HEADER_COUNT) { + logger.verbose("HV+ csv recieved."); + return { + hasBeginnerAndLegg: true, + }; + } + + logger.info(`Invalid CSV header count of ${headers.length} received.`); + throw new ScoreImportFatalError( + 400, + "Invalid CSV provided. CSV does not have the correct amount of headers." + ); +} + +export function NaiveCSVParse(csvBuffer: Buffer, logger: KtLogger) { + const csvString = csvBuffer.toString("utf-8"); + + const csvData = csvString.split("\n"); + + const rawHeaders = []; + let headerLen = 0; + let curStr = ""; + + // looks like we're doing it like this. + for (const char of csvData[0]) { + headerLen++; + + // safety checks to avoid getting DOS'd + if (headerLen > 1000) { + throw new ScoreImportFatalError(400, "Headers were longer than 1000 characters long."); + } else if (rawHeaders.length >= 50) { + // this does not *really* do what it seems. + // because there's inevitably something left in curStr in this fn + // this means that the above check is actually > 50 headers. Not + // >= 50. + throw new ScoreImportFatalError(400, "Too many CSV headers."); + } + + if (char === ",") { + rawHeaders.push(curStr); + curStr = ""; + } else { + curStr += char; + } + } + + rawHeaders.push(curStr); + + const { hasBeginnerAndLegg } = ResolveHeaders(rawHeaders, logger); + + const diffs = hasBeginnerAndLegg + ? (["beginner", "normal", "hyper", "another", "leggendaria"] as const) + : (["normal", "hyper", "another"] as const); + + const iterableData = []; + + let gameVersion = 0; + + for (let i = 1; i < csvData.length; i++) { + const data = csvData[i]; + + // @security: This should probably be safetied from DOSing + const cells = data.split(","); + + // weirdly enough, an empty string split on "," is an array with + // one empty value. + // regardless, this line skips empty rows + if (cells.length === 1) { + logger.verbose(`Skipped empty row ${i}.`); + continue; + } + + if (cells.length !== rawHeaders.length) { + logger.info( + `eamusement-iidx csv has row (${i}) with invalid cell count of ${cells.length}, rejecting.`, + { + data, + } + ); + throw new ScoreImportFatalError( + 400, + `Row ${i} has an invalid amount of cells (${cells.length}, expected ${rawHeaders.length}).` + ); + } + + const version = cells[0]; + const title = cells[1].trim(); // konmai quality + const timestamp = cells[rawHeaders.length - 1].trim(); + + // wtf typescript?? what's the point of enums? + const versionNum = EAM_VERSION_NAMES[version as keyof typeof EAM_VERSION_NAMES]; + + if (!versionNum) { + logger.info(`Invalid/Unsupported EAM_VERSION_NAME ${version}.`); + throw new ScoreImportFatalError( + 400, + `Invalid/Unsupported Eamusement Version Name ${version}.` + ); + } + + if (versionNum > gameVersion) { + gameVersion = versionNum; + } + + const scores: EamusementScoreData[] = []; + + for (let d = 0; d < diffs.length; d++) { + const diff = diffs[d]; + const di = 5 + d * 7; + + scores.push({ + difficulty: diff.toUpperCase() as Uppercase, + bp: cells[di + 4], + exscore: cells[di + 1], + pgreat: cells[di + 2], + great: cells[di + 3], + lamp: cells[di + 5], + level: cells[di], + }); + } + + iterableData.push( + ...scores.map((e) => ({ + score: e, + timestamp, + title, + })) + ); + } + + if (!["26", "27", "28"].includes(gameVersion.toString())) { + throw new ScoreImportFatalError(400, `Only versions 26, 27 and 28 are supported.`); + } + + return { + iterableData, + version: gameVersion.toString() as "26" | "27" | "28", + hasBeginnerAndLegg, + }; +} + +/** + * Parses a buffer of EamusementCSV data. + * @param fileData - The buffer to parse. + * @param body - The request body that made this file import request. Used to infer playtype. + */ +function GenericParseEamIIDXCSV( + fileData: Express.Multer.File, + body: Record, + service: string, + logger: KtLogger +): ParserFunctionReturns { + let playtype: "SP" | "DP"; + + if (body.playtype === "SP") { + playtype = "SP"; + } else if (body.playtype === "DP") { + playtype = "DP"; + } else { + logger.info(`Invalid playtype of ${body.playtype} passed to ParseEamusementCSV.`); + throw new ScoreImportFatalError( + 400, + `Invalid playtype of ${body.playtype ?? "Nothing"} given.` + ); + } + + const lowercaseFilename = fileData.originalname.toLowerCase(); + + // prettier pls + if ( + !body.assertPlaytypeCorrect && + ((lowercaseFilename.includes("sp") && playtype === "DP") || + (lowercaseFilename.includes("dp") && playtype === "SP")) + ) { + logger.info( + `File was uploaded with filename ${fileData.originalname}, but this was set as a ${playtype} import. Sanity check refusing.` + ); + + throw new ScoreImportFatalError( + 400, + `Safety Triggered: Filename contained '${ + playtype === "SP" ? "DP" : "SP" + }', but was marked as a ${playtype} import. Are you *absolutely* sure this is right?` + ); + } + + const { hasBeginnerAndLegg, version, iterableData } = NaiveCSVParse(fileData.buffer, logger); + + logger.verbose("Successfully parsed CSV."); + + const context: IIDXEamusementCSVContext = { + playtype, + importVersion: version, + hasBeginnerAndLegg, + service, + }; + + logger.verbose(`Successfully Parsed with ${iterableData.length} results.`); + + return { + iterable: iterableData, + context, + game: "iidx", + classHandler: null, + }; +} + +export default GenericParseEamIIDXCSV; diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts index 02b819c7a..9608596ca 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts @@ -1,36 +1,36 @@ -import { GPTSupportedVersions } from "tachi-common"; - -export interface EamusementScoreData { - difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"; - lamp: string; - exscore: string; - pgreat: string; - great: string; - bp: string; - level: string; -} - -type BaseProps = { - title: string; - timestamp: string; -}; - -export type IIDXEamusementCSVData = { - score: EamusementScoreData; -} & BaseProps; - -type Props = "exscore" | "lamp" | "pgreat" | "great" | "bp" | "level"; - -type RawPropKeys = `${"beginner" | "normal" | "hyper" | "another" | "leggendaria"}-${Props}`; - -export type RawIIDXEamusementCSVData = { - [K in RawPropKeys]: unknown; -} & - BaseProps & { [index: string]: unknown }; - -export interface IIDXEamusementCSVContext { - playtype: "SP" | "DP"; - importVersion: GPTSupportedVersions["iidx:SP" | "iidx:DP"]; - hasBeginnerAndLegg: boolean; - service: string; -} +import { GPTSupportedVersions } from "tachi-common"; + +export interface EamusementScoreData { + difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"; + lamp: string; + exscore: string; + pgreat: string; + great: string; + bp: string; + level: string; +} + +type BaseProps = { + title: string; + timestamp: string; +}; + +export type IIDXEamusementCSVData = { + score: EamusementScoreData; +} & BaseProps; + +type Props = "exscore" | "lamp" | "pgreat" | "great" | "bp" | "level"; + +type RawPropKeys = `${"beginner" | "normal" | "hyper" | "another" | "leggendaria"}-${Props}`; + +export type RawIIDXEamusementCSVData = { + [K in RawPropKeys]: unknown; +} & + BaseProps & { [index: string]: unknown }; + +export interface IIDXEamusementCSVContext { + playtype: "SP" | "DP"; + importVersion: GPTSupportedVersions["iidx:SP" | "iidx:DP"]; + hasBeginnerAndLegg: boolean; + service: string; +} diff --git a/server/src/lib/score-import/import-types/file/eamusement-iidx-csv/parser.ts b/server/src/lib/score-import/import-types/file/eamusement-iidx-csv/parser.ts index 83c084311..a6d781208 100644 --- a/server/src/lib/score-import/import-types/file/eamusement-iidx-csv/parser.ts +++ b/server/src/lib/score-import/import-types/file/eamusement-iidx-csv/parser.ts @@ -1,17 +1,17 @@ -import { KtLogger } from "lib/logger/logger"; -import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser"; -import { - IIDXEamusementCSVContext, - IIDXEamusementCSVData, -} from "../../common/eamusement-iidx-csv/types"; -import { ParserFunctionReturns } from "../../common/types"; - -function ParseEamusementIIDXCSV( - fileData: Express.Multer.File, - body: Record, - logger: KtLogger -): ParserFunctionReturns { - return GenericParseEamIIDXCSV(fileData, body, "e-amusement", logger); -} - -export default ParseEamusementIIDXCSV; +import { KtLogger } from "lib/logger/logger"; +import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser"; +import { + IIDXEamusementCSVContext, + IIDXEamusementCSVData, +} from "../../common/eamusement-iidx-csv/types"; +import { ParserFunctionReturns } from "../../common/types"; + +function ParseEamusementIIDXCSV( + fileData: Express.Multer.File, + body: Record, + logger: KtLogger +): ParserFunctionReturns { + return GenericParseEamIIDXCSV(fileData, body, "e-amusement", logger); +} + +export default ParseEamusementIIDXCSV; diff --git a/server/src/lib/score-import/import-types/file/pli-iidx-csv/parser.ts b/server/src/lib/score-import/import-types/file/pli-iidx-csv/parser.ts index ce9aca60d..43259ad68 100644 --- a/server/src/lib/score-import/import-types/file/pli-iidx-csv/parser.ts +++ b/server/src/lib/score-import/import-types/file/pli-iidx-csv/parser.ts @@ -1,17 +1,17 @@ -import { KtLogger } from "lib/logger/logger"; -import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser"; -import { - IIDXEamusementCSVContext, - IIDXEamusementCSVData, -} from "../../common/eamusement-iidx-csv/types"; -import { ParserFunctionReturns } from "../../common/types"; - -function ParsePLIIIDXCSV( - fileData: Express.Multer.File, - body: Record, - logger: KtLogger -): ParserFunctionReturns { - return GenericParseEamIIDXCSV(fileData, body, "PLI", logger); -} - -export default ParsePLIIIDXCSV; +import { KtLogger } from "lib/logger/logger"; +import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser"; +import { + IIDXEamusementCSVContext, + IIDXEamusementCSVData, +} from "../../common/eamusement-iidx-csv/types"; +import { ParserFunctionReturns } from "../../common/types"; + +function ParsePLIIIDXCSV( + fileData: Express.Multer.File, + body: Record, + logger: KtLogger +): ParserFunctionReturns { + return GenericParseEamIIDXCSV(fileData, body, "PLI", logger); +} + +export default ParsePLIIIDXCSV; diff --git a/server/src/main.ts b/server/src/main.ts index 3fe8afd9b..60d26a40e 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -1,115 +1,115 @@ -// Before we run anything, set a global to indicate to the code that -// we're running as a server, and not as a job runner or score worker. -process.env.IS_SERVER = "true"; - -import { spawn } from "child_process"; -import db, { monkDB } from "external/mongo/db"; -import { SetIndexesIfNoneSet } from "external/mongo/indexes"; -import { InitSequenceDocs } from "external/mongo/sequence-docs"; -import fs from "fs"; -import https from "https"; -import { LoadDefaultClients } from "lib/builtin-clients/builtin-clients"; -import CreateLogCtx from "lib/logger/logger"; -import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; -import path from "path"; -import server from "server/server"; -import { InitaliseFolderChartLookup } from "utils/folder"; -import { FormatVersion } from "./lib/constants/version"; -import fetch from "utils/fetch"; -import http from "http"; -import { CloseRedisConnection } from "external/redis/redis"; -import { CloseScoreImportQueue } from "lib/score-import/worker/queue"; -import { HandleSIGTERMGracefully } from "lib/handlers/sigterm"; - -const logger = CreateLogCtx(__filename); - -logger.info(`Booting ${TachiConfig.NAME} - ${FormatVersion()} [ENV: ${Environment.nodeEnv}]`, { - bootInfo: true, -}); -logger.info(`Log level is set to ${ServerConfig.LOGGER_CONFIG.LOG_LEVEL}.`, { bootInfo: true }); - -logger.info(`Loading sequence documents...`, { bootInfo: true }); - -async function RunOnInit() { - await InitSequenceDocs(); - await SetIndexesIfNoneSet(); - - await db["folder-chart-lookup"].findOne().then((r) => { - // If there are no folder chart lookups, initialise them. - if (!r) { - InitaliseFolderChartLookup(); - } - }); - - await LoadDefaultClients(); - - try { - await fetch("https://example.com"); - } catch (err) { - logger.crit( - `Cannot send HTTPS request to https://example.com. This instance of tachi-server cannot access the internet?`, - err - ); - process.exit(1); - } -} - -RunOnInit(); - -let instance: http.Server | https.Server; - -if (ServerConfig.ENABLE_SERVER_HTTPS) { - logger.warn( - "HTTPS Mode is enabled. This should not be used in production, and you should instead run behind a reverse proxy.", - { bootInfo: true } - ); - const privateKey = fs.readFileSync("./cert/key.pem"); - const certificate = fs.readFileSync("./cert/cert.pem"); - - const httpsServer = https.createServer({ key: privateKey, cert: certificate }, server); - - instance = httpsServer.listen(Environment.port); - logger.info(`HTTPS Listening on port ${Environment.port}`, { bootInfo: true }); -} else { - instance = server.listen(Environment.port); - logger.info(`HTTP Listening on port ${Environment.port}`, { bootInfo: true }); -} - -process.on("SIGTERM", () => HandleSIGTERMGracefully(instance)); - -if (process.env.INVOKE_JOB_RUNNER) { - logger.info(`Spawning a tachi-server job runner inline.`, { bootInfo: true }); - - if (Environment.nodeEnv === "production") { - logger.warn( - `Spawning inline tachi-server job runner in production. Is this actually what you want? You should run a tool like Ofelia to manage this.`, - { bootInfo: true } - ); - } - - // Spawn as a separate process to avoid hogging the main thread. - const jobProcess = spawn( - "ts-node", - [ - // Note: Can't use -r tsconfig-paths/register here - // because that is rejected by some library called - // arg. - // I'm not sure why. - "--require=tsconfig-paths/register", - path.join(__dirname, "../src/lib/jobs/job-runner.ts"), - ], - { - stdio: "inherit", - } - ); - - jobProcess.on("error", (err) => { - logger.crit(`Failed to spawn job runner. Terminating process.`, { err }); - process.exit(1); - }); - - process.on("beforeExit", () => { - logger.info(`Killing Job Runner.`); - jobProcess.kill(); - }); -} +// Before we run anything, set a global to indicate to the code that +// we're running as a server, and not as a job runner or score worker. +process.env.IS_SERVER = "true"; + +import { spawn } from "child_process"; +import db, { monkDB } from "external/mongo/db"; +import { SetIndexesIfNoneSet } from "external/mongo/indexes"; +import { InitSequenceDocs } from "external/mongo/sequence-docs"; +import fs from "fs"; +import https from "https"; +import { LoadDefaultClients } from "lib/builtin-clients/builtin-clients"; +import CreateLogCtx from "lib/logger/logger"; +import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; +import path from "path"; +import server from "server/server"; +import { InitaliseFolderChartLookup } from "utils/folder"; +import { FormatVersion } from "./lib/constants/version"; +import fetch from "utils/fetch"; +import http from "http"; +import { CloseRedisConnection } from "external/redis/redis"; +import { CloseScoreImportQueue } from "lib/score-import/worker/queue"; +import { HandleSIGTERMGracefully } from "lib/handlers/sigterm"; + +const logger = CreateLogCtx(__filename); + +logger.info(`Booting ${TachiConfig.NAME} - ${FormatVersion()} [ENV: ${Environment.nodeEnv}]`, { + bootInfo: true, +}); +logger.info(`Log level is set to ${ServerConfig.LOGGER_CONFIG.LOG_LEVEL}.`, { bootInfo: true }); + +logger.info(`Loading sequence documents...`, { bootInfo: true }); + +async function RunOnInit() { + await InitSequenceDocs(); + await SetIndexesIfNoneSet(); + + await db["folder-chart-lookup"].findOne().then((r) => { + // If there are no folder chart lookups, initialise them. + if (!r) { + InitaliseFolderChartLookup(); + } + }); + + await LoadDefaultClients(); + + try { + await fetch("https://example.com"); + } catch (err) { + logger.crit( + `Cannot send HTTPS request to https://example.com. This instance of tachi-server cannot access the internet?`, + err + ); + process.exit(1); + } +} + +RunOnInit(); + +let instance: http.Server | https.Server; + +if (ServerConfig.ENABLE_SERVER_HTTPS) { + logger.warn( + "HTTPS Mode is enabled. This should not be used in production, and you should instead run behind a reverse proxy.", + { bootInfo: true } + ); + const privateKey = fs.readFileSync("./cert/key.pem"); + const certificate = fs.readFileSync("./cert/cert.pem"); + + const httpsServer = https.createServer({ key: privateKey, cert: certificate }, server); + + instance = httpsServer.listen(Environment.port); + logger.info(`HTTPS Listening on port ${Environment.port}`, { bootInfo: true }); +} else { + instance = server.listen(Environment.port); + logger.info(`HTTP Listening on port ${Environment.port}`, { bootInfo: true }); +} + +process.on("SIGTERM", () => HandleSIGTERMGracefully(instance)); + +if (process.env.INVOKE_JOB_RUNNER) { + logger.info(`Spawning a tachi-server job runner inline.`, { bootInfo: true }); + + if (Environment.nodeEnv === "production") { + logger.warn( + `Spawning inline tachi-server job runner in production. Is this actually what you want? You should run a tool like Ofelia to manage this.`, + { bootInfo: true } + ); + } + + // Spawn as a separate process to avoid hogging the main thread. + const jobProcess = spawn( + "ts-node", + [ + // Note: Can't use -r tsconfig-paths/register here + // because that is rejected by some library called + // arg. + // I'm not sure why. + "--require=tsconfig-paths/register", + path.join(__dirname, "../src/lib/jobs/job-runner.ts"), + ], + { + stdio: "inherit", + } + ); + + jobProcess.on("error", (err) => { + logger.crit(`Failed to spawn job runner. Terminating process.`, { err }); + process.exit(1); + }); + + process.on("beforeExit", () => { + logger.info(`Killing Job Runner.`); + jobProcess.kill(); + }); +} diff --git a/server/src/server/middleware/prudence-validate.test.ts b/server/src/server/middleware/prudence-validate.test.ts index 43fc54f11..902ce4531 100644 --- a/server/src/server/middleware/prudence-validate.test.ts +++ b/server/src/server/middleware/prudence-validate.test.ts @@ -1,143 +1,143 @@ -import t from "tap"; -import expMiddlewareMock from "express-request-mock"; -import prValidate from "./prudence-validate"; -import Prudence from "prudence"; - -t.test("#PrudenceMiddleware", (t) => { - const mw = prValidate({ foo: Prudence.regex(/^baz$/u) }, { foo: "example error message" }); - - t.test("Should return 400 on invalid prudence validation", async (t) => { - const { res } = await expMiddlewareMock(mw, { - query: { - foo: "bar", - }, - }); - - t.equal(res.statusCode, 400, "Status code should be 400"); - - const json = res._getJSONData(); - t.equal( - json.description, - "[foo] example error message (Received bar)", - "Should return error message" - ); - - t.end(); - }); - - t.test("Should return 'nothing' instead of undefined for missing fields", async (t) => { - const { res } = await expMiddlewareMock(mw, { - query: {}, - }); - - t.equal(res.statusCode, 400, "Status code should be 400"); - - const json = res._getJSONData(); - t.equal( - json.description, - "[foo] example error message (Received nothing)", - "Should return error message with recieved nothing" - ); - - t.end(); - }); - - t.test("Should allow valid prudence data.", async (t) => { - const { res } = await expMiddlewareMock(mw, { - query: { - foo: "baz", - }, - }); - - t.equal(res.statusCode, 200, "Should stay as 200"); - - // no body -- not returned. - t.equal(res._isJSON(), false, "Should not have any data set"); - - t.end(); - }); - - t.test("Should allow valid bodies on non-GET requests", async (t) => { - const { res } = await expMiddlewareMock(mw, { - method: "POST", - body: { - foo: "baz", - }, - }); - - t.equal(res.statusCode, 200, "Should stay as 200"); - - // no body -- not returned. - t.equal(res._isJSON(), false, "Should not have any data set"); - - t.end(); - }); - - t.test("Should return 400 on invalid prudence validation for non-GET requests", async (t) => { - const { res } = await expMiddlewareMock(mw, { - method: "POST", - body: { - foo: "bar", - }, - }); - - t.equal(res.statusCode, 400, "Status code should be 400"); - - const json = res._getJSONData(); - t.equal( - json.description, - "[foo] example error message (Received bar)", - "Should return error message" - ); - - t.end(); - }); - - const mwWithPassword = prValidate({ password: "string" }, { password: "invalid password" }); - - t.test( - "Should not return the contents of the error message if the field matches /password/", - async (t) => { - const { res } = await expMiddlewareMock(mwWithPassword, { - query: { - password: 123, - }, - }); - - t.equal(res.statusCode, 400, "Status code should be 400"); - - const json = res._getJSONData(); - t.equal( - json.description, - "[password] invalid password (Received ****)", - "Should return obscured error message" - ); - - t.end(); - } - ); - - t.test( - "Should not return the contents of the error message if the field matches /password/", - async (t) => { - const { res } = await expMiddlewareMock(mwWithPassword, { - query: { - password: undefined, - }, - }); - - t.equal(res.statusCode, 400, "Status code should be 400"); - - const json = res._getJSONData(); - t.equal( - json.description, - "[password] invalid password (Received nothing)", - "Should indicate if no data was sent in obscured error message" - ); - - t.end(); - } - ); - - t.end(); -}); +import t from "tap"; +import expMiddlewareMock from "express-request-mock"; +import prValidate from "./prudence-validate"; +import Prudence from "prudence"; + +t.test("#PrudenceMiddleware", (t) => { + const mw = prValidate({ foo: Prudence.regex(/^baz$/u) }, { foo: "example error message" }); + + t.test("Should return 400 on invalid prudence validation", async (t) => { + const { res } = await expMiddlewareMock(mw, { + query: { + foo: "bar", + }, + }); + + t.equal(res.statusCode, 400, "Status code should be 400"); + + const json = res._getJSONData(); + t.equal( + json.description, + "[foo] example error message (Received bar)", + "Should return error message" + ); + + t.end(); + }); + + t.test("Should return 'nothing' instead of undefined for missing fields", async (t) => { + const { res } = await expMiddlewareMock(mw, { + query: {}, + }); + + t.equal(res.statusCode, 400, "Status code should be 400"); + + const json = res._getJSONData(); + t.equal( + json.description, + "[foo] example error message (Received nothing)", + "Should return error message with recieved nothing" + ); + + t.end(); + }); + + t.test("Should allow valid prudence data.", async (t) => { + const { res } = await expMiddlewareMock(mw, { + query: { + foo: "baz", + }, + }); + + t.equal(res.statusCode, 200, "Should stay as 200"); + + // no body -- not returned. + t.equal(res._isJSON(), false, "Should not have any data set"); + + t.end(); + }); + + t.test("Should allow valid bodies on non-GET requests", async (t) => { + const { res } = await expMiddlewareMock(mw, { + method: "POST", + body: { + foo: "baz", + }, + }); + + t.equal(res.statusCode, 200, "Should stay as 200"); + + // no body -- not returned. + t.equal(res._isJSON(), false, "Should not have any data set"); + + t.end(); + }); + + t.test("Should return 400 on invalid prudence validation for non-GET requests", async (t) => { + const { res } = await expMiddlewareMock(mw, { + method: "POST", + body: { + foo: "bar", + }, + }); + + t.equal(res.statusCode, 400, "Status code should be 400"); + + const json = res._getJSONData(); + t.equal( + json.description, + "[foo] example error message (Received bar)", + "Should return error message" + ); + + t.end(); + }); + + const mwWithPassword = prValidate({ password: "string" }, { password: "invalid password" }); + + t.test( + "Should not return the contents of the error message if the field matches /password/", + async (t) => { + const { res } = await expMiddlewareMock(mwWithPassword, { + query: { + password: 123, + }, + }); + + t.equal(res.statusCode, 400, "Status code should be 400"); + + const json = res._getJSONData(); + t.equal( + json.description, + "[password] invalid password (Received ****)", + "Should return obscured error message" + ); + + t.end(); + } + ); + + t.test( + "Should not return the contents of the error message if the field matches /password/", + async (t) => { + const { res } = await expMiddlewareMock(mwWithPassword, { + query: { + password: undefined, + }, + }); + + t.equal(res.statusCode, 400, "Status code should be 400"); + + const json = res._getJSONData(); + t.equal( + json.description, + "[password] invalid password (Received nothing)", + "Should indicate if no data was sent in obscured error message" + ); + + t.end(); + } + ); + + t.end(); +}); diff --git a/server/src/server/middleware/prudence-validate.ts b/server/src/server/middleware/prudence-validate.ts index f162a3d64..e45f50f31 100644 --- a/server/src/server/middleware/prudence-validate.ts +++ b/server/src/server/middleware/prudence-validate.ts @@ -1,63 +1,63 @@ -import Prudence, { - MiddlewareErrorHandler, - PrudenceSchema, - ErrorMessages, - PrudenceOptions, -} from "prudence"; -import { RequestHandler } from "express-serve-static-core"; -import CreateLogCtx from "lib/logger/logger"; - -const logger = CreateLogCtx(__filename); - -const printf = (message: string, stringVal: string | null, keychain: string | null) => - `[${keychain}] ${message}${stringVal ? ` (Received ${stringVal})` : ""}`; - -const API_ERR_HANDLER = - (logLevel: TachiLogLevels): MiddlewareErrorHandler => - (req, res, next, error) => { - let stringVal = error.userVal; - if (error.keychain && error.keychain.includes("password") && error.userVal) { - stringVal = "****"; - } - - if (typeof stringVal === "object" && stringVal !== null && !stringVal.toString) { - // this is probably null-prototype - stringVal = null; - } else if (stringVal === undefined) { - stringVal = "nothing"; - } else { - stringVal = String(stringVal); - } - - logger[logLevel]( - `Prudence rejection: ${error.message}, ${stringVal} [K:${error.keychain}]`, - { - userVal: error.userVal, - } - ); - - return res.status(400).json({ - success: false, - description: printf(error.message, stringVal as string | null, error.keychain), - }); - }; - -// Cache all of the possible API_ERROR_HANDLERS to avoid function creation -// overhead at runtime. -const API_ERROR_HANDLERS = Object.fromEntries( - (["crit", "severe", "error", "warn", "info", "verbose", "debug"] as const).map((e) => [ - e, - API_ERR_HANDLER(e), - ]) -) as Record; - -type TachiLogLevels = "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug"; - -const prValidate = ( - s: PrudenceSchema, - errorMessage?: ErrorMessages, - options?: Partial, - level: TachiLogLevels = "info" -): RequestHandler => Prudence.CurryMiddleware(API_ERROR_HANDLERS[level])(s, errorMessage, options); - -export default prValidate; +import Prudence, { + MiddlewareErrorHandler, + PrudenceSchema, + ErrorMessages, + PrudenceOptions, +} from "prudence"; +import { RequestHandler } from "express-serve-static-core"; +import CreateLogCtx from "lib/logger/logger"; + +const logger = CreateLogCtx(__filename); + +const printf = (message: string, stringVal: string | null, keychain: string | null) => + `[${keychain}] ${message}${stringVal ? ` (Received ${stringVal})` : ""}`; + +const API_ERR_HANDLER = + (logLevel: TachiLogLevels): MiddlewareErrorHandler => + (req, res, next, error) => { + let stringVal = error.userVal; + if (error.keychain && error.keychain.includes("password") && error.userVal) { + stringVal = "****"; + } + + if (typeof stringVal === "object" && stringVal !== null && !stringVal.toString) { + // this is probably null-prototype + stringVal = null; + } else if (stringVal === undefined) { + stringVal = "nothing"; + } else { + stringVal = String(stringVal); + } + + logger[logLevel]( + `Prudence rejection: ${error.message}, ${stringVal} [K:${error.keychain}]`, + { + userVal: error.userVal, + } + ); + + return res.status(400).json({ + success: false, + description: printf(error.message, stringVal as string | null, error.keychain), + }); + }; + +// Cache all of the possible API_ERROR_HANDLERS to avoid function creation +// overhead at runtime. +const API_ERROR_HANDLERS = Object.fromEntries( + (["crit", "severe", "error", "warn", "info", "verbose", "debug"] as const).map((e) => [ + e, + API_ERR_HANDLER(e), + ]) +) as Record; + +type TachiLogLevels = "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug"; + +const prValidate = ( + s: PrudenceSchema, + errorMessage?: ErrorMessages, + options?: Partial, + level: TachiLogLevels = "info" +): RequestHandler => Prudence.CurryMiddleware(API_ERROR_HANDLERS[level])(s, errorMessage, options); + +export default prValidate; diff --git a/server/src/server/middleware/sanitise-qstring.test.ts b/server/src/server/middleware/sanitise-qstring.test.ts index 9a2c8f9f5..bd8d4d3ae 100644 --- a/server/src/server/middleware/sanitise-qstring.test.ts +++ b/server/src/server/middleware/sanitise-qstring.test.ts @@ -1,35 +1,35 @@ -import SanitiseQString from "./sanitise-qstring"; -import t from "tap"; -import expMiddlewareMock from "express-request-mock"; - -t.test("#SanitiseQString", (t) => { - t.test("Should allow GET requests with valid data.", async (t) => { - const { res } = await expMiddlewareMock(SanitiseQString, { - method: "GET", - query: { - foo: "bar", - }, - }); - - t.not(res.statusCode, 400, "Status code should NOT be 400"); - - t.end(); - }); - - t.test("Should disallow GET requests with nested data.", async (t) => { - const { res } = await expMiddlewareMock(SanitiseQString, { - method: "GET", - query: { - foo: { - bar: "baz", - }, - }, - }); - - t.equal(res.statusCode, 400, "Status code should be 400"); - - t.end(); - }); - - t.end(); -}); +import SanitiseQString from "./sanitise-qstring"; +import t from "tap"; +import expMiddlewareMock from "express-request-mock"; + +t.test("#SanitiseQString", (t) => { + t.test("Should allow GET requests with valid data.", async (t) => { + const { res } = await expMiddlewareMock(SanitiseQString, { + method: "GET", + query: { + foo: "bar", + }, + }); + + t.not(res.statusCode, 400, "Status code should NOT be 400"); + + t.end(); + }); + + t.test("Should disallow GET requests with nested data.", async (t) => { + const { res } = await expMiddlewareMock(SanitiseQString, { + method: "GET", + query: { + foo: { + bar: "baz", + }, + }, + }); + + t.equal(res.statusCode, 400, "Status code should be 400"); + + t.end(); + }); + + t.end(); +}); diff --git a/server/src/server/middleware/sanitise-qstring.ts b/server/src/server/middleware/sanitise-qstring.ts index 3b769d1bf..6b7cc3603 100644 --- a/server/src/server/middleware/sanitise-qstring.ts +++ b/server/src/server/middleware/sanitise-qstring.ts @@ -1,21 +1,21 @@ -import { NextFunction, Request, Response } from "express"; - -/** - * Disallow complex query strings by enforcing single key:value pairs. - */ -function SanitiseQString(req: Request, res: Response, next: NextFunction) { - if (req.method === "GET") { - for (const data in req.query) { - if (typeof req.query[data] !== "string") { - return res.status(400).json({ - success: false, - description: `Invalid request with key ${data}`, - }); - } - } - } - - next(); -} - -export default SanitiseQString; +import { NextFunction, Request, Response } from "express"; + +/** + * Disallow complex query strings by enforcing single key:value pairs. + */ +function SanitiseQString(req: Request, res: Response, next: NextFunction) { + if (req.method === "GET") { + for (const data in req.query) { + if (typeof req.query[data] !== "string") { + return res.status(400).json({ + success: false, + description: `Invalid request with key ${data}`, + }); + } + } + } + + next(); +} + +export default SanitiseQString; diff --git a/server/src/server/router/api/v1/auth/auth.test.ts b/server/src/server/router/api/v1/auth/auth.test.ts index 587356a36..631064eac 100644 --- a/server/src/server/router/api/v1/auth/auth.test.ts +++ b/server/src/server/router/api/v1/auth/auth.test.ts @@ -1,73 +1,73 @@ -import { AddNewInvite, ReinstateInvite, ValidateCaptcha } from "./auth"; -import t from "tap"; -import db from "external/mongo/db"; -import ResetDBState from "test-utils/resets"; -import { MockBasicFetch } from "test-utils/mock-fetch"; - -t.test("#ReinstateInvite", (t) => { - t.beforeEach(ResetDBState); - - t.test("Should change the 'consumed' property of an invite to true.", async (t) => { - // mock insert - const inviteDoc = await db.invites.insert({ - code: "foobar", - consumed: true, - createdBy: 1, - createdAt: 1, - consumedAt: 2, - consumedBy: 2, - }); - - const response = await ReinstateInvite(inviteDoc.code); - - t.equal(response.nModified, 1, "Should modify one document"); - - const invite2 = await db.invites.findOne({ - code: inviteDoc.code, // lol - }); - - t.equal(invite2!.consumed, false, "Should no longer be consumed"); - t.equal(invite2!.consumedAt, null, "Should revoke when it was consumed."); - t.equal(invite2!.consumedBy, null, "Should revoke who it was consumed by."); - - t.end(); - }); - - t.end(); -}); - -t.test("#AddNewInvite", (t) => { - t.beforeEach(ResetDBState); - - t.test("Should create a new invite from a given user", async (t) => { - const userDoc = await db.users.findOne({ id: 1 }); - - const result = await AddNewInvite(userDoc!); - - t.equal(result.createdBy, userDoc!.id, "Invite should be created by the requesting user."); - t.equal(result.consumed, false, "Invite should not be consumed."); - - // was created +/- 6 seconds from now. This is perhaps too lenient, but we're only really testing its just around now ish. - t.ok(Math.abs(result.createdAt - Date.now()) <= 6000, "Invite was created roughly now."); - - t.match(result.code, /^[0-9a-f]{40}$/u, "Invite code should be a 40 character hex string."); - }); - - t.end(); -}); - -t.test("#ValidateCaptcha", async (t) => { - t.equal( - await ValidateCaptcha("200", "bar", MockBasicFetch({ status: 200 })), - true, - "Validates captcha when status return is 200" - ); - - t.equal( - await ValidateCaptcha("400", "bar", MockBasicFetch({ status: 400 })), - false, - "Invalidates captcha when status return is not 200" - ); - - t.end(); -}); +import { AddNewInvite, ReinstateInvite, ValidateCaptcha } from "./auth"; +import t from "tap"; +import db from "external/mongo/db"; +import ResetDBState from "test-utils/resets"; +import { MockBasicFetch } from "test-utils/mock-fetch"; + +t.test("#ReinstateInvite", (t) => { + t.beforeEach(ResetDBState); + + t.test("Should change the 'consumed' property of an invite to true.", async (t) => { + // mock insert + const inviteDoc = await db.invites.insert({ + code: "foobar", + consumed: true, + createdBy: 1, + createdAt: 1, + consumedAt: 2, + consumedBy: 2, + }); + + const response = await ReinstateInvite(inviteDoc.code); + + t.equal(response.nModified, 1, "Should modify one document"); + + const invite2 = await db.invites.findOne({ + code: inviteDoc.code, // lol + }); + + t.equal(invite2!.consumed, false, "Should no longer be consumed"); + t.equal(invite2!.consumedAt, null, "Should revoke when it was consumed."); + t.equal(invite2!.consumedBy, null, "Should revoke who it was consumed by."); + + t.end(); + }); + + t.end(); +}); + +t.test("#AddNewInvite", (t) => { + t.beforeEach(ResetDBState); + + t.test("Should create a new invite from a given user", async (t) => { + const userDoc = await db.users.findOne({ id: 1 }); + + const result = await AddNewInvite(userDoc!); + + t.equal(result.createdBy, userDoc!.id, "Invite should be created by the requesting user."); + t.equal(result.consumed, false, "Invite should not be consumed."); + + // was created +/- 6 seconds from now. This is perhaps too lenient, but we're only really testing its just around now ish. + t.ok(Math.abs(result.createdAt - Date.now()) <= 6000, "Invite was created roughly now."); + + t.match(result.code, /^[0-9a-f]{40}$/u, "Invite code should be a 40 character hex string."); + }); + + t.end(); +}); + +t.test("#ValidateCaptcha", async (t) => { + t.equal( + await ValidateCaptcha("200", "bar", MockBasicFetch({ status: 200 })), + true, + "Validates captcha when status return is 200" + ); + + t.equal( + await ValidateCaptcha("400", "bar", MockBasicFetch({ status: 400 })), + false, + "Invalidates captcha when status return is not 200" + ); + + t.end(); +}); diff --git a/server/src/server/router/api/v1/auth/auth.ts b/server/src/server/router/api/v1/auth/auth.ts index ea1dd9889..0a4346400 100644 --- a/server/src/server/router/api/v1/auth/auth.ts +++ b/server/src/server/router/api/v1/auth/auth.ts @@ -1,178 +1,178 @@ -import bcrypt from "bcryptjs"; -import { - integer, - PrivateUserInfoDocument, - PublicUserDocument, - UserAuthLevels, - UserSettings, -} from "tachi-common"; -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; -import { FormatUserDoc } from "utils/user"; -import nodeFetch from "utils/fetch"; -import { Random20Hex } from "utils/misc"; -import { Environment, ServerConfig } from "lib/setup/config"; -import { CreateURLWithParams } from "utils/url"; - -const logger = CreateLogCtx(__filename); - -const BCRYPT_SALT_ROUNDS = 12; - -export const ValidatePassword = (self: unknown) => - (typeof self === "string" && self.length >= 8) || "Passwords must be 8 characters or more."; - -/** - * Compares a plaintext string of a users password to a hash. - * @param plaintext The provided user input. - * @param password The hash to compare against. - */ -export function PasswordCompare(plaintext: string, password: string) { - return bcrypt.compare(plaintext, password); -} - -export function ReinstateInvite(code: string) { - logger.info(`Reinstated Invite ${code}`); - return db.invites.update( - { - code, - }, - { - $set: { - consumed: false, - consumedAt: null, - consumedBy: null, - }, - } - ); -} - -export async function AddNewInvite(user: PublicUserDocument) { - const code = Random20Hex(); - - const result = await db.invites.insert({ - code, - consumed: false, - createdBy: user.id, - createdAt: Date.now(), - consumedAt: null, - consumedBy: null, - }); - - logger.info(`User ${FormatUserDoc(user)} created an invite.`); - - if (!result) { - logger.error( - `Fatal error in creating ${FormatUserDoc( - user - )}'s invite code. Database refused key ${code}.` - ); - throw new Error( - `Fatal error in creating ${FormatUserDoc( - user - )}'s invite code. Database refused key ${code}.` - ); - } - - return result; -} - -export const DEFAULT_USER_SETTINGS: UserSettings["preferences"] = { - developerMode: false, - advancedMode: false, - invisible: false, - contentiousContent: false, -}; - -export function HashPassword(plaintext: string) { - return bcrypt.hash(plaintext, BCRYPT_SALT_ROUNDS); -} - -export async function AddNewUser( - username: string, - plaintext: string, - email: string, - userID: integer -) { - const hashedPassword = await HashPassword(plaintext); - - logger.verbose(`Hashed password for ${username}.`); - - const userDoc: PublicUserDocument = { - id: userID, - username, - usernameLowercase: username.toLowerCase(), - about: "I'm a fairly nondescript person.", - clan: null, - socialMedia: {}, - status: null, - customBannerLocation: null, - customPfpLocation: null, - joinDate: Date.now(), - lastSeen: Date.now(), - authLevel: UserAuthLevels.USER, - badges: [], - }; - - const res = await db.users.insert(userDoc); - - const settingsRes = await InsertDefaultUserSettings(userID); - - await InsertPrivateUserInfo(userID, hashedPassword, email); - - return { newUser: res, newSettings: settingsRes }; -} - -export function InsertPrivateUserInfo(userID: integer, hashedPassword: string, email: string) { - const privateInfo: PrivateUserInfoDocument = { - userID, - email, - password: hashedPassword, - }; - - return db["user-private-information"].insert(privateInfo); -} - -export function InsertDefaultUserSettings(userID: integer) { - logger.verbose(`Inserting default settings for ${userID}.`); - const userSettings: UserSettings = { - userID, - preferences: DEFAULT_USER_SETTINGS, - }; - - return db["user-settings"].insert(userSettings); -} - -export async function ValidateCaptcha( - recaptcha: string, - remoteAddr: string | undefined, - fetch = nodeFetch -) { - const url = CreateURLWithParams(`https://www.google.com/recaptcha/api/siteverify`, { - secret: ServerConfig.CAPTCHA_SECRET_KEY, - response: recaptcha, - remoteip: remoteAddr ?? "", - }); - - const r = await fetch(url.href); - - if (r.status !== 200) { - logger.verbose(`Failed GCaptcha response ${r.status}, ${r.body}`); - return false; - } - - return true; -} - -export function MountAuthCookie( - req: Express.Request, - user: PublicUserDocument, - settings: UserSettings -) { - req.session.tachi = { - user, - settings, - }; - - req.session.cookie.maxAge = 3.154e10; - req.session.cookie.secure = Environment.nodeEnv === "production"; -} +import bcrypt from "bcryptjs"; +import { + integer, + PrivateUserInfoDocument, + PublicUserDocument, + UserAuthLevels, + UserSettings, +} from "tachi-common"; +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import { FormatUserDoc } from "utils/user"; +import nodeFetch from "utils/fetch"; +import { Random20Hex } from "utils/misc"; +import { Environment, ServerConfig } from "lib/setup/config"; +import { CreateURLWithParams } from "utils/url"; + +const logger = CreateLogCtx(__filename); + +const BCRYPT_SALT_ROUNDS = 12; + +export const ValidatePassword = (self: unknown) => + (typeof self === "string" && self.length >= 8) || "Passwords must be 8 characters or more."; + +/** + * Compares a plaintext string of a users password to a hash. + * @param plaintext The provided user input. + * @param password The hash to compare against. + */ +export function PasswordCompare(plaintext: string, password: string) { + return bcrypt.compare(plaintext, password); +} + +export function ReinstateInvite(code: string) { + logger.info(`Reinstated Invite ${code}`); + return db.invites.update( + { + code, + }, + { + $set: { + consumed: false, + consumedAt: null, + consumedBy: null, + }, + } + ); +} + +export async function AddNewInvite(user: PublicUserDocument) { + const code = Random20Hex(); + + const result = await db.invites.insert({ + code, + consumed: false, + createdBy: user.id, + createdAt: Date.now(), + consumedAt: null, + consumedBy: null, + }); + + logger.info(`User ${FormatUserDoc(user)} created an invite.`); + + if (!result) { + logger.error( + `Fatal error in creating ${FormatUserDoc( + user + )}'s invite code. Database refused key ${code}.` + ); + throw new Error( + `Fatal error in creating ${FormatUserDoc( + user + )}'s invite code. Database refused key ${code}.` + ); + } + + return result; +} + +export const DEFAULT_USER_SETTINGS: UserSettings["preferences"] = { + developerMode: false, + advancedMode: false, + invisible: false, + contentiousContent: false, +}; + +export function HashPassword(plaintext: string) { + return bcrypt.hash(plaintext, BCRYPT_SALT_ROUNDS); +} + +export async function AddNewUser( + username: string, + plaintext: string, + email: string, + userID: integer +) { + const hashedPassword = await HashPassword(plaintext); + + logger.verbose(`Hashed password for ${username}.`); + + const userDoc: PublicUserDocument = { + id: userID, + username, + usernameLowercase: username.toLowerCase(), + about: "I'm a fairly nondescript person.", + clan: null, + socialMedia: {}, + status: null, + customBannerLocation: null, + customPfpLocation: null, + joinDate: Date.now(), + lastSeen: Date.now(), + authLevel: UserAuthLevels.USER, + badges: [], + }; + + const res = await db.users.insert(userDoc); + + const settingsRes = await InsertDefaultUserSettings(userID); + + await InsertPrivateUserInfo(userID, hashedPassword, email); + + return { newUser: res, newSettings: settingsRes }; +} + +export function InsertPrivateUserInfo(userID: integer, hashedPassword: string, email: string) { + const privateInfo: PrivateUserInfoDocument = { + userID, + email, + password: hashedPassword, + }; + + return db["user-private-information"].insert(privateInfo); +} + +export function InsertDefaultUserSettings(userID: integer) { + logger.verbose(`Inserting default settings for ${userID}.`); + const userSettings: UserSettings = { + userID, + preferences: DEFAULT_USER_SETTINGS, + }; + + return db["user-settings"].insert(userSettings); +} + +export async function ValidateCaptcha( + recaptcha: string, + remoteAddr: string | undefined, + fetch = nodeFetch +) { + const url = CreateURLWithParams(`https://www.google.com/recaptcha/api/siteverify`, { + secret: ServerConfig.CAPTCHA_SECRET_KEY, + response: recaptcha, + remoteip: remoteAddr ?? "", + }); + + const r = await fetch(url.href); + + if (r.status !== 200) { + logger.verbose(`Failed GCaptcha response ${r.status}, ${r.body}`); + return false; + } + + return true; +} + +export function MountAuthCookie( + req: Express.Request, + user: PublicUserDocument, + settings: UserSettings +) { + req.session.tachi = { + user, + settings, + }; + + req.session.cookie.maxAge = 3.154e10; + req.session.cookie.secure = Environment.nodeEnv === "production"; +} diff --git a/server/src/server/router/api/v1/auth/router.ts b/server/src/server/router/api/v1/auth/router.ts index 103c4ccd8..0deb3ac25 100644 --- a/server/src/server/router/api/v1/auth/router.ts +++ b/server/src/server/router/api/v1/auth/router.ts @@ -1,521 +1,521 @@ -import { Router } from "express"; -import Prudence from "prudence"; -import { - AddNewUser, - PasswordCompare, - ReinstateInvite, - ValidatePassword, - ValidateCaptcha, - MountAuthCookie, - InsertDefaultUserSettings, - HashPassword, -} from "./auth"; -import { - CheckIfEmailInUse, - FormatUserDoc, - GetSettingsForUser, - GetUserCaseInsensitive, - GetUserPrivateInfo, - GetUserWithID, -} from "utils/user"; -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; -import prValidate from "server/middleware/prudence-validate"; -import { DecrementCounterValue, GetNextCounterValue } from "utils/db"; -import { SendEmail } from "lib/email/client"; -import { EmailFormatResetPassword, EmailFormatVerifyEmail } from "lib/email/formats"; -import { Random20Hex } from "utils/misc"; -import { Environment, ServerConfig } from "lib/setup/config"; -import { integer } from "tachi-common"; - -const logger = CreateLogCtx(__filename); - -const router: Router = Router({ mergeParams: true }); - -const LAZY_EMAIL_REGEX = /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/u; - -/** - * Logs in a user. - * @name POST /api/v1/auth/login - */ -router.post( - "/login", - prValidate( - { - username: Prudence.regex(/^[a-zA-Z_-][a-zA-Z0-9_-]{2,20}$/u), - password: ValidatePassword, - captcha: "string", - }, - { - username: - "Invalid username. Usernames cannot start with a number, and must be between 2 and 20 characters.", - captcha: "Please fill out the captcha.", - }, - undefined, - "verbose" - ), - async (req, res) => { - if (req.session.tachi?.user.id) { - logger.info(`Dual log-in attempted from ${req.session.tachi.user.id}`); - return res.status(409).json({ - success: false, - description: `You are already logged in as someone.`, - }); - } - - logger.verbose(`Recieved login request with username ${req.body.username} (${req.ip})`); - - /* istanbul ignore next */ - if (Environment.nodeEnv === "production" || Environment.nodeEnv === "staging") { - logger.verbose("Validating captcha..."); - const validCaptcha = await ValidateCaptcha( - req.body.recaptcha, - req.socket.remoteAddress - ); - - if (!validCaptcha) { - logger.verbose("Captcha failed."); - return res.status(400).json({ - success: false, - description: `Captcha failed.`, - }); - } - - logger.verbose("Captcha validated!"); - } else { - logger.verbose("Skipped captcha check because not in production."); - } - - const requestedUser = await GetUserCaseInsensitive(req.body.username); - - if (!requestedUser) { - logger.verbose(`Invalid username for login ${req.body.username}.`); - return res.status(404).json({ - success: false, - description: `This user does not exist.`, - }); - } - - const privateInfo = await GetUserPrivateInfo(requestedUser.id); - - if (!privateInfo) { - logger.severe( - `State desync for user ${FormatUserDoc( - requestedUser - )}. This user has no password/email information?`, - { requestedUser } - ); - - return res.status(500).json({ - success: false, - description: `An internal server error has occured.`, - }); - } - - const passwordMatch = await PasswordCompare(req.body.password, privateInfo.password); - - if (!passwordMatch) { - logger.verbose("Invalid password provided."); - return res.status(401).json({ - success: false, - description: `Invalid password.`, - }); - } - - const user = await GetUserWithID(requestedUser.id); - - if (!user) { - logger.severe(`User logged in as someone who does not exist?`, { requestedUser }); - return res.status(500).json({ - success: false, - description: `An internal server error has occured.`, - }); - } - - let settings = await GetSettingsForUser(requestedUser.id); - - if (!settings) { - logger.warn(`User ${FormatUserDoc(user)} has no settings. Inserting default settings.`); - settings = await InsertDefaultUserSettings(user.id); - } - - MountAuthCookie(req, user, settings); - - logger.verbose(`${FormatUserDoc(requestedUser)} Logged in.`); - - return res.status(200).json({ - success: true, - description: `Successfully logged in as ${FormatUserDoc(requestedUser)}`, - body: { - userID: requestedUser.id, - }, - }); - } -); - -/** - * Registers a new user. - * @name POST /api/v1/auth/register - */ -router.post( - "/register", - prValidate( - { - username: Prudence.regex(/^[a-zA-Z_-][a-zA-Z0-9_-]{2,20}$/u), - password: ValidatePassword, - email: Prudence.regex(LAZY_EMAIL_REGEX), - inviteCode: "*string", - captcha: "string", - }, - { - username: - "Usernames must be between 3 and 20 characters long, can only contain alphanumeric characters and cannot start with a number.", - email: "Invalid email.", - inviteCode: "Invalid invite code.", - captcha: "Please fill out the captcha.", - }, - undefined, - "verbose" - ), - async (req, res) => { - logger.verbose(`Recieved register request with username ${req.body.username} (${req.ip})`); - - /* istanbul ignore next */ - if (Environment.nodeEnv === "production" || Environment.nodeEnv === "staging") { - logger.verbose("Validating captcha..."); - const validCaptcha = await ValidateCaptcha( - req.body.recaptcha, - req.socket.remoteAddress - ); - - if (!validCaptcha) { - logger.verbose("Captcha failed."); - return res.status(400).json({ - success: false, - description: `Captcha failed.`, - }); - } - - logger.verbose("Captcha validated."); - } else { - logger.warn("Skipped captcha check because not in production."); - } - - const existingUser = await GetUserCaseInsensitive(req.body.username); - - if (existingUser) { - logger.verbose(`Invalid username ${req.body.username}, already in use.`); - return res.status(409).json({ - success: false, - description: "This username is already in use.", - }); - } - - const existingEmail = await CheckIfEmailInUse(req.body.email); - - if (existingEmail) { - logger.info(`User attempted to sign up with email that was already in use.`); - return res.status(409).json({ - success: false, - description: `This email is already in use.`, - }); - } - - let hasInsertedUserID: integer | null = null; - - try { - const userID = await GetNextCounterValue("users"); - - if (ServerConfig.INVITE_CODE_CONFIG) { - const inviteCodeDoc = await db.invites.findOneAndUpdate( - { - code: req.body.inviteCode, - consumed: false, - }, - { - $set: { - consumed: true, - consumedAt: Date.now(), - consumedBy: userID, - }, - } - ); - - if (!inviteCodeDoc) { - logger.info(`Invalid invite code given: ${req.body.inviteCode}.`); - return res.status(401).json({ - success: false, - description: `This invite code is not valid.`, - }); - } - - logger.info(`Consumed invite ${inviteCodeDoc.code}.`); - } - - // if we get to this point, We're good to create the user. - - const { newUser, newSettings } = await AddNewUser( - req.body.username, - req.body.password, - req.body.email, - userID - ); - - if (!newUser) { - throw new Error("AddNewUser failed to create a user."); - } - - hasInsertedUserID = newUser.id; - - // re-fetch the user like this so we guaranteeably omit the private fields. - const user = await GetUserWithID(newUser.id); - - MountAuthCookie(req, user!, newSettings); - - const resetEmailCode = Random20Hex(); - - await db["verify-email-codes"].insert({ - code: resetEmailCode, - userID: userID, - email: req.body.email, - }); - - const { text, html } = EmailFormatVerifyEmail(user!.username, resetEmailCode); - - SendEmail(req.body.email, "Email Verification", html, text); - - return res.status(200).json({ - success: true, - description: `Successfully created account ${req.body.username}!`, - body: user, - }); - } catch (err) { - logger.error(`Bailed on user creation ${req.body.username}.`, { err }); - - if (ServerConfig.INVITE_CODE_CONFIG) { - await ReinstateInvite(req.body.inviteCode); - } - - if (hasInsertedUserID !== null) { - logger.warn( - `Removing user ${req.body.username} (#${hasInsertedUserID}), as their document was created, but creation still failed.` - ); - await db.users.remove({ username: req.body.username }); - await db["user-settings"].remove({ userID: hasInsertedUserID }); - await db["user-private-information"].remove({ userID: hasInsertedUserID }); - } - - await DecrementCounterValue("users"); - - return res.status(500).json({ - success: false, - description: "An internal server error has occured.", - }); - } - } -); - -/** - * Verifies the provided email according to the code provided. - * - * @param code - The emailCode set in the /register function. - * - * @name POST /api/v1/auth/verify-email - */ -router.post( - "/verify-email", - prValidate({ - code: "string", - }), - async (req, res) => { - const code = await db["verify-email-codes"].findOne({ - code: req.body.code, - }); - - if (!code) { - return res.status(400).json({ - success: false, - description: `This email code is invalid.`, - }); - } - - await db["verify-email-codes"].remove({ - code: req.body.code, - }); - - return res.status(200).json({ - success: true, - description: `Verified email!`, - body: {}, - }); - } -); - -/** - * Resend a verification email, for when they fall through the - * cracks. - * - * @param email - The email to send a verification email to. - * - * @name POST /api/v1/auth/resend-verify-email - */ -router.post("/resend-verify-email", prValidate({ email: "string" }), async (req, res) => { - // Immediately send a response so the existence of emails - // cannot be timing attacked out. - res.status(200).json({ - success: true, - description: `Sent an email if the email address has not been verified.`, - body: {}, - }); - - const verifyInfo = await db["verify-email-codes"].findOne({ email: req.body.email }); - - if (!verifyInfo) { - logger.warn( - `Attempted to send reset email to ${req.body.email}, but no verifyInfo was set for them.` - ); - return; - } - - const user = await GetUserWithID(verifyInfo.userID); - - if (!user) { - logger.severe(`Email verifyInfo belongs to user that no longer exists?`, verifyInfo); - return; - } - - // Send the email again. - - const { text, html } = EmailFormatVerifyEmail(user!.username, verifyInfo.code); - - SendEmail(req.body.email, "Email Verification", html, text); -}); - -/** - * Logs out the requesting user. - * @name POST /api/v1/auth/logout - */ -router.post("/logout", (req, res) => { - if (!req.session?.tachi?.user.id) { - return res.status(409).json({ - success: false, - description: `You are not logged in.`, - }); - } - - req.session.destroy(() => 0); - - return res.status(200).json({ - success: true, - description: `Logged Out.`, - body: {}, - }); -}); - -/** - * Creates a password reset code for a user. The user will then - * be able to trigger POST /reset-password with that code. - * - * @param email - The email associated with the account you want to reset. - * - * @name POST /api/v1/auth/forgot-password - */ -router.post("/forgot-password", prValidate({ email: "string" }), async (req, res) => { - if (!ServerConfig.EMAIL_CONFIG && Environment.nodeEnv !== "test") { - return res.status(501).json({ - success: false, - description: `This server does not support password resets.`, - }); - } - - logger.debug(`Recieved password reset request for ${req.body.email}.`); - // For timing attack and infosec reasons, we can't do anything but **immediately** return here. - res.status(202).json({ - success: true, - description: "A code has been sent to your email.", - body: {}, - }); - - const userPrivateInfo = await db["user-private-information"].findOne({ email: req.body.email }); - - if (userPrivateInfo) { - const user = await db.users.findOne({ id: userPrivateInfo.userID }); - - if (!user) { - logger.severe( - `User ${userPrivateInfo.userID} has private information but no real account.` - ); - return; - } - - const code = `M${Random20Hex()}`; - - logger.verbose(`Created password reset code for ${FormatUserDoc(user)}.`); - - await db["password-reset-codes"].insert({ - code, - userID: user.id, - createdOn: Date.now(), - }); - - const { html, text } = EmailFormatResetPassword(user.username, code, req.ip); - - SendEmail(userPrivateInfo.email, "Reset Password", html, text); - } else { - logger.info( - `Silently rejected password reset request for ${req.body.email}, as no user has this email.` - ); - } -}); - -/** - * Takes a code generated from /forgot-password, a new password, - * and performs the reset for the user. - * - * @param password - The users new password. - * @param code - The code to use to reset this password. - * - * @name POST /api/v1/auth/reset-password - */ -router.post( - "/reset-password", - prValidate({ - code: "string", - password: ValidatePassword, - }), - async (req, res) => { - const code = await db["password-reset-codes"].findOneAndDelete({ - code: req.body.code, - }); - - if (!code) { - return res.status(404).json({ - success: false, - description: `Invalid Reset Code.`, - }); - } - - const encryptedPassword = await HashPassword(req.body.password); - - await db["user-private-information"].update( - { - userID: code.userID, - }, - { - $set: { - password: encryptedPassword, - }, - } - ); - - logger.info(`User ${code.userID} reset their password.`); - - return res.status(200).json({ - success: true, - description: `Reset your password.`, - body: {}, - }); - } -); - -export default router; +import { Router } from "express"; +import Prudence from "prudence"; +import { + AddNewUser, + PasswordCompare, + ReinstateInvite, + ValidatePassword, + ValidateCaptcha, + MountAuthCookie, + InsertDefaultUserSettings, + HashPassword, +} from "./auth"; +import { + CheckIfEmailInUse, + FormatUserDoc, + GetSettingsForUser, + GetUserCaseInsensitive, + GetUserPrivateInfo, + GetUserWithID, +} from "utils/user"; +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import prValidate from "server/middleware/prudence-validate"; +import { DecrementCounterValue, GetNextCounterValue } from "utils/db"; +import { SendEmail } from "lib/email/client"; +import { EmailFormatResetPassword, EmailFormatVerifyEmail } from "lib/email/formats"; +import { Random20Hex } from "utils/misc"; +import { Environment, ServerConfig } from "lib/setup/config"; +import { integer } from "tachi-common"; + +const logger = CreateLogCtx(__filename); + +const router: Router = Router({ mergeParams: true }); + +const LAZY_EMAIL_REGEX = /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/u; + +/** + * Logs in a user. + * @name POST /api/v1/auth/login + */ +router.post( + "/login", + prValidate( + { + username: Prudence.regex(/^[a-zA-Z_-][a-zA-Z0-9_-]{2,20}$/u), + password: ValidatePassword, + captcha: "string", + }, + { + username: + "Invalid username. Usernames cannot start with a number, and must be between 2 and 20 characters.", + captcha: "Please fill out the captcha.", + }, + undefined, + "verbose" + ), + async (req, res) => { + if (req.session.tachi?.user.id) { + logger.info(`Dual log-in attempted from ${req.session.tachi.user.id}`); + return res.status(409).json({ + success: false, + description: `You are already logged in as someone.`, + }); + } + + logger.verbose(`Recieved login request with username ${req.body.username} (${req.ip})`); + + /* istanbul ignore next */ + if (Environment.nodeEnv === "production" || Environment.nodeEnv === "staging") { + logger.verbose("Validating captcha..."); + const validCaptcha = await ValidateCaptcha( + req.body.recaptcha, + req.socket.remoteAddress + ); + + if (!validCaptcha) { + logger.verbose("Captcha failed."); + return res.status(400).json({ + success: false, + description: `Captcha failed.`, + }); + } + + logger.verbose("Captcha validated!"); + } else { + logger.verbose("Skipped captcha check because not in production."); + } + + const requestedUser = await GetUserCaseInsensitive(req.body.username); + + if (!requestedUser) { + logger.verbose(`Invalid username for login ${req.body.username}.`); + return res.status(404).json({ + success: false, + description: `This user does not exist.`, + }); + } + + const privateInfo = await GetUserPrivateInfo(requestedUser.id); + + if (!privateInfo) { + logger.severe( + `State desync for user ${FormatUserDoc( + requestedUser + )}. This user has no password/email information?`, + { requestedUser } + ); + + return res.status(500).json({ + success: false, + description: `An internal server error has occured.`, + }); + } + + const passwordMatch = await PasswordCompare(req.body.password, privateInfo.password); + + if (!passwordMatch) { + logger.verbose("Invalid password provided."); + return res.status(401).json({ + success: false, + description: `Invalid password.`, + }); + } + + const user = await GetUserWithID(requestedUser.id); + + if (!user) { + logger.severe(`User logged in as someone who does not exist?`, { requestedUser }); + return res.status(500).json({ + success: false, + description: `An internal server error has occured.`, + }); + } + + let settings = await GetSettingsForUser(requestedUser.id); + + if (!settings) { + logger.warn(`User ${FormatUserDoc(user)} has no settings. Inserting default settings.`); + settings = await InsertDefaultUserSettings(user.id); + } + + MountAuthCookie(req, user, settings); + + logger.verbose(`${FormatUserDoc(requestedUser)} Logged in.`); + + return res.status(200).json({ + success: true, + description: `Successfully logged in as ${FormatUserDoc(requestedUser)}`, + body: { + userID: requestedUser.id, + }, + }); + } +); + +/** + * Registers a new user. + * @name POST /api/v1/auth/register + */ +router.post( + "/register", + prValidate( + { + username: Prudence.regex(/^[a-zA-Z_-][a-zA-Z0-9_-]{2,20}$/u), + password: ValidatePassword, + email: Prudence.regex(LAZY_EMAIL_REGEX), + inviteCode: "*string", + captcha: "string", + }, + { + username: + "Usernames must be between 3 and 20 characters long, can only contain alphanumeric characters and cannot start with a number.", + email: "Invalid email.", + inviteCode: "Invalid invite code.", + captcha: "Please fill out the captcha.", + }, + undefined, + "verbose" + ), + async (req, res) => { + logger.verbose(`Recieved register request with username ${req.body.username} (${req.ip})`); + + /* istanbul ignore next */ + if (Environment.nodeEnv === "production" || Environment.nodeEnv === "staging") { + logger.verbose("Validating captcha..."); + const validCaptcha = await ValidateCaptcha( + req.body.recaptcha, + req.socket.remoteAddress + ); + + if (!validCaptcha) { + logger.verbose("Captcha failed."); + return res.status(400).json({ + success: false, + description: `Captcha failed.`, + }); + } + + logger.verbose("Captcha validated."); + } else { + logger.warn("Skipped captcha check because not in production."); + } + + const existingUser = await GetUserCaseInsensitive(req.body.username); + + if (existingUser) { + logger.verbose(`Invalid username ${req.body.username}, already in use.`); + return res.status(409).json({ + success: false, + description: "This username is already in use.", + }); + } + + const existingEmail = await CheckIfEmailInUse(req.body.email); + + if (existingEmail) { + logger.info(`User attempted to sign up with email that was already in use.`); + return res.status(409).json({ + success: false, + description: `This email is already in use.`, + }); + } + + let hasInsertedUserID: integer | null = null; + + try { + const userID = await GetNextCounterValue("users"); + + if (ServerConfig.INVITE_CODE_CONFIG) { + const inviteCodeDoc = await db.invites.findOneAndUpdate( + { + code: req.body.inviteCode, + consumed: false, + }, + { + $set: { + consumed: true, + consumedAt: Date.now(), + consumedBy: userID, + }, + } + ); + + if (!inviteCodeDoc) { + logger.info(`Invalid invite code given: ${req.body.inviteCode}.`); + return res.status(401).json({ + success: false, + description: `This invite code is not valid.`, + }); + } + + logger.info(`Consumed invite ${inviteCodeDoc.code}.`); + } + + // if we get to this point, We're good to create the user. + + const { newUser, newSettings } = await AddNewUser( + req.body.username, + req.body.password, + req.body.email, + userID + ); + + if (!newUser) { + throw new Error("AddNewUser failed to create a user."); + } + + hasInsertedUserID = newUser.id; + + // re-fetch the user like this so we guaranteeably omit the private fields. + const user = await GetUserWithID(newUser.id); + + MountAuthCookie(req, user!, newSettings); + + const resetEmailCode = Random20Hex(); + + await db["verify-email-codes"].insert({ + code: resetEmailCode, + userID: userID, + email: req.body.email, + }); + + const { text, html } = EmailFormatVerifyEmail(user!.username, resetEmailCode); + + SendEmail(req.body.email, "Email Verification", html, text); + + return res.status(200).json({ + success: true, + description: `Successfully created account ${req.body.username}!`, + body: user, + }); + } catch (err) { + logger.error(`Bailed on user creation ${req.body.username}.`, { err }); + + if (ServerConfig.INVITE_CODE_CONFIG) { + await ReinstateInvite(req.body.inviteCode); + } + + if (hasInsertedUserID !== null) { + logger.warn( + `Removing user ${req.body.username} (#${hasInsertedUserID}), as their document was created, but creation still failed.` + ); + await db.users.remove({ username: req.body.username }); + await db["user-settings"].remove({ userID: hasInsertedUserID }); + await db["user-private-information"].remove({ userID: hasInsertedUserID }); + } + + await DecrementCounterValue("users"); + + return res.status(500).json({ + success: false, + description: "An internal server error has occured.", + }); + } + } +); + +/** + * Verifies the provided email according to the code provided. + * + * @param code - The emailCode set in the /register function. + * + * @name POST /api/v1/auth/verify-email + */ +router.post( + "/verify-email", + prValidate({ + code: "string", + }), + async (req, res) => { + const code = await db["verify-email-codes"].findOne({ + code: req.body.code, + }); + + if (!code) { + return res.status(400).json({ + success: false, + description: `This email code is invalid.`, + }); + } + + await db["verify-email-codes"].remove({ + code: req.body.code, + }); + + return res.status(200).json({ + success: true, + description: `Verified email!`, + body: {}, + }); + } +); + +/** + * Resend a verification email, for when they fall through the + * cracks. + * + * @param email - The email to send a verification email to. + * + * @name POST /api/v1/auth/resend-verify-email + */ +router.post("/resend-verify-email", prValidate({ email: "string" }), async (req, res) => { + // Immediately send a response so the existence of emails + // cannot be timing attacked out. + res.status(200).json({ + success: true, + description: `Sent an email if the email address has not been verified.`, + body: {}, + }); + + const verifyInfo = await db["verify-email-codes"].findOne({ email: req.body.email }); + + if (!verifyInfo) { + logger.warn( + `Attempted to send reset email to ${req.body.email}, but no verifyInfo was set for them.` + ); + return; + } + + const user = await GetUserWithID(verifyInfo.userID); + + if (!user) { + logger.severe(`Email verifyInfo belongs to user that no longer exists?`, verifyInfo); + return; + } + + // Send the email again. + + const { text, html } = EmailFormatVerifyEmail(user!.username, verifyInfo.code); + + SendEmail(req.body.email, "Email Verification", html, text); +}); + +/** + * Logs out the requesting user. + * @name POST /api/v1/auth/logout + */ +router.post("/logout", (req, res) => { + if (!req.session?.tachi?.user.id) { + return res.status(409).json({ + success: false, + description: `You are not logged in.`, + }); + } + + req.session.destroy(() => 0); + + return res.status(200).json({ + success: true, + description: `Logged Out.`, + body: {}, + }); +}); + +/** + * Creates a password reset code for a user. The user will then + * be able to trigger POST /reset-password with that code. + * + * @param email - The email associated with the account you want to reset. + * + * @name POST /api/v1/auth/forgot-password + */ +router.post("/forgot-password", prValidate({ email: "string" }), async (req, res) => { + if (!ServerConfig.EMAIL_CONFIG && Environment.nodeEnv !== "test") { + return res.status(501).json({ + success: false, + description: `This server does not support password resets.`, + }); + } + + logger.debug(`Recieved password reset request for ${req.body.email}.`); + // For timing attack and infosec reasons, we can't do anything but **immediately** return here. + res.status(202).json({ + success: true, + description: "A code has been sent to your email.", + body: {}, + }); + + const userPrivateInfo = await db["user-private-information"].findOne({ email: req.body.email }); + + if (userPrivateInfo) { + const user = await db.users.findOne({ id: userPrivateInfo.userID }); + + if (!user) { + logger.severe( + `User ${userPrivateInfo.userID} has private information but no real account.` + ); + return; + } + + const code = `M${Random20Hex()}`; + + logger.verbose(`Created password reset code for ${FormatUserDoc(user)}.`); + + await db["password-reset-codes"].insert({ + code, + userID: user.id, + createdOn: Date.now(), + }); + + const { html, text } = EmailFormatResetPassword(user.username, code, req.ip); + + SendEmail(userPrivateInfo.email, "Reset Password", html, text); + } else { + logger.info( + `Silently rejected password reset request for ${req.body.email}, as no user has this email.` + ); + } +}); + +/** + * Takes a code generated from /forgot-password, a new password, + * and performs the reset for the user. + * + * @param password - The users new password. + * @param code - The code to use to reset this password. + * + * @name POST /api/v1/auth/reset-password + */ +router.post( + "/reset-password", + prValidate({ + code: "string", + password: ValidatePassword, + }), + async (req, res) => { + const code = await db["password-reset-codes"].findOneAndDelete({ + code: req.body.code, + }); + + if (!code) { + return res.status(404).json({ + success: false, + description: `Invalid Reset Code.`, + }); + } + + const encryptedPassword = await HashPassword(req.body.password); + + await db["user-private-information"].update( + { + userID: code.userID, + }, + { + $set: { + password: encryptedPassword, + }, + } + ); + + logger.info(`User ${code.userID} reset their password.`); + + return res.status(200).json({ + success: true, + description: `Reset your password.`, + body: {}, + }); + } +); + +export default router; diff --git a/server/src/server/router/api/v1/import/router.test.ts b/server/src/server/router/api/v1/import/router.test.ts index 86581fa91..2510bb6e1 100644 --- a/server/src/server/router/api/v1/import/router.test.ts +++ b/server/src/server/router/api/v1/import/router.test.ts @@ -1,403 +1,403 @@ -import t from "tap"; -import mockApi from "test-utils/mock-api"; -import { - GetKTDataBuffer, - LoadTachiIIDXData, - TestingIIDXEamusementCSV26, - TestingIIDXEamusementCSV27, -} from "test-utils/test-data"; - -import { RequireAuthPerms } from "test-utils/api-common"; -import { CreateFakeAuthCookie } from "test-utils/fake-auth"; -import ResetDBState from "test-utils/resets"; -import db from "external/mongo/db"; -import { rootLogger } from "lib/logger/logger"; - -t.test("POST /api/v1/import/file", async (t) => { - const cookie = await CreateFakeAuthCookie(mockApi); - - t.beforeEach(ResetDBState); - - RequireAuthPerms("/api/v1/import/file", "submit_score", "POST"); - - t.test("file/eamusement-iidx-csv", (t) => { - t.beforeEach(LoadTachiIIDXData); - - t.test("Mini HV import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach( - "scoreData", - GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"), - "my_csv.csv" - ) - .field("importType", "file/eamusement-iidx-csv") - .field("playtype", "SP"); - - t.equal(res.body.success, true, "Should be successful."); - - t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores."); - - t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.test("Valid Rootage CSV import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv") - .field("importType", "file/eamusement-iidx-csv") - .field("playtype", "SP"); - - t.equal(res.body.success, true, "Should be successful."); - - t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.test("Valid Heroic Verse CSV import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv") - .field("importType", "file/eamusement-iidx-csv") - .field("playtype", "SP"); - - t.equal(res.body.success, true, "Should be successful."); - - t.strictSame(res.body.body.errors, [], "Should have 0 failed scores."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.end(); - }); - - // thats right i literally just copied it - t.test("file/pli-iidx-csv", (t) => { - t.beforeEach(LoadTachiIIDXData); - - t.test("Mini HV import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach( - "scoreData", - GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"), - "my_csv.csv" - ) - .field("importType", "file/pli-iidx-csv") - .field("playtype", "SP"); - - t.equal(res.body.success, true, "Should be successful."); - - t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores."); - - t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.test("Valid Rootage CSV import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv") - .field("importType", "file/pli-iidx-csv") - .field("playtype", "SP"); - - t.equal(res.body.success, true, "Should be successful."); - - t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.test("Valid Heroic Verse CSV import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv") - .field("importType", "file/pli-iidx-csv") - .field("playtype", "SP"); - - t.equal(res.body.success, true, "Should be successful."); - - t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.end(); - }); - - t.test("file/batch-manual", (t) => { - t.test("Empty import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach( - "scoreData", - GetKTDataBuffer("./batch-manual/empty-file.json"), - "empty-file.json" - ) - .field("importType", "file/batch-manual"); - - t.equal(res.body.success, true, "Should be successful."); - - t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); - - t.equal(res.body.body.scoreIDs.length, 0, "Should have 0 successful scores."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.test("Invalid JSON", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", Buffer.from("{invalid JSON")) - .field("importType", "file/batch-manual"); - - t.equal(res.body.success, false, "Should not be successful."); - - t.equal(res.statusCode, 400); - - t.end(); - }); - - t.test("Single import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach( - "scoreData", - GetKTDataBuffer("./batch-manual/small-file.json"), - "small-file.json" - ) - .field("importType", "file/batch-manual"); - - t.equal(res.body.success, true, "Should be successful."); - - t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); - - t.equal(res.body.body.scoreIDs.length, 1, "Should have 1 successful score."); - - const scoreCount = await db.scores.find({ - scoreID: { $in: res.body.body.scoreIDs }, - }); - - t.equal( - scoreCount.length, - res.body.body.scoreIDs.length, - "All returned scoreIDs should be inserted to the DB." - ); - - t.end(); - }); - - t.end(); - }); - - t.test("file/mer-iidx", (t) => { - t.beforeEach(LoadTachiIIDXData); - - t.test("Example Import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", GetKTDataBuffer("./mer/base.json"), "base.json") - .field("importType", "file/mer-iidx"); - - t.equal(res.body.success, true, "Should be successful"); - - t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); - - t.equal(res.body.body.scoreIDs.length, 3, "Should have 3 successful scores."); - - t.end(); - }); - - t.test("Example Import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", GetKTDataBuffer("./mer/large.json"), "base.json") - .field("importType", "file/mer-iidx"); - - t.equal(res.body.success, true, "Should be successful"); - - t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); - - t.equal(res.body.body.scoreIDs.length, 627, "Should have 627 successful scores."); - - t.end(); - }); - - t.end(); - }); - - t.skip("file/solid-state-squad", (t) => { - t.beforeEach(LoadTachiIIDXData); - - t.test("Large Import", async (t) => { - const res = await mockApi - .post("/api/v1/import/file") - .set("Cookie", cookie) - .attach("scoreData", GetKTDataBuffer("./s3/large-example.xml"), "large.xml") - .field("importType", "file/solid-state-squad"); - - t.equal(res.body.success, true, "Should be successful"); - t.equal(res.body.body.scoreIDs.length, null, "Should parse N scores."); - - t.end(); - }); - - t.end(); - }); - - t.end(); -}); - -t.test("POST /api/v1/import/orphans", async (t) => { - const cookie = await CreateFakeAuthCookie(mockApi); - - t.beforeEach(ResetDBState); - - t.test("Should force a reprocessing of orphan scores.", async (t) => { - await db["orphan-scores"].insert([ - { - userID: 1, - timeInserted: 1000, - orphanID: "asdf", - importType: "ir/direct-manual", - errMsg: "foo", - context: { - game: "iidx", - playtype: "SP", - service: "foo", - version: null, - }, - data: { - score: 500, - lamp: "HARD CLEAR", - matchType: "songTitle", - identifier: "5.1.1.", - difficulty: "ANOTHER", - }, - game: "iidx", - }, - { - userID: 1, - timeInserted: 1000, - orphanID: "asdf2", - importType: "ir/direct-manual", - errMsg: "foo", - context: { - game: "iidx", - playtype: "SP", - service: "foo", - version: null, - }, - data: { - score: 500, - lamp: "HARD CLEAR", - matchType: "songTitle", - identifier: "TITLE NOBODY WILL USE", - difficulty: "ANOTHER", - }, - game: "iidx", - }, - ]); - - const res = await mockApi.post("/api/v1/import/orphans").set("Cookie", cookie); - - t.equal(res.statusCode, 200, "Should return 200."); - - t.equal(res.body.body.success, 1, "Should successfully reprocess one orphan."); - t.equal(res.body.body.done, 2, "Should reprocess two orphans."); - t.equal(res.body.body.failed, 1, "Should fail in de-orphaning one orphan."); - - const dbCount = await db["orphan-scores"].count({}); - - t.equal(dbCount, 1, "Should only leave one orphan-score in the database."); - - t.end(); - }); - - t.end(); -}); +import t from "tap"; +import mockApi from "test-utils/mock-api"; +import { + GetKTDataBuffer, + LoadTachiIIDXData, + TestingIIDXEamusementCSV26, + TestingIIDXEamusementCSV27, +} from "test-utils/test-data"; + +import { RequireAuthPerms } from "test-utils/api-common"; +import { CreateFakeAuthCookie } from "test-utils/fake-auth"; +import ResetDBState from "test-utils/resets"; +import db from "external/mongo/db"; +import { rootLogger } from "lib/logger/logger"; + +t.test("POST /api/v1/import/file", async (t) => { + const cookie = await CreateFakeAuthCookie(mockApi); + + t.beforeEach(ResetDBState); + + RequireAuthPerms("/api/v1/import/file", "submit_score", "POST"); + + t.test("file/eamusement-iidx-csv", (t) => { + t.beforeEach(LoadTachiIIDXData); + + t.test("Mini HV import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach( + "scoreData", + GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"), + "my_csv.csv" + ) + .field("importType", "file/eamusement-iidx-csv") + .field("playtype", "SP"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores."); + + t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.test("Valid Rootage CSV import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv") + .field("importType", "file/eamusement-iidx-csv") + .field("playtype", "SP"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.test("Valid Heroic Verse CSV import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv") + .field("importType", "file/eamusement-iidx-csv") + .field("playtype", "SP"); + + t.equal(res.body.success, true, "Should be successful."); + + t.strictSame(res.body.body.errors, [], "Should have 0 failed scores."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.end(); + }); + + // thats right i literally just copied it + t.test("file/pli-iidx-csv", (t) => { + t.beforeEach(LoadTachiIIDXData); + + t.test("Mini HV import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach( + "scoreData", + GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"), + "my_csv.csv" + ) + .field("importType", "file/pli-iidx-csv") + .field("playtype", "SP"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores."); + + t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.test("Valid Rootage CSV import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv") + .field("importType", "file/pli-iidx-csv") + .field("playtype", "SP"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.test("Valid Heroic Verse CSV import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv") + .field("importType", "file/pli-iidx-csv") + .field("playtype", "SP"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.end(); + }); + + t.test("file/batch-manual", (t) => { + t.test("Empty import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach( + "scoreData", + GetKTDataBuffer("./batch-manual/empty-file.json"), + "empty-file.json" + ) + .field("importType", "file/batch-manual"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); + + t.equal(res.body.body.scoreIDs.length, 0, "Should have 0 successful scores."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.test("Invalid JSON", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", Buffer.from("{invalid JSON")) + .field("importType", "file/batch-manual"); + + t.equal(res.body.success, false, "Should not be successful."); + + t.equal(res.statusCode, 400); + + t.end(); + }); + + t.test("Single import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach( + "scoreData", + GetKTDataBuffer("./batch-manual/small-file.json"), + "small-file.json" + ) + .field("importType", "file/batch-manual"); + + t.equal(res.body.success, true, "Should be successful."); + + t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); + + t.equal(res.body.body.scoreIDs.length, 1, "Should have 1 successful score."); + + const scoreCount = await db.scores.find({ + scoreID: { $in: res.body.body.scoreIDs }, + }); + + t.equal( + scoreCount.length, + res.body.body.scoreIDs.length, + "All returned scoreIDs should be inserted to the DB." + ); + + t.end(); + }); + + t.end(); + }); + + t.test("file/mer-iidx", (t) => { + t.beforeEach(LoadTachiIIDXData); + + t.test("Example Import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", GetKTDataBuffer("./mer/base.json"), "base.json") + .field("importType", "file/mer-iidx"); + + t.equal(res.body.success, true, "Should be successful"); + + t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); + + t.equal(res.body.body.scoreIDs.length, 3, "Should have 3 successful scores."); + + t.end(); + }); + + t.test("Example Import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", GetKTDataBuffer("./mer/large.json"), "base.json") + .field("importType", "file/mer-iidx"); + + t.equal(res.body.success, true, "Should be successful"); + + t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores."); + + t.equal(res.body.body.scoreIDs.length, 627, "Should have 627 successful scores."); + + t.end(); + }); + + t.end(); + }); + + t.skip("file/solid-state-squad", (t) => { + t.beforeEach(LoadTachiIIDXData); + + t.test("Large Import", async (t) => { + const res = await mockApi + .post("/api/v1/import/file") + .set("Cookie", cookie) + .attach("scoreData", GetKTDataBuffer("./s3/large-example.xml"), "large.xml") + .field("importType", "file/solid-state-squad"); + + t.equal(res.body.success, true, "Should be successful"); + t.equal(res.body.body.scoreIDs.length, null, "Should parse N scores."); + + t.end(); + }); + + t.end(); + }); + + t.end(); +}); + +t.test("POST /api/v1/import/orphans", async (t) => { + const cookie = await CreateFakeAuthCookie(mockApi); + + t.beforeEach(ResetDBState); + + t.test("Should force a reprocessing of orphan scores.", async (t) => { + await db["orphan-scores"].insert([ + { + userID: 1, + timeInserted: 1000, + orphanID: "asdf", + importType: "ir/direct-manual", + errMsg: "foo", + context: { + game: "iidx", + playtype: "SP", + service: "foo", + version: null, + }, + data: { + score: 500, + lamp: "HARD CLEAR", + matchType: "songTitle", + identifier: "5.1.1.", + difficulty: "ANOTHER", + }, + game: "iidx", + }, + { + userID: 1, + timeInserted: 1000, + orphanID: "asdf2", + importType: "ir/direct-manual", + errMsg: "foo", + context: { + game: "iidx", + playtype: "SP", + service: "foo", + version: null, + }, + data: { + score: 500, + lamp: "HARD CLEAR", + matchType: "songTitle", + identifier: "TITLE NOBODY WILL USE", + difficulty: "ANOTHER", + }, + game: "iidx", + }, + ]); + + const res = await mockApi.post("/api/v1/import/orphans").set("Cookie", cookie); + + t.equal(res.statusCode, 200, "Should return 200."); + + t.equal(res.body.body.success, 1, "Should successfully reprocess one orphan."); + t.equal(res.body.body.done, 2, "Should reprocess two orphans."); + t.equal(res.body.body.failed, 1, "Should fail in de-orphaning one orphan."); + + const dbCount = await db["orphan-scores"].count({}); + + t.equal(dbCount, 1, "Should only leave one orphan-score in the database."); + + t.end(); + }); + + t.end(); +}); diff --git a/server/src/server/router/api/v1/import/router.ts b/server/src/server/router/api/v1/import/router.ts index 04ca97032..5cba19143 100644 --- a/server/src/server/router/api/v1/import/router.ts +++ b/server/src/server/router/api/v1/import/router.ts @@ -1,214 +1,214 @@ -import { Router } from "express"; -import db from "external/mongo/db"; -import { SIXTEEN_MEGABTYES } from "lib/constants/filesize"; -import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; -import CreateLogCtx from "lib/logger/logger"; -import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; -import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; -import { MakeScoreImport } from "lib/score-import/framework/score-import"; -import { ScoreImportJobData } from "lib/score-import/worker/types"; -import { ServerConfig, TachiConfig } from "lib/setup/config"; -import Prudence from "prudence"; -import { RequirePermissions } from "server/middleware/auth"; -import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload"; -import prValidate from "server/middleware/prudence-validate"; -import { APIImportTypes, FileUploadImportTypes } from "tachi-common"; -import { Random20Hex } from "utils/misc"; -import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user"; - -const logger = CreateLogCtx(__filename); - -const router: Router = Router({ mergeParams: true }); - -const ParseMultipartScoredata = CreateMulterSingleUploadMiddleware( - "scoreData", - SIXTEEN_MEGABTYES, - logger -); - -const fileImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("file/")); -const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/")); - -/** - * Import scores from a file. Expects the post request to be multipart, and to provide a scoreData file. - * - * @param importType - The import type for this file. - * @param file - The actual file. Should be passed as multipart. - * - * @name POST /api/v1/import/file - */ -router.post( - "/file", - RequirePermissions("submit_score"), - ParseMultipartScoredata, - prValidate( - { - importType: Prudence.isIn(fileImportTypes), - }, - {}, - { allowExcessKeys: true } - ), - async (req, res) => { - if (!req.file) { - return res.status(400).json({ - success: false, - description: `No file provided.`, - }); - } - - const importType = req.body.importType as FileUploadImportTypes; - - const userIntent = !!req.header("X-User-Intent"); - - if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { - const importID = Random20Hex(); - - const job: ScoreImportJobData = { - importID, - userID: req[SYMBOL_TachiAPIAuth].userID!, - userIntent, - importType, - parserArguments: [req.file, req.body], - }; - - // Fire the score import, but make no guarantees about its state. - MakeScoreImport(job); - - return res.status(202).json({ - success: true, - description: - "Import loaded into queue. You can poll the provided URL for information on when its complete.", - body: { - url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`, - importID, - }, - }); - } else { - // Fire the score import and wait for it to finish! - const importResponse = await ExpressWrappedScoreImportMain( - req[SYMBOL_TachiAPIAuth].userID!, - userIntent, - importType, - [req.file, req.body] - ); - - return res.status(importResponse.statusCode).json(importResponse.body); - } - } -); - -/** - * Import scores from another API. This typically will perform a full sync. - * @name POST /api/v1/import/from-api - */ -router.post( - "/from-api", - RequirePermissions("submit_score"), - prValidate( - { - importType: Prudence.isIn(apiImportTypes), - }, - {}, - { allowExcessKeys: true } - ), - async (req, res) => { - const importType = req.body.importType as APIImportTypes; - - const importID = Random20Hex(); - - const userID = req[SYMBOL_TachiAPIAuth].userID!; - - const userIntent = !!req.header("X-User-Intent"); - - if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { - const job: ScoreImportJobData = { - importID, - userID, - userIntent, - importType, - parserArguments: [userID], - }; - - // Fire the score import, but make no guarantees about its state. - MakeScoreImport(job); - - return res.status(202).json({ - success: true, - description: - "Import loaded into queue. You can poll the provided URL for information on when its complete.", - body: { - url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`, - importID, - }, - }); - } else { - // Fire the score import and wait for it to finish! - const importResponse = await ExpressWrappedScoreImportMain( - userID, - userIntent, - importType, - [userID] - ); - - return res.status(importResponse.statusCode).json(importResponse.body); - } - } -); - -/** - * Force Tachi to reprocess your orphanned scores. This is automatically done - * daily, but this endpoint allows users to speed that up. - * - * @name POST /api/v1/import/orphans - */ -router.post("/orphans", RequirePermissions("submit_score"), async (req, res) => { - const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id); - - logger.info(`User ${FormatUserDoc(userDoc)} forced an orphan sync.`); - - const orphans = await db["orphan-scores"].find({ - userID: userDoc.id, - }); - - // ScoreIDs are essentially userID dependent, so this is fine. - const blacklist = (await db["score-blacklist"].find({ userID: userDoc.id })).map( - (e) => e.scoreID - ); - - let done = 0; - let failed = 0; - let success = 0; - let removed = 0; - - await Promise.all( - orphans.map((or) => - ReprocessOrphan(or, blacklist, logger).then((r) => { - done++; - if (r === null) { - removed++; - } else if (r === false) { - failed++; - } else { - success++; - } - }) - ) - ); - - logger.info(`Finished attempting deorphaning.`); - - logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`); - - return res.status(200).json({ - success: true, - description: `Reprocessed ${done} orphan scores.`, - body: { - done, - failed, - success, - removed, - }, - }); -}); - -export default router; +import { Router } from "express"; +import db from "external/mongo/db"; +import { SIXTEEN_MEGABTYES } from "lib/constants/filesize"; +import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; +import CreateLogCtx from "lib/logger/logger"; +import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; +import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; +import { MakeScoreImport } from "lib/score-import/framework/score-import"; +import { ScoreImportJobData } from "lib/score-import/worker/types"; +import { ServerConfig, TachiConfig } from "lib/setup/config"; +import Prudence from "prudence"; +import { RequirePermissions } from "server/middleware/auth"; +import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload"; +import prValidate from "server/middleware/prudence-validate"; +import { APIImportTypes, FileUploadImportTypes } from "tachi-common"; +import { Random20Hex } from "utils/misc"; +import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user"; + +const logger = CreateLogCtx(__filename); + +const router: Router = Router({ mergeParams: true }); + +const ParseMultipartScoredata = CreateMulterSingleUploadMiddleware( + "scoreData", + SIXTEEN_MEGABTYES, + logger +); + +const fileImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("file/")); +const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/")); + +/** + * Import scores from a file. Expects the post request to be multipart, and to provide a scoreData file. + * + * @param importType - The import type for this file. + * @param file - The actual file. Should be passed as multipart. + * + * @name POST /api/v1/import/file + */ +router.post( + "/file", + RequirePermissions("submit_score"), + ParseMultipartScoredata, + prValidate( + { + importType: Prudence.isIn(fileImportTypes), + }, + {}, + { allowExcessKeys: true } + ), + async (req, res) => { + if (!req.file) { + return res.status(400).json({ + success: false, + description: `No file provided.`, + }); + } + + const importType = req.body.importType as FileUploadImportTypes; + + const userIntent = !!req.header("X-User-Intent"); + + if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { + const importID = Random20Hex(); + + const job: ScoreImportJobData = { + importID, + userID: req[SYMBOL_TachiAPIAuth].userID!, + userIntent, + importType, + parserArguments: [req.file, req.body], + }; + + // Fire the score import, but make no guarantees about its state. + MakeScoreImport(job); + + return res.status(202).json({ + success: true, + description: + "Import loaded into queue. You can poll the provided URL for information on when its complete.", + body: { + url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`, + importID, + }, + }); + } else { + // Fire the score import and wait for it to finish! + const importResponse = await ExpressWrappedScoreImportMain( + req[SYMBOL_TachiAPIAuth].userID!, + userIntent, + importType, + [req.file, req.body] + ); + + return res.status(importResponse.statusCode).json(importResponse.body); + } + } +); + +/** + * Import scores from another API. This typically will perform a full sync. + * @name POST /api/v1/import/from-api + */ +router.post( + "/from-api", + RequirePermissions("submit_score"), + prValidate( + { + importType: Prudence.isIn(apiImportTypes), + }, + {}, + { allowExcessKeys: true } + ), + async (req, res) => { + const importType = req.body.importType as APIImportTypes; + + const importID = Random20Hex(); + + const userID = req[SYMBOL_TachiAPIAuth].userID!; + + const userIntent = !!req.header("X-User-Intent"); + + if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) { + const job: ScoreImportJobData = { + importID, + userID, + userIntent, + importType, + parserArguments: [userID], + }; + + // Fire the score import, but make no guarantees about its state. + MakeScoreImport(job); + + return res.status(202).json({ + success: true, + description: + "Import loaded into queue. You can poll the provided URL for information on when its complete.", + body: { + url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`, + importID, + }, + }); + } else { + // Fire the score import and wait for it to finish! + const importResponse = await ExpressWrappedScoreImportMain( + userID, + userIntent, + importType, + [userID] + ); + + return res.status(importResponse.statusCode).json(importResponse.body); + } + } +); + +/** + * Force Tachi to reprocess your orphanned scores. This is automatically done + * daily, but this endpoint allows users to speed that up. + * + * @name POST /api/v1/import/orphans + */ +router.post("/orphans", RequirePermissions("submit_score"), async (req, res) => { + const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id); + + logger.info(`User ${FormatUserDoc(userDoc)} forced an orphan sync.`); + + const orphans = await db["orphan-scores"].find({ + userID: userDoc.id, + }); + + // ScoreIDs are essentially userID dependent, so this is fine. + const blacklist = (await db["score-blacklist"].find({ userID: userDoc.id })).map( + (e) => e.scoreID + ); + + let done = 0; + let failed = 0; + let success = 0; + let removed = 0; + + await Promise.all( + orphans.map((or) => + ReprocessOrphan(or, blacklist, logger).then((r) => { + done++; + if (r === null) { + removed++; + } else if (r === false) { + failed++; + } else { + success++; + } + }) + ) + ); + + logger.info(`Finished attempting deorphaning.`); + + logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`); + + return res.status(200).json({ + success: true, + description: `Reprocessed ${done} orphan scores.`, + body: { + done, + failed, + success, + removed, + }, + }); +}); + +export default router; diff --git a/server/src/server/router/api/v1/router.ts b/server/src/server/router/api/v1/router.ts index b1a2a16f0..614c0a180 100644 --- a/server/src/server/router/api/v1/router.ts +++ b/server/src/server/router/api/v1/router.ts @@ -1,44 +1,44 @@ -import { Router } from "express"; -import adminRouter from "./admin/router"; -import authRouter from "./auth/router"; -import clientsRouter from "./clients/router"; -import configRouter from "./config/router"; -import gamesRouter from "./games/router"; -import importRouter from "./import/router"; -import importsRouter from "./imports/router"; -import oauthRouter from "./oauth/router"; -import scoresRouter from "./scores/router"; -import searchRouter from "./search/router"; -import sessionsRouter from "./sessions/router"; -import statusRouter from "./status/router"; -import usersRouter from "./users/router"; - -const router: Router = Router({ mergeParams: true }); - -router.use("/admin", adminRouter); -router.use("/auth", authRouter); -router.use("/status", statusRouter); -router.use("/import", importRouter); -router.use("/imports", importsRouter); -router.use("/users", usersRouter); -router.use("/games", gamesRouter); -router.use("/search", searchRouter); -router.use("/scores", scoresRouter); -router.use("/sessions", sessionsRouter); -router.use("/oauth", oauthRouter); -router.use("/clients", clientsRouter); -router.use("/config", configRouter); - -/** - * Return a JSON 404 response if an endpoint is hit that does not exist. - * - * @name ALL /api/v1/* - */ -router.all("*", (req, res) => - res.status(404).json({ - success: false, - description: "Endpoint Not Found.", - }) -); - -export default router; +import { Router } from "express"; +import adminRouter from "./admin/router"; +import authRouter from "./auth/router"; +import clientsRouter from "./clients/router"; +import configRouter from "./config/router"; +import gamesRouter from "./games/router"; +import importRouter from "./import/router"; +import importsRouter from "./imports/router"; +import oauthRouter from "./oauth/router"; +import scoresRouter from "./scores/router"; +import searchRouter from "./search/router"; +import sessionsRouter from "./sessions/router"; +import statusRouter from "./status/router"; +import usersRouter from "./users/router"; + +const router: Router = Router({ mergeParams: true }); + +router.use("/admin", adminRouter); +router.use("/auth", authRouter); +router.use("/status", statusRouter); +router.use("/import", importRouter); +router.use("/imports", importsRouter); +router.use("/users", usersRouter); +router.use("/games", gamesRouter); +router.use("/search", searchRouter); +router.use("/scores", scoresRouter); +router.use("/sessions", sessionsRouter); +router.use("/oauth", oauthRouter); +router.use("/clients", clientsRouter); +router.use("/config", configRouter); + +/** + * Return a JSON 404 response if an endpoint is hit that does not exist. + * + * @name ALL /api/v1/* + */ +router.all("*", (req, res) => + res.status(404).json({ + success: false, + description: "Endpoint Not Found.", + }) +); + +export default router; diff --git a/server/src/server/server.ts b/server/src/server/server.ts index b49868fa1..c80603ad3 100644 --- a/server/src/server/server.ts +++ b/server/src/server/server.ts @@ -1,170 +1,170 @@ -import express, { Express } from "express"; -import "express-async-errors"; -import expressSession from "express-session"; -import { integer } from "tachi-common"; -import { RedisClient } from "external/redis/redis"; -import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; -import connectRedis from "connect-redis"; -import helmet from "helmet"; -import CreateLogCtx from "lib/logger/logger"; - -const logger = CreateLogCtx(__filename); - -let store; - -if (Environment.nodeEnv !== "test") { - logger.info("Connecting ExpressSession to Redis."); - const RedisStore = connectRedis(expressSession); - store = new RedisStore({ - host: "localhost", - port: 6379, - client: RedisClient, - prefix: TachiConfig.NAME, - }); -} - -const userSessionMiddleware = expressSession({ - // append node_env onto the end of the session name - // so we can separate tokens under the same URL. - // say for staging.kamaitachi.xyz - name: `${TachiConfig.NAME.replace(/ /gu, "_")}_SESSION`, - secret: ServerConfig.SESSION_SECRET, - store, - resave: true, - saveUninitialized: false, - cookie: { - secure: Environment.nodeEnv === "production" || ServerConfig.ENABLE_SERVER_HTTPS, - sameSite: "lax", // Very important. Without this, we're vulnerable to CSRF! - }, -}); - -const app: Express = express(); - -if (Environment.nodeEnv !== "production" && ServerConfig.CLIENT_DEV_SERVER) { - logger.warn(`Enabling CORS requests from ${ServerConfig.CLIENT_DEV_SERVER}.`, { - bootInfo: true, - }); - - // Allow CORS requests from another server (since we have our dev server hosted separately). - app.use((req, res, next) => { - res.header("Access-Control-Allow-Origin", ServerConfig.CLIENT_DEV_SERVER!); - res.header( - "Access-Control-Allow-Headers", - "Origin, X-Requested-With, Content-Type, Accept, X-User-Intent" - ); - res.header("Access-Control-Allow-Credentials", "true"); - res.header("Access-Control-Allow-Methods", "GET,POST,PATCH,PUT,DELETE,OPTIONS"); - next(); - }); - - // hack to allow all OPTIONS requests. Remember that this setting should not be on in production! - if (ServerConfig.OPTIONS_ALWAYS_SUCCEEDS) { - app.options("*", (req, res) => res.send()); - } -} else { - if (Environment.nodeEnv !== "test") { - logger.info("Enabling Helmet, as no CLIENT_DEV_SERVER was set, or we are in production.", { - bootInfo: true, - }); - } - app.use(helmet()); -} - -app.use(userSessionMiddleware); - -// Most of these options are leveraged from KTAPI - -// Pass the IP of the user up our increasingly insane chain of nginx/docker nonsense -app.set("trust proxy", ["loopback", "linklocal", "uniquelocal"]); - -// we don't allow nesting in query strings. -app.set("query parser", "simple"); - -// taken from https://nodejs.org/api/process.html#process_event_unhandledrejection -// to avoid future deprecation. -process.on("unhandledRejection", (reason, promise) => { - // @ts-expect-error reason is an error, and the logger can handle errors - // it just refuses. - logger.error(reason, { promise }); -}); - -// enable reading json bodies -// limit them so as not to choke the api -app.use(express.json({ limit: "4mb" })); - -app.use((req, res, next) => { - if (req.method !== "GET" && !req.body) { - req.body = {}; - } - - return next(); -}); - -import mainRouter from "./router/router"; -import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; -import { RequestLoggerMiddleware } from "./middleware/request-logger"; - -app.use(RequestLoggerMiddleware); - -app.use("/", mainRouter); - -// The SERVE_OWN_CDN option means that our /cdn path has to be hosted by us. In production, -// this is not the case (we have a dedicated nginx box for it running in a separate process). -// In dev, this is a pain to setup, so we can just run it locally. -if ( - ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM" && - ServerConfig.CDN_CONFIG.SAVE_LOCATION.SERVE_OWN_CDN -) { - if (Environment.nodeEnv === "production") { - logger.warn( - `Running LOCAL_FILESYSTEM OWN_CDN in production. Consider making a separate process handle your CDN for performance.`, - { bootInfo: true } - ); - } - - logger.info(`Running own CDN at ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION}.`, { - bootInfo: true, - }); - - app.use("/cdn", express.static(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION)); - app.get("/cdn/*", (req, res) => res.status(404).send("No content here.")); -} - -// completely stolen from ktapi error handler -interface ExpressJSONErr extends SyntaxError { - status: integer; - message: string; -} - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const MAIN_ERR_HANDLER: express.ErrorRequestHandler = (err, req, res, next) => { - if (err instanceof SyntaxError) { - const expErr: ExpressJSONErr = err as ExpressJSONErr; - if (expErr.status === 400 && "body" in expErr) { - logger.info(`JSON Parsing Error?`, { - url: req.originalUrl, - userID: req[SYMBOL_TachiAPIAuth]?.userID, - }); - return res.status(400).send({ success: false, description: err.message }); - } - - // else, this isn't a JSON parsing error - } - - if (err.type === "entity.too.large") { - return res.status(413).json({ - success: false, - description: "Your request body was too large. The limit is 4MB.", - }); - } - - logger.error(err, req.route); - return res.status(500).json({ - success: false, - description: "A fatal internal server error has occured.", - }); -}; - -app.use(MAIN_ERR_HANDLER); - -export default app; +import express, { Express } from "express"; +import "express-async-errors"; +import expressSession from "express-session"; +import { integer } from "tachi-common"; +import { RedisClient } from "external/redis/redis"; +import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; +import connectRedis from "connect-redis"; +import helmet from "helmet"; +import CreateLogCtx from "lib/logger/logger"; + +const logger = CreateLogCtx(__filename); + +let store; + +if (Environment.nodeEnv !== "test") { + logger.info("Connecting ExpressSession to Redis."); + const RedisStore = connectRedis(expressSession); + store = new RedisStore({ + host: "localhost", + port: 6379, + client: RedisClient, + prefix: TachiConfig.NAME, + }); +} + +const userSessionMiddleware = expressSession({ + // append node_env onto the end of the session name + // so we can separate tokens under the same URL. + // say for staging.kamaitachi.xyz + name: `${TachiConfig.NAME.replace(/ /gu, "_")}_SESSION`, + secret: ServerConfig.SESSION_SECRET, + store, + resave: true, + saveUninitialized: false, + cookie: { + secure: Environment.nodeEnv === "production" || ServerConfig.ENABLE_SERVER_HTTPS, + sameSite: "lax", // Very important. Without this, we're vulnerable to CSRF! + }, +}); + +const app: Express = express(); + +if (Environment.nodeEnv !== "production" && ServerConfig.CLIENT_DEV_SERVER) { + logger.warn(`Enabling CORS requests from ${ServerConfig.CLIENT_DEV_SERVER}.`, { + bootInfo: true, + }); + + // Allow CORS requests from another server (since we have our dev server hosted separately). + app.use((req, res, next) => { + res.header("Access-Control-Allow-Origin", ServerConfig.CLIENT_DEV_SERVER!); + res.header( + "Access-Control-Allow-Headers", + "Origin, X-Requested-With, Content-Type, Accept, X-User-Intent" + ); + res.header("Access-Control-Allow-Credentials", "true"); + res.header("Access-Control-Allow-Methods", "GET,POST,PATCH,PUT,DELETE,OPTIONS"); + next(); + }); + + // hack to allow all OPTIONS requests. Remember that this setting should not be on in production! + if (ServerConfig.OPTIONS_ALWAYS_SUCCEEDS) { + app.options("*", (req, res) => res.send()); + } +} else { + if (Environment.nodeEnv !== "test") { + logger.info("Enabling Helmet, as no CLIENT_DEV_SERVER was set, or we are in production.", { + bootInfo: true, + }); + } + app.use(helmet()); +} + +app.use(userSessionMiddleware); + +// Most of these options are leveraged from KTAPI + +// Pass the IP of the user up our increasingly insane chain of nginx/docker nonsense +app.set("trust proxy", ["loopback", "linklocal", "uniquelocal"]); + +// we don't allow nesting in query strings. +app.set("query parser", "simple"); + +// taken from https://nodejs.org/api/process.html#process_event_unhandledrejection +// to avoid future deprecation. +process.on("unhandledRejection", (reason, promise) => { + // @ts-expect-error reason is an error, and the logger can handle errors + // it just refuses. + logger.error(reason, { promise }); +}); + +// enable reading json bodies +// limit them so as not to choke the api +app.use(express.json({ limit: "4mb" })); + +app.use((req, res, next) => { + if (req.method !== "GET" && !req.body) { + req.body = {}; + } + + return next(); +}); + +import mainRouter from "./router/router"; +import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; +import { RequestLoggerMiddleware } from "./middleware/request-logger"; + +app.use(RequestLoggerMiddleware); + +app.use("/", mainRouter); + +// The SERVE_OWN_CDN option means that our /cdn path has to be hosted by us. In production, +// this is not the case (we have a dedicated nginx box for it running in a separate process). +// In dev, this is a pain to setup, so we can just run it locally. +if ( + ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM" && + ServerConfig.CDN_CONFIG.SAVE_LOCATION.SERVE_OWN_CDN +) { + if (Environment.nodeEnv === "production") { + logger.warn( + `Running LOCAL_FILESYSTEM OWN_CDN in production. Consider making a separate process handle your CDN for performance.`, + { bootInfo: true } + ); + } + + logger.info(`Running own CDN at ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION}.`, { + bootInfo: true, + }); + + app.use("/cdn", express.static(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION)); + app.get("/cdn/*", (req, res) => res.status(404).send("No content here.")); +} + +// completely stolen from ktapi error handler +interface ExpressJSONErr extends SyntaxError { + status: integer; + message: string; +} + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const MAIN_ERR_HANDLER: express.ErrorRequestHandler = (err, req, res, next) => { + if (err instanceof SyntaxError) { + const expErr: ExpressJSONErr = err as ExpressJSONErr; + if (expErr.status === 400 && "body" in expErr) { + logger.info(`JSON Parsing Error?`, { + url: req.originalUrl, + userID: req[SYMBOL_TachiAPIAuth]?.userID, + }); + return res.status(400).send({ success: false, description: err.message }); + } + + // else, this isn't a JSON parsing error + } + + if (err.type === "entity.too.large") { + return res.status(413).json({ + success: false, + description: "Your request body was too large. The limit is 4MB.", + }); + } + + logger.error(err, req.route); + return res.status(500).json({ + success: false, + description: "A fatal internal server error has occured.", + }); +}; + +app.use(MAIN_ERR_HANDLER); + +export default app; diff --git a/server/src/test-utils/mock-api.ts b/server/src/test-utils/mock-api.ts index 345f5a1cd..a30f23fab 100644 --- a/server/src/test-utils/mock-api.ts +++ b/server/src/test-utils/mock-api.ts @@ -1,18 +1,18 @@ -import supertest from "supertest"; -import CreateLogCtx from "lib/logger/logger"; - -const logger = CreateLogCtx(__filename); - -import server from "../server/server"; - -logger.verbose("Creating Mock Server Connection..."); -const connection = server.listen(); - -logger.verbose("Connecting to Supertest..."); -const mockApi = supertest(connection); - -export function CloseServerConnection() { - connection.close(); -} - -export default mockApi; +import supertest from "supertest"; +import CreateLogCtx from "lib/logger/logger"; + +const logger = CreateLogCtx(__filename); + +import server from "../server/server"; + +logger.verbose("Creating Mock Server Connection..."); +const connection = server.listen(); + +logger.verbose("Connecting to Supertest..."); +const mockApi = supertest(connection); + +export function CloseServerConnection() { + connection.close(); +} + +export default mockApi; diff --git a/server/src/test-utils/mock-fetch.ts b/server/src/test-utils/mock-fetch.ts index e16a8ec78..16f4bf380 100644 --- a/server/src/test-utils/mock-fetch.ts +++ b/server/src/test-utils/mock-fetch.ts @@ -1,32 +1,32 @@ -// We know! these are mock functions. -/* eslint-disable require-await */ -import { Response } from "node-fetch"; -import { NodeFetch } from "utils/fetch"; - -/** - * Creates a basic Fetch function used for statusCode checks. - */ -export function MockBasicFetch(data: Partial) { - return (async () => data) as unknown as NodeFetch; -} - -/** - * Creates a fake "fetch" function which takes a url and returns - * a fake JSON -> data function at that key. - */ -export function MockJSONFetch(urlDataMap: Record) { - return (async (url: string) => { - if (urlDataMap[url]) { - return { - status: 200, - json: async () => urlDataMap[url], - }; - } - - throw new Error( - `Unexpected url ${url} - No Data Present? Valid urls are ${Object.keys(urlDataMap).join( - ", " - )}` - ); - }) as NodeFetch; -} +// We know! these are mock functions. +/* eslint-disable require-await */ +import { Response } from "node-fetch"; +import { NodeFetch } from "utils/fetch"; + +/** + * Creates a basic Fetch function used for statusCode checks. + */ +export function MockBasicFetch(data: Partial) { + return (async () => data) as unknown as NodeFetch; +} + +/** + * Creates a fake "fetch" function which takes a url and returns + * a fake JSON -> data function at that key. + */ +export function MockJSONFetch(urlDataMap: Record) { + return (async (url: string) => { + if (urlDataMap[url]) { + return { + status: 200, + json: async () => urlDataMap[url], + }; + } + + throw new Error( + `Unexpected url ${url} - No Data Present? Valid urls are ${Object.keys(urlDataMap).join( + ", " + )}` + ); + }) as NodeFetch; +} diff --git a/server/src/test-utils/resets.ts b/server/src/test-utils/resets.ts index e5ef362e1..0362e6923 100644 --- a/server/src/test-utils/resets.ts +++ b/server/src/test-utils/resets.ts @@ -1,113 +1,113 @@ -/** - * Resets the state of the database. - */ -import db from "external/mongo/db"; -import fs from "fs"; -import path from "path"; -import CreateLogCtx from "lib/logger/logger"; -// im installing an entire library for rm rf... -import rimraf from "rimraf"; -import { SetIndexes } from "external/mongo/indexes"; -import { Environment, ServerConfig } from "lib/setup/config"; - -if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { - throw new Error( - `Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)` - ); -} - -const logger = CreateLogCtx(__filename); - -const DATA_DIR = path.join(__dirname, "./mock-db"); - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const CACHE: Record = {}; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -async function ResetState(data: any[], collection: any) { - await collection.remove({}); - - await collection.insert(data); -} - -function GetAndCache(filename: string, fileLoc: string) { - let collection; - if (filename.startsWith("songs-")) { - // @ts-expect-error it's right, but we know what we're doing! - collection = db.songs[filename.split("-")[1]]; - } else if (filename.startsWith("charts-")) { - // @ts-expect-error see above - collection = db.charts[filename.split("-")[1]]; - } else { - // @ts-expect-error see above - collection = db[filename]; - } - - if (CACHE[filename]) { - return { data: CACHE[filename], collection }; - } - - const data = JSON.parse(fs.readFileSync(fileLoc, "utf-8")); - - if (!Array.isArray(data)) { - throw new Error(`Panic, ${filename} not JSONArray?`); - } - - CACHE[filename] = data; - - return { data, collection }; -} - -let CACHE_FILENAMES: string[]; - -export default async function ResetDBState() { - let files; - if (CACHE_FILENAMES) { - files = CACHE_FILENAMES; - } else { - files = fs.readdirSync(DATA_DIR); - CACHE_FILENAMES = files; - } - - const promises = []; - - for (const file of files) { - const filename = path.basename(file, ".json"); - const fileLoc = path.join(DATA_DIR, file); - - const { data, collection } = GetAndCache(filename, fileLoc); - - promises.push(ResetState(data, collection)); - } - - await Promise.all(promises); -} - -export function ResetCDN() { - return new Promise((resolve, reject) => { - if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { - throw new Error( - `Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)` - ); - } - - rimraf(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION, (err) => { - if (err) { - reject(err); - } - resolve(); - }); - }); -} - -export async function SetIndexesForDB() { - await ResetDBState(); - const url = `${Environment.mongoUrl}/testingdb`; - - logger.info(`Setting indexes for ${url}`); - - await SetIndexes(url, true); - - logger.info(`Done.`); - return true; -} +/** + * Resets the state of the database. + */ +import db from "external/mongo/db"; +import fs from "fs"; +import path from "path"; +import CreateLogCtx from "lib/logger/logger"; +// im installing an entire library for rm rf... +import rimraf from "rimraf"; +import { SetIndexes } from "external/mongo/indexes"; +import { Environment, ServerConfig } from "lib/setup/config"; + +if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { + throw new Error( + `Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)` + ); +} + +const logger = CreateLogCtx(__filename); + +const DATA_DIR = path.join(__dirname, "./mock-db"); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const CACHE: Record = {}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function ResetState(data: any[], collection: any) { + await collection.remove({}); + + await collection.insert(data); +} + +function GetAndCache(filename: string, fileLoc: string) { + let collection; + if (filename.startsWith("songs-")) { + // @ts-expect-error it's right, but we know what we're doing! + collection = db.songs[filename.split("-")[1]]; + } else if (filename.startsWith("charts-")) { + // @ts-expect-error see above + collection = db.charts[filename.split("-")[1]]; + } else { + // @ts-expect-error see above + collection = db[filename]; + } + + if (CACHE[filename]) { + return { data: CACHE[filename], collection }; + } + + const data = JSON.parse(fs.readFileSync(fileLoc, "utf-8")); + + if (!Array.isArray(data)) { + throw new Error(`Panic, ${filename} not JSONArray?`); + } + + CACHE[filename] = data; + + return { data, collection }; +} + +let CACHE_FILENAMES: string[]; + +export default async function ResetDBState() { + let files; + if (CACHE_FILENAMES) { + files = CACHE_FILENAMES; + } else { + files = fs.readdirSync(DATA_DIR); + CACHE_FILENAMES = files; + } + + const promises = []; + + for (const file of files) { + const filename = path.basename(file, ".json"); + const fileLoc = path.join(DATA_DIR, file); + + const { data, collection } = GetAndCache(filename, fileLoc); + + promises.push(ResetState(data, collection)); + } + + await Promise.all(promises); +} + +export function ResetCDN() { + return new Promise((resolve, reject) => { + if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") { + throw new Error( + `Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)` + ); + } + + rimraf(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION, (err) => { + if (err) { + reject(err); + } + resolve(); + }); + }); +} + +export async function SetIndexesForDB() { + await ResetDBState(); + const url = `${Environment.mongoUrl}/testingdb`; + + logger.info(`Setting indexes for ${url}`); + + await SetIndexes(url, true); + + logger.info(`Done.`); + return true; +} diff --git a/server/src/test-utils/server.ts b/server/src/test-utils/server.ts index fce93ab1f..f8806cf6f 100644 --- a/server/src/test-utils/server.ts +++ b/server/src/test-utils/server.ts @@ -1,7 +1,7 @@ -import server from "../server/server"; -import supertest from "supertest"; - -const mockServer = supertest(server); - -// mock the server for requests -export default mockServer; +import server from "../server/server"; +import supertest from "supertest"; + +const mockServer = supertest(server); + +// mock the server for requests +export default mockServer; diff --git a/server/src/test-utils/test-data.ts b/server/src/test-utils/test-data.ts index d25677325..cd44da7e2 100644 --- a/server/src/test-utils/test-data.ts +++ b/server/src/test-utils/test-data.ts @@ -1,241 +1,241 @@ -import db from "external/mongo/db"; -import fs from "fs"; -import { - USC_DEFAULT_HOLD, - USC_DEFAULT_MISS, - USC_DEFAULT_NEAR, - USC_DEFAULT_PERFECT, - USC_DEFAULT_SLAM, -} from "lib/constants/usc-ir"; -import { DryScore } from "lib/score-import/framework/common/types"; -import { BarbatosScore } from "lib/score-import/import-types/ir/barbatos/types"; -import { KsHookSV3CScore } from "lib/score-import/import-types/ir/kshook-sv3c/types"; -import path from "path"; -import { USCClientScore } from "server/router/ir/usc/_playtype/types"; -import { - ChartDocument, - FolderDocument, - GoalDocument, - MilestoneDocument, - PBScoreDocument, - ScoreDocument, - SongDocument, - UserGoalDocument, -} from "tachi-common"; - -const file = (name: string) => path.join(__dirname, "/test-data", name); - -export const GetKTDataJSON = (name: string) => JSON.parse(fs.readFileSync(file(name), "utf-8")); -export const GetKTDataBuffer = (name: string) => fs.readFileSync(file(name)); - -export const TestingIIDXSPDryScore = GetKTDataJSON( - "./tachi/iidx-dryscore.json" -) as DryScore<"iidx:SP">; - -export const TestingIIDXSPScorePB = GetKTDataJSON( - "./tachi/iidx-scorepb.json" -) as PBScoreDocument<"iidx:SP">; -export const TestingIIDXSPScore = GetKTDataJSON( - "./tachi/iidx-score.json" -) as ScoreDocument<"iidx:SP">; - -export const TestingKsHookSV3CScore = GetKTDataJSON("./kshook-sv3c/base.json") as KsHookSV3CScore; - -export const Testing511SPA = GetKTDataJSON("./tachi/iidx-511spa.json") as ChartDocument<"iidx:SP">; - -export const Testing511Song = GetKTDataJSON("./tachi/iidx-511-song.json") as SongDocument<"iidx">; - -export const TestingAlbidaADV = GetKTDataJSON( - "./tachi/sdvx-albida-chart.json" -) as ChartDocument<"sdvx:Single">; - -export const TestingSDVXAlbidaSong = GetKTDataJSON( - "./tachi/sdvx-albida-song.json" -) as SongDocument<"sdvx">; - -export const TestingDoraChart = GetKTDataJSON( - "./tachi/gitadora-ithinkabtyou.json" -) as ChartDocument<"gitadora:Dora">; - -export const TestingSDVXSingleDryScore = GetKTDataJSON( - "./tachi/sdvx-dryscore.json" -) as DryScore<"sdvx:Single">; - -export const TestingGITADORADoraDryScore = GetKTDataJSON( - "./tachi/gitadora-dryscore.json" -) as DryScore<"gitadora:Dora">; - -export const TestingIIDXEamusementCSV26 = GetKTDataBuffer( - "./eamusement-iidx-csv/pre-leggendaria.csv" -); -export const TestingIIDXEamusementCSV27 = GetKTDataBuffer( - "./eamusement-iidx-csv/post-leggendaria.csv" -); - -export const TestingBarbatosScore: BarbatosScore = GetKTDataJSON("./barbatos/base.json"); - -export const HC511Goal: GoalDocument = { - charts: { - type: "single", - data: Testing511SPA.chartID, - }, - createdBy: 1, - game: "iidx", - goalID: "mock_goalID", - playtype: "SP", - timeAdded: 0, - title: "HC 5.1.1. SPA", - criteria: { - mode: "single", - value: 5, - key: "scoreData.lampIndex", - }, -}; - -export const HC511UserGoal: UserGoalDocument = { - achieved: false, - timeAchieved: null, - game: "iidx", - playtype: "SP", - goalID: "mock_goalID", - lastInteraction: null, - outOf: 5, - outOfHuman: "HARD CLEAR", - progress: null, - progressHuman: "NO DATA", - timeSet: 0, - userID: 1, -}; - -export const TestingIIDXFolderSP10: FolderDocument = { - title: "Level 10", - game: "iidx", - playtype: "SP", - type: "charts", - folderID: "ed9d8c734447ce67d7135c0067441a98cc81aeaf", - data: { - level: "10", - "flags¬IN BASE GAME": true, - }, - searchTerms: [], - inactive: false, -}; - -export const TestingIIDXSPMilestone: MilestoneDocument = { - createdBy: 1, - criteria: { - type: "all", - }, - desc: "testing milestone", - game: "iidx", - playtype: "SP", - group: "ExampleGroup", - groupIndex: 1, - milestoneID: "example_milestone_id", - name: "Example Milestone", - milestoneData: [ - { - title: "Group1", - desc: "Foo", - goals: [ - { - goalID: "eg_goal_1", - }, - { - goalID: "eg_goal_2", - }, - ], - }, - { - title: "Group2", - desc: "Bar", - goals: [ - { - goalID: "eg_goal_3", - }, - { - goalID: "eg_goal_4", - }, - ], - }, - ], -}; - -let KTDATA_CACHE: { songs: unknown[]; charts: unknown[] } | undefined; - -export async function LoadTachiIIDXData() { - let songs; - let charts; - - if (KTDATA_CACHE) { - songs = KTDATA_CACHE.songs; - charts = KTDATA_CACHE.charts; - } else { - songs = GetKTDataJSON("./tachi/tachi-songs-iidx.json"); - charts = GetKTDataJSON("./tachi/tachi-charts-iidx.json"); - KTDATA_CACHE = { songs, charts }; - } - - await db.songs.iidx.remove({}); - await db.songs.iidx.insert(songs); - await db.charts.iidx.remove({}); - await db.charts.iidx.insert(charts); -} - -export const barbScore: BarbatosScore = { - clear_type: 2, - did_fail: false, - difficulty: 1, - critical: 100, - error: 5, - near_total: 50, - near_fast: 40, - near_slow: 10, - gauge_type: 1, - is_skill_analyser: false, - level: 10, - max_chain: 100, - percent: 90, - score: 9_000_000, - song_id: 1, -}; - -export const uscChart: ChartDocument<"usc:Controller" | "usc:Keyboard"> = { - rgcID: null, - chartID: "USC_CHART_ID", - difficulty: "NOV", - songID: 1, - playtype: "Keyboard", - levelNum: 1, - level: "1", - data: { - hashSHA1: "USC_CHART_HASH", - isOfficial: false, - }, - tierlistInfo: {}, - isPrimary: true, - versions: [], -}; - -export const uscScore: USCClientScore = { - crit: 100, - error: 5, - near: 50, - gauge: 0.8, - options: { - autoFlags: 0, - gaugeOpt: 0, - gaugeType: 1, - mirror: true, - random: false, - }, - score: 9_000_000, - timestamp: 0, - windows: { - good: USC_DEFAULT_NEAR, - perfect: USC_DEFAULT_PERFECT, - hold: USC_DEFAULT_HOLD, - miss: USC_DEFAULT_MISS, - slam: USC_DEFAULT_SLAM, - }, -}; +import db from "external/mongo/db"; +import fs from "fs"; +import { + USC_DEFAULT_HOLD, + USC_DEFAULT_MISS, + USC_DEFAULT_NEAR, + USC_DEFAULT_PERFECT, + USC_DEFAULT_SLAM, +} from "lib/constants/usc-ir"; +import { DryScore } from "lib/score-import/framework/common/types"; +import { BarbatosScore } from "lib/score-import/import-types/ir/barbatos/types"; +import { KsHookSV3CScore } from "lib/score-import/import-types/ir/kshook-sv3c/types"; +import path from "path"; +import { USCClientScore } from "server/router/ir/usc/_playtype/types"; +import { + ChartDocument, + FolderDocument, + GoalDocument, + MilestoneDocument, + PBScoreDocument, + ScoreDocument, + SongDocument, + UserGoalDocument, +} from "tachi-common"; + +const file = (name: string) => path.join(__dirname, "/test-data", name); + +export const GetKTDataJSON = (name: string) => JSON.parse(fs.readFileSync(file(name), "utf-8")); +export const GetKTDataBuffer = (name: string) => fs.readFileSync(file(name)); + +export const TestingIIDXSPDryScore = GetKTDataJSON( + "./tachi/iidx-dryscore.json" +) as DryScore<"iidx:SP">; + +export const TestingIIDXSPScorePB = GetKTDataJSON( + "./tachi/iidx-scorepb.json" +) as PBScoreDocument<"iidx:SP">; +export const TestingIIDXSPScore = GetKTDataJSON( + "./tachi/iidx-score.json" +) as ScoreDocument<"iidx:SP">; + +export const TestingKsHookSV3CScore = GetKTDataJSON("./kshook-sv3c/base.json") as KsHookSV3CScore; + +export const Testing511SPA = GetKTDataJSON("./tachi/iidx-511spa.json") as ChartDocument<"iidx:SP">; + +export const Testing511Song = GetKTDataJSON("./tachi/iidx-511-song.json") as SongDocument<"iidx">; + +export const TestingAlbidaADV = GetKTDataJSON( + "./tachi/sdvx-albida-chart.json" +) as ChartDocument<"sdvx:Single">; + +export const TestingSDVXAlbidaSong = GetKTDataJSON( + "./tachi/sdvx-albida-song.json" +) as SongDocument<"sdvx">; + +export const TestingDoraChart = GetKTDataJSON( + "./tachi/gitadora-ithinkabtyou.json" +) as ChartDocument<"gitadora:Dora">; + +export const TestingSDVXSingleDryScore = GetKTDataJSON( + "./tachi/sdvx-dryscore.json" +) as DryScore<"sdvx:Single">; + +export const TestingGITADORADoraDryScore = GetKTDataJSON( + "./tachi/gitadora-dryscore.json" +) as DryScore<"gitadora:Dora">; + +export const TestingIIDXEamusementCSV26 = GetKTDataBuffer( + "./eamusement-iidx-csv/pre-leggendaria.csv" +); +export const TestingIIDXEamusementCSV27 = GetKTDataBuffer( + "./eamusement-iidx-csv/post-leggendaria.csv" +); + +export const TestingBarbatosScore: BarbatosScore = GetKTDataJSON("./barbatos/base.json"); + +export const HC511Goal: GoalDocument = { + charts: { + type: "single", + data: Testing511SPA.chartID, + }, + createdBy: 1, + game: "iidx", + goalID: "mock_goalID", + playtype: "SP", + timeAdded: 0, + title: "HC 5.1.1. SPA", + criteria: { + mode: "single", + value: 5, + key: "scoreData.lampIndex", + }, +}; + +export const HC511UserGoal: UserGoalDocument = { + achieved: false, + timeAchieved: null, + game: "iidx", + playtype: "SP", + goalID: "mock_goalID", + lastInteraction: null, + outOf: 5, + outOfHuman: "HARD CLEAR", + progress: null, + progressHuman: "NO DATA", + timeSet: 0, + userID: 1, +}; + +export const TestingIIDXFolderSP10: FolderDocument = { + title: "Level 10", + game: "iidx", + playtype: "SP", + type: "charts", + folderID: "ed9d8c734447ce67d7135c0067441a98cc81aeaf", + data: { + level: "10", + "flags¬IN BASE GAME": true, + }, + searchTerms: [], + inactive: false, +}; + +export const TestingIIDXSPMilestone: MilestoneDocument = { + createdBy: 1, + criteria: { + type: "all", + }, + desc: "testing milestone", + game: "iidx", + playtype: "SP", + group: "ExampleGroup", + groupIndex: 1, + milestoneID: "example_milestone_id", + name: "Example Milestone", + milestoneData: [ + { + title: "Group1", + desc: "Foo", + goals: [ + { + goalID: "eg_goal_1", + }, + { + goalID: "eg_goal_2", + }, + ], + }, + { + title: "Group2", + desc: "Bar", + goals: [ + { + goalID: "eg_goal_3", + }, + { + goalID: "eg_goal_4", + }, + ], + }, + ], +}; + +let KTDATA_CACHE: { songs: unknown[]; charts: unknown[] } | undefined; + +export async function LoadTachiIIDXData() { + let songs; + let charts; + + if (KTDATA_CACHE) { + songs = KTDATA_CACHE.songs; + charts = KTDATA_CACHE.charts; + } else { + songs = GetKTDataJSON("./tachi/tachi-songs-iidx.json"); + charts = GetKTDataJSON("./tachi/tachi-charts-iidx.json"); + KTDATA_CACHE = { songs, charts }; + } + + await db.songs.iidx.remove({}); + await db.songs.iidx.insert(songs); + await db.charts.iidx.remove({}); + await db.charts.iidx.insert(charts); +} + +export const barbScore: BarbatosScore = { + clear_type: 2, + did_fail: false, + difficulty: 1, + critical: 100, + error: 5, + near_total: 50, + near_fast: 40, + near_slow: 10, + gauge_type: 1, + is_skill_analyser: false, + level: 10, + max_chain: 100, + percent: 90, + score: 9_000_000, + song_id: 1, +}; + +export const uscChart: ChartDocument<"usc:Controller" | "usc:Keyboard"> = { + rgcID: null, + chartID: "USC_CHART_ID", + difficulty: "NOV", + songID: 1, + playtype: "Keyboard", + levelNum: 1, + level: "1", + data: { + hashSHA1: "USC_CHART_HASH", + isOfficial: false, + }, + tierlistInfo: {}, + isPrimary: true, + versions: [], +}; + +export const uscScore: USCClientScore = { + crit: 100, + error: 5, + near: 50, + gauge: 0.8, + options: { + autoFlags: 0, + gaugeOpt: 0, + gaugeType: 1, + mirror: true, + random: false, + }, + score: 9_000_000, + timestamp: 0, + windows: { + good: USC_DEFAULT_NEAR, + perfect: USC_DEFAULT_PERFECT, + hold: USC_DEFAULT_HOLD, + miss: USC_DEFAULT_MISS, + slam: USC_DEFAULT_SLAM, + }, +}; diff --git a/server/src/utils/db.test.ts b/server/src/utils/db.test.ts index 4f4bbd71e..96f38414a 100644 --- a/server/src/utils/db.test.ts +++ b/server/src/utils/db.test.ts @@ -1,53 +1,53 @@ -import db from "external/mongo/db"; -import t from "tap"; -import ResetDBState from "test-utils/resets"; -import { DecrementCounterValue, GetNextCounterValue } from "./db"; - -t.test("#GetNextCounterValue", (t) => { - t.beforeEach(ResetDBState); - - t.test("Increments on valid counter hit", async (t) => { - const response = await GetNextCounterValue("real-counter"); - - // database starts with this at one - t.equal(response, 2, "Counter should return the current number stored"); - - const dbData = await db.counters.findOne({ - counterName: "real-counter", - }); - - t.equal(dbData!.value, 3, "Counter should increment after being hit"); - }); - - t.rejects( - () => GetNextCounterValue("fake-counter"), - "Could not find sequence document for fake-counter." - ); - - t.end(); -}); - -t.test("#DecrementCounterValue", (t) => { - t.beforeEach(ResetDBState); - - t.test("Should decrease a counter.", async (t) => { - await db.counters.insert({ counterName: "foo", value: 3 }); - - const res = await DecrementCounterValue("foo"); - - t.equal(res, 2); - - const doc = await db.counters.findOne({ counterName: "foo" }); - - t.equal(doc?.value, 2); - - t.end(); - }); - - t.rejects( - () => GetNextCounterValue("fake-counter"), - "Could not find sequence document for fake-counter." - ); - - t.end(); -}); +import db from "external/mongo/db"; +import t from "tap"; +import ResetDBState from "test-utils/resets"; +import { DecrementCounterValue, GetNextCounterValue } from "./db"; + +t.test("#GetNextCounterValue", (t) => { + t.beforeEach(ResetDBState); + + t.test("Increments on valid counter hit", async (t) => { + const response = await GetNextCounterValue("real-counter"); + + // database starts with this at one + t.equal(response, 2, "Counter should return the current number stored"); + + const dbData = await db.counters.findOne({ + counterName: "real-counter", + }); + + t.equal(dbData!.value, 3, "Counter should increment after being hit"); + }); + + t.rejects( + () => GetNextCounterValue("fake-counter"), + "Could not find sequence document for fake-counter." + ); + + t.end(); +}); + +t.test("#DecrementCounterValue", (t) => { + t.beforeEach(ResetDBState); + + t.test("Should decrease a counter.", async (t) => { + await db.counters.insert({ counterName: "foo", value: 3 }); + + const res = await DecrementCounterValue("foo"); + + t.equal(res, 2); + + const doc = await db.counters.findOne({ counterName: "foo" }); + + t.equal(doc?.value, 2); + + t.end(); + }); + + t.rejects( + () => GetNextCounterValue("fake-counter"), + "Could not find sequence document for fake-counter." + ); + + t.end(); +}); diff --git a/server/src/utils/db.ts b/server/src/utils/db.ts index c4f81aaf7..0ec029c08 100644 --- a/server/src/utils/db.ts +++ b/server/src/utils/db.ts @@ -1,70 +1,70 @@ -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; -const logger = CreateLogCtx(__filename); -import { integer, Game, ScoreDocument, PBScoreDocument } from "tachi-common"; - -export async function GetNextCounterValue(counterName: string): Promise { - const sequenceDoc = await db.counters.findOneAndUpdate( - { - counterName, - }, - { - $inc: { - value: 1, - }, - }, - { - // this is marked as deprecated, but it shouldn't be, as returnDocument: "before" - // does nothing. - returnOriginal: true, - } - ); - - if (!sequenceDoc) { - logger.error(`Could not find sequence document for ${counterName}`); - throw new Error(`Could not find sequence document for ${counterName}.`); - } - - return sequenceDoc.value; -} - -export async function DecrementCounterValue(counterName: string): Promise { - logger.verbose(`Decrementing Counter Value ${counterName}.`); - - const sequenceDoc = await db.counters.findOneAndUpdate( - { - counterName, - }, - { - $inc: { - value: -1, - }, - }, - { - returnOriginal: false, - } - ); - - if (!sequenceDoc) { - logger.error(`Could not find sequence document for ${counterName}`); - throw new Error(`Could not find sequence document for ${counterName}.`); - } - - return sequenceDoc.value; -} - -export async function GetRelevantSongsAndCharts( - scores: (ScoreDocument | PBScoreDocument)[], - game: Game -) { - const [songs, charts] = await Promise.all([ - db.songs[game].find({ - id: { $in: scores.map((e) => e.songID) }, - }), - db.charts[game].find({ - chartID: { $in: scores.map((e) => e.chartID) }, - }), - ]); - - return { songs, charts }; -} +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +const logger = CreateLogCtx(__filename); +import { integer, Game, ScoreDocument, PBScoreDocument } from "tachi-common"; + +export async function GetNextCounterValue(counterName: string): Promise { + const sequenceDoc = await db.counters.findOneAndUpdate( + { + counterName, + }, + { + $inc: { + value: 1, + }, + }, + { + // this is marked as deprecated, but it shouldn't be, as returnDocument: "before" + // does nothing. + returnOriginal: true, + } + ); + + if (!sequenceDoc) { + logger.error(`Could not find sequence document for ${counterName}`); + throw new Error(`Could not find sequence document for ${counterName}.`); + } + + return sequenceDoc.value; +} + +export async function DecrementCounterValue(counterName: string): Promise { + logger.verbose(`Decrementing Counter Value ${counterName}.`); + + const sequenceDoc = await db.counters.findOneAndUpdate( + { + counterName, + }, + { + $inc: { + value: -1, + }, + }, + { + returnOriginal: false, + } + ); + + if (!sequenceDoc) { + logger.error(`Could not find sequence document for ${counterName}`); + throw new Error(`Could not find sequence document for ${counterName}.`); + } + + return sequenceDoc.value; +} + +export async function GetRelevantSongsAndCharts( + scores: (ScoreDocument | PBScoreDocument)[], + game: Game +) { + const [songs, charts] = await Promise.all([ + db.songs[game].find({ + id: { $in: scores.map((e) => e.songID) }, + }), + db.charts[game].find({ + chartID: { $in: scores.map((e) => e.chartID) }, + }), + ]); + + return { songs, charts }; +} diff --git a/server/src/utils/fetch.ts b/server/src/utils/fetch.ts index bd1ef7f8b..3f6aafffc 100644 --- a/server/src/utils/fetch.ts +++ b/server/src/utils/fetch.ts @@ -1,13 +1,13 @@ -import { Environment } from "lib/setup/config"; -import nodeFetch from "node-fetch"; - -const fetch = - Environment.nodeEnv === "test" - ? () => { - throw new Error("Cannot use real fetch inside testing env!"); - } - : nodeFetch; - -export type NodeFetch = (url: RequestInfo, init?: RequestInit | undefined) => Promise; - -export default fetch as unknown as NodeFetch; +import { Environment } from "lib/setup/config"; +import nodeFetch from "node-fetch"; + +const fetch = + Environment.nodeEnv === "test" + ? () => { + throw new Error("Cannot use real fetch inside testing env!"); + } + : nodeFetch; + +export type NodeFetch = (url: RequestInfo, init?: RequestInit | undefined) => Promise; + +export default fetch as unknown as NodeFetch; diff --git a/server/src/utils/queries/charts.ts b/server/src/utils/queries/charts.ts index 475653920..615f9e52b 100644 --- a/server/src/utils/queries/charts.ts +++ b/server/src/utils/queries/charts.ts @@ -1,319 +1,319 @@ -import db from "external/mongo/db"; -import { FilterQuery } from "mongodb"; -import { - ChartDocument, - Difficulties, - Game, - GPTSupportedVersions, - IDStrings, - integer, - Playtypes, -} from "tachi-common"; - -export function FindChartWithChartID(game: Game, chartID: string) { - return db.charts[game].findOne({ chartID }); -} - -/** - * Find chart with PlaytypeDifficulty. This only finds charts that have `isPrimary` set to true. - * If you want to find charts that are not primary, you need to use PTDFVersion. - * @see FindChartWithPTDFVersion - */ -export function FindChartWithPTDF< - G extends Game = Game, - P extends Playtypes[G] = Playtypes[G], - I extends IDStrings = IDStrings ->(game: G, songID: integer, playtype: P, difficulty: Difficulties[I]) { - return db.charts[game].findOne({ - songID: songID, - playtype: playtype, - difficulty: difficulty, - isPrimary: true, - }); -} - -/** - * Find chart with Playtype, Difficulty and a given version. This does not necessarily return a chart that has - * `isPrimary` set. - */ -export function FindChartWithPTDFVersion< - G extends Game = Game, - P extends Playtypes[G] = Playtypes[G], - I extends IDStrings = IDStrings ->( - game: G, - songID: integer, - playtype: P, - difficulty: Difficulties[I], - version: GPTSupportedVersions[I] -) { - return db.charts[game].findOne({ - songID: songID, - playtype: playtype, - difficulty: difficulty, - versions: version, - }); -} - -/** - * Finds a DDR Chart based on its "song hash". - * Songs in DDR have a consistent checksum-like identifier used on the e-amusement website. - * We can use this to locate a chart by combining it with a playtype and difficulty. - * - * Despite the potentially confusing name of "songHash", songs are NOT meant to store lookup-like tokens. - * This is just for simplification reasons. - * @param songHash The identifier for the song. - * @param playtype The playtype for the chart. - * @param difficulty The difficulty for the chart. - */ -export function FindDDRChartOnSongHash( - songHash: string, - // Technically both of these should be "ddr" instead of Game, but it proves hard to work with. - playtype: Playtypes[Game], - difficulty: Difficulties[IDStrings] -) { - // note: this only works on accident because monk - // allows any strings like "foo.bar". - // We *should* normally cast this to ChartDocument<"ddr:SP" | "ddr:DP"> - return db.charts.ddr.findOne({ - "data.songHash": songHash, - playtype, - difficulty, - isPrimary: true, - }); -} - -/** - * Find a BMS chart on either its md5sum or its sha256sum. - * @param hash The md5 or sha256 hash to look for. - */ -export function FindBMSChartOnHash(hash: string) { - return db.charts.bms.findOne({ - $or: [{ "data.hashMD5": hash }, { "data.hashSHA256": hash }], - }); -} - -/** - * Find a chart on its in-game-ID, playtype and difficulty. - */ -export function FindChartOnInGameID( - game: Game, - inGameID: number, - playtype: Playtypes[Game], - difficulty: Difficulties[IDStrings] -) { - if (game === "bms" || game === "usc") { - throw new Error(`Cannot call FindChartOnInGameID for game ${game}.`); - } - - return db.charts[game].findOne({ - "data.inGameID": inGameID, - playtype, - difficulty, - }); -} - -/** - * Finds a non-custom chart on its in-game-ID, playtype and difficulty. - * This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate. - */ -export function FindIIDXChartOnInGameID( - inGameID: number, - playtype: Playtypes[Game], - difficulty: Difficulties[IDStrings] -) { - return db.charts.iidx.findOne({ - "data.inGameID": inGameID, - "data.2dxtraSet": null, - isPrimary: true, - playtype, - difficulty, - }); -} - -/** - * Finds a non-custom chart on its in-game-ID, playtype and difficulty. - * This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate. - */ -export function FindIIDXChartOnInGameIDVersion( - inGameID: number, - playtype: Playtypes[Game], - difficulty: Difficulties[IDStrings], - version: GPTSupportedVersions[IDStrings] -) { - return db.charts.iidx.findOne({ - "data.inGameID": inGameID, - "data.2dxtraSet": null, - playtype, - difficulty, - versions: version, - }); -} - -/** - * Find a chart on its in-game-ID, playtype, difficulty and version. - */ -export function FindChartOnInGameIDVersion( - game: Game, - inGameID: number, - playtype: Playtypes[Game], - difficulty: Difficulties[I], - version: GPTSupportedVersions[I] -) { - return db.charts[game].findOne({ - "data.inGameID": inGameID, - versions: version, - playtype, - difficulty, - }); -} - -/** - * Finds an IIDX chart on its 2dxtra hash, which is the sha256 of the .1 buffer. - */ -export function FindIIDXChartWith2DXtraHash(hash: string) { - return db.charts.iidx.findOne({ - "data.hashSHA256": hash, - }); -} - -/** - * Find an SDVX Chart on its in game ID. This exists to handle - * oddities with SDVX difficulties - If "ANY_INF" is sent, it actually - * refers to any of INF, GRV, HVN or VVD. This is because some services treat - * all of those as the same difficulty, but we do not. - */ -export function FindSDVXChartOnInGameID( - inGameID: number, - difficulty: Difficulties["sdvx:Single"] | "ANY_INF" -) { - const diffQuery = - difficulty === "ANY_INF" - ? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] } - : difficulty; - - return db.charts.sdvx.findOne({ - "data.inGameID": inGameID, - difficulty: diffQuery, - isPrimary: true, - }); -} - -export function FindSDVXChartOnInGameIDVersion( - inGameID: number, - difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF", - version: GPTSupportedVersions["sdvx:Single"] -) { - const diffQuery = - difficulty === "ANY_INF" - ? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] } - : difficulty; - - return db.charts.sdvx.findOne({ - "data.inGameID": inGameID, - difficulty: diffQuery, - versions: version, - }); -} - -export function FindSDVXChartOnDFVersion( - songID: integer, - difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF", - version: GPTSupportedVersions["sdvx:Single"] -) { - const diffQuery = - difficulty === "ANY_INF" - ? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] } - : difficulty; - - return db.charts.sdvx.findOne({ - songID, - difficulty: diffQuery, - versions: version, - }); -} - -export function FindChartOnSHA256(game: Game, hash: string) { - if (game !== "bms" && game !== "usc" && game !== "iidx") { - throw new Error(`Cannot call FindChartOnSHA256 for game ${game}.`); - } - - return db.charts[game].findOne({ - "data.hashSHA256": hash, - }); -} - -export function FindChartOnARCID(game: "iidx" | "ddr" | "jubeat" | "sdvx", arcID: string) { - return db.charts[game].findOne({ - "data.arcChartID": arcID, - }); -} - -/** - * Returns the N most popular charts for this game + playtype. - * Popularity is determined by how many scores match in the score - * collection. - */ -export async function FindChartsOnPopularity( - game: Game, - playtype: Playtypes[Game], - songIDs?: integer[], - skip = 0, - limit = 100, - scoreCollection: "personal-bests" | "scores" = "personal-bests" -): Promise<(ChartDocument & { __playcount: integer })[]> { - const matchQuery: FilterQuery = { - playtype, - }; - - if (songIDs) { - matchQuery.songID = { $in: songIDs }; - } - - // MongoDB is a hard beast to wield. - // This code might look very inefficient, but originally this *was* - // a single aggregate pipeline. - // - // We've split it up into multiple queries as this is an order of - // magnitude faster. - // Not entirely sure why, but $lookup is incredibly inefficient, - // and you should just avoid it. - const charts = (await db.charts[game].find(matchQuery)) as unknown as (ChartDocument & { - __playcount: integer; - })[]; - - const scoreCounts = (await db[scoreCollection].aggregate([ - { - $match: { chartID: { $in: charts.map((e) => e.chartID) } }, - }, - { - $group: { - _id: "$chartID", - count: { $sum: 1 }, - }, - }, - { - $sort: { - count: -1, - }, - }, - { - $skip: skip, - }, - { - $limit: limit, - }, - ])) as { _id: string; count: integer }[]; - - const scoreCountMap = new Map(); - - for (const sc of scoreCounts) { - scoreCountMap.set(sc._id, sc.count); - } - - for (const chart of charts) { - chart.__playcount = scoreCountMap.get(chart.chartID) ?? 0; - } - - return charts.sort((a, b) => b.__playcount - a.__playcount).slice(skip, skip + limit); -} +import db from "external/mongo/db"; +import { FilterQuery } from "mongodb"; +import { + ChartDocument, + Difficulties, + Game, + GPTSupportedVersions, + IDStrings, + integer, + Playtypes, +} from "tachi-common"; + +export function FindChartWithChartID(game: Game, chartID: string) { + return db.charts[game].findOne({ chartID }); +} + +/** + * Find chart with PlaytypeDifficulty. This only finds charts that have `isPrimary` set to true. + * If you want to find charts that are not primary, you need to use PTDFVersion. + * @see FindChartWithPTDFVersion + */ +export function FindChartWithPTDF< + G extends Game = Game, + P extends Playtypes[G] = Playtypes[G], + I extends IDStrings = IDStrings +>(game: G, songID: integer, playtype: P, difficulty: Difficulties[I]) { + return db.charts[game].findOne({ + songID: songID, + playtype: playtype, + difficulty: difficulty, + isPrimary: true, + }); +} + +/** + * Find chart with Playtype, Difficulty and a given version. This does not necessarily return a chart that has + * `isPrimary` set. + */ +export function FindChartWithPTDFVersion< + G extends Game = Game, + P extends Playtypes[G] = Playtypes[G], + I extends IDStrings = IDStrings +>( + game: G, + songID: integer, + playtype: P, + difficulty: Difficulties[I], + version: GPTSupportedVersions[I] +) { + return db.charts[game].findOne({ + songID: songID, + playtype: playtype, + difficulty: difficulty, + versions: version, + }); +} + +/** + * Finds a DDR Chart based on its "song hash". + * Songs in DDR have a consistent checksum-like identifier used on the e-amusement website. + * We can use this to locate a chart by combining it with a playtype and difficulty. + * + * Despite the potentially confusing name of "songHash", songs are NOT meant to store lookup-like tokens. + * This is just for simplification reasons. + * @param songHash The identifier for the song. + * @param playtype The playtype for the chart. + * @param difficulty The difficulty for the chart. + */ +export function FindDDRChartOnSongHash( + songHash: string, + // Technically both of these should be "ddr" instead of Game, but it proves hard to work with. + playtype: Playtypes[Game], + difficulty: Difficulties[IDStrings] +) { + // note: this only works on accident because monk + // allows any strings like "foo.bar". + // We *should* normally cast this to ChartDocument<"ddr:SP" | "ddr:DP"> + return db.charts.ddr.findOne({ + "data.songHash": songHash, + playtype, + difficulty, + isPrimary: true, + }); +} + +/** + * Find a BMS chart on either its md5sum or its sha256sum. + * @param hash The md5 or sha256 hash to look for. + */ +export function FindBMSChartOnHash(hash: string) { + return db.charts.bms.findOne({ + $or: [{ "data.hashMD5": hash }, { "data.hashSHA256": hash }], + }); +} + +/** + * Find a chart on its in-game-ID, playtype and difficulty. + */ +export function FindChartOnInGameID( + game: Game, + inGameID: number, + playtype: Playtypes[Game], + difficulty: Difficulties[IDStrings] +) { + if (game === "bms" || game === "usc") { + throw new Error(`Cannot call FindChartOnInGameID for game ${game}.`); + } + + return db.charts[game].findOne({ + "data.inGameID": inGameID, + playtype, + difficulty, + }); +} + +/** + * Finds a non-custom chart on its in-game-ID, playtype and difficulty. + * This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate. + */ +export function FindIIDXChartOnInGameID( + inGameID: number, + playtype: Playtypes[Game], + difficulty: Difficulties[IDStrings] +) { + return db.charts.iidx.findOne({ + "data.inGameID": inGameID, + "data.2dxtraSet": null, + isPrimary: true, + playtype, + difficulty, + }); +} + +/** + * Finds a non-custom chart on its in-game-ID, playtype and difficulty. + * This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate. + */ +export function FindIIDXChartOnInGameIDVersion( + inGameID: number, + playtype: Playtypes[Game], + difficulty: Difficulties[IDStrings], + version: GPTSupportedVersions[IDStrings] +) { + return db.charts.iidx.findOne({ + "data.inGameID": inGameID, + "data.2dxtraSet": null, + playtype, + difficulty, + versions: version, + }); +} + +/** + * Find a chart on its in-game-ID, playtype, difficulty and version. + */ +export function FindChartOnInGameIDVersion( + game: Game, + inGameID: number, + playtype: Playtypes[Game], + difficulty: Difficulties[I], + version: GPTSupportedVersions[I] +) { + return db.charts[game].findOne({ + "data.inGameID": inGameID, + versions: version, + playtype, + difficulty, + }); +} + +/** + * Finds an IIDX chart on its 2dxtra hash, which is the sha256 of the .1 buffer. + */ +export function FindIIDXChartWith2DXtraHash(hash: string) { + return db.charts.iidx.findOne({ + "data.hashSHA256": hash, + }); +} + +/** + * Find an SDVX Chart on its in game ID. This exists to handle + * oddities with SDVX difficulties - If "ANY_INF" is sent, it actually + * refers to any of INF, GRV, HVN or VVD. This is because some services treat + * all of those as the same difficulty, but we do not. + */ +export function FindSDVXChartOnInGameID( + inGameID: number, + difficulty: Difficulties["sdvx:Single"] | "ANY_INF" +) { + const diffQuery = + difficulty === "ANY_INF" + ? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] } + : difficulty; + + return db.charts.sdvx.findOne({ + "data.inGameID": inGameID, + difficulty: diffQuery, + isPrimary: true, + }); +} + +export function FindSDVXChartOnInGameIDVersion( + inGameID: number, + difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF", + version: GPTSupportedVersions["sdvx:Single"] +) { + const diffQuery = + difficulty === "ANY_INF" + ? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] } + : difficulty; + + return db.charts.sdvx.findOne({ + "data.inGameID": inGameID, + difficulty: diffQuery, + versions: version, + }); +} + +export function FindSDVXChartOnDFVersion( + songID: integer, + difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF", + version: GPTSupportedVersions["sdvx:Single"] +) { + const diffQuery = + difficulty === "ANY_INF" + ? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] } + : difficulty; + + return db.charts.sdvx.findOne({ + songID, + difficulty: diffQuery, + versions: version, + }); +} + +export function FindChartOnSHA256(game: Game, hash: string) { + if (game !== "bms" && game !== "usc" && game !== "iidx") { + throw new Error(`Cannot call FindChartOnSHA256 for game ${game}.`); + } + + return db.charts[game].findOne({ + "data.hashSHA256": hash, + }); +} + +export function FindChartOnARCID(game: "iidx" | "ddr" | "jubeat" | "sdvx", arcID: string) { + return db.charts[game].findOne({ + "data.arcChartID": arcID, + }); +} + +/** + * Returns the N most popular charts for this game + playtype. + * Popularity is determined by how many scores match in the score + * collection. + */ +export async function FindChartsOnPopularity( + game: Game, + playtype: Playtypes[Game], + songIDs?: integer[], + skip = 0, + limit = 100, + scoreCollection: "personal-bests" | "scores" = "personal-bests" +): Promise<(ChartDocument & { __playcount: integer })[]> { + const matchQuery: FilterQuery = { + playtype, + }; + + if (songIDs) { + matchQuery.songID = { $in: songIDs }; + } + + // MongoDB is a hard beast to wield. + // This code might look very inefficient, but originally this *was* + // a single aggregate pipeline. + // + // We've split it up into multiple queries as this is an order of + // magnitude faster. + // Not entirely sure why, but $lookup is incredibly inefficient, + // and you should just avoid it. + const charts = (await db.charts[game].find(matchQuery)) as unknown as (ChartDocument & { + __playcount: integer; + })[]; + + const scoreCounts = (await db[scoreCollection].aggregate([ + { + $match: { chartID: { $in: charts.map((e) => e.chartID) } }, + }, + { + $group: { + _id: "$chartID", + count: { $sum: 1 }, + }, + }, + { + $sort: { + count: -1, + }, + }, + { + $skip: skip, + }, + { + $limit: limit, + }, + ])) as { _id: string; count: integer }[]; + + const scoreCountMap = new Map(); + + for (const sc of scoreCounts) { + scoreCountMap.set(sc._id, sc.count); + } + + for (const chart of charts) { + chart.__playcount = scoreCountMap.get(chart.chartID) ?? 0; + } + + return charts.sort((a, b) => b.__playcount - a.__playcount).slice(skip, skip + limit); +} diff --git a/server/src/utils/queries/songs.ts b/server/src/utils/queries/songs.ts index 3634d2725..746a497a0 100644 --- a/server/src/utils/queries/songs.ts +++ b/server/src/utils/queries/songs.ts @@ -1,75 +1,75 @@ -import db from "external/mongo/db"; -import { KtLogger } from "lib/logger/logger"; -import { InternalFailure } from "lib/score-import/framework/common/converter-failures"; -import { FindOneResult } from "monk"; -import { Game, integer, SongDocument } from "tachi-common"; -import { EscapeStringRegexp } from "../misc"; - -/** - * Finds a song document for the given game with the given title (or alt-title). - * This is NOT the preferred way to find a song, as encodings, and typos, make this - * rather difficult. Prefer other functions! - * @param game - The game to search upon. - * @param title - The song title to match. - * @returns SongDocument - */ -export function FindSongOnTitle(game: Game, title: string): Promise> { - // @optimisable: Performance should be tested here by having a utility field for all-titles. - return db.songs[game].findOne({ - $or: [ - { - title: title, - }, - { - altTitles: title, - }, - ], - }); -} - -/** - * Finds a song on a song title case-insensitively. - * This is needed for services that provide horrifically mutated string titles. - */ -export function FindSongOnTitleInsensitive( - game: Game, - title: string -): Promise> { - // @optimisable: Performance should be tested here by having a utility field for all-titles. - - const regex = new RegExp(`^${EscapeStringRegexp(title)}$`, "iu"); - return db.songs[game].findOne({ - $or: [ - { - title: { $regex: regex }, - }, - { - altTitles: { $regex: regex }, - }, - ], - }); -} - -/** - * Finds a song document based on the Tachi songID. Depending on the database this might - * also be the in-game-ID. - * @param game - The game to search upon. - * @param songID - The song ID to match. - * @returns SongDocument - */ -export function FindSongOnID(game: Game, songID: integer): Promise> { - return db.songs[game].findOne({ - id: songID, - }); -} - -export async function FindSongOnIDGuaranteed(game: Game, songID: integer, logger: KtLogger) { - const song = await FindSongOnID(game, songID); - - if (!song) { - logger.severe(`Song-Chart desync for ${songID}. Has charts, but no song.`); - throw new InternalFailure(`Song-Chart desync for ${songID}. Has charts, but no song.`); - } - - return song; -} +import db from "external/mongo/db"; +import { KtLogger } from "lib/logger/logger"; +import { InternalFailure } from "lib/score-import/framework/common/converter-failures"; +import { FindOneResult } from "monk"; +import { Game, integer, SongDocument } from "tachi-common"; +import { EscapeStringRegexp } from "../misc"; + +/** + * Finds a song document for the given game with the given title (or alt-title). + * This is NOT the preferred way to find a song, as encodings, and typos, make this + * rather difficult. Prefer other functions! + * @param game - The game to search upon. + * @param title - The song title to match. + * @returns SongDocument + */ +export function FindSongOnTitle(game: Game, title: string): Promise> { + // @optimisable: Performance should be tested here by having a utility field for all-titles. + return db.songs[game].findOne({ + $or: [ + { + title: title, + }, + { + altTitles: title, + }, + ], + }); +} + +/** + * Finds a song on a song title case-insensitively. + * This is needed for services that provide horrifically mutated string titles. + */ +export function FindSongOnTitleInsensitive( + game: Game, + title: string +): Promise> { + // @optimisable: Performance should be tested here by having a utility field for all-titles. + + const regex = new RegExp(`^${EscapeStringRegexp(title)}$`, "iu"); + return db.songs[game].findOne({ + $or: [ + { + title: { $regex: regex }, + }, + { + altTitles: { $regex: regex }, + }, + ], + }); +} + +/** + * Finds a song document based on the Tachi songID. Depending on the database this might + * also be the in-game-ID. + * @param game - The game to search upon. + * @param songID - The song ID to match. + * @returns SongDocument + */ +export function FindSongOnID(game: Game, songID: integer): Promise> { + return db.songs[game].findOne({ + id: songID, + }); +} + +export async function FindSongOnIDGuaranteed(game: Game, songID: integer, logger: KtLogger) { + const song = await FindSongOnID(game, songID); + + if (!song) { + logger.severe(`Song-Chart desync for ${songID}. Has charts, but no song.`); + throw new InternalFailure(`Song-Chart desync for ${songID}. Has charts, but no song.`); + } + + return song; +} diff --git a/server/src/utils/types.ts b/server/src/utils/types.ts index 748e79584..f168d1b02 100644 --- a/server/src/utils/types.ts +++ b/server/src/utils/types.ts @@ -1,67 +1,67 @@ -import { - FolderDocument, - TableDocument, - SessionDocument, - ScoreDocument, - ChartDocument, - PublicUserDocument, - UserGameStats, - Game, - Playtypes, - SongDocument, - UserSettings, - TachiAPIClientDocument, -} from "tachi-common"; - -declare module "express-session" { - // Inject additional properties on express-session - interface SessionData { - tachi: TachiSessionData; - } -} - -export interface TachiSessionData { - user: PublicUserDocument; - settings: UserSettings; -} - -export interface TachiAPIFailResponse { - success: false; - description: string; -} - -export interface TachiAPISuccessResponse { - success: true; - description: string; - body: Record; -} - -export type TachiAPIReponse = TachiAPIFailResponse | TachiAPISuccessResponse; - -/** - * Clarity type for empty objects - such as in context. - */ -export type EmptyObject = Record; - -/** - * Data that may be monkey-patched onto req.tachi. This holds things such as middleware results. - */ -export interface TachiRequestData { - uscChartDoc?: ChartDocument<"usc:Controller" | "usc:Keyboard">; - - beatorajaChartDoc?: ChartDocument<"bms:7K" | "bms:14K">; - - requestedUser?: PublicUserDocument; - requestedUserGameStats?: UserGameStats; - game?: Game; - playtype?: Playtypes[Game]; - - chartDoc?: ChartDocument; - songDoc?: SongDocument; - scoreDoc?: ScoreDocument; - sessionDoc?: SessionDocument; - tableDoc?: TableDocument; - folderDoc?: FolderDocument; - - apiClientDoc: Omit; -} +import { + FolderDocument, + TableDocument, + SessionDocument, + ScoreDocument, + ChartDocument, + PublicUserDocument, + UserGameStats, + Game, + Playtypes, + SongDocument, + UserSettings, + TachiAPIClientDocument, +} from "tachi-common"; + +declare module "express-session" { + // Inject additional properties on express-session + interface SessionData { + tachi: TachiSessionData; + } +} + +export interface TachiSessionData { + user: PublicUserDocument; + settings: UserSettings; +} + +export interface TachiAPIFailResponse { + success: false; + description: string; +} + +export interface TachiAPISuccessResponse { + success: true; + description: string; + body: Record; +} + +export type TachiAPIReponse = TachiAPIFailResponse | TachiAPISuccessResponse; + +/** + * Clarity type for empty objects - such as in context. + */ +export type EmptyObject = Record; + +/** + * Data that may be monkey-patched onto req.tachi. This holds things such as middleware results. + */ +export interface TachiRequestData { + uscChartDoc?: ChartDocument<"usc:Controller" | "usc:Keyboard">; + + beatorajaChartDoc?: ChartDocument<"bms:7K" | "bms:14K">; + + requestedUser?: PublicUserDocument; + requestedUserGameStats?: UserGameStats; + game?: Game; + playtype?: Playtypes[Game]; + + chartDoc?: ChartDocument; + songDoc?: SongDocument; + scoreDoc?: ScoreDocument; + sessionDoc?: SessionDocument; + tableDoc?: TableDocument; + folderDoc?: FolderDocument; + + apiClientDoc: Omit; +} diff --git a/server/src/utils/user.test.ts b/server/src/utils/user.test.ts index dafdbe753..a2fd92a42 100644 --- a/server/src/utils/user.test.ts +++ b/server/src/utils/user.test.ts @@ -1,59 +1,59 @@ -import t from "tap"; -import { DatabaseSchemas } from "external/mongo/schemas"; -import { prAssert } from "test-utils/asserts"; -import { GetUserCaseInsensitive, FormatUserDoc } from "./user"; -import { PublicUserDocument } from "tachi-common"; - -import ResetDBState from "test-utils/resets"; - -t.test("#GetUserCaseInsensitive", (t) => { - t.beforeEach(ResetDBState); - - t.test("Should return the user for an exact username", async (t) => { - const result = await GetUserCaseInsensitive("test_zkldi"); - - t.not(result, null, "Should not return null"); - - t.equal(result!.username, "test_zkldi", "Should return test_zkldi"); - - t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument"); - - // @ts-expect-error yeah - t.equal(result.password, undefined, "Should not return password"); - // @ts-expect-error yeah - t.equal(result.email, undefined, "Should not return email"); - }); - - t.test("Should return the user for an incorrectly cased username", async (t) => { - const result = await GetUserCaseInsensitive("tesT_ZkLdi"); - - t.not(result, null, "Should not return null"); - - t.equal(result!.username, "test_zkldi", "Should return test_zkldi"); - - t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument"); - - // @ts-expect-error yeah - t.equal(result.password, undefined, "Should not return password"); - // @ts-expect-error yeah - t.equal(result.email, undefined, "Should not return email"); - }); - - t.test("Should not return the user for a username that does not exist", async (t) => { - const result = await GetUserCaseInsensitive("foobar"); - - t.equal(result, null, "Should return null"); - }); - - t.end(); -}); - -t.test("#FormatUserDoc", (t) => { - t.equal( - FormatUserDoc({ username: "zkldi", id: 123 } as PublicUserDocument), - "zkldi (#123)", - "Should format a user document into username #id format." - ); - - t.end(); -}); +import t from "tap"; +import { DatabaseSchemas } from "external/mongo/schemas"; +import { prAssert } from "test-utils/asserts"; +import { GetUserCaseInsensitive, FormatUserDoc } from "./user"; +import { PublicUserDocument } from "tachi-common"; + +import ResetDBState from "test-utils/resets"; + +t.test("#GetUserCaseInsensitive", (t) => { + t.beforeEach(ResetDBState); + + t.test("Should return the user for an exact username", async (t) => { + const result = await GetUserCaseInsensitive("test_zkldi"); + + t.not(result, null, "Should not return null"); + + t.equal(result!.username, "test_zkldi", "Should return test_zkldi"); + + t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument"); + + // @ts-expect-error yeah + t.equal(result.password, undefined, "Should not return password"); + // @ts-expect-error yeah + t.equal(result.email, undefined, "Should not return email"); + }); + + t.test("Should return the user for an incorrectly cased username", async (t) => { + const result = await GetUserCaseInsensitive("tesT_ZkLdi"); + + t.not(result, null, "Should not return null"); + + t.equal(result!.username, "test_zkldi", "Should return test_zkldi"); + + t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument"); + + // @ts-expect-error yeah + t.equal(result.password, undefined, "Should not return password"); + // @ts-expect-error yeah + t.equal(result.email, undefined, "Should not return email"); + }); + + t.test("Should not return the user for a username that does not exist", async (t) => { + const result = await GetUserCaseInsensitive("foobar"); + + t.equal(result, null, "Should return null"); + }); + + t.end(); +}); + +t.test("#FormatUserDoc", (t) => { + t.equal( + FormatUserDoc({ username: "zkldi", id: 123 } as PublicUserDocument), + "zkldi (#123)", + "Should format a user document into username #id format." + ); + + t.end(); +}); diff --git a/server/src/utils/user.ts b/server/src/utils/user.ts index 40c81aaa1..8a0c66639 100644 --- a/server/src/utils/user.ts +++ b/server/src/utils/user.ts @@ -1,224 +1,224 @@ -import { - integer, - PublicUserDocument, - UserGameStats, - GetGamePTConfig, - Game, - Playtypes, - UGSRatingsLookup, - IDStrings, -} from "tachi-common"; -import { FindOneResult } from "monk"; -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; - -const logger = CreateLogCtx(__filename); - -/** - * Returns a user's username from their ID. Throws if no user with that ID exists. - */ -export async function GetUsernameFromUserID(userID: integer): Promise { - const partialDoc = await db.users.findOne( - { - id: userID, - }, - { - projection: { - username: 1, - }, - } - ); - - if (!partialDoc) { - throw new Error(`Could not find username for userID ${userID}.`); - } - - return partialDoc.username; -} - -/** - * Gets a user based on their username case-insensitively. - */ -export function GetUserCaseInsensitive( - username: string -): Promise> { - return db.users.findOne({ - usernameLowercase: username.toLowerCase(), - }) as Promise>; -} - -export async function CheckIfEmailInUse(email: string) { - const doc = await db["user-private-information"].findOne({ email }); - - return !!doc; -} - -export function GetUserPrivateInfo(userID: integer) { - return db["user-private-information"].findOne({ userID }); -} - -/** - * Gets a user from their userID. - */ -export function GetUserWithID(userID: integer): Promise> { - return db.users.findOne({ - id: userID, - }) as Promise>; -} - -export function GetSettingsForUser(userID: integer) { - return db["user-settings"].findOne({ - userID: userID, - }); -} - -/** - * Gets the users for these user IDs. - */ -export function GetUsersWithIDs(userIDs: integer[]) { - return db.users.find({ - id: { $in: userIDs }, - }); -} - -/** - * Retrieve a user document that is expected to exist. - * If the user document is not found, a severe error is logged, and this - * function throws. - */ -export async function GetUserWithIDGuaranteed(userID: integer): Promise { - const userDoc = await GetUserWithID(userID); - - if (!userDoc) { - logger.severe( - `User ${userID} does not have an associated user document, but one was expected.` - ); - throw new Error( - `User ${userID} does not have an associated user document, but one was expected.` - ); - } - - return userDoc; -} - -/** - * Gets a user based on either their username case-insensitively, or a direct lookup of their ID. - * This is used in URLs to resolve the passed user. - */ -export function ResolveUser(usernameOrID: string) { - // user ID passed - if (usernameOrID.match(/^[0-9]+$/u)) { - const intID = Number(usernameOrID); - - return GetUserWithID(intID); - } - - return GetUserCaseInsensitive(usernameOrID); -} - -/** - * Returns a formatted string indicating the user. This is used for logging. - */ -export function FormatUserDoc(userdoc: PublicUserDocument) { - return `${userdoc.username} (#${userdoc.id})`; -} - -export async function GetUsersRanking(stats: UserGameStats) { - const gptConfig = GetGamePTConfig(stats.game, stats.playtype); - - const aggRes = await db["game-stats"].aggregate([ - { - $match: { - game: stats.game, - playtype: stats.playtype, - }, - }, - { - $group: { - _id: null, - ranking: { - $sum: { - $cond: { - if: { - $gt: [ - `$ratings.${gptConfig.defaultProfileRatingAlg}`, - stats.ratings[gptConfig.defaultProfileRatingAlg], - ], - }, - then: 1, - else: 0, - }, - }, - }, - }, - }, - ]); - - return (aggRes[0].ranking + 1) as integer; -} - -export function GetUGPTPlaycount(userID: integer, game: Game, playtype: Playtypes[Game]) { - return db.scores.count({ userID, game, playtype }); -} - -export async function GetAllRankings(stats: UserGameStats) { - const gptConfig = GetGamePTConfig(stats.game, stats.playtype); - - const entries = await Promise.all( - gptConfig.profileRatingAlgs.map((k) => - GetUsersRankingAndOutOf(stats, k).then((r) => [k, r]) - ) - ); - - return Object.fromEntries(entries); -} - -export async function GetUsersRankingAndOutOf( - stats: UserGameStats, - alg?: UGSRatingsLookup[IDStrings] -) { - const gptConfig = GetGamePTConfig(stats.game, stats.playtype); - - const ratingAlg = alg ?? gptConfig.defaultProfileRatingAlg; - - const aggRes = await db["game-stats"].aggregate([ - { - $match: { - game: stats.game, - playtype: stats.playtype, - }, - }, - { - $group: { - _id: null, - outOf: { $sum: 1 }, - ranking: { - $sum: { - $cond: { - if: { - $gt: [`$ratings.${ratingAlg}`, stats.ratings[ratingAlg]], - }, - then: 1, - else: 0, - }, - }, - }, - }, - }, - ]); - - return { - ranking: (aggRes[0].ranking + 1) as integer, - outOf: aggRes[0].outOf as integer, - }; -} - -const FIVE_MINUTES = 1000 * 60 * 5; - -/** - * Returns the cutoff point for "being online" in tachi. This means the user - * has made any page request in the past 5 minutes. - */ -export function GetOnlineCutoff() { - return Date.now() - FIVE_MINUTES; -} +import { + integer, + PublicUserDocument, + UserGameStats, + GetGamePTConfig, + Game, + Playtypes, + UGSRatingsLookup, + IDStrings, +} from "tachi-common"; +import { FindOneResult } from "monk"; +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; + +const logger = CreateLogCtx(__filename); + +/** + * Returns a user's username from their ID. Throws if no user with that ID exists. + */ +export async function GetUsernameFromUserID(userID: integer): Promise { + const partialDoc = await db.users.findOne( + { + id: userID, + }, + { + projection: { + username: 1, + }, + } + ); + + if (!partialDoc) { + throw new Error(`Could not find username for userID ${userID}.`); + } + + return partialDoc.username; +} + +/** + * Gets a user based on their username case-insensitively. + */ +export function GetUserCaseInsensitive( + username: string +): Promise> { + return db.users.findOne({ + usernameLowercase: username.toLowerCase(), + }) as Promise>; +} + +export async function CheckIfEmailInUse(email: string) { + const doc = await db["user-private-information"].findOne({ email }); + + return !!doc; +} + +export function GetUserPrivateInfo(userID: integer) { + return db["user-private-information"].findOne({ userID }); +} + +/** + * Gets a user from their userID. + */ +export function GetUserWithID(userID: integer): Promise> { + return db.users.findOne({ + id: userID, + }) as Promise>; +} + +export function GetSettingsForUser(userID: integer) { + return db["user-settings"].findOne({ + userID: userID, + }); +} + +/** + * Gets the users for these user IDs. + */ +export function GetUsersWithIDs(userIDs: integer[]) { + return db.users.find({ + id: { $in: userIDs }, + }); +} + +/** + * Retrieve a user document that is expected to exist. + * If the user document is not found, a severe error is logged, and this + * function throws. + */ +export async function GetUserWithIDGuaranteed(userID: integer): Promise { + const userDoc = await GetUserWithID(userID); + + if (!userDoc) { + logger.severe( + `User ${userID} does not have an associated user document, but one was expected.` + ); + throw new Error( + `User ${userID} does not have an associated user document, but one was expected.` + ); + } + + return userDoc; +} + +/** + * Gets a user based on either their username case-insensitively, or a direct lookup of their ID. + * This is used in URLs to resolve the passed user. + */ +export function ResolveUser(usernameOrID: string) { + // user ID passed + if (usernameOrID.match(/^[0-9]+$/u)) { + const intID = Number(usernameOrID); + + return GetUserWithID(intID); + } + + return GetUserCaseInsensitive(usernameOrID); +} + +/** + * Returns a formatted string indicating the user. This is used for logging. + */ +export function FormatUserDoc(userdoc: PublicUserDocument) { + return `${userdoc.username} (#${userdoc.id})`; +} + +export async function GetUsersRanking(stats: UserGameStats) { + const gptConfig = GetGamePTConfig(stats.game, stats.playtype); + + const aggRes = await db["game-stats"].aggregate([ + { + $match: { + game: stats.game, + playtype: stats.playtype, + }, + }, + { + $group: { + _id: null, + ranking: { + $sum: { + $cond: { + if: { + $gt: [ + `$ratings.${gptConfig.defaultProfileRatingAlg}`, + stats.ratings[gptConfig.defaultProfileRatingAlg], + ], + }, + then: 1, + else: 0, + }, + }, + }, + }, + }, + ]); + + return (aggRes[0].ranking + 1) as integer; +} + +export function GetUGPTPlaycount(userID: integer, game: Game, playtype: Playtypes[Game]) { + return db.scores.count({ userID, game, playtype }); +} + +export async function GetAllRankings(stats: UserGameStats) { + const gptConfig = GetGamePTConfig(stats.game, stats.playtype); + + const entries = await Promise.all( + gptConfig.profileRatingAlgs.map((k) => + GetUsersRankingAndOutOf(stats, k).then((r) => [k, r]) + ) + ); + + return Object.fromEntries(entries); +} + +export async function GetUsersRankingAndOutOf( + stats: UserGameStats, + alg?: UGSRatingsLookup[IDStrings] +) { + const gptConfig = GetGamePTConfig(stats.game, stats.playtype); + + const ratingAlg = alg ?? gptConfig.defaultProfileRatingAlg; + + const aggRes = await db["game-stats"].aggregate([ + { + $match: { + game: stats.game, + playtype: stats.playtype, + }, + }, + { + $group: { + _id: null, + outOf: { $sum: 1 }, + ranking: { + $sum: { + $cond: { + if: { + $gt: [`$ratings.${ratingAlg}`, stats.ratings[ratingAlg]], + }, + then: 1, + else: 0, + }, + }, + }, + }, + }, + ]); + + return { + ranking: (aggRes[0].ranking + 1) as integer, + outOf: aggRes[0].outOf as integer, + }; +} + +const FIVE_MINUTES = 1000 * 60 * 5; + +/** + * Returns the cutoff point for "being online" in tachi. This means the user + * has made any page request in the past 5 minutes. + */ +export function GetOnlineCutoff() { + return Date.now() - FIVE_MINUTES; +}