Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-05 05:18:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

Change all CRLF to LF and standardise on LF

Browse Source
This commit is contained in:
zkldi
2021-11-16 12:44:28 +00:00
parent 6aeaf4b853
commit b9be948290
49 changed files with 7248 additions and 7248 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/.eslintrc
+93 -93
View File
@@ -1,94 +1,94 @@
{
"root": true,
"env": {
"node": true,
"es6": true
},
"plugins": [
"@typescript-eslint",
"import"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"sourceType": "module",
"ecmaVersion": 2020
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended",
"plugin:prettier/recommended",
"plugin:import/typescript",
"plugin:import/warnings",
"plugin:import/errors"
],
"rules": {
"prettier/prettier": [
"error",
{
"printWidth": 100,
"tabWidth": 4,
"semi": true,
"singleQuote": false,
"trailingComma": "es5",
"bracketSpacing": true,
"endOfLine": "auto",
"useTabs": true
}
],
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/no-non-null-assertion": "off",
"@typescript-eslint/no-unused-vars": [
"warn",
{
"argsIgnorePattern": "logger"
}
],
// breaks with baseUrl
"import/no-unresolved": "off",
"require-await": "warn",
"no-await-in-loop": "warn",
"no-unused-labels": "error",
"prefer-const": "error",
"no-console": "error",
"eqeqeq": "error",
"no-eval": "error",
"no-var": "error",
"no-new-object": "error",
"quote-props": [
"error",
"as-needed"
],
"no-array-constructor": "error",
"prefer-template": "error",
"prefer-promise-reject-errors": "error",
"no-useless-escape": "error",
"no-loop-func": "error",
"prefer-rest-params": "error",
"default-param-last": "error",
"no-new-func": "error",
"arrow-body-style": "error",
"no-useless-constructor": "error",
"no-duplicate-imports": "error",
"dot-notation": "error",
"space-before-blocks": "error",
"no-prototype-builtins": "off",
"prefer-arrow-callback": "error",
"curly": "error",
"no-template-curly-in-string": "error",
"no-implied-eval": "error",
"yoda": "error",
"no-labels": "error",
"no-useless-concat": "error",
"radix": "error",
"no-octal-escape": "error",
"no-param-reassign": "error",
"no-proto": "error",
"no-return-await": "error",
"no-throw-literal": "error",
"no-unmodified-loop-condition": "error",
"no-unused-expressions": "error",
"no-useless-call": "error",
"no-useless-return": "error",
"require-unicode-regexp": "error"
}
{
"root": true,
"env": {
"node": true,
"es6": true
},
"plugins": [
"@typescript-eslint",
"import"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"sourceType": "module",
"ecmaVersion": 2020
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended",
"plugin:prettier/recommended",
"plugin:import/typescript",
"plugin:import/warnings",
"plugin:import/errors"
],
"rules": {
"prettier/prettier": [
"error",
{
"printWidth": 100,
"tabWidth": 4,
"semi": true,
"singleQuote": false,
"trailingComma": "es5",
"bracketSpacing": true,
"endOfLine": "lf",
"useTabs": true
}
],
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/no-non-null-assertion": "off",
"@typescript-eslint/no-unused-vars": [
"warn",
{
"argsIgnorePattern": "logger"
}
],
// breaks with baseUrl
"import/no-unresolved": "off",
"require-await": "warn",
"no-await-in-loop": "warn",
"no-unused-labels": "error",
"prefer-const": "error",
"no-console": "error",
"eqeqeq": "error",
"no-eval": "error",
"no-var": "error",
"no-new-object": "error",
"quote-props": [
"error",
"as-needed"
],
"no-array-constructor": "error",
"prefer-template": "error",
"prefer-promise-reject-errors": "error",
"no-useless-escape": "error",
"no-loop-func": "error",
"prefer-rest-params": "error",
"default-param-last": "error",
"no-new-func": "error",
"arrow-body-style": "error",
"no-useless-constructor": "error",
"no-duplicate-imports": "error",
"dot-notation": "error",
"space-before-blocks": "error",
"no-prototype-builtins": "off",
"prefer-arrow-callback": "error",
"curly": "error",
"no-template-curly-in-string": "error",
"no-implied-eval": "error",
"yoda": "error",
"no-labels": "error",
"no-useless-concat": "error",
"radix": "error",
"no-octal-escape": "error",
"no-param-reassign": "error",
"no-proto": "error",
"no-return-await": "error",
"no-throw-literal": "error",
"no-unmodified-loop-condition": "error",
"no-unused-expressions": "error",
"no-useless-call": "error",
"no-useless-return": "error",
"require-unicode-regexp": "error"
}
}
server/src/external/mongo/db.ts
Vendored
+230 -230
View File
@@ -1,230 +1,230 @@
import { ONE_MINUTE } from "lib/constants/time";
import CreateLogCtx from "lib/logger/logger";
import { OrphanScoreDocument } from "lib/score-import/import-types/common/types";
import { Environment, ServerConfig } from "lib/setup/config";
import monk, { TMiddleware } from "monk";
import {
APITokenDocument,
ARCSavedProfileDocument,
BMSCourseDocument,
ChartDocument,
ClassAchievementDocument,
CounterDocument,
FervidexSettingsDocument,
FolderChartLookup,
FolderDocument,
Game,
GoalDocument,
IIDXBPIData,
ImportDocument,
ImportLockDocument,
ImportTimingsDocument,
integer,
InviteCodeDocument,
KaiAuthDocument,
MilestoneDocument,
OrphanChart,
PBScoreDocument,
PrivateUserInfoDocument,
PublicUserDocument,
ScoreDocument,
SessionDocument,
SessionViewDocument,
SongDocument,
TableDocument,
TachiAPIClientDocument,
UGPTSettings,
UserGameStats,
UserGameStatsSnapshot,
UserGoalDocument,
UserMilestoneDocument,
UserSettings,
} from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
const logger = CreateLogCtx(__filename);
let dbName = ServerConfig.MONGO_DATABASE_NAME;
/* istanbul ignore next */
if (Environment.nodeEnv === "test") {
dbName = `testingdb`;
}
logger.info(`Connecting to database ${Environment.mongoUrl}/${dbName}...`, { bootInfo: true });
const dbtime = process.hrtime.bigint();
// By default the connectTimeoutMS is 30 seconds. This has been upped to 2 minutes, due to poor performance
// inside githubs test runners.
export const monkDB = monk(`${Environment.mongoUrl}/${dbName}`, {
serverSelectionTimeoutMS: ONE_MINUTE * 2,
});
/* istanbul ignore next */
monkDB
.then(() => {
logger.info(`Database connection successful: took ${GetMillisecondsSince(dbtime)}ms`, {
bootInfo: true,
});
})
.catch((err) => {
logger.crit(err);
process.exit(1);
});
const RemoveIDMiddleware: TMiddleware =
() =>
(next) =>
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(args: any, method) => {
if ((method === "find" || method === "findOne") && !args.options.projectID) {
if (args.options.projection) {
args.options.projection._id = 0;
} else {
args.options.projection = { _id: 0 };
}
}
return next(args, method);
};
// a bug in monks types means that :any has to be used here. Maybe we'll make a PR for this?
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const StripIDMiddleware: TMiddleware = () => (next) => (args: any, method) => {
if (method === "insert") {
if (Array.isArray(args.data)) {
for (const d of args.data) {
delete d._id;
}
} else {
delete args.data._id;
}
}
return next(args, method);
};
monkDB.addMiddleware(StripIDMiddleware);
monkDB.addMiddleware(RemoveIDMiddleware);
export async function CloseMongoConnection() {
await monkDB.close();
}
const songs = {
bms: monkDB.get<SongDocument>(`songs-bms`),
chunithm: monkDB.get<SongDocument>(`songs-chunithm`),
ddr: monkDB.get<SongDocument>(`songs-ddr`),
gitadora: monkDB.get<SongDocument>(`songs-gitadora`),
iidx: monkDB.get<SongDocument>(`songs-iidx`),
jubeat: monkDB.get<SongDocument>(`songs-jubeat`),
maimai: monkDB.get<SongDocument>(`songs-maimai`),
museca: monkDB.get<SongDocument>(`songs-museca`),
popn: monkDB.get<SongDocument>(`songs-popn`),
sdvx: monkDB.get<SongDocument>(`songs-sdvx`),
usc: monkDB.get<SongDocument>(`songs-usc`),
};
const charts = {
bms: monkDB.get<ChartDocument>(`charts-bms`),
chunithm: monkDB.get<ChartDocument>(`charts-chunithm`),
ddr: monkDB.get<ChartDocument>(`charts-ddr`),
gitadora: monkDB.get<ChartDocument>(`charts-gitadora`),
iidx: monkDB.get<ChartDocument>(`charts-iidx`),
jubeat: monkDB.get<ChartDocument>(`charts-jubeat`),
maimai: monkDB.get<ChartDocument>(`charts-maimai`),
museca: monkDB.get<ChartDocument>(`charts-museca`),
popn: monkDB.get<ChartDocument>(`charts-popn`),
sdvx: monkDB.get<ChartDocument>(`charts-sdvx`),
usc: monkDB.get<ChartDocument>(`charts-usc`),
};
const db = {
// i have to handwrite this out for TS... :(
// dont worry, it was all macro'd.
songs,
charts,
scores: monkDB.get<ScoreDocument>("scores"),
"personal-bests": monkDB.get<PBScoreDocument>("personal-bests"),
folders: monkDB.get<FolderDocument>("folders"),
"folder-chart-lookup": monkDB.get<FolderChartLookup>("folder-chart-lookup"),
goals: monkDB.get<GoalDocument>("goals"),
"user-goals": monkDB.get<UserGoalDocument>("user-goals"),
milestones: monkDB.get<MilestoneDocument>("milestones"),
"user-milestones": monkDB.get<UserMilestoneDocument>("user-milestones"),
users: monkDB.get<PublicUserDocument>("users"),
imports: monkDB.get<ImportDocument>("imports"),
"import-timings": monkDB.get<ImportTimingsDocument>("import-timings"),
sessions: monkDB.get<SessionDocument>("sessions"),
"iidx-bpi-data": monkDB.get<IIDXBPIData>("iidx-bpi-data"),
invites: monkDB.get<InviteCodeDocument>("invites"),
counters: monkDB.get<CounterDocument>("counters"),
"game-stats": monkDB.get<UserGameStats>("game-stats"),
"kai-auth-tokens": monkDB.get<KaiAuthDocument>("kai-auth-tokens"),
"bms-course-lookup": monkDB.get<BMSCourseDocument>("bms-course-lookup"),
"api-tokens": monkDB.get<APITokenDocument>("api-tokens"),
"orphan-scores": monkDB.get<OrphanScoreDocument>("orphan-scores"),
"import-locks": monkDB.get<ImportLockDocument>("import-locks"),
tables: monkDB.get<TableDocument>("tables"),
"game-settings": monkDB.get<UGPTSettings>("game-settings"),
"game-stats-snapshots": monkDB.get<UserGameStatsSnapshot>("game-stats-snapshots"),
"session-view-cache": monkDB.get<SessionViewDocument>("session-view-cache"),
"arc-saved-profiles": monkDB.get<ARCSavedProfileDocument>("arc-saved-profiles"),
"user-settings": monkDB.get<UserSettings>("user-settings"),
"user-private-information": monkDB.get<PrivateUserInfoDocument>("user-private-information"),
"api-clients": monkDB.get<TachiAPIClientDocument>("api-clients"),
"oauth2-auth-codes":
// i've inlined this one because i don't see it appearing anywhere else.
monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"),
"fer-settings": monkDB.get<FervidexSettingsDocument>("fer-settings"),
"orphan-chart-queue": monkDB.get<OrphanChart>("orphan-chart-queue"),
"password-reset-codes": monkDB.get<{
code: string;
userID: integer;
createdOn: number;
}>("password-reset-codes"),
"class-achievements": monkDB.get<ClassAchievementDocument>("class-achievements"),
"score-blacklist":
monkDB.get<{ scoreID: string; userID: integer; score: ScoreDocument }>("score-blacklist"),
"verify-email-codes":
monkDB.get<{ userID: integer; code: string; email: string }>("verify-email-codes"),
};
export type StaticDatabases =
| "sessions"
| "session-view-cache"
| "folders"
| "folder-chart-lookup"
| "scores"
| "personal-bests"
| "imports"
| "import-timings"
| "goals"
| "user-goals"
| "user-milestones"
| "milestones"
| "game-stats"
| "game-settings"
| "users"
| "kai-auth-tokens"
| "bms-course-lookup"
| "api-tokens"
| "import-locks"
| "tables"
| "game-stats-snapshots"
| "arc-saved-profiles"
| "user-private-information"
| "api-clients"
| "oauth2-auth-codes"
| "fer-settings"
| "orphan-chart-queue"
| "password-reset-codes"
| "user-settings"
| "counters"
| "score-blacklist"
| "verify-email-codes"
| "class-achievements";
export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`;
export default db;
import { ONE_MINUTE } from "lib/constants/time";
import CreateLogCtx from "lib/logger/logger";
import { OrphanScoreDocument } from "lib/score-import/import-types/common/types";
import { Environment, ServerConfig } from "lib/setup/config";
import monk, { TMiddleware } from "monk";
import {
APITokenDocument,
ARCSavedProfileDocument,
BMSCourseDocument,
ChartDocument,
ClassAchievementDocument,
CounterDocument,
FervidexSettingsDocument,
FolderChartLookup,
FolderDocument,
Game,
GoalDocument,
IIDXBPIData,
ImportDocument,
ImportLockDocument,
ImportTimingsDocument,
integer,
InviteCodeDocument,
KaiAuthDocument,
MilestoneDocument,
OrphanChart,
PBScoreDocument,
PrivateUserInfoDocument,
PublicUserDocument,
ScoreDocument,
SessionDocument,
SessionViewDocument,
SongDocument,
TableDocument,
TachiAPIClientDocument,
UGPTSettings,
UserGameStats,
UserGameStatsSnapshot,
UserGoalDocument,
UserMilestoneDocument,
UserSettings,
} from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
const logger = CreateLogCtx(__filename);
let dbName = ServerConfig.MONGO_DATABASE_NAME;
/* istanbul ignore next */
if (Environment.nodeEnv === "test") {
dbName = `testingdb`;
}
logger.info(`Connecting to database ${Environment.mongoUrl}/${dbName}...`, { bootInfo: true });
const dbtime = process.hrtime.bigint();
// By default the connectTimeoutMS is 30 seconds. This has been upped to 2 minutes, due to poor performance
// inside githubs test runners.
export const monkDB = monk(`${Environment.mongoUrl}/${dbName}`, {
serverSelectionTimeoutMS: ONE_MINUTE * 2,
});
/* istanbul ignore next */
monkDB
.then(() => {
logger.info(`Database connection successful: took ${GetMillisecondsSince(dbtime)}ms`, {
bootInfo: true,
});
})
.catch((err) => {
logger.crit(err);
process.exit(1);
});
const RemoveIDMiddleware: TMiddleware =
() =>
(next) =>
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(args: any, method) => {
if ((method === "find" || method === "findOne") && !args.options.projectID) {
if (args.options.projection) {
args.options.projection._id = 0;
} else {
args.options.projection = { _id: 0 };
}
}
return next(args, method);
};
// a bug in monks types means that :any has to be used here. Maybe we'll make a PR for this?
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const StripIDMiddleware: TMiddleware = () => (next) => (args: any, method) => {
if (method === "insert") {
if (Array.isArray(args.data)) {
for (const d of args.data) {
delete d._id;
}
} else {
delete args.data._id;
}
}
return next(args, method);
};
monkDB.addMiddleware(StripIDMiddleware);
monkDB.addMiddleware(RemoveIDMiddleware);
export async function CloseMongoConnection() {
await monkDB.close();
}
const songs = {
bms: monkDB.get<SongDocument>(`songs-bms`),
chunithm: monkDB.get<SongDocument>(`songs-chunithm`),
ddr: monkDB.get<SongDocument>(`songs-ddr`),
gitadora: monkDB.get<SongDocument>(`songs-gitadora`),
iidx: monkDB.get<SongDocument>(`songs-iidx`),
jubeat: monkDB.get<SongDocument>(`songs-jubeat`),
maimai: monkDB.get<SongDocument>(`songs-maimai`),
museca: monkDB.get<SongDocument>(`songs-museca`),
popn: monkDB.get<SongDocument>(`songs-popn`),
sdvx: monkDB.get<SongDocument>(`songs-sdvx`),
usc: monkDB.get<SongDocument>(`songs-usc`),
};
const charts = {
bms: monkDB.get<ChartDocument>(`charts-bms`),
chunithm: monkDB.get<ChartDocument>(`charts-chunithm`),
ddr: monkDB.get<ChartDocument>(`charts-ddr`),
gitadora: monkDB.get<ChartDocument>(`charts-gitadora`),
iidx: monkDB.get<ChartDocument>(`charts-iidx`),
jubeat: monkDB.get<ChartDocument>(`charts-jubeat`),
maimai: monkDB.get<ChartDocument>(`charts-maimai`),
museca: monkDB.get<ChartDocument>(`charts-museca`),
popn: monkDB.get<ChartDocument>(`charts-popn`),
sdvx: monkDB.get<ChartDocument>(`charts-sdvx`),
usc: monkDB.get<ChartDocument>(`charts-usc`),
};
const db = {
// i have to handwrite this out for TS... :(
// dont worry, it was all macro'd.
songs,
charts,
scores: monkDB.get<ScoreDocument>("scores"),
"personal-bests": monkDB.get<PBScoreDocument>("personal-bests"),
folders: monkDB.get<FolderDocument>("folders"),
"folder-chart-lookup": monkDB.get<FolderChartLookup>("folder-chart-lookup"),
goals: monkDB.get<GoalDocument>("goals"),
"user-goals": monkDB.get<UserGoalDocument>("user-goals"),
milestones: monkDB.get<MilestoneDocument>("milestones"),
"user-milestones": monkDB.get<UserMilestoneDocument>("user-milestones"),
users: monkDB.get<PublicUserDocument>("users"),
imports: monkDB.get<ImportDocument>("imports"),
"import-timings": monkDB.get<ImportTimingsDocument>("import-timings"),
sessions: monkDB.get<SessionDocument>("sessions"),
"iidx-bpi-data": monkDB.get<IIDXBPIData>("iidx-bpi-data"),
invites: monkDB.get<InviteCodeDocument>("invites"),
counters: monkDB.get<CounterDocument>("counters"),
"game-stats": monkDB.get<UserGameStats>("game-stats"),
"kai-auth-tokens": monkDB.get<KaiAuthDocument>("kai-auth-tokens"),
"bms-course-lookup": monkDB.get<BMSCourseDocument>("bms-course-lookup"),
"api-tokens": monkDB.get<APITokenDocument>("api-tokens"),
"orphan-scores": monkDB.get<OrphanScoreDocument>("orphan-scores"),
"import-locks": monkDB.get<ImportLockDocument>("import-locks"),
tables: monkDB.get<TableDocument>("tables"),
"game-settings": monkDB.get<UGPTSettings>("game-settings"),
"game-stats-snapshots": monkDB.get<UserGameStatsSnapshot>("game-stats-snapshots"),
"session-view-cache": monkDB.get<SessionViewDocument>("session-view-cache"),
"arc-saved-profiles": monkDB.get<ARCSavedProfileDocument>("arc-saved-profiles"),
"user-settings": monkDB.get<UserSettings>("user-settings"),
"user-private-information": monkDB.get<PrivateUserInfoDocument>("user-private-information"),
"api-clients": monkDB.get<TachiAPIClientDocument>("api-clients"),
"oauth2-auth-codes":
// i've inlined this one because i don't see it appearing anywhere else.
monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"),
"fer-settings": monkDB.get<FervidexSettingsDocument>("fer-settings"),
"orphan-chart-queue": monkDB.get<OrphanChart>("orphan-chart-queue"),
"password-reset-codes": monkDB.get<{
code: string;
userID: integer;
createdOn: number;
}>("password-reset-codes"),
"class-achievements": monkDB.get<ClassAchievementDocument>("class-achievements"),
"score-blacklist":
monkDB.get<{ scoreID: string; userID: integer; score: ScoreDocument }>("score-blacklist"),
"verify-email-codes":
monkDB.get<{ userID: integer; code: string; email: string }>("verify-email-codes"),
};
export type StaticDatabases =
| "sessions"
| "session-view-cache"
| "folders"
| "folder-chart-lookup"
| "scores"
| "personal-bests"
| "imports"
| "import-timings"
| "goals"
| "user-goals"
| "user-milestones"
| "milestones"
| "game-stats"
| "game-settings"
| "users"
| "kai-auth-tokens"
| "bms-course-lookup"
| "api-tokens"
| "import-locks"
| "tables"
| "game-stats-snapshots"
| "arc-saved-profiles"
| "user-private-information"
| "api-clients"
| "oauth2-auth-codes"
| "fer-settings"
| "orphan-chart-queue"
| "password-reset-codes"
| "user-settings"
| "counters"
| "score-blacklist"
| "verify-email-codes"
| "class-achievements";
export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`;
export default db;
server/src/lib/logger/logger.ts
+253 -253
View File
@@ -1,253 +1,253 @@
import { Transport as SeqTransport } from "@valuabletouch/winston-seq";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import SafeJSONStringify from "safe-json-stringify";
import { SeqLogLevel } from "seq-logging";
import { EscapeStringRegexp } from "utils/misc";
import winston, { format, LeveledLogMethod, Logger, transports } from "winston";
import "winston-daily-rotate-file";
import DiscordWinstonTransport from "./discord-transport";
export type KtLogger = Logger & { severe: LeveledLogMethod };
const level = process.env.LOG_LEVEL ?? ServerConfig.LOGGER_CONFIG.LOG_LEVEL;
const formatExcessProperties = (meta: Record<string, unknown>, limit = false) => {
let i = 0;
for (const key in meta) {
const val = meta[key];
if (val instanceof Error) {
meta[key] = { message: val.message, stack: val.stack };
}
i++;
}
if (!i) {
return "";
}
const content = SafeJSONStringify(meta);
return ` ${limit ? StrCap(content) : content}`;
};
function StrCap(string: string) {
if (string.length > 300) {
return `${string.slice(0, 297)}...`;
}
return string;
}
const formatExcessPropertiesNoStack = (
meta: Record<string, unknown>,
omitKeys: string[] = [],
limit = false
) => {
const realMeta: Record<string, unknown> = {};
for (const key in meta) {
if (omitKeys.includes(key)) {
continue;
}
const val = meta[key];
if (val instanceof Error) {
realMeta[key] = { message: val.message };
} else if (!key.startsWith("__") && !key.startsWith("!")) {
realMeta[key] = val;
}
}
if (Object.keys(realMeta).length === 0) {
return "";
}
const content = SafeJSONStringify(realMeta);
return ` ${limit ? StrCap(content) : content}`;
};
const replicaInfo = Environment.replicaIdentity ? ` (${Environment.replicaIdentity})` : "";
const tachiPrintf = format.printf(
({ level, message, context = "tachi-root", timestamp, ...meta }) =>
`${timestamp}${replicaInfo} [${
Array.isArray(context) ? context.join(" | ") : context
}] ${level}: ${message}${formatExcessProperties(meta)}`
);
const tachiConsolePrintf = format.printf(
({ level, message, context = "tachi-root", timestamp, hideFromConsole, ...meta }) =>
`${timestamp}${replicaInfo} [${
Array.isArray(context) ? context.join(" | ") : context
}] ${level}: ${message}${formatExcessPropertiesNoStack(meta, hideFromConsole, true)}`
);
winston.addColors({
crit: ["bgRed", "black"],
severe: ["bgBrightRed", "black"],
error: ["red"],
warn: ["yellow"],
info: ["blue"],
verbose: ["cyan"],
debug: ["white"],
});
const baseFormatRoute = format.combine(
format.timestamp({
format: "YYYY-MM-DD HH:mm:ss",
})
);
const defaultFormatRoute = format.combine(
baseFormatRoute,
format.errors({ stack: false }),
tachiPrintf
);
const consoleFormatRoute = format.combine(
baseFormatRoute,
format.errors({ stack: false }),
tachiConsolePrintf,
format.colorize({
all: true,
})
);
const tports: winston.transport[] = [];
if (ServerConfig.LOGGER_CONFIG.FILE) {
tports.push(
new transports.DailyRotateFile({
filename: "logs/tachi-%DATE%.log",
datePattern: "YYYY-MM-DD-HH",
zippedArchive: true,
maxSize: "20m",
maxFiles: "14d",
createSymlink: true,
symlinkName: "tachi.log",
format: defaultFormatRoute,
})
);
}
if (ServerConfig.LOGGER_CONFIG.CONSOLE) {
tports.push(
new transports.Console({
format: consoleFormatRoute,
})
);
}
if (ServerConfig.LOGGER_CONFIG.DISCORD) {
tports.push(
new DiscordWinstonTransport({
webhook: ServerConfig.LOGGER_CONFIG.DISCORD.WEBHOOK_URL,
level: "warn",
})
);
}
if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) {
// Turns winston log levels into seq format.
const levelMap: Record<string, SeqLogLevel> = {
crit: "Fatal",
severe: "Error",
error: "Error",
warn: "Warning",
info: "Information",
// Note that Seq interprets these in reverse,
// however, it's easier to read this code if I just
// use the same levels, instead of the right onesQ.
verbose: "Verbose",
debug: "Debug",
};
tports.push(
new SeqTransport({
apiKey: ServerConfig.LOGGER_CONFIG.SEQ_API_KEY,
serverUrl: Environment.seqUrl,
levelMapper(level = "") {
return levelMap[level] ?? "information";
},
})
);
}
export const rootLogger = winston.createLogger({
levels: {
crit: 0, // entire process termination is necessary
severe: 1, // something is wrong, and more than one function is affected (such as a failed assertion that is definitely expected to be true).
error: 2, // function call (or related process) has failed unexpectedly
warn: 3, // function call has hit something it didn't want, but can recover
info: 4, // something has happened that is expected, but worth logging
verbose: 5, // something has happened
debug: 6, // glorified console.log debugging
},
level,
format: defaultFormatRoute,
transports: tports,
defaultMeta: {
__ServerName: TachiConfig.NAME,
__Worker: !!process.env.IS_WORKER,
__ReplicaID: Environment.replicaIdentity,
},
});
if (!!ServerConfig.LOGGER_CONFIG.SEQ_API_KEY !== !!Environment.seqUrl) {
rootLogger.warn(
`Only one of SEQ_API_KEY (conf.json5) and SEQ_URL (Environment) were set. Not sending logs to Seq, as both must be provided.`
);
}
if (tports.length === 0) {
// eslint-disable-next-line no-console
console.warn(
"You have no transports set. Absolutely no logs will be saved. This is a terrible idea!"
);
}
function CreateLogCtx(filename: string, lg = rootLogger): KtLogger {
const replacedFilename = filename.replace(
new RegExp(`^${EscapeStringRegexp(process.cwd())}/((js|src)/)?`, "u"),
""
);
const logger = lg.child({
context: [replacedFilename],
}) as KtLogger;
logger.defaultMeta = Object.assign({}, logger.defaultMeta ?? {}, {
context: [replacedFilename],
});
return logger;
}
export function AppendLogCtx(context: string, lg: KtLogger): KtLogger {
const newContext = [...lg.defaultMeta.context, context];
return lg.child({ context: newContext }) as KtLogger;
}
export function ChangeRootLogLevel(
level: "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug"
) {
rootLogger.info(`Changing log level to ${level}.`);
for (const tp of tports) {
tp.level = level;
}
}
export function GetLogLevel() {
return (
tports.map((e) => e.level).find((e) => typeof e === "string") ??
ServerConfig.LOGGER_CONFIG.LOG_LEVEL
);
}
export const Transports = tports;
export default CreateLogCtx;
import { Transport as SeqTransport } from "@valuabletouch/winston-seq";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import SafeJSONStringify from "safe-json-stringify";
import { SeqLogLevel } from "seq-logging";
import { EscapeStringRegexp } from "utils/misc";
import winston, { format, LeveledLogMethod, Logger, transports } from "winston";
import "winston-daily-rotate-file";
import DiscordWinstonTransport from "./discord-transport";
export type KtLogger = Logger & { severe: LeveledLogMethod };
const level = process.env.LOG_LEVEL ?? ServerConfig.LOGGER_CONFIG.LOG_LEVEL;
const formatExcessProperties = (meta: Record<string, unknown>, limit = false) => {
let i = 0;
for (const key in meta) {
const val = meta[key];
if (val instanceof Error) {
meta[key] = { message: val.message, stack: val.stack };
}
i++;
}
if (!i) {
return "";
}
const content = SafeJSONStringify(meta);
return ` ${limit ? StrCap(content) : content}`;
};
function StrCap(string: string) {
if (string.length > 300) {
return `${string.slice(0, 297)}...`;
}
return string;
}
const formatExcessPropertiesNoStack = (
meta: Record<string, unknown>,
omitKeys: string[] = [],
limit = false
) => {
const realMeta: Record<string, unknown> = {};
for (const key in meta) {
if (omitKeys.includes(key)) {
continue;
}
const val = meta[key];
if (val instanceof Error) {
realMeta[key] = { message: val.message };
} else if (!key.startsWith("__") && !key.startsWith("!")) {
realMeta[key] = val;
}
}
if (Object.keys(realMeta).length === 0) {
return "";
}
const content = SafeJSONStringify(realMeta);
return ` ${limit ? StrCap(content) : content}`;
};
const replicaInfo = Environment.replicaIdentity ? ` (${Environment.replicaIdentity})` : "";
const tachiPrintf = format.printf(
({ level, message, context = "tachi-root", timestamp, ...meta }) =>
`${timestamp}${replicaInfo} [${
Array.isArray(context) ? context.join(" | ") : context
}] ${level}: ${message}${formatExcessProperties(meta)}`
);
const tachiConsolePrintf = format.printf(
({ level, message, context = "tachi-root", timestamp, hideFromConsole, ...meta }) =>
`${timestamp}${replicaInfo} [${
Array.isArray(context) ? context.join(" | ") : context
}] ${level}: ${message}${formatExcessPropertiesNoStack(meta, hideFromConsole, true)}`
);
winston.addColors({
crit: ["bgRed", "black"],
severe: ["bgBrightRed", "black"],
error: ["red"],
warn: ["yellow"],
info: ["blue"],
verbose: ["cyan"],
debug: ["white"],
});
const baseFormatRoute = format.combine(
format.timestamp({
format: "YYYY-MM-DD HH:mm:ss",
})
);
const defaultFormatRoute = format.combine(
baseFormatRoute,
format.errors({ stack: false }),
tachiPrintf
);
const consoleFormatRoute = format.combine(
baseFormatRoute,
format.errors({ stack: false }),
tachiConsolePrintf,
format.colorize({
all: true,
})
);
const tports: winston.transport[] = [];
if (ServerConfig.LOGGER_CONFIG.FILE) {
tports.push(
new transports.DailyRotateFile({
filename: "logs/tachi-%DATE%.log",
datePattern: "YYYY-MM-DD-HH",
zippedArchive: true,
maxSize: "20m",
maxFiles: "14d",
createSymlink: true,
symlinkName: "tachi.log",
format: defaultFormatRoute,
})
);
}
if (ServerConfig.LOGGER_CONFIG.CONSOLE) {
tports.push(
new transports.Console({
format: consoleFormatRoute,
})
);
}
if (ServerConfig.LOGGER_CONFIG.DISCORD) {
tports.push(
new DiscordWinstonTransport({
webhook: ServerConfig.LOGGER_CONFIG.DISCORD.WEBHOOK_URL,
level: "warn",
})
);
}
if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) {
// Turns winston log levels into seq format.
const levelMap: Record<string, SeqLogLevel> = {
crit: "Fatal",
severe: "Error",
error: "Error",
warn: "Warning",
info: "Information",
// Note that Seq interprets these in reverse,
// however, it's easier to read this code if I just
// use the same levels, instead of the right onesQ.
verbose: "Verbose",
debug: "Debug",
};
tports.push(
new SeqTransport({
apiKey: ServerConfig.LOGGER_CONFIG.SEQ_API_KEY,
serverUrl: Environment.seqUrl,
levelMapper(level = "") {
return levelMap[level] ?? "information";
},
})
);
}
export const rootLogger = winston.createLogger({
levels: {
crit: 0, // entire process termination is necessary
severe: 1, // something is wrong, and more than one function is affected (such as a failed assertion that is definitely expected to be true).
error: 2, // function call (or related process) has failed unexpectedly
warn: 3, // function call has hit something it didn't want, but can recover
info: 4, // something has happened that is expected, but worth logging
verbose: 5, // something has happened
debug: 6, // glorified console.log debugging
},
level,
format: defaultFormatRoute,
transports: tports,
defaultMeta: {
__ServerName: TachiConfig.NAME,
__Worker: !!process.env.IS_WORKER,
__ReplicaID: Environment.replicaIdentity,
},
});
if (!!ServerConfig.LOGGER_CONFIG.SEQ_API_KEY !== !!Environment.seqUrl) {
rootLogger.warn(
`Only one of SEQ_API_KEY (conf.json5) and SEQ_URL (Environment) were set. Not sending logs to Seq, as both must be provided.`
);
}
if (tports.length === 0) {
// eslint-disable-next-line no-console
console.warn(
"You have no transports set. Absolutely no logs will be saved. This is a terrible idea!"
);
}
function CreateLogCtx(filename: string, lg = rootLogger): KtLogger {
const replacedFilename = filename.replace(
new RegExp(`^${EscapeStringRegexp(process.cwd())}/((js|src)/)?`, "u"),
""
);
const logger = lg.child({
context: [replacedFilename],
}) as KtLogger;
logger.defaultMeta = Object.assign({}, logger.defaultMeta ?? {}, {
context: [replacedFilename],
});
return logger;
}
export function AppendLogCtx(context: string, lg: KtLogger): KtLogger {
const newContext = [...lg.defaultMeta.context, context];
return lg.child({ context: newContext }) as KtLogger;
}
export function ChangeRootLogLevel(
level: "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug"
) {
rootLogger.info(`Changing log level to ${level}.`);
for (const tp of tports) {
tp.level = level;
}
}
export function GetLogLevel() {
return (
tports.map((e) => e.level).find((e) => typeof e === "string") ??
ServerConfig.LOGGER_CONFIG.LOG_LEVEL
);
}
export const Transports = tports;
export default CreateLogCtx;
server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts
+414 -414
View File
@@ -1,414 +1,414 @@
import CreateLogCtx from "lib/logger/logger";
import p from "prudence";
import { ChartDocument, ScoreDocument } from "tachi-common";
import t from "tap";
import { prAssert } from "test-utils/asserts";
import {
Testing511SPA,
TestingDoraChart,
TestingGITADORADoraDryScore,
TestingIIDXSPDryScore,
TestingSDVXSingleDryScore,
} from "test-utils/test-data";
import { CalculateDataForGamePT, CreateCalculatedData } from "./calculated-data";
const logger = CreateLogCtx(__filename);
t.test("#CreateCalculatedData", async (t) => {
const res = await CreateCalculatedData(TestingIIDXSPDryScore, Testing511SPA, 30, logger);
prAssert(
res,
{
ktRating: p.aprx(2.65),
ktLampRating: p.equalTo(10),
BPI: "?number",
},
"Should correctly produce calculatedData"
);
const gitadoraRes = await CreateCalculatedData(
TestingGITADORADoraDryScore,
TestingDoraChart,
30,
logger
);
prAssert(
gitadoraRes,
{
skill: p.isPositiveNonZero,
},
"Should correctly call rating function overrides for different games"
);
const uscRes = await CreateCalculatedData(
{ game: "usc", playtype: "Controller" } as ScoreDocument,
{ data: { isOfficial: false }, playtype: "Controller" } as ChartDocument,
null,
logger
);
t.strictSame(uscRes, { VF6: null }, "Should return null if chart was not an official.");
const uscKbRes = await CreateCalculatedData(
{ game: "usc", playtype: "Keyboard" } as ScoreDocument,
{ data: { isOfficial: false }, playtype: "Keyboard" } as ChartDocument,
null,
logger
);
t.strictSame(
uscKbRes,
{ VF6: null },
"Should return null if chart was not an official (Keyboard)."
);
t.end();
});
/**
* These tests only check that the right properties are assigned.
*/
t.test("#CalculateDataForGamePT", (t) => {
t.test("IIDX:SP", async (t) => {
const res = await CalculateDataForGamePT(
"iidx",
"SP",
Testing511SPA,
TestingIIDXSPDryScore,
30,
logger
);
prAssert(
res,
{
ktRating: "?number",
ktLampRating: "?number",
BPI: "?number",
},
"Response should contain keys for IIDX:SP"
);
t.end();
});
t.test("IIDX:DP", async (t) => {
const res = await CalculateDataForGamePT(
"iidx",
"DP",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an SP score. but we're testing
30,
logger
);
prAssert(
res,
{
ktRating: "?number",
ktLampRating: "?number",
BPI: "?number",
},
"Response should contain keys for IIDX:DP"
);
t.end();
});
t.test("SDVX:Single", async (t) => {
const res = await CalculateDataForGamePT(
"sdvx",
"Single",
Testing511SPA,
TestingSDVXSingleDryScore,
null,
logger
);
prAssert(
res,
{
VF6: p.nullable(p.isPositive),
},
"Response should contain keys for SDVX:Single"
);
t.end();
});
t.test("DDR:SP", async (t) => {
const res = await CalculateDataForGamePT(
"ddr",
"SP",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
MFCP: p.nullable(p.isPositiveInteger),
ktRating: "?number",
},
"Response should contain keys for DDR:SP"
);
t.end();
});
t.test("DDR:DP", async (t) => {
const res = await CalculateDataForGamePT(
"ddr",
"DP",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
MFCP: "null",
ktRating: "?number",
},
"Response should contain nulled keys for DDR:DP"
);
t.end();
});
t.test("chunithm:Single", async (t) => {
const res = await CalculateDataForGamePT(
"chunithm",
"Single",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
rating: "?number",
},
"Response should contain nulled keys for chunithm:Single"
);
t.end();
});
t.test("maimai:Single", async (t) => {
const res = await CalculateDataForGamePT(
"maimai",
"Single",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
ktRating: "?number",
},
"Response should contain nulled keys for maimai:Single"
);
t.end();
});
t.test("museca:Single", async (t) => {
const res = await CalculateDataForGamePT(
"museca",
"Single",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
ktRating: "?number",
},
"Response should contain nulled keys for museca:Single"
);
t.end();
});
t.test("bms:7K", async (t) => {
const res = await CalculateDataForGamePT(
"bms",
"7K",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
sieglinde: "?number",
},
"Response should contain nulled keys for bms:7K"
);
t.end();
});
t.test("bms:14K", async (t) => {
const res = await CalculateDataForGamePT(
"bms",
"14K",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
sieglinde: "?number",
},
"Response should contain nulled keys for bms:14K"
);
t.end();
});
t.test("gitadora:Gita", async (t) => {
const res = await CalculateDataForGamePT(
"gitadora",
"Gita",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
skill: "?number",
},
"Response should contain nulled keys for gitadora:Gita"
);
t.end();
});
t.test("gitadora:Dora", async (t) => {
const res = await CalculateDataForGamePT(
"gitadora",
"Dora",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
skill: "?number",
},
"Response should contain nulled keys for gitadora:Dora"
);
t.end();
});
t.test("usc:Controller", async (t) => {
const res = await CalculateDataForGamePT(
"usc",
"Controller",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
VF6: "?number",
},
"Response should contain nulled keys for usc:Controller"
);
t.end();
});
t.test("usc:Keyboard", async (t) => {
const res = await CalculateDataForGamePT(
"usc",
"Keyboard",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
VF6: "?number",
},
"Response should contain nulled keys for usc:Keyboard"
);
t.end();
});
// t.test("jubeat:Single", async (t) => {
// const res = await CalculateDataForGamePT(
// "jubeat",
// "Single",
// Testing511SPA,
// TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
// null,
//
// logger
// );
// prAssert(
// res,
// {
// jubility: "?number",
// },
// "Response should contain nulled keys for jubeat:Single"
// );
// t.end();
// });
// t.test("popn:9B", async (t) => {
// const res = await CalculateDataForGamePT(
// "popn",
// "9B",
// Testing511SPA,
// TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
// null,
//
// logger
// );
// prAssert(res, {}, "Response should contain nulled keys for popn:9B");
// t.end();
// });
t.end();
});
import CreateLogCtx from "lib/logger/logger";
import p from "prudence";
import { ChartDocument, ScoreDocument } from "tachi-common";
import t from "tap";
import { prAssert } from "test-utils/asserts";
import {
Testing511SPA,
TestingDoraChart,
TestingGITADORADoraDryScore,
TestingIIDXSPDryScore,
TestingSDVXSingleDryScore,
} from "test-utils/test-data";
import { CalculateDataForGamePT, CreateCalculatedData } from "./calculated-data";
const logger = CreateLogCtx(__filename);
t.test("#CreateCalculatedData", async (t) => {
const res = await CreateCalculatedData(TestingIIDXSPDryScore, Testing511SPA, 30, logger);
prAssert(
res,
{
ktRating: p.aprx(2.65),
ktLampRating: p.equalTo(10),
BPI: "?number",
},
"Should correctly produce calculatedData"
);
const gitadoraRes = await CreateCalculatedData(
TestingGITADORADoraDryScore,
TestingDoraChart,
30,
logger
);
prAssert(
gitadoraRes,
{
skill: p.isPositiveNonZero,
},
"Should correctly call rating function overrides for different games"
);
const uscRes = await CreateCalculatedData(
{ game: "usc", playtype: "Controller" } as ScoreDocument,
{ data: { isOfficial: false }, playtype: "Controller" } as ChartDocument,
null,
logger
);
t.strictSame(uscRes, { VF6: null }, "Should return null if chart was not an official.");
const uscKbRes = await CreateCalculatedData(
{ game: "usc", playtype: "Keyboard" } as ScoreDocument,
{ data: { isOfficial: false }, playtype: "Keyboard" } as ChartDocument,
null,
logger
);
t.strictSame(
uscKbRes,
{ VF6: null },
"Should return null if chart was not an official (Keyboard)."
);
t.end();
});
/**
* These tests only check that the right properties are assigned.
*/
t.test("#CalculateDataForGamePT", (t) => {
t.test("IIDX:SP", async (t) => {
const res = await CalculateDataForGamePT(
"iidx",
"SP",
Testing511SPA,
TestingIIDXSPDryScore,
30,
logger
);
prAssert(
res,
{
ktRating: "?number",
ktLampRating: "?number",
BPI: "?number",
},
"Response should contain keys for IIDX:SP"
);
t.end();
});
t.test("IIDX:DP", async (t) => {
const res = await CalculateDataForGamePT(
"iidx",
"DP",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an SP score. but we're testing
30,
logger
);
prAssert(
res,
{
ktRating: "?number",
ktLampRating: "?number",
BPI: "?number",
},
"Response should contain keys for IIDX:DP"
);
t.end();
});
t.test("SDVX:Single", async (t) => {
const res = await CalculateDataForGamePT(
"sdvx",
"Single",
Testing511SPA,
TestingSDVXSingleDryScore,
null,
logger
);
prAssert(
res,
{
VF6: p.nullable(p.isPositive),
},
"Response should contain keys for SDVX:Single"
);
t.end();
});
t.test("DDR:SP", async (t) => {
const res = await CalculateDataForGamePT(
"ddr",
"SP",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
MFCP: p.nullable(p.isPositiveInteger),
ktRating: "?number",
},
"Response should contain keys for DDR:SP"
);
t.end();
});
t.test("DDR:DP", async (t) => {
const res = await CalculateDataForGamePT(
"ddr",
"DP",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
MFCP: "null",
ktRating: "?number",
},
"Response should contain nulled keys for DDR:DP"
);
t.end();
});
t.test("chunithm:Single", async (t) => {
const res = await CalculateDataForGamePT(
"chunithm",
"Single",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
rating: "?number",
},
"Response should contain nulled keys for chunithm:Single"
);
t.end();
});
t.test("maimai:Single", async (t) => {
const res = await CalculateDataForGamePT(
"maimai",
"Single",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
ktRating: "?number",
},
"Response should contain nulled keys for maimai:Single"
);
t.end();
});
t.test("museca:Single", async (t) => {
const res = await CalculateDataForGamePT(
"museca",
"Single",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
ktRating: "?number",
},
"Response should contain nulled keys for museca:Single"
);
t.end();
});
t.test("bms:7K", async (t) => {
const res = await CalculateDataForGamePT(
"bms",
"7K",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
sieglinde: "?number",
},
"Response should contain nulled keys for bms:7K"
);
t.end();
});
t.test("bms:14K", async (t) => {
const res = await CalculateDataForGamePT(
"bms",
"14K",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
sieglinde: "?number",
},
"Response should contain nulled keys for bms:14K"
);
t.end();
});
t.test("gitadora:Gita", async (t) => {
const res = await CalculateDataForGamePT(
"gitadora",
"Gita",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
skill: "?number",
},
"Response should contain nulled keys for gitadora:Gita"
);
t.end();
});
t.test("gitadora:Dora", async (t) => {
const res = await CalculateDataForGamePT(
"gitadora",
"Dora",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
skill: "?number",
},
"Response should contain nulled keys for gitadora:Dora"
);
t.end();
});
t.test("usc:Controller", async (t) => {
const res = await CalculateDataForGamePT(
"usc",
"Controller",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
VF6: "?number",
},
"Response should contain nulled keys for usc:Controller"
);
t.end();
});
t.test("usc:Keyboard", async (t) => {
const res = await CalculateDataForGamePT(
"usc",
"Keyboard",
Testing511SPA,
TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
null,
logger
);
prAssert(
res,
{
VF6: "?number",
},
"Response should contain nulled keys for usc:Keyboard"
);
t.end();
});
// t.test("jubeat:Single", async (t) => {
// const res = await CalculateDataForGamePT(
// "jubeat",
// "Single",
// Testing511SPA,
// TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
// null,
//
// logger
// );
// prAssert(
// res,
// {
// jubility: "?number",
// },
// "Response should contain nulled keys for jubeat:Single"
// );
// t.end();
// });
// t.test("popn:9B", async (t) => {
// const res = await CalculateDataForGamePT(
// "popn",
// "9B",
// Testing511SPA,
// TestingIIDXSPDryScore, // fake! this is an iidx score. but we're testing
// null,
//
// logger
// );
// prAssert(res, {}, "Response should contain nulled keys for popn:9B");
// t.end();
// });
t.end();
});
server/src/lib/score-import/framework/calculated-data/calculated-data.ts
+291 -291
View File
@@ -1,291 +1,291 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import {
ChartDocument,
Game,
Grades,
IDStrings,
Lamps,
Playtypes,
ScoreDocument,
} from "tachi-common";
import { HasOwnProperty } from "utils/misc";
import { DryScore } from "../common/types";
import {
CalculateBPI,
CalculateCHUNITHMRating,
CalculateGITADORASkill,
CalculateKTLampRating,
CalculateKTRating,
CalculateMFCP,
CalculateVF6,
} from "./stats";
export async function CreateCalculatedData(
dryScore: DryScore,
chart: ChartDocument,
esd: number | null,
logger: KtLogger
): Promise<ScoreDocument["calculatedData"]> {
const game = dryScore.game;
const playtype = chart.playtype;
const calculatedData = await CalculateDataForGamePT(
game,
playtype,
chart,
dryScore,
esd,
logger
);
return calculatedData;
}
type CalculatedDataFunctions = {
[G in Game]: {
[P in Playtypes[G]]: (
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
) => Promise<ScoreDocument["calculatedData"]> | ScoreDocument["calculatedData"];
};
};
const CalculatedDataFunctions: CalculatedDataFunctions = {
iidx: {
SP: CalculateDataIIDXSP,
DP: CalculateDataIIDXDP,
},
sdvx: {
Single: CalculateDataSDVXorUSC,
},
// popn: {
// "9B": () => ({}),
// },
museca: {
Single: CalculateDataMuseca,
},
chunithm: {
Single: CalculateDataCHUNITHM,
},
maimai: {
Single: CalculateDataMaimai,
},
gitadora: {
Gita: CalculateDataGitadora,
Dora: CalculateDataGitadora,
},
bms: {
"7K": CalculateDataBMS7K,
"14K": CalculateDataBMS14K,
},
ddr: {
SP: CalculateDataDDR,
DP: CalculateDataDDR,
},
// jubeat: {
// Single: CalculateDataJubeat,
// },
usc: {
Controller: CalculateDataSDVXorUSC,
Keyboard: CalculateDataSDVXorUSC,
},
};
// Creates Game-Specific calculatedData for the provided game & playtype.
// eslint-disable-next-line require-await
export async function CalculateDataForGamePT<G extends Game>(
game: G,
playtype: Playtypes[G],
chart: ChartDocument,
dryScore: DryScore,
// ESD gets specially passed through because it's not part of the DryScore, but
// can be used for statistics anyway.
esd: number | null,
logger: KtLogger
): Promise<ScoreDocument["calculatedData"]> {
const GameRatingFns = CalculatedDataFunctions[game];
if (!HasOwnProperty(GameRatingFns, playtype)) {
logger.error(
`Invalid playtype of ${playtype} given for game ${game} in CalculateDataForGamePT, returning an empty object.`
);
return {};
}
// @ts-expect-error standard game->pt stuff.
return GameRatingFns[playtype](dryScore, chart, logger);
}
type CalculatedData<I extends IDStrings> = Required<ScoreDocument<I>["calculatedData"]>;
async function CalculateDataIIDXSP(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"iidx:SP">> {
const BPIData = await db["iidx-bpi-data"].findOne({
chartID: chart.chartID,
});
let bpi;
if (BPIData) {
bpi = CalculateBPI(
BPIData.kavg,
BPIData.wr,
dryScore.scoreData.score,
(chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2,
BPIData.coef
);
// kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled
} else {
bpi = null;
}
return {
BPI: bpi,
ktRating: await CalculateKTRating(dryScore, "iidx", "SP", chart, logger),
ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "SP", chart),
};
}
async function CalculateDataIIDXDP(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"iidx:DP">> {
const BPIData = await db["iidx-bpi-data"].findOne({
chartID: chart.chartID,
});
let bpi;
if (BPIData) {
bpi = CalculateBPI(
BPIData.kavg,
BPIData.wr,
dryScore.scoreData.score,
(chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2,
BPIData.coef
);
// kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled
} else {
bpi = null;
}
return {
BPI: bpi,
ktRating: await CalculateKTRating(dryScore, "iidx", "DP", chart, logger),
ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "DP", chart),
};
}
function CalculateDataSDVXorUSC(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): CalculatedData<"sdvx:Single" | "usc:Keyboard" | "usc:Controller"> {
// for usc, unofficial charts currently have no VF6 value.
if (
dryScore.game === "usc" &&
!(chart as ChartDocument<"usc:Controller" | "usc:Keyboard">).data.isOfficial
) {
return { VF6: null };
}
const VF6 = CalculateVF6(
dryScore.scoreData.grade as Grades["sdvx:Single"],
dryScore.scoreData.lamp as Lamps["sdvx:Single"],
dryScore.scoreData.percent,
chart.levelNum,
logger
);
return {
VF6,
};
}
async function CalculateDataMuseca(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"museca:Single">> {
return {
ktRating: await CalculateKTRating(dryScore, "museca", "Single", chart, logger),
};
}
function CalculateDataCHUNITHM(
dryScore: DryScore,
chart: ChartDocument
): CalculatedData<"chunithm:Single"> {
return {
rating: CalculateCHUNITHMRating(dryScore, chart),
};
}
async function CalculateDataMaimai(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"maimai:Single">> {
// @todo #373 Add maimai rating algorithms.
return {
ktRating: 0,
};
}
function CalculateDataGitadora(
dryScore: DryScore,
chart: ChartDocument
): CalculatedData<"gitadora:Gita" | "gitadora:Dora"> {
return {
skill: CalculateGITADORASkill(dryScore, chart),
};
}
async function CalculateDataDDR(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"ddr:SP" | "ddr:DP">> {
return {
MFCP: CalculateMFCP(dryScore, chart, logger),
ktRating: await CalculateKTRating(dryScore, "ddr", chart.playtype, chart, logger),
};
}
export async function CalculateDataBMS14K(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"bms:14K">> {
return {
sieglinde: 0, // @todo #33
};
}
export async function CalculateDataBMS7K(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"bms:7K">> {
return {
sieglinde: 0, // @todo #33
};
}
// async function CalculateDataJubeat(
// dryScore: DryScore,
// chart: ChartDocument,
// logger: KtLogger
// ): Promise<CalculatedData<"jubeat:Single">> {
// return {
// jubility: 0, // @todo #163 Jubeat Jubility
// };
// }
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import {
ChartDocument,
Game,
Grades,
IDStrings,
Lamps,
Playtypes,
ScoreDocument,
} from "tachi-common";
import { HasOwnProperty } from "utils/misc";
import { DryScore } from "../common/types";
import {
CalculateBPI,
CalculateCHUNITHMRating,
CalculateGITADORASkill,
CalculateKTLampRating,
CalculateKTRating,
CalculateMFCP,
CalculateVF6,
} from "./stats";
export async function CreateCalculatedData(
dryScore: DryScore,
chart: ChartDocument,
esd: number | null,
logger: KtLogger
): Promise<ScoreDocument["calculatedData"]> {
const game = dryScore.game;
const playtype = chart.playtype;
const calculatedData = await CalculateDataForGamePT(
game,
playtype,
chart,
dryScore,
esd,
logger
);
return calculatedData;
}
type CalculatedDataFunctions = {
[G in Game]: {
[P in Playtypes[G]]: (
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
) => Promise<ScoreDocument["calculatedData"]> | ScoreDocument["calculatedData"];
};
};
const CalculatedDataFunctions: CalculatedDataFunctions = {
iidx: {
SP: CalculateDataIIDXSP,
DP: CalculateDataIIDXDP,
},
sdvx: {
Single: CalculateDataSDVXorUSC,
},
// popn: {
// "9B": () => ({}),
// },
museca: {
Single: CalculateDataMuseca,
},
chunithm: {
Single: CalculateDataCHUNITHM,
},
maimai: {
Single: CalculateDataMaimai,
},
gitadora: {
Gita: CalculateDataGitadora,
Dora: CalculateDataGitadora,
},
bms: {
"7K": CalculateDataBMS7K,
"14K": CalculateDataBMS14K,
},
ddr: {
SP: CalculateDataDDR,
DP: CalculateDataDDR,
},
// jubeat: {
// Single: CalculateDataJubeat,
// },
usc: {
Controller: CalculateDataSDVXorUSC,
Keyboard: CalculateDataSDVXorUSC,
},
};
// Creates Game-Specific calculatedData for the provided game & playtype.
// eslint-disable-next-line require-await
export async function CalculateDataForGamePT<G extends Game>(
game: G,
playtype: Playtypes[G],
chart: ChartDocument,
dryScore: DryScore,
// ESD gets specially passed through because it's not part of the DryScore, but
// can be used for statistics anyway.
esd: number | null,
logger: KtLogger
): Promise<ScoreDocument["calculatedData"]> {
const GameRatingFns = CalculatedDataFunctions[game];
if (!HasOwnProperty(GameRatingFns, playtype)) {
logger.error(
`Invalid playtype of ${playtype} given for game ${game} in CalculateDataForGamePT, returning an empty object.`
);
return {};
}
// @ts-expect-error standard game->pt stuff.
return GameRatingFns[playtype](dryScore, chart, logger);
}
type CalculatedData<I extends IDStrings> = Required<ScoreDocument<I>["calculatedData"]>;
async function CalculateDataIIDXSP(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"iidx:SP">> {
const BPIData = await db["iidx-bpi-data"].findOne({
chartID: chart.chartID,
});
let bpi;
if (BPIData) {
bpi = CalculateBPI(
BPIData.kavg,
BPIData.wr,
dryScore.scoreData.score,
(chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2,
BPIData.coef
);
// kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled
} else {
bpi = null;
}
return {
BPI: bpi,
ktRating: await CalculateKTRating(dryScore, "iidx", "SP", chart, logger),
ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "SP", chart),
};
}
async function CalculateDataIIDXDP(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"iidx:DP">> {
const BPIData = await db["iidx-bpi-data"].findOne({
chartID: chart.chartID,
});
let bpi;
if (BPIData) {
bpi = CalculateBPI(
BPIData.kavg,
BPIData.wr,
dryScore.scoreData.score,
(chart as ChartDocument<"iidx:DP" | "iidx:SP">).data.notecount * 2,
BPIData.coef
);
// kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled
} else {
bpi = null;
}
return {
BPI: bpi,
ktRating: await CalculateKTRating(dryScore, "iidx", "DP", chart, logger),
ktLampRating: await CalculateKTLampRating(dryScore, "iidx", "DP", chart),
};
}
function CalculateDataSDVXorUSC(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): CalculatedData<"sdvx:Single" | "usc:Keyboard" | "usc:Controller"> {
// for usc, unofficial charts currently have no VF6 value.
if (
dryScore.game === "usc" &&
!(chart as ChartDocument<"usc:Controller" | "usc:Keyboard">).data.isOfficial
) {
return { VF6: null };
}
const VF6 = CalculateVF6(
dryScore.scoreData.grade as Grades["sdvx:Single"],
dryScore.scoreData.lamp as Lamps["sdvx:Single"],
dryScore.scoreData.percent,
chart.levelNum,
logger
);
return {
VF6,
};
}
async function CalculateDataMuseca(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"museca:Single">> {
return {
ktRating: await CalculateKTRating(dryScore, "museca", "Single", chart, logger),
};
}
function CalculateDataCHUNITHM(
dryScore: DryScore,
chart: ChartDocument
): CalculatedData<"chunithm:Single"> {
return {
rating: CalculateCHUNITHMRating(dryScore, chart),
};
}
async function CalculateDataMaimai(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"maimai:Single">> {
// @todo #373 Add maimai rating algorithms.
return {
ktRating: 0,
};
}
function CalculateDataGitadora(
dryScore: DryScore,
chart: ChartDocument
): CalculatedData<"gitadora:Gita" | "gitadora:Dora"> {
return {
skill: CalculateGITADORASkill(dryScore, chart),
};
}
async function CalculateDataDDR(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"ddr:SP" | "ddr:DP">> {
return {
MFCP: CalculateMFCP(dryScore, chart, logger),
ktRating: await CalculateKTRating(dryScore, "ddr", chart.playtype, chart, logger),
};
}
export async function CalculateDataBMS14K(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"bms:14K">> {
return {
sieglinde: 0, // @todo #33
};
}
export async function CalculateDataBMS7K(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): Promise<CalculatedData<"bms:7K">> {
return {
sieglinde: 0, // @todo #33
};
}
// async function CalculateDataJubeat(
// dryScore: DryScore,
// chart: ChartDocument,
// logger: KtLogger
// ): Promise<CalculatedData<"jubeat:Single">> {
// return {
// jubility: 0, // @todo #163 Jubeat Jubility
// };
// }
server/src/lib/score-import/framework/calculated-data/stats.test.ts
+361 -361
View File
@@ -1,361 +1,361 @@
import deepmerge from "deepmerge";
import CreateLogCtx from "lib/logger/logger";
import { ChartDocument, Difficulties, Lamps, ScoreDocument } from "tachi-common";
import t from "tap";
import { isApproximately } from "test-utils/asserts";
import { GetKTDataJSON, Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data";
import { DryScore } from "../common/types";
import {
CalculateBPI,
CalculateCHUNITHMRating,
CalculateGITADORASkill,
CalculateKTRating,
CalculateMFCP,
CalculateVF6,
} from "./stats";
t.test("#CalculateBPI", (t) => {
t.test("AA BPI tests", (t) => {
// data accurate as of 14/04/2021
const WR_AA = 3650;
const KAVG_AA = 3204;
const MAX_AA = 1834 * 2;
const COEF_AA = 1.25945;
// AA, the song.
function AA_BPI(yourEx: number) {
return CalculateBPI(KAVG_AA, WR_AA, yourEx, MAX_AA, COEF_AA);
}
t.equal(AA_BPI(KAVG_AA), 0, "A score of KAVG should be exactly 0BPI");
isApproximately(AA_BPI(3393), 10.02, "A score with 3393 should be approximately 10.02BPI");
isApproximately(AA_BPI(3481), 20.09, "A score with 3481 should be approximately 20.09BPI");
isApproximately(AA_BPI(3535), 30.01, "A score with 3535 should be approximately 30.01BPI");
isApproximately(AA_BPI(3572), 40.18, "A score with 3572 should be approximately 40.18BPI");
isApproximately(AA_BPI(3597), 50.1, "A score with 3597 should be approximately 50.1BPI");
isApproximately(AA_BPI(3615), 60.12, "A score with 3615 should be approximately 60.12BPI");
isApproximately(AA_BPI(3628), 70.11, "A score with 3628 should be approximately 70.11BPI");
isApproximately(AA_BPI(3638), 80.62, "A score with 3638 should be approximately 80.62BPI");
isApproximately(AA_BPI(3645), 90.59, "A score with 3645 should be approximately 90.59BPI");
t.equal(AA_BPI(WR_AA), 100, "A score of WR should be exactly 100BPI");
isApproximately(AA_BPI(3041), -5, "A score with 3628 should be approximately -5BPI");
isApproximately(AA_BPI(2886), -9.99, "A score with 3638 should be approximately -9.99BPI");
t.equal(AA_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI");
isApproximately(AA_BPI(MAX_AA), 244.56, "A score of MAX should be approximately 244.56BPI");
t.end();
});
t.test("COLOSSEUM BPI Tests", (t) => {
const WR_AFT = 2312;
const KAVG_CL = 2131;
const MAX_CL = 2318;
const COEF_CL = -1;
function CL_BPI(yourEx: number) {
return CalculateBPI(KAVG_CL, WR_AFT, yourEx, MAX_CL, COEF_CL);
}
isApproximately(CL_BPI(2307), 79.63);
t.end();
});
// This song has both no co-efficient and is an sp11, so it serves
// as another interesting test.
t.test("Afterimage d'automne BPI tests", (t) => {
// data accurate as of 14/04/2021
const WR_AFT = 2891;
const KAVG_AFT = 2497;
const MAX_AFT = 1480 * 2;
const COEF_AFT = null;
function AFT_BPI(yourEx: number) {
return CalculateBPI(KAVG_AFT, WR_AFT, yourEx, MAX_AFT, COEF_AFT);
}
t.equal(AFT_BPI(KAVG_AFT), 0, "A score of KAVG should be exactly 0BPI");
isApproximately(AFT_BPI(2606), 10, "A score with 2606 should be approximately 10BPI");
isApproximately(AFT_BPI(2675), 20.07, "A score with 2675 should be approximately 20.07BPI");
isApproximately(AFT_BPI(2727), 30.18, "A score with 2727 should be approximately 30.18BPI");
isApproximately(AFT_BPI(2767), 40.12, "A score with 2767 should be approximately 40.12BPI");
isApproximately(AFT_BPI(2799), 50.06, "A score with 2799 should be approximately 50.06BPI");
isApproximately(AFT_BPI(2826), 60.43, "A score with 2826 should be approximately 60.43BPI");
isApproximately(AFT_BPI(2847), 70.3, "A score with 2847 should be approximately 70.3BPI");
isApproximately(AFT_BPI(2865), 80.57, "A score with 2865 should be approximately 80.57BPI");
isApproximately(AFT_BPI(2879), 90.18, "A score with 2879 should be approximately 90.18BPI");
t.equal(AFT_BPI(WR_AFT), 100, "A score of WR should be exactly 100BPI");
isApproximately(AFT_BPI(2423), -4.99, "A score with 2423 should be approximately -4.99BPI");
isApproximately(
AFT_BPI(2354),
-10.04,
"A score with 2354 should be approximately -10.04BPI"
);
t.equal(AFT_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI");
isApproximately(
AFT_BPI(MAX_AFT),
431.57,
"A score of MAX should be approximately 431.57BPI"
);
t.end();
});
t.end();
});
t.test("#CalculateGITADORARating", (t) => {
function TestGitadoraRating(percent: number, levelNum: number) {
return CalculateGITADORASkill(
{ scoreData: { percent } } as DryScore,
{ levelNum } as ChartDocument
);
}
t.equal(
TestGitadoraRating(89.48, 3.4),
60.84,
"Test GITADORA Rating function aligns with game (1)"
);
t.equal(
TestGitadoraRating(70.76, 5.8),
82.08,
"Test GITADORA Rating function aligns with game (2)"
);
t.end();
});
const logger = CreateLogCtx(__filename);
t.test("#CalculateMFCP", (t) => {
function TestMFCP(
lamp: Lamps["ddr:SP" | "ddr:DP"],
levelNum: number,
difficulty: Difficulties["ddr:DP" | "ddr:SP"]
) {
return CalculateMFCP(
{
scoreData: {
lamp,
},
} as ScoreDocument,
{
difficulty,
levelNum,
} as ChartDocument,
logger
);
}
t.equal(TestMFCP("FAILED", 10, "EXPERT"), null, "Should return null for non-mfcs");
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 10, "BASIC"),
null,
"Should reject charts on BASIC difficulty."
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 10, "BEGINNER"),
null,
"Should reject charts on BEGINNER difficulty."
);
t.test("Should return null for charts with level less than 8", (t) => {
for (let i = 1; i <= 7; i++) {
t.equal(
TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"),
null,
`Should return null for charts with level ${i}`
);
}
t.end();
});
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 8, "EXPERT"),
1,
"Should return 1 for charts with level 8"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 9, "EXPERT"),
1,
"Should return 1 for charts with level 9"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 10, "EXPERT"),
1,
"Should return 1 for charts with level 10"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 11, "EXPERT"),
2,
"Should return 2 for charts with level 11"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 12, "EXPERT"),
2,
"Should return 2 for charts with level 12"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 13, "EXPERT"),
4,
"Should return 4 for charts with level 13"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 14, "EXPERT"),
8,
"Should return 8 for charts with level 14"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 15, "EXPERT"),
15,
"Should return 15 for charts with level 15"
);
t.test("Should return 25 for charts with level 16-20", (t) => {
for (let i = 16; i <= 20; i++) {
t.equal(
TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"),
25,
`Should return 25 for charts with level ${i}`
);
}
t.end();
});
t.equal(
TestMFCP("MARVELOUS FULL COMBO", NaN, "EXPERT"),
null,
"Invalid level triggers failsafe."
);
t.end();
});
const bbkk = GetKTDataJSON("./tachi/chunithm-bbkk-chart.json");
// unit testing a mathematical function is a square-round-hole problem.
t.test("#CalculateCHUNITHMRating", (t) => {
// cutoffs
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_010_000 } } as DryScore, bbkk), 5.0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_005_000 } } as DryScore, bbkk), 4.5);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_000_000 } } as DryScore, bbkk), 4);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 975_000 } } as DryScore, bbkk), 3);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 925_000 } } as DryScore, bbkk), 0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 900_000 } } as DryScore, bbkk), 0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 800_000 } } as DryScore, bbkk), 0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 0 } } as DryScore, bbkk), 0);
// inbetweens
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 987_000 } } as DryScore, bbkk), 3.48);
t.end();
});
// Random assertions plucked from bemaniwiki.
t.test("#CalculateVF6", (t) => {
t.equal(CalculateVF6("S", "CLEAR", 99, 16, logger), 0.332);
t.equal(CalculateVF6("AAA+", "ULTIMATE CHAIN", 98, 17, logger), 0.356);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 16, logger), 0.369);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 17, logger), 0.392);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 18, logger), 0.415);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 19, logger), 0.438);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 20, logger), 0.462);
t.end();
});
t.test("#CalculateRating", (t) => {
t.test("Should call the success calculator if percent > pivotPercent", async (t) => {
const r = await CalculateKTRating(
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 80 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.ok(r > 10, "Should return rating greater than the levelNum of the chart.");
t.end();
});
t.test("Should call the fail calculator if percent > pivotPercent", async (t) => {
const r = await CalculateKTRating(
TestingIIDXSPDryScore,
"iidx",
"SP",
Testing511SPA,
logger
);
t.ok(r < 10, "Should return rating less than the levelNum of the chart.");
t.end();
});
t.test("Should call levelNum if percent === pivotPercent", async (t) => {
const r = await CalculateKTRating(
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 77.7777 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.equal(
// hack for approximate tests
parseFloat(r.toFixed(2)),
10,
"Should return rating exactly that of the levelNum of the chart."
);
t.end();
});
t.test(
"Should trigger safety if completely invalid percent somehow gets through",
async (t) => {
let r = await CalculateKTRating(
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 1000000000 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.equal(r, 0, "Should safely return 0 and log a warning.");
r = await CalculateKTRating(
// not high enough to be non-finite but high enough to be > 1000
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 200 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.equal(r, 0, "Should safely return 0 and log a warning.");
t.end();
}
);
t.end();
});
import deepmerge from "deepmerge";
import CreateLogCtx from "lib/logger/logger";
import { ChartDocument, Difficulties, Lamps, ScoreDocument } from "tachi-common";
import t from "tap";
import { isApproximately } from "test-utils/asserts";
import { GetKTDataJSON, Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data";
import { DryScore } from "../common/types";
import {
CalculateBPI,
CalculateCHUNITHMRating,
CalculateGITADORASkill,
CalculateKTRating,
CalculateMFCP,
CalculateVF6,
} from "./stats";
t.test("#CalculateBPI", (t) => {
t.test("AA BPI tests", (t) => {
// data accurate as of 14/04/2021
const WR_AA = 3650;
const KAVG_AA = 3204;
const MAX_AA = 1834 * 2;
const COEF_AA = 1.25945;
// AA, the song.
function AA_BPI(yourEx: number) {
return CalculateBPI(KAVG_AA, WR_AA, yourEx, MAX_AA, COEF_AA);
}
t.equal(AA_BPI(KAVG_AA), 0, "A score of KAVG should be exactly 0BPI");
isApproximately(AA_BPI(3393), 10.02, "A score with 3393 should be approximately 10.02BPI");
isApproximately(AA_BPI(3481), 20.09, "A score with 3481 should be approximately 20.09BPI");
isApproximately(AA_BPI(3535), 30.01, "A score with 3535 should be approximately 30.01BPI");
isApproximately(AA_BPI(3572), 40.18, "A score with 3572 should be approximately 40.18BPI");
isApproximately(AA_BPI(3597), 50.1, "A score with 3597 should be approximately 50.1BPI");
isApproximately(AA_BPI(3615), 60.12, "A score with 3615 should be approximately 60.12BPI");
isApproximately(AA_BPI(3628), 70.11, "A score with 3628 should be approximately 70.11BPI");
isApproximately(AA_BPI(3638), 80.62, "A score with 3638 should be approximately 80.62BPI");
isApproximately(AA_BPI(3645), 90.59, "A score with 3645 should be approximately 90.59BPI");
t.equal(AA_BPI(WR_AA), 100, "A score of WR should be exactly 100BPI");
isApproximately(AA_BPI(3041), -5, "A score with 3628 should be approximately -5BPI");
isApproximately(AA_BPI(2886), -9.99, "A score with 3638 should be approximately -9.99BPI");
t.equal(AA_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI");
isApproximately(AA_BPI(MAX_AA), 244.56, "A score of MAX should be approximately 244.56BPI");
t.end();
});
t.test("COLOSSEUM BPI Tests", (t) => {
const WR_AFT = 2312;
const KAVG_CL = 2131;
const MAX_CL = 2318;
const COEF_CL = -1;
function CL_BPI(yourEx: number) {
return CalculateBPI(KAVG_CL, WR_AFT, yourEx, MAX_CL, COEF_CL);
}
isApproximately(CL_BPI(2307), 79.63);
t.end();
});
// This song has both no co-efficient and is an sp11, so it serves
// as another interesting test.
t.test("Afterimage d'automne BPI tests", (t) => {
// data accurate as of 14/04/2021
const WR_AFT = 2891;
const KAVG_AFT = 2497;
const MAX_AFT = 1480 * 2;
const COEF_AFT = null;
function AFT_BPI(yourEx: number) {
return CalculateBPI(KAVG_AFT, WR_AFT, yourEx, MAX_AFT, COEF_AFT);
}
t.equal(AFT_BPI(KAVG_AFT), 0, "A score of KAVG should be exactly 0BPI");
isApproximately(AFT_BPI(2606), 10, "A score with 2606 should be approximately 10BPI");
isApproximately(AFT_BPI(2675), 20.07, "A score with 2675 should be approximately 20.07BPI");
isApproximately(AFT_BPI(2727), 30.18, "A score with 2727 should be approximately 30.18BPI");
isApproximately(AFT_BPI(2767), 40.12, "A score with 2767 should be approximately 40.12BPI");
isApproximately(AFT_BPI(2799), 50.06, "A score with 2799 should be approximately 50.06BPI");
isApproximately(AFT_BPI(2826), 60.43, "A score with 2826 should be approximately 60.43BPI");
isApproximately(AFT_BPI(2847), 70.3, "A score with 2847 should be approximately 70.3BPI");
isApproximately(AFT_BPI(2865), 80.57, "A score with 2865 should be approximately 80.57BPI");
isApproximately(AFT_BPI(2879), 90.18, "A score with 2879 should be approximately 90.18BPI");
t.equal(AFT_BPI(WR_AFT), 100, "A score of WR should be exactly 100BPI");
isApproximately(AFT_BPI(2423), -4.99, "A score with 2423 should be approximately -4.99BPI");
isApproximately(
AFT_BPI(2354),
-10.04,
"A score with 2354 should be approximately -10.04BPI"
);
t.equal(AFT_BPI(0), -15, "Excessively bad (vs kavg) scores should cap at -15BPI");
isApproximately(
AFT_BPI(MAX_AFT),
431.57,
"A score of MAX should be approximately 431.57BPI"
);
t.end();
});
t.end();
});
t.test("#CalculateGITADORARating", (t) => {
function TestGitadoraRating(percent: number, levelNum: number) {
return CalculateGITADORASkill(
{ scoreData: { percent } } as DryScore,
{ levelNum } as ChartDocument
);
}
t.equal(
TestGitadoraRating(89.48, 3.4),
60.84,
"Test GITADORA Rating function aligns with game (1)"
);
t.equal(
TestGitadoraRating(70.76, 5.8),
82.08,
"Test GITADORA Rating function aligns with game (2)"
);
t.end();
});
const logger = CreateLogCtx(__filename);
t.test("#CalculateMFCP", (t) => {
function TestMFCP(
lamp: Lamps["ddr:SP" | "ddr:DP"],
levelNum: number,
difficulty: Difficulties["ddr:DP" | "ddr:SP"]
) {
return CalculateMFCP(
{
scoreData: {
lamp,
},
} as ScoreDocument,
{
difficulty,
levelNum,
} as ChartDocument,
logger
);
}
t.equal(TestMFCP("FAILED", 10, "EXPERT"), null, "Should return null for non-mfcs");
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 10, "BASIC"),
null,
"Should reject charts on BASIC difficulty."
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 10, "BEGINNER"),
null,
"Should reject charts on BEGINNER difficulty."
);
t.test("Should return null for charts with level less than 8", (t) => {
for (let i = 1; i <= 7; i++) {
t.equal(
TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"),
null,
`Should return null for charts with level ${i}`
);
}
t.end();
});
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 8, "EXPERT"),
1,
"Should return 1 for charts with level 8"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 9, "EXPERT"),
1,
"Should return 1 for charts with level 9"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 10, "EXPERT"),
1,
"Should return 1 for charts with level 10"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 11, "EXPERT"),
2,
"Should return 2 for charts with level 11"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 12, "EXPERT"),
2,
"Should return 2 for charts with level 12"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 13, "EXPERT"),
4,
"Should return 4 for charts with level 13"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 14, "EXPERT"),
8,
"Should return 8 for charts with level 14"
);
t.equal(
TestMFCP("MARVELOUS FULL COMBO", 15, "EXPERT"),
15,
"Should return 15 for charts with level 15"
);
t.test("Should return 25 for charts with level 16-20", (t) => {
for (let i = 16; i <= 20; i++) {
t.equal(
TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"),
25,
`Should return 25 for charts with level ${i}`
);
}
t.end();
});
t.equal(
TestMFCP("MARVELOUS FULL COMBO", NaN, "EXPERT"),
null,
"Invalid level triggers failsafe."
);
t.end();
});
const bbkk = GetKTDataJSON("./tachi/chunithm-bbkk-chart.json");
// unit testing a mathematical function is a square-round-hole problem.
t.test("#CalculateCHUNITHMRating", (t) => {
// cutoffs
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_010_000 } } as DryScore, bbkk), 5.0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_005_000 } } as DryScore, bbkk), 4.5);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 1_000_000 } } as DryScore, bbkk), 4);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 975_000 } } as DryScore, bbkk), 3);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 925_000 } } as DryScore, bbkk), 0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 900_000 } } as DryScore, bbkk), 0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 800_000 } } as DryScore, bbkk), 0);
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 0 } } as DryScore, bbkk), 0);
// inbetweens
t.equal(CalculateCHUNITHMRating({ scoreData: { score: 987_000 } } as DryScore, bbkk), 3.48);
t.end();
});
// Random assertions plucked from bemaniwiki.
t.test("#CalculateVF6", (t) => {
t.equal(CalculateVF6("S", "CLEAR", 99, 16, logger), 0.332);
t.equal(CalculateVF6("AAA+", "ULTIMATE CHAIN", 98, 17, logger), 0.356);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 16, logger), 0.369);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 17, logger), 0.392);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 18, logger), 0.415);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 19, logger), 0.438);
t.equal(CalculateVF6("S", "PERFECT ULTIMATE CHAIN", 100, 20, logger), 0.462);
t.end();
});
t.test("#CalculateRating", (t) => {
t.test("Should call the success calculator if percent > pivotPercent", async (t) => {
const r = await CalculateKTRating(
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 80 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.ok(r > 10, "Should return rating greater than the levelNum of the chart.");
t.end();
});
t.test("Should call the fail calculator if percent > pivotPercent", async (t) => {
const r = await CalculateKTRating(
TestingIIDXSPDryScore,
"iidx",
"SP",
Testing511SPA,
logger
);
t.ok(r < 10, "Should return rating less than the levelNum of the chart.");
t.end();
});
t.test("Should call levelNum if percent === pivotPercent", async (t) => {
const r = await CalculateKTRating(
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 77.7777 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.equal(
// hack for approximate tests
parseFloat(r.toFixed(2)),
10,
"Should return rating exactly that of the levelNum of the chart."
);
t.end();
});
t.test(
"Should trigger safety if completely invalid percent somehow gets through",
async (t) => {
let r = await CalculateKTRating(
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 1000000000 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.equal(r, 0, "Should safely return 0 and log a warning.");
r = await CalculateKTRating(
// not high enough to be non-finite but high enough to be > 1000
deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 200 } }),
"iidx",
"SP",
Testing511SPA,
logger
);
t.equal(r, 0, "Should safely return 0 and log a warning.");
t.end();
}
);
t.end();
});
server/src/lib/score-import/framework/calculated-data/stats.ts
+404 -404
View File
@@ -1,404 +1,404 @@
import { KtLogger } from "lib/logger/logger";
import {
ChartDocument,
Game,
GetGamePTConfig,
Grades,
integer,
Lamps,
Playtypes,
} from "tachi-common";
import { DryScore } from "../common/types";
/**
* Calculates the in-game CHUNITHM rating for a score.
*/
export function CalculateCHUNITHMRating(dryScore: DryScore, chartData: ChartDocument) {
const score = dryScore.scoreData.score;
const levelBase = chartData.levelNum * 100;
let val = 0;
if (score >= 1_007_500) {
val = levelBase + 200;
} else if (score >= 1_005_000) {
val = levelBase + 150 + ((score - 1_005_000) * 10) / 500;
} else if (score >= 1_000_000) {
val = levelBase + 100 + ((score - 1_000_000) * 5) / 100;
} else if (score >= 975_000) {
val = levelBase + ((score - 975_000) * 2) / 500;
} else if (score >= 925_000) {
val = levelBase - 300 + ((score - 925_000) * 3) / 500;
} else if (score >= 900_000) {
val = levelBase - 500 + ((score - 900_000) * 4) / 500;
} else if (score >= 800_000) {
val = (levelBase - 500) / 2 + ((score - 800_000) * ((levelBase - 500) / 2)) / 100_000;
}
return Math.max(Math.floor(val) / 100, 0);
}
/**
* Calculates the in-game GITADORA rating for a score.
*/
export function CalculateGITADORASkill(dryScore: DryScore, chartData: ChartDocument) {
const trueRating = (dryScore.scoreData.percent / 100) * chartData.levelNum * 20;
const flooredRating = Math.floor(trueRating * 100) / 100;
return flooredRating;
}
/**
* Calculates the PikaGreatFunction, used in BPI. I have no idea what this does.
* @returns
*/
function BPIPikaGreatFn(score: integer, max: integer) {
return score === max ? max * 0.8 : 1 + (score / max - 0.5) / (1 - score / max);
}
/**
* Oh boy.
*
* Calculates the "Beat Performance Index" of an IIDX score. This algorithm has many issues,
* but is a direct port of Poyashi's implementation for consistencies sake.
* https://github.com/potakusan/iidx_score_manager/blob/f21ba6b85fcc0bf8b7ca888fa2239a3951a9c9c2/src/components/bpi/index.tsx#L120
*
* @param kaidenEx The kaiden average EX score.
* @param wrEx The world record's EX score.
* @param yourEx Your EX score.
* @param max The maximum amount of EX achievable on this chart.
* @param powCoef What power the BPI should be raised to. This is arbitrary, and assigned on a per-song basis. Defaults to 1.175.
* @returns A number between -15 and 100. Unless your score is better than the world record, in which case
* returns can be above 100.
*/
export function CalculateBPI(
kaidenEx: integer,
wrEx: integer,
yourEx: integer,
max: integer,
pc: number | null
) {
let powCoef = pc ?? 1.175;
if (powCoef === -1) {
powCoef = 1.175;
}
const yourPGF = BPIPikaGreatFn(yourEx, max);
const kaidenPGF = BPIPikaGreatFn(kaidenEx, max);
const wrPGF = BPIPikaGreatFn(wrEx, max);
// no idea what these var names are
const _s_ = yourPGF / kaidenPGF;
const _z_ = wrPGF / kaidenPGF;
const isBetterThanKavg = yourEx >= kaidenEx;
// this line of code isn't mine, and that's why it's *really* bad here.
const bpi =
Math.round(
(isBetterThanKavg ? 100 : -100) *
Math.pow(
(isBetterThanKavg ? Math.log(_s_) : -Math.log(_s_)) / Math.log(_z_),
powCoef
) *
100
) / 100;
if (bpi < -15) {
return -15;
}
return bpi;
}
/**
* Calculate Marvelous Full Combo Points. This algorithm
* is used in LIFE4, and described here:
* https://life4ddr.com/requirements/#mfcpoints
* @returns Null if this score was not eligible, a number otherwise.
*/
export function CalculateMFCP(dryScore: DryScore, chartData: ChartDocument, logger: KtLogger) {
if (dryScore.scoreData.lamp !== "MARVELOUS FULL COMBO") {
return null;
}
// Beginner and BASIC scores are explicitly excluded.
if (chartData.difficulty === "BEGINNER" || chartData.difficulty === "BASIC") {
return null;
}
if (chartData.levelNum < 8) {
return null;
} else if (chartData.levelNum <= 10) {
return 1;
} else if (chartData.levelNum <= 12) {
return 2;
} else if (chartData.levelNum === 13) {
return 4;
} else if (chartData.levelNum === 14) {
return 8;
} else if (chartData.levelNum === 15) {
return 15;
} else if (chartData.levelNum >= 16) {
return 25;
}
logger.warn(
`Invalid levelNum passed to MFCP ${chartData.levelNum}. ChartID ${chartData.chartID}.`
);
// failsafe
return null;
}
const VF5GradeCoefficients = {
PUC: 1.05,
S: 1.05,
"AAA+": 1.02,
AAA: 1.0,
"AA+": 0.97,
AA: 0.94,
"A+": 0.91, // everything below this point (incl. this) is marked with a (?) in bemaniwiki.
A: 0.88,
B: 0.85,
C: 0.82,
D: 0.8,
};
const VF5LampCoefficients = {
"PERFECT ULTIMATE CHAIN": 1.1,
"ULTIMATE CHAIN": 1.05,
"EXCESSIVE CLEAR": 1.02,
CLEAR: 1.0,
FAILED: 0.5,
};
function FloorToNDP(number: number, dp: integer) {
const mul = 10 ** dp;
return Math.floor(number * mul) / mul;
}
export function CalculateVF6(
grade: Grades["sdvx:Single"],
lamp: Lamps["sdvx:Single"],
per: number,
levelNum: number,
logger: KtLogger
) {
const gradeCoefficient = VF5GradeCoefficients[grade];
const lampCoefficient = VF5LampCoefficients[lamp];
if (!lampCoefficient) {
logger.warn(`Invalid lamp of ${lamp} passed to CalculateVF5. Returning null.`);
return null;
}
if (!gradeCoefficient) {
logger.warn(`Invalid grade of ${grade} passed to CalculateVF5. Returning null.`);
return null;
}
const percent = per / 100;
if (!levelNum || !percent) {
return 0;
}
const realVF6 = (levelNum * 2 * percent * gradeCoefficient * lampCoefficient) / 100;
return FloorToNDP(realVF6, 3);
}
// function CalculateJubility(
// dryScore: DryScore<"jubeat:Single">,
// chartData: ChartDocument,
// logger: KtLogger
// ) {
// let rate = dryScore.calculatedData.musicRate; eurgh, this is hard.
// }
interface RatingParameters {
failHarshnessMultiplier: number;
pivotPercent: number;
clearExpMultiplier: number;
}
// Generic Rating Calc that is guaranteed to work for everything. This is unspecialised, and not great.
function KTRatingCalcV1(
percent: number,
levelNum: number,
parameters: RatingParameters,
logger: KtLogger
) {
const percentDiv100 = percent / 100;
if (percentDiv100 < parameters.pivotPercent) {
return RatingCalcV0Fail(percentDiv100, levelNum, parameters);
}
return RatingCalcV1Clear(percentDiv100, levelNum, parameters, logger);
}
function RatingCalcV1Clear(
percentDiv100: number,
levelNum: number,
parameters: RatingParameters,
logger: KtLogger
) {
// https://www.desmos.com/calculator/hn7uxjmjkc
const rating =
Math.cosh(
parameters.clearExpMultiplier * levelNum * (percentDiv100 - parameters.pivotPercent)
) +
(levelNum - 1);
// checks for Infinity or NaN. I'm not sure how this would happen, but it's a failsafe.
if (!Number.isFinite(rating)) {
logger.warn(
`Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid. Defaulting to 0.`
);
return 0;
} else if (rating > 1000) {
logger.warn(
`Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid (> 1000). Defaulting to 0.`
);
return 0;
}
return rating;
}
function RatingCalcV0Fail(percentDiv100: number, levelNum: number, parameters: RatingParameters) {
// https://www.desmos.com/calculator/hn7uxjmjkc
return (
percentDiv100 ** (parameters.failHarshnessMultiplier * levelNum) *
(levelNum / parameters.pivotPercent ** (parameters.failHarshnessMultiplier * levelNum))
);
}
const ratingParameters = {
iidx: {
failHarshnessMultiplier: 0.3,
pivotPercent: 0.7777, // Grade: AA
clearExpMultiplier: 1,
},
bms: {
failHarshnessMultiplier: 0.5,
pivotPercent: 0.7777, // Grade: AA
clearExpMultiplier: 0.75,
},
museca: {
failHarshnessMultiplier: 1,
pivotPercent: 0.8, // grade: not fail
clearExpMultiplier: 1, // no real reason
},
maimai: {
failHarshnessMultiplier: 1,
pivotPercent: 0.8,
clearExpMultiplier: 1,
},
ddr: {
failHarshnessMultiplier: 0.9,
pivotPercent: 0.9,
clearExpMultiplier: 1,
},
};
export function CalculateKTRating(
dryScore: DryScore,
game: "iidx" | "ddr" | "museca",
playtype: Playtypes[Game],
chart: ChartDocument,
logger: KtLogger
) {
const parameters = ratingParameters[game];
const levelNum = chart.levelNum;
return KTRatingCalcV1(dryScore.scoreData.percent, levelNum, parameters, logger);
}
export function CalculateKTLampRating(
dryScore: DryScore,
game: Game,
playtype: Playtypes[Game],
chart: ChartDocument
) {
return LampRatingNoTierlistInfo(dryScore, game, playtype, chart);
}
function LampRatingNoTierlistInfo(
dryScore: DryScore,
game: Game,
playtype: Playtypes[Game],
chart: ChartDocument
) {
const gptConfig = GetGamePTConfig(game, playtype);
const lamps = gptConfig.lamps;
const CLEAR_LAMP_INDEX = lamps.indexOf(gptConfig.clearLamp);
// if this is a clear
if (lamps.indexOf(dryScore.scoreData.lamp) >= CLEAR_LAMP_INDEX) {
// return this chart's numeric level as the lamp rating
return chart.levelNum;
}
// else, this score is worth 0.
return 0;
}
// deprecated calcs
// const VF4GradeCoefficients = {
// S: 1.0,
// "AAA+": 0.99,
// AAA: 0.98,
// "AA+": 0.97,
// AA: 0.96,
// "A+": 0.95,
// A: 0.94,
// B: 0.93,
// C: 0.92,
// D: 0.91,
// };
// export function CalculateVF4(
// grade: Grades["sdvx:Single"],
// per: number,
// levelNum: number,
// logger: KtLogger
// ) {
// const multiplier = 25;
// const gradeCoefficient = VF4GradeCoefficients[grade];
// if (!gradeCoefficient) {
// logger.warn(`Invalid grade of ${grade} passed to CalculateVF4. Returning null.`);
// return null;
// }
// const percent = per / 100;
// if (!levelNum || !percent) {
// return 0;
// }
// return Math.floor(multiplier * (levelNum + 1) * percent * gradeCoefficient);
// }
// // VF5 is just VF6 but to three decimal places instead of four.
// export function CalculateVF5(
// grade: Grades["sdvx:Single"],
// lamp: Lamps["sdvx:Single"],
// per: number,
// levelNum: number,
// logger: KtLogger
// ) {
// const vf6 = CalculateVF6(grade, lamp, per, levelNum, logger);
// if (vf6 === null) {
// return null;
// }
// return FloorToNDP(vf6, 2);
// }
import { KtLogger } from "lib/logger/logger";
import {
ChartDocument,
Game,
GetGamePTConfig,
Grades,
integer,
Lamps,
Playtypes,
} from "tachi-common";
import { DryScore } from "../common/types";
/**
* Calculates the in-game CHUNITHM rating for a score.
*/
export function CalculateCHUNITHMRating(dryScore: DryScore, chartData: ChartDocument) {
const score = dryScore.scoreData.score;
const levelBase = chartData.levelNum * 100;
let val = 0;
if (score >= 1_007_500) {
val = levelBase + 200;
} else if (score >= 1_005_000) {
val = levelBase + 150 + ((score - 1_005_000) * 10) / 500;
} else if (score >= 1_000_000) {
val = levelBase + 100 + ((score - 1_000_000) * 5) / 100;
} else if (score >= 975_000) {
val = levelBase + ((score - 975_000) * 2) / 500;
} else if (score >= 925_000) {
val = levelBase - 300 + ((score - 925_000) * 3) / 500;
} else if (score >= 900_000) {
val = levelBase - 500 + ((score - 900_000) * 4) / 500;
} else if (score >= 800_000) {
val = (levelBase - 500) / 2 + ((score - 800_000) * ((levelBase - 500) / 2)) / 100_000;
}
return Math.max(Math.floor(val) / 100, 0);
}
/**
* Calculates the in-game GITADORA rating for a score.
*/
export function CalculateGITADORASkill(dryScore: DryScore, chartData: ChartDocument) {
const trueRating = (dryScore.scoreData.percent / 100) * chartData.levelNum * 20;
const flooredRating = Math.floor(trueRating * 100) / 100;
return flooredRating;
}
/**
* Calculates the PikaGreatFunction, used in BPI. I have no idea what this does.
* @returns
*/
function BPIPikaGreatFn(score: integer, max: integer) {
return score === max ? max * 0.8 : 1 + (score / max - 0.5) / (1 - score / max);
}
/**
* Oh boy.
*
* Calculates the "Beat Performance Index" of an IIDX score. This algorithm has many issues,
* but is a direct port of Poyashi's implementation for consistencies sake.
* https://github.com/potakusan/iidx_score_manager/blob/f21ba6b85fcc0bf8b7ca888fa2239a3951a9c9c2/src/components/bpi/index.tsx#L120
*
* @param kaidenEx The kaiden average EX score.
* @param wrEx The world record's EX score.
* @param yourEx Your EX score.
* @param max The maximum amount of EX achievable on this chart.
* @param powCoef What power the BPI should be raised to. This is arbitrary, and assigned on a per-song basis. Defaults to 1.175.
* @returns A number between -15 and 100. Unless your score is better than the world record, in which case
* returns can be above 100.
*/
export function CalculateBPI(
kaidenEx: integer,
wrEx: integer,
yourEx: integer,
max: integer,
pc: number | null
) {
let powCoef = pc ?? 1.175;
if (powCoef === -1) {
powCoef = 1.175;
}
const yourPGF = BPIPikaGreatFn(yourEx, max);
const kaidenPGF = BPIPikaGreatFn(kaidenEx, max);
const wrPGF = BPIPikaGreatFn(wrEx, max);
// no idea what these var names are
const _s_ = yourPGF / kaidenPGF;
const _z_ = wrPGF / kaidenPGF;
const isBetterThanKavg = yourEx >= kaidenEx;
// this line of code isn't mine, and that's why it's *really* bad here.
const bpi =
Math.round(
(isBetterThanKavg ? 100 : -100) *
Math.pow(
(isBetterThanKavg ? Math.log(_s_) : -Math.log(_s_)) / Math.log(_z_),
powCoef
) *
100
) / 100;
if (bpi < -15) {
return -15;
}
return bpi;
}
/**
* Calculate Marvelous Full Combo Points. This algorithm
* is used in LIFE4, and described here:
* https://life4ddr.com/requirements/#mfcpoints
* @returns Null if this score was not eligible, a number otherwise.
*/
export function CalculateMFCP(dryScore: DryScore, chartData: ChartDocument, logger: KtLogger) {
if (dryScore.scoreData.lamp !== "MARVELOUS FULL COMBO") {
return null;
}
// Beginner and BASIC scores are explicitly excluded.
if (chartData.difficulty === "BEGINNER" || chartData.difficulty === "BASIC") {
return null;
}
if (chartData.levelNum < 8) {
return null;
} else if (chartData.levelNum <= 10) {
return 1;
} else if (chartData.levelNum <= 12) {
return 2;
} else if (chartData.levelNum === 13) {
return 4;
} else if (chartData.levelNum === 14) {
return 8;
} else if (chartData.levelNum === 15) {
return 15;
} else if (chartData.levelNum >= 16) {
return 25;
}
logger.warn(
`Invalid levelNum passed to MFCP ${chartData.levelNum}. ChartID ${chartData.chartID}.`
);
// failsafe
return null;
}
const VF5GradeCoefficients = {
PUC: 1.05,
S: 1.05,
"AAA+": 1.02,
AAA: 1.0,
"AA+": 0.97,
AA: 0.94,
"A+": 0.91, // everything below this point (incl. this) is marked with a (?) in bemaniwiki.
A: 0.88,
B: 0.85,
C: 0.82,
D: 0.8,
};
const VF5LampCoefficients = {
"PERFECT ULTIMATE CHAIN": 1.1,
"ULTIMATE CHAIN": 1.05,
"EXCESSIVE CLEAR": 1.02,
CLEAR: 1.0,
FAILED: 0.5,
};
function FloorToNDP(number: number, dp: integer) {
const mul = 10 ** dp;
return Math.floor(number * mul) / mul;
}
export function CalculateVF6(
grade: Grades["sdvx:Single"],
lamp: Lamps["sdvx:Single"],
per: number,
levelNum: number,
logger: KtLogger
) {
const gradeCoefficient = VF5GradeCoefficients[grade];
const lampCoefficient = VF5LampCoefficients[lamp];
if (!lampCoefficient) {
logger.warn(`Invalid lamp of ${lamp} passed to CalculateVF5. Returning null.`);
return null;
}
if (!gradeCoefficient) {
logger.warn(`Invalid grade of ${grade} passed to CalculateVF5. Returning null.`);
return null;
}
const percent = per / 100;
if (!levelNum || !percent) {
return 0;
}
const realVF6 = (levelNum * 2 * percent * gradeCoefficient * lampCoefficient) / 100;
return FloorToNDP(realVF6, 3);
}
// function CalculateJubility(
// dryScore: DryScore<"jubeat:Single">,
// chartData: ChartDocument,
// logger: KtLogger
// ) {
// let rate = dryScore.calculatedData.musicRate; eurgh, this is hard.
// }
interface RatingParameters {
failHarshnessMultiplier: number;
pivotPercent: number;
clearExpMultiplier: number;
}
// Generic Rating Calc that is guaranteed to work for everything. This is unspecialised, and not great.
function KTRatingCalcV1(
percent: number,
levelNum: number,
parameters: RatingParameters,
logger: KtLogger
) {
const percentDiv100 = percent / 100;
if (percentDiv100 < parameters.pivotPercent) {
return RatingCalcV0Fail(percentDiv100, levelNum, parameters);
}
return RatingCalcV1Clear(percentDiv100, levelNum, parameters, logger);
}
function RatingCalcV1Clear(
percentDiv100: number,
levelNum: number,
parameters: RatingParameters,
logger: KtLogger
) {
// https://www.desmos.com/calculator/hn7uxjmjkc
const rating =
Math.cosh(
parameters.clearExpMultiplier * levelNum * (percentDiv100 - parameters.pivotPercent)
) +
(levelNum - 1);
// checks for Infinity or NaN. I'm not sure how this would happen, but it's a failsafe.
if (!Number.isFinite(rating)) {
logger.warn(
`Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid. Defaulting to 0.`
);
return 0;
} else if (rating > 1000) {
logger.warn(
`Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid (> 1000). Defaulting to 0.`
);
return 0;
}
return rating;
}
function RatingCalcV0Fail(percentDiv100: number, levelNum: number, parameters: RatingParameters) {
// https://www.desmos.com/calculator/hn7uxjmjkc
return (
percentDiv100 ** (parameters.failHarshnessMultiplier * levelNum) *
(levelNum / parameters.pivotPercent ** (parameters.failHarshnessMultiplier * levelNum))
);
}
const ratingParameters = {
iidx: {
failHarshnessMultiplier: 0.3,
pivotPercent: 0.7777, // Grade: AA
clearExpMultiplier: 1,
},
bms: {
failHarshnessMultiplier: 0.5,
pivotPercent: 0.7777, // Grade: AA
clearExpMultiplier: 0.75,
},
museca: {
failHarshnessMultiplier: 1,
pivotPercent: 0.8, // grade: not fail
clearExpMultiplier: 1, // no real reason
},
maimai: {
failHarshnessMultiplier: 1,
pivotPercent: 0.8,
clearExpMultiplier: 1,
},
ddr: {
failHarshnessMultiplier: 0.9,
pivotPercent: 0.9,
clearExpMultiplier: 1,
},
};
export function CalculateKTRating(
dryScore: DryScore,
game: "iidx" | "ddr" | "museca",
playtype: Playtypes[Game],
chart: ChartDocument,
logger: KtLogger
) {
const parameters = ratingParameters[game];
const levelNum = chart.levelNum;
return KTRatingCalcV1(dryScore.scoreData.percent, levelNum, parameters, logger);
}
export function CalculateKTLampRating(
dryScore: DryScore,
game: Game,
playtype: Playtypes[Game],
chart: ChartDocument
) {
return LampRatingNoTierlistInfo(dryScore, game, playtype, chart);
}
function LampRatingNoTierlistInfo(
dryScore: DryScore,
game: Game,
playtype: Playtypes[Game],
chart: ChartDocument
) {
const gptConfig = GetGamePTConfig(game, playtype);
const lamps = gptConfig.lamps;
const CLEAR_LAMP_INDEX = lamps.indexOf(gptConfig.clearLamp);
// if this is a clear
if (lamps.indexOf(dryScore.scoreData.lamp) >= CLEAR_LAMP_INDEX) {
// return this chart's numeric level as the lamp rating
return chart.levelNum;
}
// else, this score is worth 0.
return 0;
}
// deprecated calcs
// const VF4GradeCoefficients = {
// S: 1.0,
// "AAA+": 0.99,
// AAA: 0.98,
// "AA+": 0.97,
// AA: 0.96,
// "A+": 0.95,
// A: 0.94,
// B: 0.93,
// C: 0.92,
// D: 0.91,
// };
// export function CalculateVF4(
// grade: Grades["sdvx:Single"],
// per: number,
// levelNum: number,
// logger: KtLogger
// ) {
// const multiplier = 25;
// const gradeCoefficient = VF4GradeCoefficients[grade];
// if (!gradeCoefficient) {
// logger.warn(`Invalid grade of ${grade} passed to CalculateVF4. Returning null.`);
// return null;
// }
// const percent = per / 100;
// if (!levelNum || !percent) {
// return 0;
// }
// return Math.floor(multiplier * (levelNum + 1) * percent * gradeCoefficient);
// }
// // VF5 is just VF6 but to three decimal places instead of four.
// export function CalculateVF5(
// grade: Grades["sdvx:Single"],
// lamp: Lamps["sdvx:Single"],
// per: number,
// levelNum: number,
// logger: KtLogger
// ) {
// const vf6 = CalculateVF6(grade, lamp, per, levelNum, logger);
// if (vf6 === null) {
// return null;
// }
// return FloorToNDP(vf6, 2);
// }
server/src/lib/score-import/framework/common/converter-failures.ts
+55 -55
View File
@@ -1,55 +1,55 @@
// Various Errors that can occur during processing.
import { ImportTypes } from "tachi-common";
import { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types";
export class ConverterFailure {
message: string | null;
constructor(message: string | null) {
this.message = message;
}
}
/**
* SkipScoreFailure - This score could not be processed because we do not
* support it. This is not an error, but is not a success either.
* An example scenario would be something like a 5KEY score being imported from SSS.
*/
export class SkipScoreFailure extends ConverterFailure {}
/**
* KTDataNotFoundError - We could not find the song or chart this score
* belongs to. The identifier used to try and match this chart is stored here.
* Alongside any other fields used in the query.
*/
export class KTDataNotFoundFailure<T extends ImportTypes> extends ConverterFailure {
data: ImportTypeDataMap[T];
converterContext: ImportTypeContextMap[T];
importType: T;
constructor(
message: string | null,
importType: T,
data: ImportTypeDataMap[T],
context: ImportTypeContextMap[T]
) {
super(message);
this.importType = importType;
this.data = data;
this.converterContext = context;
}
}
/**
* InvalidScoreError - This score provided invalid data that we
* can not accept.
*/
export class InvalidScoreFailure extends ConverterFailure {}
/**
* KTInternalFailure - An unexpected, internal error has occured,
* and the score could not be processed.
*/
export class InternalFailure extends ConverterFailure {}
// Various Errors that can occur during processing.
import { ImportTypes } from "tachi-common";
import { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types";
export class ConverterFailure {
message: string | null;
constructor(message: string | null) {
this.message = message;
}
}
/**
* SkipScoreFailure - This score could not be processed because we do not
* support it. This is not an error, but is not a success either.
* An example scenario would be something like a 5KEY score being imported from SSS.
*/
export class SkipScoreFailure extends ConverterFailure {}
/**
* KTDataNotFoundError - We could not find the song or chart this score
* belongs to. The identifier used to try and match this chart is stored here.
* Alongside any other fields used in the query.
*/
export class KTDataNotFoundFailure<T extends ImportTypes> extends ConverterFailure {
data: ImportTypeDataMap[T];
converterContext: ImportTypeContextMap[T];
importType: T;
constructor(
message: string | null,
importType: T,
data: ImportTypeDataMap[T],
context: ImportTypeContextMap[T]
) {
super(message);
this.importType = importType;
this.data = data;
this.converterContext = context;
}
}
/**
* InvalidScoreError - This score provided invalid data that we
* can not accept.
*/
export class InvalidScoreFailure extends ConverterFailure {}
/**
* KTInternalFailure - An unexpected, internal error has occured,
* and the score could not be processed.
*/
export class InternalFailure extends ConverterFailure {}
server/src/lib/score-import/framework/common/import-logger.ts
+22 -22
View File
@@ -1,22 +1,22 @@
import { KtLogger, rootLogger } from "lib/logger/logger";
import { ImportTypes, PublicUserDocument } from "tachi-common";
import { Random20Hex } from "utils/misc";
import { FormatUserDoc } from "utils/user";
export function CreateScoreLogger(
user: PublicUserDocument,
importID: string,
importType: ImportTypes
): KtLogger {
const meta = {
context: ["Score Import", importType, FormatUserDoc(user)],
importID,
};
// used so appendLogCtx works
const childLogger = rootLogger.child(meta);
childLogger.defaultMeta = meta;
return childLogger as KtLogger;
}
import { KtLogger, rootLogger } from "lib/logger/logger";
import { ImportTypes, PublicUserDocument } from "tachi-common";
import { Random20Hex } from "utils/misc";
import { FormatUserDoc } from "utils/user";
export function CreateScoreLogger(
user: PublicUserDocument,
importID: string,
importType: ImportTypes
): KtLogger {
const meta = {
context: ["Score Import", importType, FormatUserDoc(user)],
importID,
};
// used so appendLogCtx works
const childLogger = rootLogger.child(meta);
childLogger.defaultMeta = meta;
return childLogger as KtLogger;
}
server/src/lib/score-import/framework/common/score-utils.ts
+163 -163
View File
@@ -1,163 +1,163 @@
import CreateLogCtx from "lib/logger/logger";
import {
ChartDocument,
ESDCore,
Game,
GameToIDStrings,
GetGamePTConfig,
Grades,
IDStrings,
Playtypes,
} from "tachi-common";
import { InternalFailure, InvalidScoreFailure } from "./converter-failures";
const logger = CreateLogCtx(__filename);
/**
* Util for getting a games' grade for a given percent.
*/
export function GetGradeFromPercent<I extends IDStrings = IDStrings>(
game: Game,
playtype: Playtypes[Game],
percent: number
): Grades[I] {
const gptConfig = GetGamePTConfig(game, playtype);
const boundaries = gptConfig.gradeBoundaries;
const grades = gptConfig.grades;
// (hey, this for loop is backwards!)
for (let i = boundaries.length; i >= 0; i--) {
if (percent >= boundaries[i]) {
return grades[i] as Grades[I];
}
}
logger.error(`Could not resolve grade for percent ${percent} on game ${game}`);
throw new InternalFailure(`Could not resolve grade for percent ${percent} on game ${game}`);
}
/**
* A Generic function for calculating a percent from a given score on
* a given game.
*/
export function GenericCalculatePercent(game: Game, score: number, chart?: ChartDocument): number {
switch (game) {
case "ddr":
case "museca":
case "chunithm":
return (score / 1_000_000) * 100;
case "sdvx":
case "usc":
return (score / 10_000_000) * 100;
// case "popn":
// return (score / 100_000) * 100;
case "gitadora":
case "maimai":
return score;
case "bms":
case "iidx":
if (!chart) {
logger.severe("No Chart passed to GenericCalcPercent but game was iidx/bms.");
throw new InternalFailure(
"No Chart passed to GenericCalcPercent but game was iidx/bms."
);
}
// Yeah, we declare it like this so the below return is actually clear.
// eslint-disable-next-line no-case-declarations
const MAX =
(chart as ChartDocument<"iidx:SP" | "bms:7K" | "bms:14K" | "iidx:DP">).data
.notecount * 2;
return (100 * score) / MAX;
default:
logger.severe(`Invalid game passed of ${game} to GenericCalcPercent.`);
throw new InternalFailure(`Invalid game passed of ${game} to GenericCalcPercent.`);
}
}
/**
* Helper utility for validating percents on a game. This throws an InvalidScoreFailure if the percent is
* invalid, and returns void on success.
*
* This exists to support maimai, as it has a dynamic "max percent".
*/
export function ValidatePercent(
game: Game,
playtype: Playtypes[Game],
percent: number,
chart: ChartDocument
) {
// i love needing a helper function for *ONE* game.
if (game === "maimai") {
const mmChart = chart as ChartDocument<"maimai:Single">;
if (percent > mmChart.data.maxPercent) {
throw new InvalidScoreFailure(
`Invalid percent - expected less than ${mmChart.data.maxPercent}.`
);
}
}
const gptConfig = GetGamePTConfig(game, playtype);
if (percent > gptConfig.percentMax) {
throw new InvalidScoreFailure(
`Invalid percent of ${percent} - expected a value less than ${gptConfig.percentMax}% (${chart.songID} ${chart.playtype} ${chart.difficulty}).`
);
}
}
/**
* Generically gets the grade and percent for a given score on a given game. This only works for games where
* grades are just percent boundaries. This will throw an InvalidScoreFailure if the percent is invalid,
* or if the grade is invalid.
*/
export function GenericGetGradeAndPercent<G extends Game>(
game: G,
score: number,
chart: ChartDocument
) {
const percent = GenericCalculatePercent(game, score, chart);
ValidatePercent(game, chart.playtype, percent, chart);
const grade = GetGradeFromPercent(game, chart.playtype, percent) as Grades[GameToIDStrings[G]];
return { percent, grade };
}
/**
* Calculates the ESD for a given game + percent combo. This function returns
* null if the game does not support support ESD.
*/
export function CalculateESDForGame(
game: Game,
playtype: Playtypes[Game],
percent: number
): number | null {
const gptConfig = GetGamePTConfig(game, playtype);
if (!gptConfig.supportsESD) {
return null;
}
return ESDCore.CalculateESD(gptConfig.judgementWindows, percent);
}
/**
* Parses and validates a date from a string.
* @returns Millisecondss from the unix epoch, or null if the initial argument was null or undefined.
*/
export function ParseDateFromString(str: string | undefined | null): number | null {
if (!str) {
return null;
}
const date = Date.parse(str);
if (Number.isNaN(date)) {
throw new InvalidScoreFailure(`Invalid/Unparsable score timestamp of ${str}.`);
}
return date;
}
import CreateLogCtx from "lib/logger/logger";
import {
ChartDocument,
ESDCore,
Game,
GameToIDStrings,
GetGamePTConfig,
Grades,
IDStrings,
Playtypes,
} from "tachi-common";
import { InternalFailure, InvalidScoreFailure } from "./converter-failures";
const logger = CreateLogCtx(__filename);
/**
* Util for getting a games' grade for a given percent.
*/
export function GetGradeFromPercent<I extends IDStrings = IDStrings>(
game: Game,
playtype: Playtypes[Game],
percent: number
): Grades[I] {
const gptConfig = GetGamePTConfig(game, playtype);
const boundaries = gptConfig.gradeBoundaries;
const grades = gptConfig.grades;
// (hey, this for loop is backwards!)
for (let i = boundaries.length; i >= 0; i--) {
if (percent >= boundaries[i]) {
return grades[i] as Grades[I];
}
}
logger.error(`Could not resolve grade for percent ${percent} on game ${game}`);
throw new InternalFailure(`Could not resolve grade for percent ${percent} on game ${game}`);
}
/**
* A Generic function for calculating a percent from a given score on
* a given game.
*/
export function GenericCalculatePercent(game: Game, score: number, chart?: ChartDocument): number {
switch (game) {
case "ddr":
case "museca":
case "chunithm":
return (score / 1_000_000) * 100;
case "sdvx":
case "usc":
return (score / 10_000_000) * 100;
// case "popn":
// return (score / 100_000) * 100;
case "gitadora":
case "maimai":
return score;
case "bms":
case "iidx":
if (!chart) {
logger.severe("No Chart passed to GenericCalcPercent but game was iidx/bms.");
throw new InternalFailure(
"No Chart passed to GenericCalcPercent but game was iidx/bms."
);
}
// Yeah, we declare it like this so the below return is actually clear.
// eslint-disable-next-line no-case-declarations
const MAX =
(chart as ChartDocument<"iidx:SP" | "bms:7K" | "bms:14K" | "iidx:DP">).data
.notecount * 2;
return (100 * score) / MAX;
default:
logger.severe(`Invalid game passed of ${game} to GenericCalcPercent.`);
throw new InternalFailure(`Invalid game passed of ${game} to GenericCalcPercent.`);
}
}
/**
* Helper utility for validating percents on a game. This throws an InvalidScoreFailure if the percent is
* invalid, and returns void on success.
*
* This exists to support maimai, as it has a dynamic "max percent".
*/
export function ValidatePercent(
game: Game,
playtype: Playtypes[Game],
percent: number,
chart: ChartDocument
) {
// i love needing a helper function for *ONE* game.
if (game === "maimai") {
const mmChart = chart as ChartDocument<"maimai:Single">;
if (percent > mmChart.data.maxPercent) {
throw new InvalidScoreFailure(
`Invalid percent - expected less than ${mmChart.data.maxPercent}.`
);
}
}
const gptConfig = GetGamePTConfig(game, playtype);
if (percent > gptConfig.percentMax) {
throw new InvalidScoreFailure(
`Invalid percent of ${percent} - expected a value less than ${gptConfig.percentMax}% (${chart.songID} ${chart.playtype} ${chart.difficulty}).`
);
}
}
/**
* Generically gets the grade and percent for a given score on a given game. This only works for games where
* grades are just percent boundaries. This will throw an InvalidScoreFailure if the percent is invalid,
* or if the grade is invalid.
*/
export function GenericGetGradeAndPercent<G extends Game>(
game: G,
score: number,
chart: ChartDocument
) {
const percent = GenericCalculatePercent(game, score, chart);
ValidatePercent(game, chart.playtype, percent, chart);
const grade = GetGradeFromPercent(game, chart.playtype, percent) as Grades[GameToIDStrings[G]];
return { percent, grade };
}
/**
* Calculates the ESD for a given game + percent combo. This function returns
* null if the game does not support support ESD.
*/
export function CalculateESDForGame(
game: Game,
playtype: Playtypes[Game],
percent: number
): number | null {
const gptConfig = GetGamePTConfig(game, playtype);
if (!gptConfig.supportsESD) {
return null;
}
return ESDCore.CalculateESD(gptConfig.judgementWindows, percent);
}
/**
* Parses and validates a date from a string.
* @returns Millisecondss from the unix epoch, or null if the initial argument was null or undefined.
*/
export function ParseDateFromString(str: string | undefined | null): number | null {
if (!str) {
return null;
}
const date = Date.parse(str);
if (Number.isNaN(date)) {
throw new InvalidScoreFailure(`Invalid/Unparsable score timestamp of ${str}.`);
}
return date;
}
server/src/lib/score-import/framework/score-importing/hydrate-score.ts
+60 -60
View File
@@ -1,60 +1,60 @@
import { ChartDocument, integer, ScoreDocument, SongDocument, GetGamePTConfig } from "tachi-common";
import { KtLogger } from "lib/logger/logger";
import { CreateCalculatedData } from "../calculated-data/calculated-data";
import { CalculateESDForGame } from "../common/score-utils";
import { DryScore } from "../common/types";
/**
* Takes an "intermediate" score and appends the rest of the data it needs.
* @param dryScore The intermediate score to make into a real score.
* @param userID The userID this score is for.
*/
export async function HydrateScore(
userID: integer,
dryScore: DryScore,
chart: ChartDocument,
song: SongDocument,
scoreID: string,
logger: KtLogger
): Promise<ScoreDocument> {
const esd = CalculateESDForGame(
dryScore.game,
chart.playtype,
dryScore.scoreData.percent / 100
);
const calculatedData = await CreateCalculatedData(dryScore, chart, esd, logger);
const { scoreData: dryScoreData, ...rest } = dryScore;
const gptConfig = GetGamePTConfig(dryScore.game, chart.playtype);
// Fill out the rest of the fields we want for scoreData
const scoreData = Object.assign(
{
lampIndex: gptConfig.lamps.indexOf(dryScore.scoreData.lamp),
gradeIndex: gptConfig.grades.indexOf(dryScore.scoreData.grade),
esd,
},
dryScoreData
);
const score: ScoreDocument = {
// extract all of the non-scoreData props from a dry score and push them here
...rest,
// then push our score data.
scoreData,
// everything below this point is sane
highlight: false,
timeAdded: Date.now(),
userID,
calculatedData,
songID: song.id,
chartID: chart.chartID,
scoreID,
playtype: chart.playtype,
isPrimary: chart.isPrimary,
};
return score;
}
import { ChartDocument, integer, ScoreDocument, SongDocument, GetGamePTConfig } from "tachi-common";
import { KtLogger } from "lib/logger/logger";
import { CreateCalculatedData } from "../calculated-data/calculated-data";
import { CalculateESDForGame } from "../common/score-utils";
import { DryScore } from "../common/types";
/**
* Takes an "intermediate" score and appends the rest of the data it needs.
* @param dryScore The intermediate score to make into a real score.
* @param userID The userID this score is for.
*/
export async function HydrateScore(
userID: integer,
dryScore: DryScore,
chart: ChartDocument,
song: SongDocument,
scoreID: string,
logger: KtLogger
): Promise<ScoreDocument> {
const esd = CalculateESDForGame(
dryScore.game,
chart.playtype,
dryScore.scoreData.percent / 100
);
const calculatedData = await CreateCalculatedData(dryScore, chart, esd, logger);
const { scoreData: dryScoreData, ...rest } = dryScore;
const gptConfig = GetGamePTConfig(dryScore.game, chart.playtype);
// Fill out the rest of the fields we want for scoreData
const scoreData = Object.assign(
{
lampIndex: gptConfig.lamps.indexOf(dryScore.scoreData.lamp),
gradeIndex: gptConfig.grades.indexOf(dryScore.scoreData.grade),
esd,
},
dryScoreData
);
const score: ScoreDocument = {
// extract all of the non-scoreData props from a dry score and push them here
...rest,
// then push our score data.
scoreData,
// everything below this point is sane
highlight: false,
timeAdded: Date.now(),
userID,
calculatedData,
songID: song.id,
chartID: chart.chartID,
scoreID,
playtype: chart.playtype,
isPrimary: chart.isPrimary,
};
return score;
}
server/src/lib/score-import/framework/score-importing/insert-score.test.ts
+152 -152
View File
@@ -1,152 +1,152 @@
import { ScoreDocument } from "tachi-common";
import t from "tap";
import db from "external/mongo/db";
import ResetDBState from "test-utils/resets";
import { InsertQueue, QueueScoreInsert } from "./insert-score";
// these two get the same tests, because they're too closely linked
t.test("#QueueScoreInsert, #InsertQueue", async (t) => {
t.beforeEach(ResetDBState);
// empty scoreDB after
t.afterEach(async () => {
await db.scores.remove({});
});
t.test("Single Queue Test", async (t) => {
// fake score doc
const res = await QueueScoreInsert({
scoreID: "foo",
userID: 1,
} as unknown as ScoreDocument);
t.equal(
res,
true,
"QueueScoreInsert should not insert a score when the queue is not full."
);
// this is the best way to get the size of the queue
const flushSize = await InsertQueue(1);
t.equal(flushSize, 1, "QueueScoreInsert should append the score to the queue.");
const dbRes = await db.scores.find({
scoreID: "foo",
});
t.equal(
dbRes.length,
1,
"InsertQueue should insert all 1 members of the queue into the database."
);
t.end();
});
let r = await InsertQueue(1); // flush queue just incase former test fails.
t.test("Queue Overflow Test", async (t) => {
for (let i = 0; i < 499; i++) {
// eslint-disable-next-line no-await-in-loop
await QueueScoreInsert({
scoreID: i,
chartID: "test",
userID: 1,
} as unknown as ScoreDocument);
}
const overflowRes = await QueueScoreInsert({
scoreID: "foo",
chartID: "test",
userID: 1,
} as unknown as ScoreDocument);
t.equal(
overflowRes,
500,
"Appending 500 items to the queue should result in them being inserted."
);
const flushRes = await InsertQueue(1);
t.equal(flushRes, 0, "The queue should now be empty.");
const dbRes = await db.scores.find({
scoreID: { $exists: true },
chartID: "test",
});
t.equal(
dbRes.length,
500,
"InsertQueue should insert all 500 members of the queue into the database."
);
t.end();
});
r = await InsertQueue(1); // flush queue just incase former test fails.
t.equal(r, 0, "Queue should be empty after test.");
t.test("Queue Dedupe Test", async (t) => {
await QueueScoreInsert({
scoreID: 1,
chartID: "foo",
userID: 1,
} as unknown as ScoreDocument);
const r2 = await QueueScoreInsert({
scoreID: 1,
chartID: "foo",
userID: 1,
} as unknown as ScoreDocument);
t.equal(r2, null, "Should return null when a duplicate scoreID is submitted");
const flushRes = await InsertQueue(1);
t.equal(flushRes, 1, "Should flush 1 score document");
const dbRes = await db.scores.find({
chartID: "foo",
});
t.equal(dbRes.length, 1, "Should only insert one document");
});
t.test("Should give separate users separate queues.", async (t) => {
await QueueScoreInsert({
scoreID: "1",
chartID: "foo",
userID: 1,
} as unknown as ScoreDocument);
await QueueScoreInsert({
scoreID: "2",
chartID: "foo",
userID: 2,
} as unknown as ScoreDocument);
const r1 = await InsertQueue(1);
t.equal(r1, 1, "Queue for userID 1 should have length 1.");
const r2 = await InsertQueue(2);
t.equal(r2, 1, "Queue for userID 2 should also have length 1.");
t.end();
});
t.test("Should not throw if InsertQueue is called on an empty queue.", async (t) => {
try {
await InsertQueue(1);
t.pass("Did not throw when inserting an empty queue.");
} catch (err) {
t.fail(err);
}
t.end();
});
t.end();
});
import { ScoreDocument } from "tachi-common";
import t from "tap";
import db from "external/mongo/db";
import ResetDBState from "test-utils/resets";
import { InsertQueue, QueueScoreInsert } from "./insert-score";
// these two get the same tests, because they're too closely linked
t.test("#QueueScoreInsert, #InsertQueue", async (t) => {
t.beforeEach(ResetDBState);
// empty scoreDB after
t.afterEach(async () => {
await db.scores.remove({});
});
t.test("Single Queue Test", async (t) => {
// fake score doc
const res = await QueueScoreInsert({
scoreID: "foo",
userID: 1,
} as unknown as ScoreDocument);
t.equal(
res,
true,
"QueueScoreInsert should not insert a score when the queue is not full."
);
// this is the best way to get the size of the queue
const flushSize = await InsertQueue(1);
t.equal(flushSize, 1, "QueueScoreInsert should append the score to the queue.");
const dbRes = await db.scores.find({
scoreID: "foo",
});
t.equal(
dbRes.length,
1,
"InsertQueue should insert all 1 members of the queue into the database."
);
t.end();
});
let r = await InsertQueue(1); // flush queue just incase former test fails.
t.test("Queue Overflow Test", async (t) => {
for (let i = 0; i < 499; i++) {
// eslint-disable-next-line no-await-in-loop
await QueueScoreInsert({
scoreID: i,
chartID: "test",
userID: 1,
} as unknown as ScoreDocument);
}
const overflowRes = await QueueScoreInsert({
scoreID: "foo",
chartID: "test",
userID: 1,
} as unknown as ScoreDocument);
t.equal(
overflowRes,
500,
"Appending 500 items to the queue should result in them being inserted."
);
const flushRes = await InsertQueue(1);
t.equal(flushRes, 0, "The queue should now be empty.");
const dbRes = await db.scores.find({
scoreID: { $exists: true },
chartID: "test",
});
t.equal(
dbRes.length,
500,
"InsertQueue should insert all 500 members of the queue into the database."
);
t.end();
});
r = await InsertQueue(1); // flush queue just incase former test fails.
t.equal(r, 0, "Queue should be empty after test.");
t.test("Queue Dedupe Test", async (t) => {
await QueueScoreInsert({
scoreID: 1,
chartID: "foo",
userID: 1,
} as unknown as ScoreDocument);
const r2 = await QueueScoreInsert({
scoreID: 1,
chartID: "foo",
userID: 1,
} as unknown as ScoreDocument);
t.equal(r2, null, "Should return null when a duplicate scoreID is submitted");
const flushRes = await InsertQueue(1);
t.equal(flushRes, 1, "Should flush 1 score document");
const dbRes = await db.scores.find({
chartID: "foo",
});
t.equal(dbRes.length, 1, "Should only insert one document");
});
t.test("Should give separate users separate queues.", async (t) => {
await QueueScoreInsert({
scoreID: "1",
chartID: "foo",
userID: 1,
} as unknown as ScoreDocument);
await QueueScoreInsert({
scoreID: "2",
chartID: "foo",
userID: 2,
} as unknown as ScoreDocument);
const r1 = await InsertQueue(1);
t.equal(r1, 1, "Queue for userID 1 should have length 1.");
const r2 = await InsertQueue(2);
t.equal(r2, 1, "Queue for userID 2 should also have length 1.");
t.end();
});
t.test("Should not throw if InsertQueue is called on an empty queue.", async (t) => {
try {
await InsertQueue(1);
t.pass("Did not throw when inserting an empty queue.");
} catch (err) {
t.fail(err);
}
t.end();
});
t.end();
});
server/src/lib/score-import/framework/score-importing/insert-score.ts
+102 -102
View File
@@ -1,102 +1,102 @@
import { ScoreDocument, integer } from "tachi-common";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
const MAX_PIPELINE_LENGTH = 500;
interface ScoreQueue {
queue: ScoreDocument[];
scoreIDSet: Set<string>;
}
const ScoreQueues: Record<integer, ScoreQueue> = {};
/**
* Returns this user's score queue. A score queue is a temporary place scores are saved
* so that they can be inserted into the database in bulk.
*
* This massively improves performance on large imports instead of constantly running single imports.
*
* If a score queue does not exist for the user, one is created.
*/
function GetOrSetScoreQueue(userID: integer) {
const queue = ScoreQueues[userID];
if (!queue) {
return SetScoreQueue(userID);
}
return queue;
}
export function GetScoreQueueMaybe(userID: integer): ScoreQueue | undefined {
return ScoreQueues[userID];
}
function SetScoreQueue(userID: integer) {
const queue: ScoreQueue = {
queue: [],
scoreIDSet: new Set(),
};
ScoreQueues[userID] = queue;
return queue;
}
/**
* Adds a new score to the given queue.
*/
function AddToScoreQueue(scoreQueue: ScoreQueue, score: ScoreDocument) {
scoreQueue.queue.push(score);
scoreQueue.scoreIDSet.add(score.scoreID);
}
export async function InsertQueue(userID: integer) {
const scoreQueue = GetOrSetScoreQueue(userID);
const queuedScores = scoreQueue.queue.splice(0);
if (queuedScores.length !== 0) {
delete ScoreQueues[userID];
try {
await db.scores.insert(queuedScores);
} catch (err) {
logger.warn(
`Triggered duplicate key protection. Race condition protected against, but this is not good.`
);
return null;
}
}
return queuedScores.length;
}
/**
* Adds a score to a queue to be inserted in batch to the database.
* @param score - The score document to queue.
* @returns True on success, The amount of scores inserted on auto-pipeline-flush, and null if
* the score provided is already loaded.
*/
export function QueueScoreInsert(score: ScoreDocument) {
const scoreQueue = GetOrSetScoreQueue(score.userID);
if (scoreQueue.scoreIDSet.has(score.scoreID)) {
// skip
logger.verbose(`Score ID ${score.scoreID} was already queued to be imported.`);
return null;
}
AddToScoreQueue(scoreQueue, score);
logger.debug(`ScoreQueue for ${score.userID} is now at ${scoreQueue.queue.length}.`);
if (scoreQueue.queue.length >= MAX_PIPELINE_LENGTH) {
logger.verbose(`Triggered pipeline flush with len ${scoreQueue.queue.length}.`);
return InsertQueue(score.userID);
}
return true;
}
import { ScoreDocument, integer } from "tachi-common";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
const MAX_PIPELINE_LENGTH = 500;
interface ScoreQueue {
queue: ScoreDocument[];
scoreIDSet: Set<string>;
}
const ScoreQueues: Record<integer, ScoreQueue> = {};
/**
* Returns this user's score queue. A score queue is a temporary place scores are saved
* so that they can be inserted into the database in bulk.
*
* This massively improves performance on large imports instead of constantly running single imports.
*
* If a score queue does not exist for the user, one is created.
*/
function GetOrSetScoreQueue(userID: integer) {
const queue = ScoreQueues[userID];
if (!queue) {
return SetScoreQueue(userID);
}
return queue;
}
export function GetScoreQueueMaybe(userID: integer): ScoreQueue | undefined {
return ScoreQueues[userID];
}
function SetScoreQueue(userID: integer) {
const queue: ScoreQueue = {
queue: [],
scoreIDSet: new Set(),
};
ScoreQueues[userID] = queue;
return queue;
}
/**
* Adds a new score to the given queue.
*/
function AddToScoreQueue(scoreQueue: ScoreQueue, score: ScoreDocument) {
scoreQueue.queue.push(score);
scoreQueue.scoreIDSet.add(score.scoreID);
}
export async function InsertQueue(userID: integer) {
const scoreQueue = GetOrSetScoreQueue(userID);
const queuedScores = scoreQueue.queue.splice(0);
if (queuedScores.length !== 0) {
delete ScoreQueues[userID];
try {
await db.scores.insert(queuedScores);
} catch (err) {
logger.warn(
`Triggered duplicate key protection. Race condition protected against, but this is not good.`
);
return null;
}
}
return queuedScores.length;
}
/**
* Adds a score to a queue to be inserted in batch to the database.
* @param score - The score document to queue.
* @returns True on success, The amount of scores inserted on auto-pipeline-flush, and null if
* the score provided is already loaded.
*/
export function QueueScoreInsert(score: ScoreDocument) {
const scoreQueue = GetOrSetScoreQueue(score.userID);
if (scoreQueue.scoreIDSet.has(score.scoreID)) {
// skip
logger.verbose(`Score ID ${score.scoreID} was already queued to be imported.`);
return null;
}
AddToScoreQueue(scoreQueue, score);
logger.debug(`ScoreQueue for ${score.userID} is now at ${scoreQueue.queue.length}.`);
if (scoreQueue.queue.length >= MAX_PIPELINE_LENGTH) {
logger.verbose(`Triggered pipeline flush with len ${scoreQueue.queue.length}.`);
return InsertQueue(score.userID);
}
return true;
}
server/src/lib/score-import/framework/score-importing/score-id.test.ts
+95 -95
View File
@@ -1,95 +1,95 @@
import { ScoreDocument } from "tachi-common";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data";
import { CreateScoreID, GetWithScoreID } from "./score-id";
t.test("#GetWithScoreID", async (t) => {
await ResetDBState();
const sc = await GetWithScoreID("TESTING_SCORE_ID");
t.not(sc, null, "Return a score for a valid score id");
const nl = await GetWithScoreID("INVALID SCORE ID");
t.equal(nl, null, "Should return null for score IDs it cannot resolve");
t.end();
});
t.test("#CreateScoreID", (t) => {
const scoreID = CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID);
t.match(
scoreID,
/^R[0-9a-f]{40}/u,
"Should return an R followed by 40 characters of lowercase hex as scoreID."
);
t.not(
scoreID,
CreateScoreID(2, TestingIIDXSPDryScore, Testing511SPA.chartID),
"Hash should be affected by userID."
);
t.equal(
scoreID,
CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID),
"ScoreIDs should be consistently generatable"
);
// duplicate score
const minimalScore = {
scoreData: {
percent: TestingIIDXSPDryScore.scoreData.percent,
score: TestingIIDXSPDryScore.scoreData.score,
grade: TestingIIDXSPDryScore.scoreData.grade,
lamp: TestingIIDXSPDryScore.scoreData.lamp,
},
};
t.equal(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should only be affected by score, percent, grade and lamp."
);
minimalScore.scoreData.percent = 0;
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if percent is different."
);
minimalScore.scoreData.percent = TestingIIDXSPDryScore.scoreData.percent;
minimalScore.scoreData.score = 0;
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if score is different."
);
minimalScore.scoreData.score = TestingIIDXSPDryScore.scoreData.score;
minimalScore.scoreData.lamp = "ASSIST CLEAR";
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if lamp is different."
);
minimalScore.scoreData.lamp = TestingIIDXSPDryScore.scoreData.lamp;
minimalScore.scoreData.grade = "F";
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if grade is different."
);
t.end();
});
import { ScoreDocument } from "tachi-common";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data";
import { CreateScoreID, GetWithScoreID } from "./score-id";
t.test("#GetWithScoreID", async (t) => {
await ResetDBState();
const sc = await GetWithScoreID("TESTING_SCORE_ID");
t.not(sc, null, "Return a score for a valid score id");
const nl = await GetWithScoreID("INVALID SCORE ID");
t.equal(nl, null, "Should return null for score IDs it cannot resolve");
t.end();
});
t.test("#CreateScoreID", (t) => {
const scoreID = CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID);
t.match(
scoreID,
/^R[0-9a-f]{40}/u,
"Should return an R followed by 40 characters of lowercase hex as scoreID."
);
t.not(
scoreID,
CreateScoreID(2, TestingIIDXSPDryScore, Testing511SPA.chartID),
"Hash should be affected by userID."
);
t.equal(
scoreID,
CreateScoreID(1, TestingIIDXSPDryScore, Testing511SPA.chartID),
"ScoreIDs should be consistently generatable"
);
// duplicate score
const minimalScore = {
scoreData: {
percent: TestingIIDXSPDryScore.scoreData.percent,
score: TestingIIDXSPDryScore.scoreData.score,
grade: TestingIIDXSPDryScore.scoreData.grade,
lamp: TestingIIDXSPDryScore.scoreData.lamp,
},
};
t.equal(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should only be affected by score, percent, grade and lamp."
);
minimalScore.scoreData.percent = 0;
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if percent is different."
);
minimalScore.scoreData.percent = TestingIIDXSPDryScore.scoreData.percent;
minimalScore.scoreData.score = 0;
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if score is different."
);
minimalScore.scoreData.score = TestingIIDXSPDryScore.scoreData.score;
minimalScore.scoreData.lamp = "ASSIST CLEAR";
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if lamp is different."
);
minimalScore.scoreData.lamp = TestingIIDXSPDryScore.scoreData.lamp;
minimalScore.scoreData.grade = "F";
t.not(
scoreID,
CreateScoreID(1, minimalScore as ScoreDocument, Testing511SPA.chartID),
"ScoreIDs should not produce the same value if grade is different."
);
t.end();
});
server/src/lib/score-import/framework/score-importing/score-id.ts
+32 -32
View File
@@ -1,32 +1,32 @@
import crypto from "crypto";
import db from "external/mongo/db";
import { integer } from "tachi-common";
import { DryScore } from "../common/types";
/**
* Performs sha256 hashing on the input data.
* @param scoreIDString - The string to sha256 hash.
* @returns A sha256 checksum in lowercase hex.
*/
function HashScoreIDString(scoreIDString: string) {
return crypto.createHash("sha256").update(scoreIDString).digest("hex");
}
/**
* Creates an identifier for this score.
* This is used to deduplicate repeated scores.
* @returns @see HashScoreIDString - prefixed with R.
*/
export function CreateScoreID(userID: integer, dryScore: DryScore, chartID: string) {
const hash = HashScoreIDString(
`${userID}|${chartID}|${dryScore.scoreData.lamp}|${dryScore.scoreData.grade}|${dryScore.scoreData.score}|${dryScore.scoreData.percent}`
);
return `R${hash}`;
}
export function GetWithScoreID(scoreID: string) {
return db.scores.findOne({
scoreID,
});
}
import crypto from "crypto";
import db from "external/mongo/db";
import { integer } from "tachi-common";
import { DryScore } from "../common/types";
/**
* Performs sha256 hashing on the input data.
* @param scoreIDString - The string to sha256 hash.
* @returns A sha256 checksum in lowercase hex.
*/
function HashScoreIDString(scoreIDString: string) {
return crypto.createHash("sha256").update(scoreIDString).digest("hex");
}
/**
* Creates an identifier for this score.
* This is used to deduplicate repeated scores.
* @returns @see HashScoreIDString - prefixed with R.
*/
export function CreateScoreID(userID: integer, dryScore: DryScore, chartID: string) {
const hash = HashScoreIDString(
`${userID}|${chartID}|${dryScore.scoreData.lamp}|${dryScore.scoreData.grade}|${dryScore.scoreData.score}|${dryScore.scoreData.percent}`
);
return `R${hash}`;
}
export function GetWithScoreID(scoreID: string) {
return db.scores.findOne({
scoreID,
});
}
server/src/lib/score-import/framework/score-importing/score-import-error.test.ts
+11 -11
View File
@@ -1,11 +1,11 @@
import t from "tap";
import ScoreImportFatalError from "./score-import-error";
t.test("#new ScoreImportFatalError()", (t) => {
const err = new ScoreImportFatalError(500, "error message");
t.equal(err.statusCode, 500, "Should store the status code passed to it.");
t.equal(err.message, "error message", "Should store description as the error message.");
t.end();
});
import t from "tap";
import ScoreImportFatalError from "./score-import-error";
t.test("#new ScoreImportFatalError()", (t) => {
const err = new ScoreImportFatalError(500, "error message");
t.equal(err.statusCode, 500, "Should store the status code passed to it.");
t.equal(err.message, "error message", "Should store description as the error message.");
t.end();
});
server/src/lib/score-import/framework/score-importing/score-import-error.ts
+16 -16
View File
@@ -1,16 +1,16 @@
/**
* Throw this if the ScoreImport needs to exit and return
* something to the user. This is used for expected errors,
* such as the user passing unparsable data. Unlike Failures,
* throwing this at any time will exit the *entire* score import
* process.
*/
export default class ScoreImportFatalError extends Error {
statusCode: number;
constructor(statusCode: number, description: string) {
super(description);
this.statusCode = statusCode;
}
}
/**
* Throw this if the ScoreImport needs to exit and return
* something to the user. This is used for expected errors,
* such as the user passing unparsable data. Unlike Failures,
* throwing this at any time will exit the *entire* score import
* process.
*/
export default class ScoreImportFatalError extends Error {
statusCode: number;
constructor(statusCode: number, description: string) {
super(description);
this.statusCode = statusCode;
}
}
server/src/lib/score-import/framework/score-importing/score-import-main.ts
+350 -350
View File
@@ -1,350 +1,350 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import {
Game,
IDStrings,
ImportDocument,
ImportProcessingInfo,
ImportTypes,
integer,
Playtypes,
PublicUserDocument,
} from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
import { GetUserWithID } from "utils/user";
import { ConverterFunction, ImportInputParser } from "../../import-types/common/types";
import { Converters } from "../../import-types/converters";
import { InternalFailure } from "../common/converter-failures";
import { CreateScoreLogger } from "../common/import-logger";
import { ScorePlaytypeMap } from "../common/types";
import { GetAndUpdateUsersGoals } from "../goals/goals";
import { GetOrSetUserLock, RemoveUserLock } from "../import-locks/lock";
import { UpdateUsersMilestones } from "../milestones/milestones";
import { ProcessPBs } from "../pb/process-pbs";
import { CreateSessions } from "../sessions/sessions";
import { ClassHandler } from "../user-game-stats/types";
import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs";
import ScoreImportFatalError from "./score-import-error";
import { ImportAllIterableData } from "./score-importing";
/**
* Performs a Score Import.
*/
export default async function ScoreImportMain<D, C>(
userID: integer,
userIntent: boolean,
importType: ImportTypes,
InputParser: ImportInputParser<D, C>,
importID: string,
providedLogger?: KtLogger
) {
const user = await GetUserWithID(userID);
if (!user) {
throw new InternalFailure(
`User with ID ${userID} does not exist, but attempted to make an import?`
);
}
// in the event of any error, we remove the user lock.
try {
const lock = await GetOrSetUserLock(user.id);
if (lock) {
// @danger
// Throwing away an import if the user already has one outgoing is *bad*, as in the case
// of degraded performance we might just start throwing scores away. This is obviously
// not great, but any other solution involves making a queue, which can't be done because
// InputParser is a very dynamic function that cannot be stored in redis or something.
//
// Under normal circumstances, there is no scenario where a user would have two ongoing
// imports at the same time - even if they were using single-score imports on a 5 second
// chart, as each score import takes only around ~10-15millisecondss.
throw new ScoreImportFatalError(409, "This user already has an ongoing import.");
}
const timeStarted = Date.now();
let logger;
if (!providedLogger) {
// If they weren't given to us -
// we create an "import logger".
// this holds a reference to the user's name, ID, and type
// of score import for any future debugging.
logger = CreateScoreLogger(user, importID, importType);
logger.debug("Received import request.");
} else {
logger = providedLogger;
}
// --- 1. Parsing ---
// We get an iterable from the provided parser function, alongside some context and a converter function.
// This iterable does not have to be an array - it's anything that's iterable, like a generator.
const parseTimeStart = process.hrtime.bigint();
const { iterable, context, game, classHandler } = await InputParser(logger);
const parseTime = GetMillisecondsSince(parseTimeStart);
logger.debug(`Parsing took ${parseTime} millisecondss.`);
// We have to cast here due to typescript generic confusions. This is guaranteed` to be correct.
const ConverterFunction = Converters[importType] as unknown as ConverterFunction<D, C>;
// --- 2. Importing ---
// ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data.
const importTimeStart = process.hrtime.bigint();
const importInfo = await ImportAllIterableData(
user.id,
importType,
iterable,
ConverterFunction,
context,
game,
logger
);
const importTime = GetMillisecondsSince(importTimeStart);
const importTimeRel = importTime / importInfo.length;
logger.debug(`Importing took ${importTime} millisecondss. (${importTimeRel}ms/doc)`);
// Steps 3-8 are handled inside here.
// This was moved inside here so the score de-orphaning process
// could hook into importing better
const {
playtypes,
scoreIDs,
errors,
sessionInfo,
classDeltas,
goalInfo,
milestoneInfo,
relativeTimes,
absoluteTimes,
} = await HandlePostImportSteps(importInfo, user, importType, game, classHandler, logger);
const { importParseTimeRel, pbTimeRel, sessionTimeRel } = relativeTimes;
const { importParseTime, sessionTime, pbTime, ugsTime, goalTime, milestoneTime } =
absoluteTimes;
// --- 9. Finalise Import Document ---
// Create and Save an import document to the database, and finish everything up!
const ImportDocument: ImportDocument = {
importType,
idStrings: playtypes.map((e) => `${game}:${e}`) as IDStrings[],
scoreIDs,
playtypes,
game,
errors,
importID,
timeFinished: Date.now(),
timeStarted,
createdSessions: sessionInfo,
userID: user.id,
classDeltas,
goalInfo,
milestoneInfo,
userIntent,
};
const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${
importInfo.length
} documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${
sessionInfo.length
}). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`;
// I only really want to log "big" imports. The others are here for debugging purposes.
if (scoreIDs.length > 500) {
logger.info(logMessage);
} else if (scoreIDs.length > 1) {
logger.verbose(logMessage);
} else {
logger.debug(logMessage);
}
await db.imports.insert(ImportDocument);
// we don't await this because we don't
// particularly care about waiting for it.
db["import-timings"].insert({
importID,
timestamp: Date.now(),
total: ImportDocument.timeFinished - timeStarted,
rel: {
import: importTimeRel,
importParse: importParseTimeRel,
pb: pbTimeRel,
session: sessionTimeRel,
},
abs: {
parse: parseTime,
import: importTime,
importParse: importParseTime,
session: sessionTime,
pb: pbTime,
ugs: ugsTime,
goal: goalTime,
milestone: milestoneTime,
},
});
return ImportDocument;
} finally {
await RemoveUserLock(user.id);
}
}
/**
* Handles every single processing step after actually loading scores
* into the database, such as updating goals, reprocessing sessions,
* and updating a users game stats.
*/
export async function HandlePostImportSteps(
importInfo: ImportProcessingInfo[],
user: PublicUserDocument,
importType: ImportTypes,
game: Game,
classHandler: ClassHandler | null,
logger: KtLogger
) {
// --- 3. ParseImportInfo ---
// ImportInfo is a relatively complex structure. We need some information from it for subsequent steps
// such as the list of chartIDs involved in this import.
const importParseTimeStart = process.hrtime.bigint();
const { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo);
const importParseTime = GetMillisecondsSince(importParseTimeStart);
const importParseTimeRel = importParseTime / importInfo.length;
logger.debug(
`Import Parsing took ${importParseTime} millisecondss. (${importParseTimeRel}ms/doc)`
);
// --- 4. Sessions ---
// We create (or update existing) sessions here. This uses the aforementioned parsed import info
// to determine what goes where.
const sessionTimeStart = process.hrtime.bigint();
const sessionInfo = await CreateSessions(user.id, importType, game, scorePlaytypeMap, logger);
const sessionTime = GetMillisecondsSince(sessionTimeStart);
const sessionTimeRel = sessionTime / sessionInfo.length;
logger.debug(`Session Processing took ${sessionTime} millisecondss (${sessionTimeRel}ms/doc).`);
// --- 5. PersonalBests ---
// We want to keep an updated reference of a users best score on a given chart.
// This function also handles conjoining different scores together (such as unioning best lamp and
// best score).
const pbTimeStart = process.hrtime.bigint();
await ProcessPBs(user.id, chartIDs, logger);
const pbTime = GetMillisecondsSince(pbTimeStart);
const pbTimeRel = pbTime / chartIDs.size;
logger.debug(`PB Processing took ${pbTime} millisecondss (${pbTimeRel}ms/doc)`);
const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][];
// --- 6. Game Stats ---
// This function updates the users "stats" for this game - such as their profile rating or their classes.
const ugsTimeStart = process.hrtime.bigint();
const classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, classHandler, logger);
const ugsTime = GetMillisecondsSince(ugsTimeStart);
logger.debug(`UGS Processing took ${ugsTime} millisecondss.`);
// --- 7. Goals ---
// Evaluate and update the users goals. This returns information about goals that have changed.
const goalTimeStart = process.hrtime.bigint();
const goalInfo = await GetAndUpdateUsersGoals(game, user.id, chartIDs, logger);
const goalTime = GetMillisecondsSince(goalTimeStart);
logger.debug(`Goal Processing took ${goalTime} millisecondss.`);
// --- 8. Milestones ---
// Evaluate and update the users milestones. This returns...
const milestoneTimeStart = process.hrtime.bigint();
const milestoneInfo = await UpdateUsersMilestones(goalInfo, game, playtypes, user.id, logger);
const milestoneTime = GetMillisecondsSince(milestoneTimeStart);
logger.debug(`Milestone Processing took ${milestoneTime} millisecondss.`);
return {
classDeltas,
milestoneInfo,
goalInfo,
playtypes,
scoreIDs,
errors,
sessionInfo,
relativeTimes: {
importParseTimeRel,
pbTimeRel,
sessionTimeRel,
},
absoluteTimes: {
importParseTime,
sessionTime,
pbTime,
ugsTime,
goalTime,
milestoneTime,
},
};
}
/**
* Calls UpdateUsersGamePlaytypeStats for every playtype in the import.
* @returns A flattened array of ClassDeltas
*/
async function UpdateUsersGameStats(
game: Game,
playtypes: Playtypes[Game][],
userID: integer,
classHandler: ClassHandler | null,
logger: KtLogger
) {
const promises = [];
for (const pt of playtypes) {
promises.push(UpdateUsersGamePlaytypeStats(game, pt, userID, classHandler, logger));
}
const r = await Promise.all(promises);
return r.flat(1);
}
/**
* Parses the return of ImportProcessingInfo into relevant information
* for the rest the import.
* @returns The list of scoreIDs used in the import, the list of errors
* A set of unique chartIDs involved in the import and the scores mapped
* on their playtype.
*/
function ParseImportInfo(importInfo: ImportProcessingInfo[]) {
const scorePlaytypeMap: ScorePlaytypeMap = Object.create(null);
const scoreIDs = [];
const errors = [];
const chartIDs: Set<string> = new Set();
for (const info of importInfo) {
if (info.success) {
scoreIDs.push(info.content.score.scoreID);
chartIDs.add(info.content.score.chartID);
if (scorePlaytypeMap[info.content.score.playtype]) {
scorePlaytypeMap[info.content.score.playtype]!.push(info.content.score);
} else {
scorePlaytypeMap[info.content.score.playtype] = [info.content.score];
}
} else {
errors.push({ type: info.type, message: info.message });
}
}
return { scoreIDs, errors, scorePlaytypeMap, chartIDs };
}
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import {
Game,
IDStrings,
ImportDocument,
ImportProcessingInfo,
ImportTypes,
integer,
Playtypes,
PublicUserDocument,
} from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
import { GetUserWithID } from "utils/user";
import { ConverterFunction, ImportInputParser } from "../../import-types/common/types";
import { Converters } from "../../import-types/converters";
import { InternalFailure } from "../common/converter-failures";
import { CreateScoreLogger } from "../common/import-logger";
import { ScorePlaytypeMap } from "../common/types";
import { GetAndUpdateUsersGoals } from "../goals/goals";
import { GetOrSetUserLock, RemoveUserLock } from "../import-locks/lock";
import { UpdateUsersMilestones } from "../milestones/milestones";
import { ProcessPBs } from "../pb/process-pbs";
import { CreateSessions } from "../sessions/sessions";
import { ClassHandler } from "../user-game-stats/types";
import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs";
import ScoreImportFatalError from "./score-import-error";
import { ImportAllIterableData } from "./score-importing";
/**
* Performs a Score Import.
*/
export default async function ScoreImportMain<D, C>(
userID: integer,
userIntent: boolean,
importType: ImportTypes,
InputParser: ImportInputParser<D, C>,
importID: string,
providedLogger?: KtLogger
) {
const user = await GetUserWithID(userID);
if (!user) {
throw new InternalFailure(
`User with ID ${userID} does not exist, but attempted to make an import?`
);
}
// in the event of any error, we remove the user lock.
try {
const lock = await GetOrSetUserLock(user.id);
if (lock) {
// @danger
// Throwing away an import if the user already has one outgoing is *bad*, as in the case
// of degraded performance we might just start throwing scores away. This is obviously
// not great, but any other solution involves making a queue, which can't be done because
// InputParser is a very dynamic function that cannot be stored in redis or something.
//
// Under normal circumstances, there is no scenario where a user would have two ongoing
// imports at the same time - even if they were using single-score imports on a 5 second
// chart, as each score import takes only around ~10-15millisecondss.
throw new ScoreImportFatalError(409, "This user already has an ongoing import.");
}
const timeStarted = Date.now();
let logger;
if (!providedLogger) {
// If they weren't given to us -
// we create an "import logger".
// this holds a reference to the user's name, ID, and type
// of score import for any future debugging.
logger = CreateScoreLogger(user, importID, importType);
logger.debug("Received import request.");
} else {
logger = providedLogger;
}
// --- 1. Parsing ---
// We get an iterable from the provided parser function, alongside some context and a converter function.
// This iterable does not have to be an array - it's anything that's iterable, like a generator.
const parseTimeStart = process.hrtime.bigint();
const { iterable, context, game, classHandler } = await InputParser(logger);
const parseTime = GetMillisecondsSince(parseTimeStart);
logger.debug(`Parsing took ${parseTime} millisecondss.`);
// We have to cast here due to typescript generic confusions. This is guaranteed` to be correct.
const ConverterFunction = Converters[importType] as unknown as ConverterFunction<D, C>;
// --- 2. Importing ---
// ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data.
const importTimeStart = process.hrtime.bigint();
const importInfo = await ImportAllIterableData(
user.id,
importType,
iterable,
ConverterFunction,
context,
game,
logger
);
const importTime = GetMillisecondsSince(importTimeStart);
const importTimeRel = importTime / importInfo.length;
logger.debug(`Importing took ${importTime} millisecondss. (${importTimeRel}ms/doc)`);
// Steps 3-8 are handled inside here.
// This was moved inside here so the score de-orphaning process
// could hook into importing better
const {
playtypes,
scoreIDs,
errors,
sessionInfo,
classDeltas,
goalInfo,
milestoneInfo,
relativeTimes,
absoluteTimes,
} = await HandlePostImportSteps(importInfo, user, importType, game, classHandler, logger);
const { importParseTimeRel, pbTimeRel, sessionTimeRel } = relativeTimes;
const { importParseTime, sessionTime, pbTime, ugsTime, goalTime, milestoneTime } =
absoluteTimes;
// --- 9. Finalise Import Document ---
// Create and Save an import document to the database, and finish everything up!
const ImportDocument: ImportDocument = {
importType,
idStrings: playtypes.map((e) => `${game}:${e}`) as IDStrings[],
scoreIDs,
playtypes,
game,
errors,
importID,
timeFinished: Date.now(),
timeStarted,
createdSessions: sessionInfo,
userID: user.id,
classDeltas,
goalInfo,
milestoneInfo,
userIntent,
};
const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${
importInfo.length
} documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${
sessionInfo.length
}). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`;
// I only really want to log "big" imports. The others are here for debugging purposes.
if (scoreIDs.length > 500) {
logger.info(logMessage);
} else if (scoreIDs.length > 1) {
logger.verbose(logMessage);
} else {
logger.debug(logMessage);
}
await db.imports.insert(ImportDocument);
// we don't await this because we don't
// particularly care about waiting for it.
db["import-timings"].insert({
importID,
timestamp: Date.now(),
total: ImportDocument.timeFinished - timeStarted,
rel: {
import: importTimeRel,
importParse: importParseTimeRel,
pb: pbTimeRel,
session: sessionTimeRel,
},
abs: {
parse: parseTime,
import: importTime,
importParse: importParseTime,
session: sessionTime,
pb: pbTime,
ugs: ugsTime,
goal: goalTime,
milestone: milestoneTime,
},
});
return ImportDocument;
} finally {
await RemoveUserLock(user.id);
}
}
/**
* Handles every single processing step after actually loading scores
* into the database, such as updating goals, reprocessing sessions,
* and updating a users game stats.
*/
export async function HandlePostImportSteps(
importInfo: ImportProcessingInfo[],
user: PublicUserDocument,
importType: ImportTypes,
game: Game,
classHandler: ClassHandler | null,
logger: KtLogger
) {
// --- 3. ParseImportInfo ---
// ImportInfo is a relatively complex structure. We need some information from it for subsequent steps
// such as the list of chartIDs involved in this import.
const importParseTimeStart = process.hrtime.bigint();
const { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo);
const importParseTime = GetMillisecondsSince(importParseTimeStart);
const importParseTimeRel = importParseTime / importInfo.length;
logger.debug(
`Import Parsing took ${importParseTime} millisecondss. (${importParseTimeRel}ms/doc)`
);
// --- 4. Sessions ---
// We create (or update existing) sessions here. This uses the aforementioned parsed import info
// to determine what goes where.
const sessionTimeStart = process.hrtime.bigint();
const sessionInfo = await CreateSessions(user.id, importType, game, scorePlaytypeMap, logger);
const sessionTime = GetMillisecondsSince(sessionTimeStart);
const sessionTimeRel = sessionTime / sessionInfo.length;
logger.debug(`Session Processing took ${sessionTime} millisecondss (${sessionTimeRel}ms/doc).`);
// --- 5. PersonalBests ---
// We want to keep an updated reference of a users best score on a given chart.
// This function also handles conjoining different scores together (such as unioning best lamp and
// best score).
const pbTimeStart = process.hrtime.bigint();
await ProcessPBs(user.id, chartIDs, logger);
const pbTime = GetMillisecondsSince(pbTimeStart);
const pbTimeRel = pbTime / chartIDs.size;
logger.debug(`PB Processing took ${pbTime} millisecondss (${pbTimeRel}ms/doc)`);
const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][];
// --- 6. Game Stats ---
// This function updates the users "stats" for this game - such as their profile rating or their classes.
const ugsTimeStart = process.hrtime.bigint();
const classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, classHandler, logger);
const ugsTime = GetMillisecondsSince(ugsTimeStart);
logger.debug(`UGS Processing took ${ugsTime} millisecondss.`);
// --- 7. Goals ---
// Evaluate and update the users goals. This returns information about goals that have changed.
const goalTimeStart = process.hrtime.bigint();
const goalInfo = await GetAndUpdateUsersGoals(game, user.id, chartIDs, logger);
const goalTime = GetMillisecondsSince(goalTimeStart);
logger.debug(`Goal Processing took ${goalTime} millisecondss.`);
// --- 8. Milestones ---
// Evaluate and update the users milestones. This returns...
const milestoneTimeStart = process.hrtime.bigint();
const milestoneInfo = await UpdateUsersMilestones(goalInfo, game, playtypes, user.id, logger);
const milestoneTime = GetMillisecondsSince(milestoneTimeStart);
logger.debug(`Milestone Processing took ${milestoneTime} millisecondss.`);
return {
classDeltas,
milestoneInfo,
goalInfo,
playtypes,
scoreIDs,
errors,
sessionInfo,
relativeTimes: {
importParseTimeRel,
pbTimeRel,
sessionTimeRel,
},
absoluteTimes: {
importParseTime,
sessionTime,
pbTime,
ugsTime,
goalTime,
milestoneTime,
},
};
}
/**
* Calls UpdateUsersGamePlaytypeStats for every playtype in the import.
* @returns A flattened array of ClassDeltas
*/
async function UpdateUsersGameStats(
game: Game,
playtypes: Playtypes[Game][],
userID: integer,
classHandler: ClassHandler | null,
logger: KtLogger
) {
const promises = [];
for (const pt of playtypes) {
promises.push(UpdateUsersGamePlaytypeStats(game, pt, userID, classHandler, logger));
}
const r = await Promise.all(promises);
return r.flat(1);
}
/**
* Parses the return of ImportProcessingInfo into relevant information
* for the rest the import.
* @returns The list of scoreIDs used in the import, the list of errors
* A set of unique chartIDs involved in the import and the scores mapped
* on their playtype.
*/
function ParseImportInfo(importInfo: ImportProcessingInfo[]) {
const scorePlaytypeMap: ScorePlaytypeMap = Object.create(null);
const scoreIDs = [];
const errors = [];
const chartIDs: Set<string> = new Set();
for (const info of importInfo) {
if (info.success) {
scoreIDs.push(info.content.score.scoreID);
chartIDs.add(info.content.score.chartID);
if (scorePlaytypeMap[info.content.score.playtype]) {
scorePlaytypeMap[info.content.score.playtype]!.push(info.content.score);
} else {
scorePlaytypeMap[info.content.score.playtype] = [info.content.score];
}
} else {
errors.push({ type: info.type, message: info.message });
}
}
return { scoreIDs, errors, scorePlaytypeMap, chartIDs };
}
server/src/lib/score-import/framework/score-importing/score-importing.ts
+339 -339
View File
@@ -1,339 +1,339 @@
import db from "external/mongo/db";
import { AppendLogCtx, KtLogger } from "lib/logger/logger";
import {
ChartDocument,
Game,
IDStrings,
ImportProcessingInfo,
ImportTypes,
integer,
ScoreDocument,
SongDocument,
} from "tachi-common";
import {
ConverterFnReturnOrFailure,
ConverterFnSuccessReturn,
ConverterFunction,
} from "../../import-types/common/types";
import {
ConverterFailure,
InternalFailure,
InvalidScoreFailure,
KTDataNotFoundFailure,
SkipScoreFailure,
} from "../common/converter-failures";
import { DryScore } from "../common/types";
import { OrphanScore } from "../orphans/orphans";
import { HydrateScore } from "./hydrate-score";
import { GetScoreQueueMaybe, InsertQueue, QueueScoreInsert } from "./insert-score";
import { CreateScoreID } from "./score-id";
/**
* Processes the iterable data into the Tachi database.
* @param userID - The user this score import was for.
* @param iterableData - The data to iterate upon.
* @param ConverterFunction - The function needed to convert the data into an IntermediateScore
* @param context - Any context the Converter may need in order to make decisions.
* @returns An array of ImportProcessInfo objects.
*/
export async function ImportAllIterableData<D, C>(
userID: integer,
importType: ImportTypes,
iterableData: Iterable<D> | AsyncIterable<D>,
ConverterFunction: ConverterFunction<D, C>,
context: C,
game: Game,
logger: KtLogger
): Promise<ImportProcessingInfo[]> {
logger.verbose("Getting Blacklist...");
// @optimisable: could filter harder with score.game and score.playtype
// stuff.
const blacklist = (
await db["score-blacklist"].find({
userID,
})
).map((e) => e.scoreID);
logger.verbose(`Starting Data Processing...`);
const processedResults = [];
// for await is used here as iterableData may be an async iterable
// An example would be making an api request after exhausting
// the first set of data.
for await (const data of iterableData) {
processedResults.push(
await ImportIterableDatapoint(
userID,
importType,
data,
ConverterFunction,
context,
game,
blacklist,
logger
)
);
}
// We need to filter out nulls, which we don't care for (these are neither successes or failures)
logger.verbose(`Finished Importing Data (${processedResults.length} datapoints).`);
logger.debug(`Removing null returns...`);
const datapoints = processedResults.filter(
(e) => e !== null
) as ImportProcessingInfo<IDStrings>[];
logger.debug(`Removed null from results.`);
logger.verbose(`Recieved ${datapoints.length} returns, from ${processedResults.length} data.`);
// Flush the score queue out after finishing most of the import. This ensures no scores get left in the
// queue.
const emptied = await InsertQueue(userID);
if (emptied) {
logger.verbose(`Emptied ${emptied} documents from score queue.`);
}
return datapoints;
}
/**
* Processes a single data object into one or many ImportProcessingInfo objects.
* @param userID - The user this score is from.
* @param data - The data to process.
* @param ConverterFunction - The processor function that takes the data and returns the partialScore(s)
* @param context - Any context the processor might need that it can not infer from the data object.
* @returns An array of ImportProcessingInfo objects, or a single ImportProcessingInfo object
*/
export async function ImportIterableDatapoint<D, C>(
userID: integer,
importType: ImportTypes,
data: D,
ConverterFunction: ConverterFunction<D, C>,
context: C,
game: Game,
blacklist: string[],
logger: KtLogger
): Promise<ImportProcessingInfo | null> {
// Converter Function Return
let cfnReturn: ConverterFnReturnOrFailure;
try {
cfnReturn = await ConverterFunction(data, context, importType, logger);
} catch (err) {
cfnReturn = err as ConverterFailure | Error;
}
// if this conversion failed, return it in the proper format
if (cfnReturn instanceof ConverterFailure) {
if (cfnReturn instanceof KTDataNotFoundFailure) {
logger.info(`KTDataNotFoundFailure: ${cfnReturn.message ?? "No message?"}`, {
cfnReturn,
hideFromConsole: ["cfnReturn"],
});
logger.debug("Inserting orphan...", { cfnReturn });
const insertOrphan = await OrphanScore(
cfnReturn.importType,
userID,
cfnReturn.data,
cfnReturn.converterContext,
cfnReturn.message,
game,
logger
);
if (insertOrphan.success) {
logger.debug("Orphan inserted successfully.", { orphanID: insertOrphan.orphanID });
return {
success: false,
type: "KTDataNotFound",
message: cfnReturn.message,
content: {
context: cfnReturn.converterContext,
data: cfnReturn.data,
orphanID: insertOrphan.orphanID,
},
};
}
logger.debug(`Orphan already exists.`, { orphanID: insertOrphan.orphanID });
return {
success: false,
type: "OrphanExists",
message: cfnReturn.message,
content: {
orphanID: insertOrphan.orphanID,
},
};
} else if (cfnReturn instanceof InvalidScoreFailure) {
logger.info(`InvalidScoreFailure: ${cfnReturn.message ?? "No message?"}`, {
cfnReturn,
hideFromConsole: ["cfnReturn"],
});
return {
success: false,
type: "InvalidDatapoint",
message: cfnReturn.message,
content: {},
};
} else if (cfnReturn instanceof InternalFailure) {
logger.error(`Internal error occured.`, { cfnReturn });
return {
success: false,
type: "InternalError",
// could return cfnReturn.message here, but we might want to hide the details of the crash.
message: "An internal error has occured.",
content: {},
};
} else if (cfnReturn instanceof SkipScoreFailure) {
return null;
} else {
logger.warn(`Unknown error returned as ConverterFailure, Ignoring.`, {
err: cfnReturn,
});
return {
success: false,
type: "InternalError",
message: "An internal service error has occured.",
content: {},
};
}
}
if (cfnReturn instanceof Error) {
logger.error(`Unknown error thrown from converter, Ignoring.`, {
err: cfnReturn,
});
return {
success: false,
type: "InternalError",
message: "An internal service error has occured.",
content: {},
};
}
return ProcessSuccessfulConverterReturn(
userID,
cfnReturn as ConverterFnSuccessReturn,
blacklist,
logger
);
}
export async function ProcessSuccessfulConverterReturn(
userID: integer,
cfnReturn: ConverterFnSuccessReturn,
blacklist: string[],
logger: KtLogger,
forceImmediateImport = false
): Promise<ImportProcessingInfo | null> {
const result = await HydrateAndInsertScore(
userID,
cfnReturn.dryScore,
cfnReturn.chart,
cfnReturn.song,
blacklist,
logger,
forceImmediateImport
);
// This used to be a ScoreExists error. However, we never actually care about
// handling ScoreExists errors (they're nobodies issue)
// so instead, the function will just return null, and we pass that on here.
if (result === null) {
return null;
}
logger.debug(`Successfully imported score: ${result.scoreID}`);
return {
success: true,
type: "ScoreImported",
message: null,
content: {
score: result,
},
};
}
/**
* Hydrates and inserts a score to the Tachi database.
* @param userID - The user this score is from.
* @param dryScore - The score that is to be hydrated and inserted.
* @param chart - The chart this score is on.
* @param song - The song this score is on.
* @param blacklist - A list of ScoreIDs to never write to the database.
*
* @param force - Whether to immediately insert the score into the database
* or not.
*/
async function HydrateAndInsertScore(
userID: integer,
dryScore: DryScore,
chart: ChartDocument,
song: SongDocument,
blacklist: string[],
importLogger: KtLogger,
force = false
): Promise<ScoreDocument | null> {
const scoreID = CreateScoreID(userID, dryScore, chart.chartID);
// sub-context the logger so the below logs are more accurate
const logger = AppendLogCtx(scoreID, importLogger);
if (blacklist.length && blacklist.includes(scoreID)) {
logger.verbose("Skipped score, as it was on the blacklist.");
return null;
}
const existingScore = await db.scores.findOne(
{
scoreID,
},
{
// micro-optimisation - mongoDB is significantly faster when returning less fields
// since we only care about whether we have a score or not here, we can minimise returned
// fields.
//
// feel free to test this yourself! - zkldi (09/04/2021)
projection: {
_id: 1,
},
}
);
if (existingScore) {
logger.verbose(`Skipped score.`);
return null;
}
// If this users score queue
if (GetScoreQueueMaybe(userID)?.scoreIDSet.has(scoreID)) {
logger.verbose(`Skipped score.`);
return null;
}
const score = await HydrateScore(userID, dryScore, chart, song, scoreID, logger);
let res;
if (force) {
res = await db.scores.insert(score);
} else {
res = await QueueScoreInsert(score);
}
// this is a last resort for avoiding doubled imports
if (res === null) {
logger.verbose(`Skipped score - Race Condition protection triggered.`);
return null;
}
return score;
}
import db from "external/mongo/db";
import { AppendLogCtx, KtLogger } from "lib/logger/logger";
import {
ChartDocument,
Game,
IDStrings,
ImportProcessingInfo,
ImportTypes,
integer,
ScoreDocument,
SongDocument,
} from "tachi-common";
import {
ConverterFnReturnOrFailure,
ConverterFnSuccessReturn,
ConverterFunction,
} from "../../import-types/common/types";
import {
ConverterFailure,
InternalFailure,
InvalidScoreFailure,
KTDataNotFoundFailure,
SkipScoreFailure,
} from "../common/converter-failures";
import { DryScore } from "../common/types";
import { OrphanScore } from "../orphans/orphans";
import { HydrateScore } from "./hydrate-score";
import { GetScoreQueueMaybe, InsertQueue, QueueScoreInsert } from "./insert-score";
import { CreateScoreID } from "./score-id";
/**
* Processes the iterable data into the Tachi database.
* @param userID - The user this score import was for.
* @param iterableData - The data to iterate upon.
* @param ConverterFunction - The function needed to convert the data into an IntermediateScore
* @param context - Any context the Converter may need in order to make decisions.
* @returns An array of ImportProcessInfo objects.
*/
export async function ImportAllIterableData<D, C>(
userID: integer,
importType: ImportTypes,
iterableData: Iterable<D> | AsyncIterable<D>,
ConverterFunction: ConverterFunction<D, C>,
context: C,
game: Game,
logger: KtLogger
): Promise<ImportProcessingInfo[]> {
logger.verbose("Getting Blacklist...");
// @optimisable: could filter harder with score.game and score.playtype
// stuff.
const blacklist = (
await db["score-blacklist"].find({
userID,
})
).map((e) => e.scoreID);
logger.verbose(`Starting Data Processing...`);
const processedResults = [];
// for await is used here as iterableData may be an async iterable
// An example would be making an api request after exhausting
// the first set of data.
for await (const data of iterableData) {
processedResults.push(
await ImportIterableDatapoint(
userID,
importType,
data,
ConverterFunction,
context,
game,
blacklist,
logger
)
);
}
// We need to filter out nulls, which we don't care for (these are neither successes or failures)
logger.verbose(`Finished Importing Data (${processedResults.length} datapoints).`);
logger.debug(`Removing null returns...`);
const datapoints = processedResults.filter(
(e) => e !== null
) as ImportProcessingInfo<IDStrings>[];
logger.debug(`Removed null from results.`);
logger.verbose(`Recieved ${datapoints.length} returns, from ${processedResults.length} data.`);
// Flush the score queue out after finishing most of the import. This ensures no scores get left in the
// queue.
const emptied = await InsertQueue(userID);
if (emptied) {
logger.verbose(`Emptied ${emptied} documents from score queue.`);
}
return datapoints;
}
/**
* Processes a single data object into one or many ImportProcessingInfo objects.
* @param userID - The user this score is from.
* @param data - The data to process.
* @param ConverterFunction - The processor function that takes the data and returns the partialScore(s)
* @param context - Any context the processor might need that it can not infer from the data object.
* @returns An array of ImportProcessingInfo objects, or a single ImportProcessingInfo object
*/
export async function ImportIterableDatapoint<D, C>(
userID: integer,
importType: ImportTypes,
data: D,
ConverterFunction: ConverterFunction<D, C>,
context: C,
game: Game,
blacklist: string[],
logger: KtLogger
): Promise<ImportProcessingInfo | null> {
// Converter Function Return
let cfnReturn: ConverterFnReturnOrFailure;
try {
cfnReturn = await ConverterFunction(data, context, importType, logger);
} catch (err) {
cfnReturn = err as ConverterFailure | Error;
}
// if this conversion failed, return it in the proper format
if (cfnReturn instanceof ConverterFailure) {
if (cfnReturn instanceof KTDataNotFoundFailure) {
logger.info(`KTDataNotFoundFailure: ${cfnReturn.message ?? "No message?"}`, {
cfnReturn,
hideFromConsole: ["cfnReturn"],
});
logger.debug("Inserting orphan...", { cfnReturn });
const insertOrphan = await OrphanScore(
cfnReturn.importType,
userID,
cfnReturn.data,
cfnReturn.converterContext,
cfnReturn.message,
game,
logger
);
if (insertOrphan.success) {
logger.debug("Orphan inserted successfully.", { orphanID: insertOrphan.orphanID });
return {
success: false,
type: "KTDataNotFound",
message: cfnReturn.message,
content: {
context: cfnReturn.converterContext,
data: cfnReturn.data,
orphanID: insertOrphan.orphanID,
},
};
}
logger.debug(`Orphan already exists.`, { orphanID: insertOrphan.orphanID });
return {
success: false,
type: "OrphanExists",
message: cfnReturn.message,
content: {
orphanID: insertOrphan.orphanID,
},
};
} else if (cfnReturn instanceof InvalidScoreFailure) {
logger.info(`InvalidScoreFailure: ${cfnReturn.message ?? "No message?"}`, {
cfnReturn,
hideFromConsole: ["cfnReturn"],
});
return {
success: false,
type: "InvalidDatapoint",
message: cfnReturn.message,
content: {},
};
} else if (cfnReturn instanceof InternalFailure) {
logger.error(`Internal error occured.`, { cfnReturn });
return {
success: false,
type: "InternalError",
// could return cfnReturn.message here, but we might want to hide the details of the crash.
message: "An internal error has occured.",
content: {},
};
} else if (cfnReturn instanceof SkipScoreFailure) {
return null;
} else {
logger.warn(`Unknown error returned as ConverterFailure, Ignoring.`, {
err: cfnReturn,
});
return {
success: false,
type: "InternalError",
message: "An internal service error has occured.",
content: {},
};
}
}
if (cfnReturn instanceof Error) {
logger.error(`Unknown error thrown from converter, Ignoring.`, {
err: cfnReturn,
});
return {
success: false,
type: "InternalError",
message: "An internal service error has occured.",
content: {},
};
}
return ProcessSuccessfulConverterReturn(
userID,
cfnReturn as ConverterFnSuccessReturn,
blacklist,
logger
);
}
export async function ProcessSuccessfulConverterReturn(
userID: integer,
cfnReturn: ConverterFnSuccessReturn,
blacklist: string[],
logger: KtLogger,
forceImmediateImport = false
): Promise<ImportProcessingInfo | null> {
const result = await HydrateAndInsertScore(
userID,
cfnReturn.dryScore,
cfnReturn.chart,
cfnReturn.song,
blacklist,
logger,
forceImmediateImport
);
// This used to be a ScoreExists error. However, we never actually care about
// handling ScoreExists errors (they're nobodies issue)
// so instead, the function will just return null, and we pass that on here.
if (result === null) {
return null;
}
logger.debug(`Successfully imported score: ${result.scoreID}`);
return {
success: true,
type: "ScoreImported",
message: null,
content: {
score: result,
},
};
}
/**
* Hydrates and inserts a score to the Tachi database.
* @param userID - The user this score is from.
* @param dryScore - The score that is to be hydrated and inserted.
* @param chart - The chart this score is on.
* @param song - The song this score is on.
* @param blacklist - A list of ScoreIDs to never write to the database.
*
* @param force - Whether to immediately insert the score into the database
* or not.
*/
async function HydrateAndInsertScore(
userID: integer,
dryScore: DryScore,
chart: ChartDocument,
song: SongDocument,
blacklist: string[],
importLogger: KtLogger,
force = false
): Promise<ScoreDocument | null> {
const scoreID = CreateScoreID(userID, dryScore, chart.chartID);
// sub-context the logger so the below logs are more accurate
const logger = AppendLogCtx(scoreID, importLogger);
if (blacklist.length && blacklist.includes(scoreID)) {
logger.verbose("Skipped score, as it was on the blacklist.");
return null;
}
const existingScore = await db.scores.findOne(
{
scoreID,
},
{
// micro-optimisation - mongoDB is significantly faster when returning less fields
// since we only care about whether we have a score or not here, we can minimise returned
// fields.
//
// feel free to test this yourself! - zkldi (09/04/2021)
projection: {
_id: 1,
},
}
);
if (existingScore) {
logger.verbose(`Skipped score.`);
return null;
}
// If this users score queue
if (GetScoreQueueMaybe(userID)?.scoreIDSet.has(scoreID)) {
logger.verbose(`Skipped score.`);
return null;
}
const score = await HydrateScore(userID, dryScore, chart, song, scoreID, logger);
let res;
if (force) {
res = await db.scores.insert(score);
} else {
res = await QueueScoreInsert(score);
}
// this is a last resort for avoiding doubled imports
if (res === null) {
logger.verbose(`Skipped score - Race Condition protection triggered.`);
return null;
}
return score;
}
server/src/lib/score-import/import-types/common/eamusement-iidx-csv/converter.ts
+178 -178
View File
@@ -1,178 +1,178 @@
import { Lamps, ChartDocument } from "tachi-common";
import { FindChartWithPTDFVersion } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
import {
KTDataNotFoundFailure,
InvalidScoreFailure,
SkipScoreFailure,
} from "../../../framework/common/converter-failures";
import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../framework/common/score-utils";
import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts";
import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../types";
const EAMUSEMENT_LAMP_RESOLVER: Map<string, Lamps["iidx:SP" | "iidx:DP"]> = new Map([
["NO PLAY", "NO PLAY"],
["FAILED", "FAILED"],
["FULLCOMBO CLEAR", "FULL COMBO"],
["EX HARD CLEAR", "EX HARD CLEAR"],
["HARD CLEAR", "HARD CLEAR"],
["CLEAR", "CLEAR"],
["EASY CLEAR", "EASY CLEAR"],
["ASSIST CLEAR", "ASSIST CLEAR"],
]);
const NINE_HOURS = 1000 * 60 * 60 * 9;
const ConvertEamIIDXCSV: ConverterFunction<IIDXEamusementCSVData, IIDXEamusementCSVContext> =
async (data, context, importType, logger) => {
const eamScore = data.score;
if (eamScore.exscore === "0") {
// skip scores with an exscore of 0
// This also skips things like score resets.
throw new SkipScoreFailure("Score has an exscore of 0.");
}
if (!eamScore.level || eamScore.level === "0") {
// charts that dont exist in the format have a level of 0
throw new SkipScoreFailure("Chart has a level of 0.");
}
let isLegacyLeggendaria = false;
// if pre-HV, leggendarias were stored in a wacky form.
if (!context.hasBeginnerAndLegg) {
// hack fix for legacy LEGGENDARIA titles
if (data.title.match(/(†|†LEGGENDARIA)$/u)) {
data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd();
isLegacyLeggendaria = true;
}
}
// @optimisable - This is actually a multi-fetch. Since eam-csv scores
// are batched up into (song, chart1, chart2, chart3 ...) rows
// we actually already have fetched this song a second ago.
const tachiSong = await FindSongOnTitle("iidx", data.title);
if (!tachiSong) {
throw new KTDataNotFoundFailure(
`Could not find song for ${data.title}.`,
importType,
data,
context
);
}
const HUMANISED_CHART_TITLE = `${tachiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`;
if (isLegacyLeggendaria) {
eamScore.difficulty = "LEGGENDARIA";
}
const tachiChart = (await FindChartWithPTDFVersion(
"iidx",
tachiSong.id,
context.playtype,
eamScore.difficulty,
context.importVersion
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
if (!tachiChart) {
throw new KTDataNotFoundFailure(
`Could not find chart for ${HUMANISED_CHART_TITLE}`,
"file/eamusement-iidx-csv",
data,
context
);
}
const exscore = AssertStrAsPositiveInt(
eamScore.exscore,
`${HUMANISED_CHART_TITLE} - Invalid EX score of ${eamScore.exscore}`
);
const MAX_EX = tachiChart.data.notecount * 2;
if (exscore > MAX_EX) {
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was greater than max chart ex of ${MAX_EX}).`
);
}
const pgreat = AssertStrAsPositiveInt(
eamScore.pgreat,
`${HUMANISED_CHART_TITLE} - Invalid PGreats of ${eamScore.pgreat}`
);
const great = AssertStrAsPositiveInt(
eamScore.great,
`${HUMANISED_CHART_TITLE} - Invalid Greats of ${eamScore.pgreat}`
);
if (pgreat * 2 + great !== exscore) {
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - PGreats * 2 + Greats did not equal EXScore (${pgreat} * 2 + ${great} != ${exscore}).`
);
}
const lamp = EAMUSEMENT_LAMP_RESOLVER.get(eamScore.lamp);
if (!lamp) {
logger.info(`Invalid lamp of ${eamScore.lamp} provided.`);
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - Invalid Lamp of ${eamScore.lamp}.`
);
}
const { percent, grade } = GenericGetGradeAndPercent("iidx", exscore, tachiChart);
const timestamp = ParseDateFromString(data.timestamp);
const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = {
service: context.service,
comment: null,
game: "iidx",
importType,
scoreData: {
score: exscore,
lamp,
judgements: {
pgreat,
great,
},
hitMeta: {},
percent,
grade,
},
scoreMeta: {},
// japan is gmt+9
timeAchieved: timestamp ? timestamp - NINE_HOURS : null,
};
const numBP = Number(eamScore.bp);
if (!Number.isNaN(numBP)) {
if (!Number.isInteger(numBP) || numBP < 0 || numBP > 9999) {
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - Invalid BP of ${eamScore.bp}.`
);
}
dryScore.scoreData.hitMeta.bp = numBP;
} else if (eamScore.bp === "---") {
logger.debug(
`Skipped assigning BP for score as it had expected null value of ${eamScore.bp}.`
);
} else {
logger.info(`Skipped assigning BP for score. Had unexpected value of ${eamScore.bp}.`);
}
// ts thinks tachiSong might be null. It's not, though!
return { chart: tachiChart, dryScore, song: tachiSong };
};
export default ConvertEamIIDXCSV;
import { Lamps, ChartDocument } from "tachi-common";
import { FindChartWithPTDFVersion } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
import {
KTDataNotFoundFailure,
InvalidScoreFailure,
SkipScoreFailure,
} from "../../../framework/common/converter-failures";
import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../framework/common/score-utils";
import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts";
import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../types";
const EAMUSEMENT_LAMP_RESOLVER: Map<string, Lamps["iidx:SP" | "iidx:DP"]> = new Map([
["NO PLAY", "NO PLAY"],
["FAILED", "FAILED"],
["FULLCOMBO CLEAR", "FULL COMBO"],
["EX HARD CLEAR", "EX HARD CLEAR"],
["HARD CLEAR", "HARD CLEAR"],
["CLEAR", "CLEAR"],
["EASY CLEAR", "EASY CLEAR"],
["ASSIST CLEAR", "ASSIST CLEAR"],
]);
const NINE_HOURS = 1000 * 60 * 60 * 9;
const ConvertEamIIDXCSV: ConverterFunction<IIDXEamusementCSVData, IIDXEamusementCSVContext> =
async (data, context, importType, logger) => {
const eamScore = data.score;
if (eamScore.exscore === "0") {
// skip scores with an exscore of 0
// This also skips things like score resets.
throw new SkipScoreFailure("Score has an exscore of 0.");
}
if (!eamScore.level || eamScore.level === "0") {
// charts that dont exist in the format have a level of 0
throw new SkipScoreFailure("Chart has a level of 0.");
}
let isLegacyLeggendaria = false;
// if pre-HV, leggendarias were stored in a wacky form.
if (!context.hasBeginnerAndLegg) {
// hack fix for legacy LEGGENDARIA titles
if (data.title.match(/(†|†LEGGENDARIA)$/u)) {
data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd();
isLegacyLeggendaria = true;
}
}
// @optimisable - This is actually a multi-fetch. Since eam-csv scores
// are batched up into (song, chart1, chart2, chart3 ...) rows
// we actually already have fetched this song a second ago.
const tachiSong = await FindSongOnTitle("iidx", data.title);
if (!tachiSong) {
throw new KTDataNotFoundFailure(
`Could not find song for ${data.title}.`,
importType,
data,
context
);
}
const HUMANISED_CHART_TITLE = `${tachiSong.title} (${context.playtype} ${eamScore.difficulty} [v${context.importVersion}])`;
if (isLegacyLeggendaria) {
eamScore.difficulty = "LEGGENDARIA";
}
const tachiChart = (await FindChartWithPTDFVersion(
"iidx",
tachiSong.id,
context.playtype,
eamScore.difficulty,
context.importVersion
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
if (!tachiChart) {
throw new KTDataNotFoundFailure(
`Could not find chart for ${HUMANISED_CHART_TITLE}`,
"file/eamusement-iidx-csv",
data,
context
);
}
const exscore = AssertStrAsPositiveInt(
eamScore.exscore,
`${HUMANISED_CHART_TITLE} - Invalid EX score of ${eamScore.exscore}`
);
const MAX_EX = tachiChart.data.notecount * 2;
if (exscore > MAX_EX) {
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - Invalid EX Score of ${eamScore.exscore} (Was greater than max chart ex of ${MAX_EX}).`
);
}
const pgreat = AssertStrAsPositiveInt(
eamScore.pgreat,
`${HUMANISED_CHART_TITLE} - Invalid PGreats of ${eamScore.pgreat}`
);
const great = AssertStrAsPositiveInt(
eamScore.great,
`${HUMANISED_CHART_TITLE} - Invalid Greats of ${eamScore.pgreat}`
);
if (pgreat * 2 + great !== exscore) {
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - PGreats * 2 + Greats did not equal EXScore (${pgreat} * 2 + ${great} != ${exscore}).`
);
}
const lamp = EAMUSEMENT_LAMP_RESOLVER.get(eamScore.lamp);
if (!lamp) {
logger.info(`Invalid lamp of ${eamScore.lamp} provided.`);
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - Invalid Lamp of ${eamScore.lamp}.`
);
}
const { percent, grade } = GenericGetGradeAndPercent("iidx", exscore, tachiChart);
const timestamp = ParseDateFromString(data.timestamp);
const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = {
service: context.service,
comment: null,
game: "iidx",
importType,
scoreData: {
score: exscore,
lamp,
judgements: {
pgreat,
great,
},
hitMeta: {},
percent,
grade,
},
scoreMeta: {},
// japan is gmt+9
timeAchieved: timestamp ? timestamp - NINE_HOURS : null,
};
const numBP = Number(eamScore.bp);
if (!Number.isNaN(numBP)) {
if (!Number.isInteger(numBP) || numBP < 0 || numBP > 9999) {
throw new InvalidScoreFailure(
`${HUMANISED_CHART_TITLE} - Invalid BP of ${eamScore.bp}.`
);
}
dryScore.scoreData.hitMeta.bp = numBP;
} else if (eamScore.bp === "---") {
logger.debug(
`Skipped assigning BP for score as it had expected null value of ${eamScore.bp}.`
);
} else {
logger.info(`Skipped assigning BP for score. Had unexpected value of ${eamScore.bp}.`);
}
// ts thinks tachiSong might be null. It's not, though!
return { chart: tachiChart, dryScore, song: tachiSong };
};
export default ConvertEamIIDXCSV;
server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts
+286 -286
View File
@@ -1,286 +1,286 @@
import { KtLogger } from "lib/logger/logger";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../types";
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
enum EAM_VERSION_NAMES {
"1st&substream" = 1,
"2nd style",
"3rd style",
"4th style",
"5th style",
"6th style",
"7th style",
"8th style",
"9th style",
"10th style",
"IIDX RED",
"HAPPY SKY",
"DistorteD",
"GOLD",
"DJ TROOPERS",
"EMPRESS",
"SIRIUS",
"Resort Anthem",
"Lincle",
"tricoro",
"SPADA",
"PENDUAL",
"copula",
"SINOBUZ",
"CANNON BALLERS",
"Rootage",
"HEROIC VERSE",
"BISTROVER",
}
const PRE_HV_HEADER_COUNT = 27;
const HV_HEADER_COUNT = 41;
// commented out for reference
// [
// "version",
// "title",
// "genre",
// "artist",
// "playcount",
// (?beginnerdata)
// "normal-level",
// "normal-exscore",
// "normal-pgreat",
// "normal-great",
// "normal-bp",
// "normal-lamp",
// "normal-grade",
// "hyper-level",
// "hyper-exscore",
// "hyper-pgreat",
// "hyper-great",
// "hyper-bp",
// "hyper-lamp",
// "hyper-grade",
// "another-level",
// "another-exscore",
// "another-pgreat",
// "another-great",
// "another-bp",
// "another-lamp",
// "another-grade",
// (?leggdata)
// "timestamp",
// ];
export function ResolveHeaders(headers: string[], logger: KtLogger) {
if (headers.length === PRE_HV_HEADER_COUNT) {
logger.verbose("PRE_HV csv recieved.");
return {
hasBeginnerAndLegg: false,
};
} else if (headers.length === HV_HEADER_COUNT) {
logger.verbose("HV+ csv recieved.");
return {
hasBeginnerAndLegg: true,
};
}
logger.info(`Invalid CSV header count of ${headers.length} received.`);
throw new ScoreImportFatalError(
400,
"Invalid CSV provided. CSV does not have the correct amount of headers."
);
}
export function NaiveCSVParse(csvBuffer: Buffer, logger: KtLogger) {
const csvString = csvBuffer.toString("utf-8");
const csvData = csvString.split("\n");
const rawHeaders = [];
let headerLen = 0;
let curStr = "";
// looks like we're doing it like this.
for (const char of csvData[0]) {
headerLen++;
// safety checks to avoid getting DOS'd
if (headerLen > 1000) {
throw new ScoreImportFatalError(400, "Headers were longer than 1000 characters long.");
} else if (rawHeaders.length >= 50) {
// this does not *really* do what it seems.
// because there's inevitably something left in curStr in this fn
// this means that the above check is actually > 50 headers. Not
// >= 50.
throw new ScoreImportFatalError(400, "Too many CSV headers.");
}
if (char === ",") {
rawHeaders.push(curStr);
curStr = "";
} else {
curStr += char;
}
}
rawHeaders.push(curStr);
const { hasBeginnerAndLegg } = ResolveHeaders(rawHeaders, logger);
const diffs = hasBeginnerAndLegg
? (["beginner", "normal", "hyper", "another", "leggendaria"] as const)
: (["normal", "hyper", "another"] as const);
const iterableData = [];
let gameVersion = 0;
for (let i = 1; i < csvData.length; i++) {
const data = csvData[i];
// @security: This should probably be safetied from DOSing
const cells = data.split(",");
// weirdly enough, an empty string split on "," is an array with
// one empty value.
// regardless, this line skips empty rows
if (cells.length === 1) {
logger.verbose(`Skipped empty row ${i}.`);
continue;
}
if (cells.length !== rawHeaders.length) {
logger.info(
`eamusement-iidx csv has row (${i}) with invalid cell count of ${cells.length}, rejecting.`,
{
data,
}
);
throw new ScoreImportFatalError(
400,
`Row ${i} has an invalid amount of cells (${cells.length}, expected ${rawHeaders.length}).`
);
}
const version = cells[0];
const title = cells[1].trim(); // konmai quality
const timestamp = cells[rawHeaders.length - 1].trim();
// wtf typescript?? what's the point of enums?
const versionNum = EAM_VERSION_NAMES[version as keyof typeof EAM_VERSION_NAMES];
if (!versionNum) {
logger.info(`Invalid/Unsupported EAM_VERSION_NAME ${version}.`);
throw new ScoreImportFatalError(
400,
`Invalid/Unsupported Eamusement Version Name ${version}.`
);
}
if (versionNum > gameVersion) {
gameVersion = versionNum;
}
const scores: EamusementScoreData[] = [];
for (let d = 0; d < diffs.length; d++) {
const diff = diffs[d];
const di = 5 + d * 7;
scores.push({
difficulty: diff.toUpperCase() as Uppercase<typeof diff>,
bp: cells[di + 4],
exscore: cells[di + 1],
pgreat: cells[di + 2],
great: cells[di + 3],
lamp: cells[di + 5],
level: cells[di],
});
}
iterableData.push(
...scores.map((e) => ({
score: e,
timestamp,
title,
}))
);
}
if (!["26", "27", "28"].includes(gameVersion.toString())) {
throw new ScoreImportFatalError(400, `Only versions 26, 27 and 28 are supported.`);
}
return {
iterableData,
version: gameVersion.toString() as "26" | "27" | "28",
hasBeginnerAndLegg,
};
}
/**
* Parses a buffer of EamusementCSV data.
* @param fileData - The buffer to parse.
* @param body - The request body that made this file import request. Used to infer playtype.
*/
function GenericParseEamIIDXCSV(
fileData: Express.Multer.File,
body: Record<string, unknown>,
service: string,
logger: KtLogger
): ParserFunctionReturns<IIDXEamusementCSVData, IIDXEamusementCSVContext> {
let playtype: "SP" | "DP";
if (body.playtype === "SP") {
playtype = "SP";
} else if (body.playtype === "DP") {
playtype = "DP";
} else {
logger.info(`Invalid playtype of ${body.playtype} passed to ParseEamusementCSV.`);
throw new ScoreImportFatalError(
400,
`Invalid playtype of ${body.playtype ?? "Nothing"} given.`
);
}
const lowercaseFilename = fileData.originalname.toLowerCase();
// prettier pls
if (
!body.assertPlaytypeCorrect &&
((lowercaseFilename.includes("sp") && playtype === "DP") ||
(lowercaseFilename.includes("dp") && playtype === "SP"))
) {
logger.info(
`File was uploaded with filename ${fileData.originalname}, but this was set as a ${playtype} import. Sanity check refusing.`
);
throw new ScoreImportFatalError(
400,
`Safety Triggered: Filename contained '${
playtype === "SP" ? "DP" : "SP"
}', but was marked as a ${playtype} import. Are you *absolutely* sure this is right?`
);
}
const { hasBeginnerAndLegg, version, iterableData } = NaiveCSVParse(fileData.buffer, logger);
logger.verbose("Successfully parsed CSV.");
const context: IIDXEamusementCSVContext = {
playtype,
importVersion: version,
hasBeginnerAndLegg,
service,
};
logger.verbose(`Successfully Parsed with ${iterableData.length} results.`);
return {
iterable: iterableData,
context,
game: "iidx",
classHandler: null,
};
}
export default GenericParseEamIIDXCSV;
import { KtLogger } from "lib/logger/logger";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../types";
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
enum EAM_VERSION_NAMES {
"1st&substream" = 1,
"2nd style",
"3rd style",
"4th style",
"5th style",
"6th style",
"7th style",
"8th style",
"9th style",
"10th style",
"IIDX RED",
"HAPPY SKY",
"DistorteD",
"GOLD",
"DJ TROOPERS",
"EMPRESS",
"SIRIUS",
"Resort Anthem",
"Lincle",
"tricoro",
"SPADA",
"PENDUAL",
"copula",
"SINOBUZ",
"CANNON BALLERS",
"Rootage",
"HEROIC VERSE",
"BISTROVER",
}
const PRE_HV_HEADER_COUNT = 27;
const HV_HEADER_COUNT = 41;
// commented out for reference
// [
// "version",
// "title",
// "genre",
// "artist",
// "playcount",
// (?beginnerdata)
// "normal-level",
// "normal-exscore",
// "normal-pgreat",
// "normal-great",
// "normal-bp",
// "normal-lamp",
// "normal-grade",
// "hyper-level",
// "hyper-exscore",
// "hyper-pgreat",
// "hyper-great",
// "hyper-bp",
// "hyper-lamp",
// "hyper-grade",
// "another-level",
// "another-exscore",
// "another-pgreat",
// "another-great",
// "another-bp",
// "another-lamp",
// "another-grade",
// (?leggdata)
// "timestamp",
// ];
export function ResolveHeaders(headers: string[], logger: KtLogger) {
if (headers.length === PRE_HV_HEADER_COUNT) {
logger.verbose("PRE_HV csv recieved.");
return {
hasBeginnerAndLegg: false,
};
} else if (headers.length === HV_HEADER_COUNT) {
logger.verbose("HV+ csv recieved.");
return {
hasBeginnerAndLegg: true,
};
}
logger.info(`Invalid CSV header count of ${headers.length} received.`);
throw new ScoreImportFatalError(
400,
"Invalid CSV provided. CSV does not have the correct amount of headers."
);
}
export function NaiveCSVParse(csvBuffer: Buffer, logger: KtLogger) {
const csvString = csvBuffer.toString("utf-8");
const csvData = csvString.split("\n");
const rawHeaders = [];
let headerLen = 0;
let curStr = "";
// looks like we're doing it like this.
for (const char of csvData[0]) {
headerLen++;
// safety checks to avoid getting DOS'd
if (headerLen > 1000) {
throw new ScoreImportFatalError(400, "Headers were longer than 1000 characters long.");
} else if (rawHeaders.length >= 50) {
// this does not *really* do what it seems.
// because there's inevitably something left in curStr in this fn
// this means that the above check is actually > 50 headers. Not
// >= 50.
throw new ScoreImportFatalError(400, "Too many CSV headers.");
}
if (char === ",") {
rawHeaders.push(curStr);
curStr = "";
} else {
curStr += char;
}
}
rawHeaders.push(curStr);
const { hasBeginnerAndLegg } = ResolveHeaders(rawHeaders, logger);
const diffs = hasBeginnerAndLegg
? (["beginner", "normal", "hyper", "another", "leggendaria"] as const)
: (["normal", "hyper", "another"] as const);
const iterableData = [];
let gameVersion = 0;
for (let i = 1; i < csvData.length; i++) {
const data = csvData[i];
// @security: This should probably be safetied from DOSing
const cells = data.split(",");
// weirdly enough, an empty string split on "," is an array with
// one empty value.
// regardless, this line skips empty rows
if (cells.length === 1) {
logger.verbose(`Skipped empty row ${i}.`);
continue;
}
if (cells.length !== rawHeaders.length) {
logger.info(
`eamusement-iidx csv has row (${i}) with invalid cell count of ${cells.length}, rejecting.`,
{
data,
}
);
throw new ScoreImportFatalError(
400,
`Row ${i} has an invalid amount of cells (${cells.length}, expected ${rawHeaders.length}).`
);
}
const version = cells[0];
const title = cells[1].trim(); // konmai quality
const timestamp = cells[rawHeaders.length - 1].trim();
// wtf typescript?? what's the point of enums?
const versionNum = EAM_VERSION_NAMES[version as keyof typeof EAM_VERSION_NAMES];
if (!versionNum) {
logger.info(`Invalid/Unsupported EAM_VERSION_NAME ${version}.`);
throw new ScoreImportFatalError(
400,
`Invalid/Unsupported Eamusement Version Name ${version}.`
);
}
if (versionNum > gameVersion) {
gameVersion = versionNum;
}
const scores: EamusementScoreData[] = [];
for (let d = 0; d < diffs.length; d++) {
const diff = diffs[d];
const di = 5 + d * 7;
scores.push({
difficulty: diff.toUpperCase() as Uppercase<typeof diff>,
bp: cells[di + 4],
exscore: cells[di + 1],
pgreat: cells[di + 2],
great: cells[di + 3],
lamp: cells[di + 5],
level: cells[di],
});
}
iterableData.push(
...scores.map((e) => ({
score: e,
timestamp,
title,
}))
);
}
if (!["26", "27", "28"].includes(gameVersion.toString())) {
throw new ScoreImportFatalError(400, `Only versions 26, 27 and 28 are supported.`);
}
return {
iterableData,
version: gameVersion.toString() as "26" | "27" | "28",
hasBeginnerAndLegg,
};
}
/**
* Parses a buffer of EamusementCSV data.
* @param fileData - The buffer to parse.
* @param body - The request body that made this file import request. Used to infer playtype.
*/
function GenericParseEamIIDXCSV(
fileData: Express.Multer.File,
body: Record<string, unknown>,
service: string,
logger: KtLogger
): ParserFunctionReturns<IIDXEamusementCSVData, IIDXEamusementCSVContext> {
let playtype: "SP" | "DP";
if (body.playtype === "SP") {
playtype = "SP";
} else if (body.playtype === "DP") {
playtype = "DP";
} else {
logger.info(`Invalid playtype of ${body.playtype} passed to ParseEamusementCSV.`);
throw new ScoreImportFatalError(
400,
`Invalid playtype of ${body.playtype ?? "Nothing"} given.`
);
}
const lowercaseFilename = fileData.originalname.toLowerCase();
// prettier pls
if (
!body.assertPlaytypeCorrect &&
((lowercaseFilename.includes("sp") && playtype === "DP") ||
(lowercaseFilename.includes("dp") && playtype === "SP"))
) {
logger.info(
`File was uploaded with filename ${fileData.originalname}, but this was set as a ${playtype} import. Sanity check refusing.`
);
throw new ScoreImportFatalError(
400,
`Safety Triggered: Filename contained '${
playtype === "SP" ? "DP" : "SP"
}', but was marked as a ${playtype} import. Are you *absolutely* sure this is right?`
);
}
const { hasBeginnerAndLegg, version, iterableData } = NaiveCSVParse(fileData.buffer, logger);
logger.verbose("Successfully parsed CSV.");
const context: IIDXEamusementCSVContext = {
playtype,
importVersion: version,
hasBeginnerAndLegg,
service,
};
logger.verbose(`Successfully Parsed with ${iterableData.length} results.`);
return {
iterable: iterableData,
context,
game: "iidx",
classHandler: null,
};
}
export default GenericParseEamIIDXCSV;
server/src/lib/score-import/import-types/common/eamusement-iidx-csv/types.ts
+36 -36
View File
@@ -1,36 +1,36 @@
import { GPTSupportedVersions } from "tachi-common";
export interface EamusementScoreData {
difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
lamp: string;
exscore: string;
pgreat: string;
great: string;
bp: string;
level: string;
}
type BaseProps = {
title: string;
timestamp: string;
};
export type IIDXEamusementCSVData = {
score: EamusementScoreData;
} & BaseProps;
type Props = "exscore" | "lamp" | "pgreat" | "great" | "bp" | "level";
type RawPropKeys = `${"beginner" | "normal" | "hyper" | "another" | "leggendaria"}-${Props}`;
export type RawIIDXEamusementCSVData = {
[K in RawPropKeys]: unknown;
} &
BaseProps & { [index: string]: unknown };
export interface IIDXEamusementCSVContext {
playtype: "SP" | "DP";
importVersion: GPTSupportedVersions["iidx:SP" | "iidx:DP"];
hasBeginnerAndLegg: boolean;
service: string;
}
import { GPTSupportedVersions } from "tachi-common";
export interface EamusementScoreData {
difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
lamp: string;
exscore: string;
pgreat: string;
great: string;
bp: string;
level: string;
}
type BaseProps = {
title: string;
timestamp: string;
};
export type IIDXEamusementCSVData = {
score: EamusementScoreData;
} & BaseProps;
type Props = "exscore" | "lamp" | "pgreat" | "great" | "bp" | "level";
type RawPropKeys = `${"beginner" | "normal" | "hyper" | "another" | "leggendaria"}-${Props}`;
export type RawIIDXEamusementCSVData = {
[K in RawPropKeys]: unknown;
} &
BaseProps & { [index: string]: unknown };
export interface IIDXEamusementCSVContext {
playtype: "SP" | "DP";
importVersion: GPTSupportedVersions["iidx:SP" | "iidx:DP"];
hasBeginnerAndLegg: boolean;
service: string;
}
server/src/lib/score-import/import-types/file/eamusement-iidx-csv/parser.ts
+17 -17
View File
@@ -1,17 +1,17 @@
import { KtLogger } from "lib/logger/logger";
import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser";
import {
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
} from "../../common/eamusement-iidx-csv/types";
import { ParserFunctionReturns } from "../../common/types";
function ParseEamusementIIDXCSV(
fileData: Express.Multer.File,
body: Record<string, unknown>,
logger: KtLogger
): ParserFunctionReturns<IIDXEamusementCSVData, IIDXEamusementCSVContext> {
return GenericParseEamIIDXCSV(fileData, body, "e-amusement", logger);
}
export default ParseEamusementIIDXCSV;
import { KtLogger } from "lib/logger/logger";
import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser";
import {
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
} from "../../common/eamusement-iidx-csv/types";
import { ParserFunctionReturns } from "../../common/types";
function ParseEamusementIIDXCSV(
fileData: Express.Multer.File,
body: Record<string, unknown>,
logger: KtLogger
): ParserFunctionReturns<IIDXEamusementCSVData, IIDXEamusementCSVContext> {
return GenericParseEamIIDXCSV(fileData, body, "e-amusement", logger);
}
export default ParseEamusementIIDXCSV;
server/src/lib/score-import/import-types/file/pli-iidx-csv/parser.ts
+17 -17
View File
@@ -1,17 +1,17 @@
import { KtLogger } from "lib/logger/logger";
import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser";
import {
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
} from "../../common/eamusement-iidx-csv/types";
import { ParserFunctionReturns } from "../../common/types";
function ParsePLIIIDXCSV(
fileData: Express.Multer.File,
body: Record<string, unknown>,
logger: KtLogger
): ParserFunctionReturns<IIDXEamusementCSVData, IIDXEamusementCSVContext> {
return GenericParseEamIIDXCSV(fileData, body, "PLI", logger);
}
export default ParsePLIIIDXCSV;
import { KtLogger } from "lib/logger/logger";
import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser";
import {
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
} from "../../common/eamusement-iidx-csv/types";
import { ParserFunctionReturns } from "../../common/types";
function ParsePLIIIDXCSV(
fileData: Express.Multer.File,
body: Record<string, unknown>,
logger: KtLogger
): ParserFunctionReturns<IIDXEamusementCSVData, IIDXEamusementCSVContext> {
return GenericParseEamIIDXCSV(fileData, body, "PLI", logger);
}
export default ParsePLIIIDXCSV;
server/src/main.ts
+115 -115
View File
@@ -1,115 +1,115 @@
// Before we run anything, set a global to indicate to the code that
// we're running as a server, and not as a job runner or score worker.
process.env.IS_SERVER = "true";
import { spawn } from "child_process";
import db, { monkDB } from "external/mongo/db";
import { SetIndexesIfNoneSet } from "external/mongo/indexes";
import { InitSequenceDocs } from "external/mongo/sequence-docs";
import fs from "fs";
import https from "https";
import { LoadDefaultClients } from "lib/builtin-clients/builtin-clients";
import CreateLogCtx from "lib/logger/logger";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import path from "path";
import server from "server/server";
import { InitaliseFolderChartLookup } from "utils/folder";
import { FormatVersion } from "./lib/constants/version";
import fetch from "utils/fetch";
import http from "http";
import { CloseRedisConnection } from "external/redis/redis";
import { CloseScoreImportQueue } from "lib/score-import/worker/queue";
import { HandleSIGTERMGracefully } from "lib/handlers/sigterm";
const logger = CreateLogCtx(__filename);
logger.info(`Booting ${TachiConfig.NAME} - ${FormatVersion()} [ENV: ${Environment.nodeEnv}]`, {
bootInfo: true,
});
logger.info(`Log level is set to ${ServerConfig.LOGGER_CONFIG.LOG_LEVEL}.`, { bootInfo: true });
logger.info(`Loading sequence documents...`, { bootInfo: true });
async function RunOnInit() {
await InitSequenceDocs();
await SetIndexesIfNoneSet();
await db["folder-chart-lookup"].findOne().then((r) => {
// If there are no folder chart lookups, initialise them.
if (!r) {
InitaliseFolderChartLookup();
}
});
await LoadDefaultClients();
try {
await fetch("https://example.com");
} catch (err) {
logger.crit(
`Cannot send HTTPS request to https://example.com. This instance of tachi-server cannot access the internet?`,
err
);
process.exit(1);
}
}
RunOnInit();
let instance: http.Server | https.Server;
if (ServerConfig.ENABLE_SERVER_HTTPS) {
logger.warn(
"HTTPS Mode is enabled. This should not be used in production, and you should instead run behind a reverse proxy.",
{ bootInfo: true }
);
const privateKey = fs.readFileSync("./cert/key.pem");
const certificate = fs.readFileSync("./cert/cert.pem");
const httpsServer = https.createServer({ key: privateKey, cert: certificate }, server);
instance = httpsServer.listen(Environment.port);
logger.info(`HTTPS Listening on port ${Environment.port}`, { bootInfo: true });
} else {
instance = server.listen(Environment.port);
logger.info(`HTTP Listening on port ${Environment.port}`, { bootInfo: true });
}
process.on("SIGTERM", () => HandleSIGTERMGracefully(instance));
if (process.env.INVOKE_JOB_RUNNER) {
logger.info(`Spawning a tachi-server job runner inline.`, { bootInfo: true });
if (Environment.nodeEnv === "production") {
logger.warn(
`Spawning inline tachi-server job runner in production. Is this actually what you want? You should run a tool like Ofelia to manage this.`,
{ bootInfo: true }
);
}
// Spawn as a separate process to avoid hogging the main thread.
const jobProcess = spawn(
"ts-node",
[
// Note: Can't use -r tsconfig-paths/register here
// because that is rejected by some library called
// arg.
// I'm not sure why.
"--require=tsconfig-paths/register",
path.join(__dirname, "../src/lib/jobs/job-runner.ts"),
],
{
stdio: "inherit",
}
);
jobProcess.on("error", (err) => {
logger.crit(`Failed to spawn job runner. Terminating process.`, { err });
process.exit(1);
});
process.on("beforeExit", () => {
logger.info(`Killing Job Runner.`);
jobProcess.kill();
});
}
// Before we run anything, set a global to indicate to the code that
// we're running as a server, and not as a job runner or score worker.
process.env.IS_SERVER = "true";
import { spawn } from "child_process";
import db, { monkDB } from "external/mongo/db";
import { SetIndexesIfNoneSet } from "external/mongo/indexes";
import { InitSequenceDocs } from "external/mongo/sequence-docs";
import fs from "fs";
import https from "https";
import { LoadDefaultClients } from "lib/builtin-clients/builtin-clients";
import CreateLogCtx from "lib/logger/logger";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import path from "path";
import server from "server/server";
import { InitaliseFolderChartLookup } from "utils/folder";
import { FormatVersion } from "./lib/constants/version";
import fetch from "utils/fetch";
import http from "http";
import { CloseRedisConnection } from "external/redis/redis";
import { CloseScoreImportQueue } from "lib/score-import/worker/queue";
import { HandleSIGTERMGracefully } from "lib/handlers/sigterm";
const logger = CreateLogCtx(__filename);
logger.info(`Booting ${TachiConfig.NAME} - ${FormatVersion()} [ENV: ${Environment.nodeEnv}]`, {
bootInfo: true,
});
logger.info(`Log level is set to ${ServerConfig.LOGGER_CONFIG.LOG_LEVEL}.`, { bootInfo: true });
logger.info(`Loading sequence documents...`, { bootInfo: true });
async function RunOnInit() {
await InitSequenceDocs();
await SetIndexesIfNoneSet();
await db["folder-chart-lookup"].findOne().then((r) => {
// If there are no folder chart lookups, initialise them.
if (!r) {
InitaliseFolderChartLookup();
}
});
await LoadDefaultClients();
try {
await fetch("https://example.com");
} catch (err) {
logger.crit(
`Cannot send HTTPS request to https://example.com. This instance of tachi-server cannot access the internet?`,
err
);
process.exit(1);
}
}
RunOnInit();
let instance: http.Server | https.Server;
if (ServerConfig.ENABLE_SERVER_HTTPS) {
logger.warn(
"HTTPS Mode is enabled. This should not be used in production, and you should instead run behind a reverse proxy.",
{ bootInfo: true }
);
const privateKey = fs.readFileSync("./cert/key.pem");
const certificate = fs.readFileSync("./cert/cert.pem");
const httpsServer = https.createServer({ key: privateKey, cert: certificate }, server);
instance = httpsServer.listen(Environment.port);
logger.info(`HTTPS Listening on port ${Environment.port}`, { bootInfo: true });
} else {
instance = server.listen(Environment.port);
logger.info(`HTTP Listening on port ${Environment.port}`, { bootInfo: true });
}
process.on("SIGTERM", () => HandleSIGTERMGracefully(instance));
if (process.env.INVOKE_JOB_RUNNER) {
logger.info(`Spawning a tachi-server job runner inline.`, { bootInfo: true });
if (Environment.nodeEnv === "production") {
logger.warn(
`Spawning inline tachi-server job runner in production. Is this actually what you want? You should run a tool like Ofelia to manage this.`,
{ bootInfo: true }
);
}
// Spawn as a separate process to avoid hogging the main thread.
const jobProcess = spawn(
"ts-node",
[
// Note: Can't use -r tsconfig-paths/register here
// because that is rejected by some library called
// arg.
// I'm not sure why.
"--require=tsconfig-paths/register",
path.join(__dirname, "../src/lib/jobs/job-runner.ts"),
],
{
stdio: "inherit",
}
);
jobProcess.on("error", (err) => {
logger.crit(`Failed to spawn job runner. Terminating process.`, { err });
process.exit(1);
});
process.on("beforeExit", () => {
logger.info(`Killing Job Runner.`);
jobProcess.kill();
});
}
server/src/server/middleware/prudence-validate.test.ts
+143 -143
View File
@@ -1,143 +1,143 @@
import t from "tap";
import expMiddlewareMock from "express-request-mock";
import prValidate from "./prudence-validate";
import Prudence from "prudence";
t.test("#PrudenceMiddleware", (t) => {
const mw = prValidate({ foo: Prudence.regex(/^baz$/u) }, { foo: "example error message" });
t.test("Should return 400 on invalid prudence validation", async (t) => {
const { res } = await expMiddlewareMock(mw, {
query: {
foo: "bar",
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[foo] example error message (Received bar)",
"Should return error message"
);
t.end();
});
t.test("Should return 'nothing' instead of undefined for missing fields", async (t) => {
const { res } = await expMiddlewareMock(mw, {
query: {},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[foo] example error message (Received nothing)",
"Should return error message with recieved nothing"
);
t.end();
});
t.test("Should allow valid prudence data.", async (t) => {
const { res } = await expMiddlewareMock(mw, {
query: {
foo: "baz",
},
});
t.equal(res.statusCode, 200, "Should stay as 200");
// no body -- not returned.
t.equal(res._isJSON(), false, "Should not have any data set");
t.end();
});
t.test("Should allow valid bodies on non-GET requests", async (t) => {
const { res } = await expMiddlewareMock(mw, {
method: "POST",
body: {
foo: "baz",
},
});
t.equal(res.statusCode, 200, "Should stay as 200");
// no body -- not returned.
t.equal(res._isJSON(), false, "Should not have any data set");
t.end();
});
t.test("Should return 400 on invalid prudence validation for non-GET requests", async (t) => {
const { res } = await expMiddlewareMock(mw, {
method: "POST",
body: {
foo: "bar",
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[foo] example error message (Received bar)",
"Should return error message"
);
t.end();
});
const mwWithPassword = prValidate({ password: "string" }, { password: "invalid password" });
t.test(
"Should not return the contents of the error message if the field matches /password/",
async (t) => {
const { res } = await expMiddlewareMock(mwWithPassword, {
query: {
password: 123,
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[password] invalid password (Received ****)",
"Should return obscured error message"
);
t.end();
}
);
t.test(
"Should not return the contents of the error message if the field matches /password/",
async (t) => {
const { res } = await expMiddlewareMock(mwWithPassword, {
query: {
password: undefined,
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[password] invalid password (Received nothing)",
"Should indicate if no data was sent in obscured error message"
);
t.end();
}
);
t.end();
});
import t from "tap";
import expMiddlewareMock from "express-request-mock";
import prValidate from "./prudence-validate";
import Prudence from "prudence";
t.test("#PrudenceMiddleware", (t) => {
const mw = prValidate({ foo: Prudence.regex(/^baz$/u) }, { foo: "example error message" });
t.test("Should return 400 on invalid prudence validation", async (t) => {
const { res } = await expMiddlewareMock(mw, {
query: {
foo: "bar",
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[foo] example error message (Received bar)",
"Should return error message"
);
t.end();
});
t.test("Should return 'nothing' instead of undefined for missing fields", async (t) => {
const { res } = await expMiddlewareMock(mw, {
query: {},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[foo] example error message (Received nothing)",
"Should return error message with recieved nothing"
);
t.end();
});
t.test("Should allow valid prudence data.", async (t) => {
const { res } = await expMiddlewareMock(mw, {
query: {
foo: "baz",
},
});
t.equal(res.statusCode, 200, "Should stay as 200");
// no body -- not returned.
t.equal(res._isJSON(), false, "Should not have any data set");
t.end();
});
t.test("Should allow valid bodies on non-GET requests", async (t) => {
const { res } = await expMiddlewareMock(mw, {
method: "POST",
body: {
foo: "baz",
},
});
t.equal(res.statusCode, 200, "Should stay as 200");
// no body -- not returned.
t.equal(res._isJSON(), false, "Should not have any data set");
t.end();
});
t.test("Should return 400 on invalid prudence validation for non-GET requests", async (t) => {
const { res } = await expMiddlewareMock(mw, {
method: "POST",
body: {
foo: "bar",
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[foo] example error message (Received bar)",
"Should return error message"
);
t.end();
});
const mwWithPassword = prValidate({ password: "string" }, { password: "invalid password" });
t.test(
"Should not return the contents of the error message if the field matches /password/",
async (t) => {
const { res } = await expMiddlewareMock(mwWithPassword, {
query: {
password: 123,
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[password] invalid password (Received ****)",
"Should return obscured error message"
);
t.end();
}
);
t.test(
"Should not return the contents of the error message if the field matches /password/",
async (t) => {
const { res } = await expMiddlewareMock(mwWithPassword, {
query: {
password: undefined,
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
const json = res._getJSONData();
t.equal(
json.description,
"[password] invalid password (Received nothing)",
"Should indicate if no data was sent in obscured error message"
);
t.end();
}
);
t.end();
});
server/src/server/middleware/prudence-validate.ts
+63 -63
View File
@@ -1,63 +1,63 @@
import Prudence, {
MiddlewareErrorHandler,
PrudenceSchema,
ErrorMessages,
PrudenceOptions,
} from "prudence";
import { RequestHandler } from "express-serve-static-core";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
const printf = (message: string, stringVal: string | null, keychain: string | null) =>
`[${keychain}] ${message}${stringVal ? ` (Received ${stringVal})` : ""}`;
const API_ERR_HANDLER =
(logLevel: TachiLogLevels): MiddlewareErrorHandler =>
(req, res, next, error) => {
let stringVal = error.userVal;
if (error.keychain && error.keychain.includes("password") && error.userVal) {
stringVal = "****";
}
if (typeof stringVal === "object" && stringVal !== null && !stringVal.toString) {
// this is probably null-prototype
stringVal = null;
} else if (stringVal === undefined) {
stringVal = "nothing";
} else {
stringVal = String(stringVal);
}
logger[logLevel](
`Prudence rejection: ${error.message}, ${stringVal} [K:${error.keychain}]`,
{
userVal: error.userVal,
}
);
return res.status(400).json({
success: false,
description: printf(error.message, stringVal as string | null, error.keychain),
});
};
// Cache all of the possible API_ERROR_HANDLERS to avoid function creation
// overhead at runtime.
const API_ERROR_HANDLERS = Object.fromEntries(
(["crit", "severe", "error", "warn", "info", "verbose", "debug"] as const).map((e) => [
e,
API_ERR_HANDLER(e),
])
) as Record<TachiLogLevels, MiddlewareErrorHandler>;
type TachiLogLevels = "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug";
const prValidate = (
s: PrudenceSchema,
errorMessage?: ErrorMessages,
options?: Partial<PrudenceOptions>,
level: TachiLogLevels = "info"
): RequestHandler => Prudence.CurryMiddleware(API_ERROR_HANDLERS[level])(s, errorMessage, options);
export default prValidate;
import Prudence, {
MiddlewareErrorHandler,
PrudenceSchema,
ErrorMessages,
PrudenceOptions,
} from "prudence";
import { RequestHandler } from "express-serve-static-core";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
const printf = (message: string, stringVal: string | null, keychain: string | null) =>
`[${keychain}] ${message}${stringVal ? ` (Received ${stringVal})` : ""}`;
const API_ERR_HANDLER =
(logLevel: TachiLogLevels): MiddlewareErrorHandler =>
(req, res, next, error) => {
let stringVal = error.userVal;
if (error.keychain && error.keychain.includes("password") && error.userVal) {
stringVal = "****";
}
if (typeof stringVal === "object" && stringVal !== null && !stringVal.toString) {
// this is probably null-prototype
stringVal = null;
} else if (stringVal === undefined) {
stringVal = "nothing";
} else {
stringVal = String(stringVal);
}
logger[logLevel](
`Prudence rejection: ${error.message}, ${stringVal} [K:${error.keychain}]`,
{
userVal: error.userVal,
}
);
return res.status(400).json({
success: false,
description: printf(error.message, stringVal as string | null, error.keychain),
});
};
// Cache all of the possible API_ERROR_HANDLERS to avoid function creation
// overhead at runtime.
const API_ERROR_HANDLERS = Object.fromEntries(
(["crit", "severe", "error", "warn", "info", "verbose", "debug"] as const).map((e) => [
e,
API_ERR_HANDLER(e),
])
) as Record<TachiLogLevels, MiddlewareErrorHandler>;
type TachiLogLevels = "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug";
const prValidate = (
s: PrudenceSchema,
errorMessage?: ErrorMessages,
options?: Partial<PrudenceOptions>,
level: TachiLogLevels = "info"
): RequestHandler => Prudence.CurryMiddleware(API_ERROR_HANDLERS[level])(s, errorMessage, options);
export default prValidate;
server/src/server/middleware/sanitise-qstring.test.ts
+35 -35
View File
@@ -1,35 +1,35 @@
import SanitiseQString from "./sanitise-qstring";
import t from "tap";
import expMiddlewareMock from "express-request-mock";
t.test("#SanitiseQString", (t) => {
t.test("Should allow GET requests with valid data.", async (t) => {
const { res } = await expMiddlewareMock(SanitiseQString, {
method: "GET",
query: {
foo: "bar",
},
});
t.not(res.statusCode, 400, "Status code should NOT be 400");
t.end();
});
t.test("Should disallow GET requests with nested data.", async (t) => {
const { res } = await expMiddlewareMock(SanitiseQString, {
method: "GET",
query: {
foo: {
bar: "baz",
},
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
t.end();
});
t.end();
});
import SanitiseQString from "./sanitise-qstring";
import t from "tap";
import expMiddlewareMock from "express-request-mock";
t.test("#SanitiseQString", (t) => {
t.test("Should allow GET requests with valid data.", async (t) => {
const { res } = await expMiddlewareMock(SanitiseQString, {
method: "GET",
query: {
foo: "bar",
},
});
t.not(res.statusCode, 400, "Status code should NOT be 400");
t.end();
});
t.test("Should disallow GET requests with nested data.", async (t) => {
const { res } = await expMiddlewareMock(SanitiseQString, {
method: "GET",
query: {
foo: {
bar: "baz",
},
},
});
t.equal(res.statusCode, 400, "Status code should be 400");
t.end();
});
t.end();
});
server/src/server/middleware/sanitise-qstring.ts
+21 -21
View File
@@ -1,21 +1,21 @@
import { NextFunction, Request, Response } from "express";
/**
* Disallow complex query strings by enforcing single key:value pairs.
*/
function SanitiseQString(req: Request, res: Response, next: NextFunction) {
if (req.method === "GET") {
for (const data in req.query) {
if (typeof req.query[data] !== "string") {
return res.status(400).json({
success: false,
description: `Invalid request with key ${data}`,
});
}
}
}
next();
}
export default SanitiseQString;
import { NextFunction, Request, Response } from "express";
/**
* Disallow complex query strings by enforcing single key:value pairs.
*/
function SanitiseQString(req: Request, res: Response, next: NextFunction) {
if (req.method === "GET") {
for (const data in req.query) {
if (typeof req.query[data] !== "string") {
return res.status(400).json({
success: false,
description: `Invalid request with key ${data}`,
});
}
}
}
next();
}
export default SanitiseQString;
server/src/server/router/api/v1/auth/auth.test.ts
+73 -73
View File
@@ -1,73 +1,73 @@
import { AddNewInvite, ReinstateInvite, ValidateCaptcha } from "./auth";
import t from "tap";
import db from "external/mongo/db";
import ResetDBState from "test-utils/resets";
import { MockBasicFetch } from "test-utils/mock-fetch";
t.test("#ReinstateInvite", (t) => {
t.beforeEach(ResetDBState);
t.test("Should change the 'consumed' property of an invite to true.", async (t) => {
// mock insert
const inviteDoc = await db.invites.insert({
code: "foobar",
consumed: true,
createdBy: 1,
createdAt: 1,
consumedAt: 2,
consumedBy: 2,
});
const response = await ReinstateInvite(inviteDoc.code);
t.equal(response.nModified, 1, "Should modify one document");
const invite2 = await db.invites.findOne({
code: inviteDoc.code, // lol
});
t.equal(invite2!.consumed, false, "Should no longer be consumed");
t.equal(invite2!.consumedAt, null, "Should revoke when it was consumed.");
t.equal(invite2!.consumedBy, null, "Should revoke who it was consumed by.");
t.end();
});
t.end();
});
t.test("#AddNewInvite", (t) => {
t.beforeEach(ResetDBState);
t.test("Should create a new invite from a given user", async (t) => {
const userDoc = await db.users.findOne({ id: 1 });
const result = await AddNewInvite(userDoc!);
t.equal(result.createdBy, userDoc!.id, "Invite should be created by the requesting user.");
t.equal(result.consumed, false, "Invite should not be consumed.");
// was created +/- 6 seconds from now. This is perhaps too lenient, but we're only really testing its just around now ish.
t.ok(Math.abs(result.createdAt - Date.now()) <= 6000, "Invite was created roughly now.");
t.match(result.code, /^[0-9a-f]{40}$/u, "Invite code should be a 40 character hex string.");
});
t.end();
});
t.test("#ValidateCaptcha", async (t) => {
t.equal(
await ValidateCaptcha("200", "bar", MockBasicFetch({ status: 200 })),
true,
"Validates captcha when status return is 200"
);
t.equal(
await ValidateCaptcha("400", "bar", MockBasicFetch({ status: 400 })),
false,
"Invalidates captcha when status return is not 200"
);
t.end();
});
import { AddNewInvite, ReinstateInvite, ValidateCaptcha } from "./auth";
import t from "tap";
import db from "external/mongo/db";
import ResetDBState from "test-utils/resets";
import { MockBasicFetch } from "test-utils/mock-fetch";
t.test("#ReinstateInvite", (t) => {
t.beforeEach(ResetDBState);
t.test("Should change the 'consumed' property of an invite to true.", async (t) => {
// mock insert
const inviteDoc = await db.invites.insert({
code: "foobar",
consumed: true,
createdBy: 1,
createdAt: 1,
consumedAt: 2,
consumedBy: 2,
});
const response = await ReinstateInvite(inviteDoc.code);
t.equal(response.nModified, 1, "Should modify one document");
const invite2 = await db.invites.findOne({
code: inviteDoc.code, // lol
});
t.equal(invite2!.consumed, false, "Should no longer be consumed");
t.equal(invite2!.consumedAt, null, "Should revoke when it was consumed.");
t.equal(invite2!.consumedBy, null, "Should revoke who it was consumed by.");
t.end();
});
t.end();
});
t.test("#AddNewInvite", (t) => {
t.beforeEach(ResetDBState);
t.test("Should create a new invite from a given user", async (t) => {
const userDoc = await db.users.findOne({ id: 1 });
const result = await AddNewInvite(userDoc!);
t.equal(result.createdBy, userDoc!.id, "Invite should be created by the requesting user.");
t.equal(result.consumed, false, "Invite should not be consumed.");
// was created +/- 6 seconds from now. This is perhaps too lenient, but we're only really testing its just around now ish.
t.ok(Math.abs(result.createdAt - Date.now()) <= 6000, "Invite was created roughly now.");
t.match(result.code, /^[0-9a-f]{40}$/u, "Invite code should be a 40 character hex string.");
});
t.end();
});
t.test("#ValidateCaptcha", async (t) => {
t.equal(
await ValidateCaptcha("200", "bar", MockBasicFetch({ status: 200 })),
true,
"Validates captcha when status return is 200"
);
t.equal(
await ValidateCaptcha("400", "bar", MockBasicFetch({ status: 400 })),
false,
"Invalidates captcha when status return is not 200"
);
t.end();
});
server/src/server/router/api/v1/auth/auth.ts
+178 -178
View File
@@ -1,178 +1,178 @@
import bcrypt from "bcryptjs";
import {
integer,
PrivateUserInfoDocument,
PublicUserDocument,
UserAuthLevels,
UserSettings,
} from "tachi-common";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { FormatUserDoc } from "utils/user";
import nodeFetch from "utils/fetch";
import { Random20Hex } from "utils/misc";
import { Environment, ServerConfig } from "lib/setup/config";
import { CreateURLWithParams } from "utils/url";
const logger = CreateLogCtx(__filename);
const BCRYPT_SALT_ROUNDS = 12;
export const ValidatePassword = (self: unknown) =>
(typeof self === "string" && self.length >= 8) || "Passwords must be 8 characters or more.";
/**
* Compares a plaintext string of a users password to a hash.
* @param plaintext The provided user input.
* @param password The hash to compare against.
*/
export function PasswordCompare(plaintext: string, password: string) {
return bcrypt.compare(plaintext, password);
}
export function ReinstateInvite(code: string) {
logger.info(`Reinstated Invite ${code}`);
return db.invites.update(
{
code,
},
{
$set: {
consumed: false,
consumedAt: null,
consumedBy: null,
},
}
);
}
export async function AddNewInvite(user: PublicUserDocument) {
const code = Random20Hex();
const result = await db.invites.insert({
code,
consumed: false,
createdBy: user.id,
createdAt: Date.now(),
consumedAt: null,
consumedBy: null,
});
logger.info(`User ${FormatUserDoc(user)} created an invite.`);
if (!result) {
logger.error(
`Fatal error in creating ${FormatUserDoc(
user
)}'s invite code. Database refused key ${code}.`
);
throw new Error(
`Fatal error in creating ${FormatUserDoc(
user
)}'s invite code. Database refused key ${code}.`
);
}
return result;
}
export const DEFAULT_USER_SETTINGS: UserSettings["preferences"] = {
developerMode: false,
advancedMode: false,
invisible: false,
contentiousContent: false,
};
export function HashPassword(plaintext: string) {
return bcrypt.hash(plaintext, BCRYPT_SALT_ROUNDS);
}
export async function AddNewUser(
username: string,
plaintext: string,
email: string,
userID: integer
) {
const hashedPassword = await HashPassword(plaintext);
logger.verbose(`Hashed password for ${username}.`);
const userDoc: PublicUserDocument = {
id: userID,
username,
usernameLowercase: username.toLowerCase(),
about: "I'm a fairly nondescript person.",
clan: null,
socialMedia: {},
status: null,
customBannerLocation: null,
customPfpLocation: null,
joinDate: Date.now(),
lastSeen: Date.now(),
authLevel: UserAuthLevels.USER,
badges: [],
};
const res = await db.users.insert(userDoc);
const settingsRes = await InsertDefaultUserSettings(userID);
await InsertPrivateUserInfo(userID, hashedPassword, email);
return { newUser: res, newSettings: settingsRes };
}
export function InsertPrivateUserInfo(userID: integer, hashedPassword: string, email: string) {
const privateInfo: PrivateUserInfoDocument = {
userID,
email,
password: hashedPassword,
};
return db["user-private-information"].insert(privateInfo);
}
export function InsertDefaultUserSettings(userID: integer) {
logger.verbose(`Inserting default settings for ${userID}.`);
const userSettings: UserSettings = {
userID,
preferences: DEFAULT_USER_SETTINGS,
};
return db["user-settings"].insert(userSettings);
}
export async function ValidateCaptcha(
recaptcha: string,
remoteAddr: string | undefined,
fetch = nodeFetch
) {
const url = CreateURLWithParams(`https://www.google.com/recaptcha/api/siteverify`, {
secret: ServerConfig.CAPTCHA_SECRET_KEY,
response: recaptcha,
remoteip: remoteAddr ?? "",
});
const r = await fetch(url.href);
if (r.status !== 200) {
logger.verbose(`Failed GCaptcha response ${r.status}, ${r.body}`);
return false;
}
return true;
}
export function MountAuthCookie(
req: Express.Request,
user: PublicUserDocument,
settings: UserSettings
) {
req.session.tachi = {
user,
settings,
};
req.session.cookie.maxAge = 3.154e10;
req.session.cookie.secure = Environment.nodeEnv === "production";
}
import bcrypt from "bcryptjs";
import {
integer,
PrivateUserInfoDocument,
PublicUserDocument,
UserAuthLevels,
UserSettings,
} from "tachi-common";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { FormatUserDoc } from "utils/user";
import nodeFetch from "utils/fetch";
import { Random20Hex } from "utils/misc";
import { Environment, ServerConfig } from "lib/setup/config";
import { CreateURLWithParams } from "utils/url";
const logger = CreateLogCtx(__filename);
const BCRYPT_SALT_ROUNDS = 12;
export const ValidatePassword = (self: unknown) =>
(typeof self === "string" && self.length >= 8) || "Passwords must be 8 characters or more.";
/**
* Compares a plaintext string of a users password to a hash.
* @param plaintext The provided user input.
* @param password The hash to compare against.
*/
export function PasswordCompare(plaintext: string, password: string) {
return bcrypt.compare(plaintext, password);
}
export function ReinstateInvite(code: string) {
logger.info(`Reinstated Invite ${code}`);
return db.invites.update(
{
code,
},
{
$set: {
consumed: false,
consumedAt: null,
consumedBy: null,
},
}
);
}
export async function AddNewInvite(user: PublicUserDocument) {
const code = Random20Hex();
const result = await db.invites.insert({
code,
consumed: false,
createdBy: user.id,
createdAt: Date.now(),
consumedAt: null,
consumedBy: null,
});
logger.info(`User ${FormatUserDoc(user)} created an invite.`);
if (!result) {
logger.error(
`Fatal error in creating ${FormatUserDoc(
user
)}'s invite code. Database refused key ${code}.`
);
throw new Error(
`Fatal error in creating ${FormatUserDoc(
user
)}'s invite code. Database refused key ${code}.`
);
}
return result;
}
export const DEFAULT_USER_SETTINGS: UserSettings["preferences"] = {
developerMode: false,
advancedMode: false,
invisible: false,
contentiousContent: false,
};
export function HashPassword(plaintext: string) {
return bcrypt.hash(plaintext, BCRYPT_SALT_ROUNDS);
}
export async function AddNewUser(
username: string,
plaintext: string,
email: string,
userID: integer
) {
const hashedPassword = await HashPassword(plaintext);
logger.verbose(`Hashed password for ${username}.`);
const userDoc: PublicUserDocument = {
id: userID,
username,
usernameLowercase: username.toLowerCase(),
about: "I'm a fairly nondescript person.",
clan: null,
socialMedia: {},
status: null,
customBannerLocation: null,
customPfpLocation: null,
joinDate: Date.now(),
lastSeen: Date.now(),
authLevel: UserAuthLevels.USER,
badges: [],
};
const res = await db.users.insert(userDoc);
const settingsRes = await InsertDefaultUserSettings(userID);
await InsertPrivateUserInfo(userID, hashedPassword, email);
return { newUser: res, newSettings: settingsRes };
}
export function InsertPrivateUserInfo(userID: integer, hashedPassword: string, email: string) {
const privateInfo: PrivateUserInfoDocument = {
userID,
email,
password: hashedPassword,
};
return db["user-private-information"].insert(privateInfo);
}
export function InsertDefaultUserSettings(userID: integer) {
logger.verbose(`Inserting default settings for ${userID}.`);
const userSettings: UserSettings = {
userID,
preferences: DEFAULT_USER_SETTINGS,
};
return db["user-settings"].insert(userSettings);
}
export async function ValidateCaptcha(
recaptcha: string,
remoteAddr: string | undefined,
fetch = nodeFetch
) {
const url = CreateURLWithParams(`https://www.google.com/recaptcha/api/siteverify`, {
secret: ServerConfig.CAPTCHA_SECRET_KEY,
response: recaptcha,
remoteip: remoteAddr ?? "",
});
const r = await fetch(url.href);
if (r.status !== 200) {
logger.verbose(`Failed GCaptcha response ${r.status}, ${r.body}`);
return false;
}
return true;
}
export function MountAuthCookie(
req: Express.Request,
user: PublicUserDocument,
settings: UserSettings
) {
req.session.tachi = {
user,
settings,
};
req.session.cookie.maxAge = 3.154e10;
req.session.cookie.secure = Environment.nodeEnv === "production";
}
server/src/server/router/api/v1/auth/router.ts
+521 -521
View File
File diff suppressed because it is too large Load Diff
server/src/server/router/api/v1/import/router.test.ts
+403 -403
View File
@@ -1,403 +1,403 @@
import t from "tap";
import mockApi from "test-utils/mock-api";
import {
GetKTDataBuffer,
LoadTachiIIDXData,
TestingIIDXEamusementCSV26,
TestingIIDXEamusementCSV27,
} from "test-utils/test-data";
import { RequireAuthPerms } from "test-utils/api-common";
import { CreateFakeAuthCookie } from "test-utils/fake-auth";
import ResetDBState from "test-utils/resets";
import db from "external/mongo/db";
import { rootLogger } from "lib/logger/logger";
t.test("POST /api/v1/import/file", async (t) => {
const cookie = await CreateFakeAuthCookie(mockApi);
t.beforeEach(ResetDBState);
RequireAuthPerms("/api/v1/import/file", "submit_score", "POST");
t.test("file/eamusement-iidx-csv", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Mini HV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"),
"my_csv.csv"
)
.field("importType", "file/eamusement-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Rootage CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv")
.field("importType", "file/eamusement-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Heroic Verse CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv")
.field("importType", "file/eamusement-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.strictSame(res.body.body.errors, [], "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.end();
});
// thats right i literally just copied it
t.test("file/pli-iidx-csv", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Mini HV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"),
"my_csv.csv"
)
.field("importType", "file/pli-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Rootage CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv")
.field("importType", "file/pli-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Heroic Verse CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv")
.field("importType", "file/pli-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.end();
});
t.test("file/batch-manual", (t) => {
t.test("Empty import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./batch-manual/empty-file.json"),
"empty-file.json"
)
.field("importType", "file/batch-manual");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 0, "Should have 0 successful scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Invalid JSON", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", Buffer.from("{invalid JSON"))
.field("importType", "file/batch-manual");
t.equal(res.body.success, false, "Should not be successful.");
t.equal(res.statusCode, 400);
t.end();
});
t.test("Single import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./batch-manual/small-file.json"),
"small-file.json"
)
.field("importType", "file/batch-manual");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 1, "Should have 1 successful score.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.end();
});
t.test("file/mer-iidx", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Example Import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", GetKTDataBuffer("./mer/base.json"), "base.json")
.field("importType", "file/mer-iidx");
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 3, "Should have 3 successful scores.");
t.end();
});
t.test("Example Import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", GetKTDataBuffer("./mer/large.json"), "base.json")
.field("importType", "file/mer-iidx");
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 627, "Should have 627 successful scores.");
t.end();
});
t.end();
});
t.skip("file/solid-state-squad", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Large Import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", GetKTDataBuffer("./s3/large-example.xml"), "large.xml")
.field("importType", "file/solid-state-squad");
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.scoreIDs.length, null, "Should parse N scores.");
t.end();
});
t.end();
});
t.end();
});
t.test("POST /api/v1/import/orphans", async (t) => {
const cookie = await CreateFakeAuthCookie(mockApi);
t.beforeEach(ResetDBState);
t.test("Should force a reprocessing of orphan scores.", async (t) => {
await db["orphan-scores"].insert([
{
userID: 1,
timeInserted: 1000,
orphanID: "asdf",
importType: "ir/direct-manual",
errMsg: "foo",
context: {
game: "iidx",
playtype: "SP",
service: "foo",
version: null,
},
data: {
score: 500,
lamp: "HARD CLEAR",
matchType: "songTitle",
identifier: "5.1.1.",
difficulty: "ANOTHER",
},
game: "iidx",
},
{
userID: 1,
timeInserted: 1000,
orphanID: "asdf2",
importType: "ir/direct-manual",
errMsg: "foo",
context: {
game: "iidx",
playtype: "SP",
service: "foo",
version: null,
},
data: {
score: 500,
lamp: "HARD CLEAR",
matchType: "songTitle",
identifier: "TITLE NOBODY WILL USE",
difficulty: "ANOTHER",
},
game: "iidx",
},
]);
const res = await mockApi.post("/api/v1/import/orphans").set("Cookie", cookie);
t.equal(res.statusCode, 200, "Should return 200.");
t.equal(res.body.body.success, 1, "Should successfully reprocess one orphan.");
t.equal(res.body.body.done, 2, "Should reprocess two orphans.");
t.equal(res.body.body.failed, 1, "Should fail in de-orphaning one orphan.");
const dbCount = await db["orphan-scores"].count({});
t.equal(dbCount, 1, "Should only leave one orphan-score in the database.");
t.end();
});
t.end();
});
import t from "tap";
import mockApi from "test-utils/mock-api";
import {
GetKTDataBuffer,
LoadTachiIIDXData,
TestingIIDXEamusementCSV26,
TestingIIDXEamusementCSV27,
} from "test-utils/test-data";
import { RequireAuthPerms } from "test-utils/api-common";
import { CreateFakeAuthCookie } from "test-utils/fake-auth";
import ResetDBState from "test-utils/resets";
import db from "external/mongo/db";
import { rootLogger } from "lib/logger/logger";
t.test("POST /api/v1/import/file", async (t) => {
const cookie = await CreateFakeAuthCookie(mockApi);
t.beforeEach(ResetDBState);
RequireAuthPerms("/api/v1/import/file", "submit_score", "POST");
t.test("file/eamusement-iidx-csv", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Mini HV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"),
"my_csv.csv"
)
.field("importType", "file/eamusement-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Rootage CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv")
.field("importType", "file/eamusement-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Heroic Verse CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv")
.field("importType", "file/eamusement-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.strictSame(res.body.body.errors, [], "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.end();
});
// thats right i literally just copied it
t.test("file/pli-iidx-csv", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Mini HV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./eamusement-iidx-csv/small-hv-file.csv"),
"my_csv.csv"
)
.field("importType", "file/pli-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Mini HV Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 2, "Should have 2 successful scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Rootage CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV26, "my_csv.csv")
.field("importType", "file/pli-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Valid Heroic Verse CSV import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", TestingIIDXEamusementCSV27, "my_csv.csv")
.field("importType", "file/pli-iidx-csv")
.field("playtype", "SP");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.end();
});
t.test("file/batch-manual", (t) => {
t.test("Empty import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./batch-manual/empty-file.json"),
"empty-file.json"
)
.field("importType", "file/batch-manual");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 0, "Should have 0 successful scores.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.test("Invalid JSON", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", Buffer.from("{invalid JSON"))
.field("importType", "file/batch-manual");
t.equal(res.body.success, false, "Should not be successful.");
t.equal(res.statusCode, 400);
t.end();
});
t.test("Single import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach(
"scoreData",
GetKTDataBuffer("./batch-manual/small-file.json"),
"small-file.json"
)
.field("importType", "file/batch-manual");
t.equal(res.body.success, true, "Should be successful.");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 1, "Should have 1 successful score.");
const scoreCount = await db.scores.find({
scoreID: { $in: res.body.body.scoreIDs },
});
t.equal(
scoreCount.length,
res.body.body.scoreIDs.length,
"All returned scoreIDs should be inserted to the DB."
);
t.end();
});
t.end();
});
t.test("file/mer-iidx", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Example Import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", GetKTDataBuffer("./mer/base.json"), "base.json")
.field("importType", "file/mer-iidx");
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 3, "Should have 3 successful scores.");
t.end();
});
t.test("Example Import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", GetKTDataBuffer("./mer/large.json"), "base.json")
.field("importType", "file/mer-iidx");
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.errors.length, 0, "Import Should have 0 failed scores.");
t.equal(res.body.body.scoreIDs.length, 627, "Should have 627 successful scores.");
t.end();
});
t.end();
});
t.skip("file/solid-state-squad", (t) => {
t.beforeEach(LoadTachiIIDXData);
t.test("Large Import", async (t) => {
const res = await mockApi
.post("/api/v1/import/file")
.set("Cookie", cookie)
.attach("scoreData", GetKTDataBuffer("./s3/large-example.xml"), "large.xml")
.field("importType", "file/solid-state-squad");
t.equal(res.body.success, true, "Should be successful");
t.equal(res.body.body.scoreIDs.length, null, "Should parse N scores.");
t.end();
});
t.end();
});
t.end();
});
t.test("POST /api/v1/import/orphans", async (t) => {
const cookie = await CreateFakeAuthCookie(mockApi);
t.beforeEach(ResetDBState);
t.test("Should force a reprocessing of orphan scores.", async (t) => {
await db["orphan-scores"].insert([
{
userID: 1,
timeInserted: 1000,
orphanID: "asdf",
importType: "ir/direct-manual",
errMsg: "foo",
context: {
game: "iidx",
playtype: "SP",
service: "foo",
version: null,
},
data: {
score: 500,
lamp: "HARD CLEAR",
matchType: "songTitle",
identifier: "5.1.1.",
difficulty: "ANOTHER",
},
game: "iidx",
},
{
userID: 1,
timeInserted: 1000,
orphanID: "asdf2",
importType: "ir/direct-manual",
errMsg: "foo",
context: {
game: "iidx",
playtype: "SP",
service: "foo",
version: null,
},
data: {
score: 500,
lamp: "HARD CLEAR",
matchType: "songTitle",
identifier: "TITLE NOBODY WILL USE",
difficulty: "ANOTHER",
},
game: "iidx",
},
]);
const res = await mockApi.post("/api/v1/import/orphans").set("Cookie", cookie);
t.equal(res.statusCode, 200, "Should return 200.");
t.equal(res.body.body.success, 1, "Should successfully reprocess one orphan.");
t.equal(res.body.body.done, 2, "Should reprocess two orphans.");
t.equal(res.body.body.failed, 1, "Should fail in de-orphaning one orphan.");
const dbCount = await db["orphan-scores"].count({});
t.equal(dbCount, 1, "Should only leave one orphan-score in the database.");
t.end();
});
t.end();
});
server/src/server/router/api/v1/import/router.ts
+214 -214
View File
@@ -1,214 +1,214 @@
import { Router } from "express";
import db from "external/mongo/db";
import { SIXTEEN_MEGABTYES } from "lib/constants/filesize";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import CreateLogCtx from "lib/logger/logger";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
import { MakeScoreImport } from "lib/score-import/framework/score-import";
import { ScoreImportJobData } from "lib/score-import/worker/types";
import { ServerConfig, TachiConfig } from "lib/setup/config";
import Prudence from "prudence";
import { RequirePermissions } from "server/middleware/auth";
import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload";
import prValidate from "server/middleware/prudence-validate";
import { APIImportTypes, FileUploadImportTypes } from "tachi-common";
import { Random20Hex } from "utils/misc";
import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user";
const logger = CreateLogCtx(__filename);
const router: Router = Router({ mergeParams: true });
const ParseMultipartScoredata = CreateMulterSingleUploadMiddleware(
"scoreData",
SIXTEEN_MEGABTYES,
logger
);
const fileImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("file/"));
const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/"));
/**
* Import scores from a file. Expects the post request to be multipart, and to provide a scoreData file.
*
* @param importType - The import type for this file.
* @param file - The actual file. Should be passed as multipart.
*
* @name POST /api/v1/import/file
*/
router.post(
"/file",
RequirePermissions("submit_score"),
ParseMultipartScoredata,
prValidate(
{
importType: Prudence.isIn(fileImportTypes),
},
{},
{ allowExcessKeys: true }
),
async (req, res) => {
if (!req.file) {
return res.status(400).json({
success: false,
description: `No file provided.`,
});
}
const importType = req.body.importType as FileUploadImportTypes;
const userIntent = !!req.header("X-User-Intent");
if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
const importID = Random20Hex();
const job: ScoreImportJobData<FileUploadImportTypes> = {
importID,
userID: req[SYMBOL_TachiAPIAuth].userID!,
userIntent,
importType,
parserArguments: [req.file, req.body],
};
// Fire the score import, but make no guarantees about its state.
MakeScoreImport<FileUploadImportTypes>(job);
return res.status(202).json({
success: true,
description:
"Import loaded into queue. You can poll the provided URL for information on when its complete.",
body: {
url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`,
importID,
},
});
} else {
// Fire the score import and wait for it to finish!
const importResponse = await ExpressWrappedScoreImportMain<FileUploadImportTypes>(
req[SYMBOL_TachiAPIAuth].userID!,
userIntent,
importType,
[req.file, req.body]
);
return res.status(importResponse.statusCode).json(importResponse.body);
}
}
);
/**
* Import scores from another API. This typically will perform a full sync.
* @name POST /api/v1/import/from-api
*/
router.post(
"/from-api",
RequirePermissions("submit_score"),
prValidate(
{
importType: Prudence.isIn(apiImportTypes),
},
{},
{ allowExcessKeys: true }
),
async (req, res) => {
const importType = req.body.importType as APIImportTypes;
const importID = Random20Hex();
const userID = req[SYMBOL_TachiAPIAuth].userID!;
const userIntent = !!req.header("X-User-Intent");
if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
const job: ScoreImportJobData<APIImportTypes> = {
importID,
userID,
userIntent,
importType,
parserArguments: [userID],
};
// Fire the score import, but make no guarantees about its state.
MakeScoreImport<APIImportTypes>(job);
return res.status(202).json({
success: true,
description:
"Import loaded into queue. You can poll the provided URL for information on when its complete.",
body: {
url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`,
importID,
},
});
} else {
// Fire the score import and wait for it to finish!
const importResponse = await ExpressWrappedScoreImportMain<APIImportTypes>(
userID,
userIntent,
importType,
[userID]
);
return res.status(importResponse.statusCode).json(importResponse.body);
}
}
);
/**
* Force Tachi to reprocess your orphanned scores. This is automatically done
* daily, but this endpoint allows users to speed that up.
*
* @name POST /api/v1/import/orphans
*/
router.post("/orphans", RequirePermissions("submit_score"), async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id);
logger.info(`User ${FormatUserDoc(userDoc)} forced an orphan sync.`);
const orphans = await db["orphan-scores"].find({
userID: userDoc.id,
});
// ScoreIDs are essentially userID dependent, so this is fine.
const blacklist = (await db["score-blacklist"].find({ userID: userDoc.id })).map(
(e) => e.scoreID
);
let done = 0;
let failed = 0;
let success = 0;
let removed = 0;
await Promise.all(
orphans.map((or) =>
ReprocessOrphan(or, blacklist, logger).then((r) => {
done++;
if (r === null) {
removed++;
} else if (r === false) {
failed++;
} else {
success++;
}
})
)
);
logger.info(`Finished attempting deorphaning.`);
logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`);
return res.status(200).json({
success: true,
description: `Reprocessed ${done} orphan scores.`,
body: {
done,
failed,
success,
removed,
},
});
});
export default router;
import { Router } from "express";
import db from "external/mongo/db";
import { SIXTEEN_MEGABTYES } from "lib/constants/filesize";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import CreateLogCtx from "lib/logger/logger";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
import { MakeScoreImport } from "lib/score-import/framework/score-import";
import { ScoreImportJobData } from "lib/score-import/worker/types";
import { ServerConfig, TachiConfig } from "lib/setup/config";
import Prudence from "prudence";
import { RequirePermissions } from "server/middleware/auth";
import { CreateMulterSingleUploadMiddleware } from "server/middleware/multer-upload";
import prValidate from "server/middleware/prudence-validate";
import { APIImportTypes, FileUploadImportTypes } from "tachi-common";
import { Random20Hex } from "utils/misc";
import { FormatUserDoc, GetUserWithIDGuaranteed } from "utils/user";
const logger = CreateLogCtx(__filename);
const router: Router = Router({ mergeParams: true });
const ParseMultipartScoredata = CreateMulterSingleUploadMiddleware(
"scoreData",
SIXTEEN_MEGABTYES,
logger
);
const fileImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("file/"));
const apiImportTypes = TachiConfig.IMPORT_TYPES.filter((e) => e.startsWith("api/"));
/**
* Import scores from a file. Expects the post request to be multipart, and to provide a scoreData file.
*
* @param importType - The import type for this file.
* @param file - The actual file. Should be passed as multipart.
*
* @name POST /api/v1/import/file
*/
router.post(
"/file",
RequirePermissions("submit_score"),
ParseMultipartScoredata,
prValidate(
{
importType: Prudence.isIn(fileImportTypes),
},
{},
{ allowExcessKeys: true }
),
async (req, res) => {
if (!req.file) {
return res.status(400).json({
success: false,
description: `No file provided.`,
});
}
const importType = req.body.importType as FileUploadImportTypes;
const userIntent = !!req.header("X-User-Intent");
if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
const importID = Random20Hex();
const job: ScoreImportJobData<FileUploadImportTypes> = {
importID,
userID: req[SYMBOL_TachiAPIAuth].userID!,
userIntent,
importType,
parserArguments: [req.file, req.body],
};
// Fire the score import, but make no guarantees about its state.
MakeScoreImport<FileUploadImportTypes>(job);
return res.status(202).json({
success: true,
description:
"Import loaded into queue. You can poll the provided URL for information on when its complete.",
body: {
url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`,
importID,
},
});
} else {
// Fire the score import and wait for it to finish!
const importResponse = await ExpressWrappedScoreImportMain<FileUploadImportTypes>(
req[SYMBOL_TachiAPIAuth].userID!,
userIntent,
importType,
[req.file, req.body]
);
return res.status(importResponse.statusCode).json(importResponse.body);
}
}
);
/**
* Import scores from another API. This typically will perform a full sync.
* @name POST /api/v1/import/from-api
*/
router.post(
"/from-api",
RequirePermissions("submit_score"),
prValidate(
{
importType: Prudence.isIn(apiImportTypes),
},
{},
{ allowExcessKeys: true }
),
async (req, res) => {
const importType = req.body.importType as APIImportTypes;
const importID = Random20Hex();
const userID = req[SYMBOL_TachiAPIAuth].userID!;
const userIntent = !!req.header("X-User-Intent");
if (ServerConfig.USE_EXTERNAL_SCORE_IMPORT_WORKER) {
const job: ScoreImportJobData<APIImportTypes> = {
importID,
userID,
userIntent,
importType,
parserArguments: [userID],
};
// Fire the score import, but make no guarantees about its state.
MakeScoreImport<APIImportTypes>(job);
return res.status(202).json({
success: true,
description:
"Import loaded into queue. You can poll the provided URL for information on when its complete.",
body: {
url: `${ServerConfig.OUR_URL}/api/v1/imports/${importID}/poll-status`,
importID,
},
});
} else {
// Fire the score import and wait for it to finish!
const importResponse = await ExpressWrappedScoreImportMain<APIImportTypes>(
userID,
userIntent,
importType,
[userID]
);
return res.status(importResponse.statusCode).json(importResponse.body);
}
}
);
/**
* Force Tachi to reprocess your orphanned scores. This is automatically done
* daily, but this endpoint allows users to speed that up.
*
* @name POST /api/v1/import/orphans
*/
router.post("/orphans", RequirePermissions("submit_score"), async (req, res) => {
const userDoc = await GetUserWithIDGuaranteed(req.session.tachi!.user.id);
logger.info(`User ${FormatUserDoc(userDoc)} forced an orphan sync.`);
const orphans = await db["orphan-scores"].find({
userID: userDoc.id,
});
// ScoreIDs are essentially userID dependent, so this is fine.
const blacklist = (await db["score-blacklist"].find({ userID: userDoc.id })).map(
(e) => e.scoreID
);
let done = 0;
let failed = 0;
let success = 0;
let removed = 0;
await Promise.all(
orphans.map((or) =>
ReprocessOrphan(or, blacklist, logger).then((r) => {
done++;
if (r === null) {
removed++;
} else if (r === false) {
failed++;
} else {
success++;
}
})
)
);
logger.info(`Finished attempting deorphaning.`);
logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`);
return res.status(200).json({
success: true,
description: `Reprocessed ${done} orphan scores.`,
body: {
done,
failed,
success,
removed,
},
});
});
export default router;
server/src/server/router/api/v1/router.ts
+44 -44
View File
@@ -1,44 +1,44 @@
import { Router } from "express";
import adminRouter from "./admin/router";
import authRouter from "./auth/router";
import clientsRouter from "./clients/router";
import configRouter from "./config/router";
import gamesRouter from "./games/router";
import importRouter from "./import/router";
import importsRouter from "./imports/router";
import oauthRouter from "./oauth/router";
import scoresRouter from "./scores/router";
import searchRouter from "./search/router";
import sessionsRouter from "./sessions/router";
import statusRouter from "./status/router";
import usersRouter from "./users/router";
const router: Router = Router({ mergeParams: true });
router.use("/admin", adminRouter);
router.use("/auth", authRouter);
router.use("/status", statusRouter);
router.use("/import", importRouter);
router.use("/imports", importsRouter);
router.use("/users", usersRouter);
router.use("/games", gamesRouter);
router.use("/search", searchRouter);
router.use("/scores", scoresRouter);
router.use("/sessions", sessionsRouter);
router.use("/oauth", oauthRouter);
router.use("/clients", clientsRouter);
router.use("/config", configRouter);
/**
* Return a JSON 404 response if an endpoint is hit that does not exist.
*
* @name ALL /api/v1/*
*/
router.all("*", (req, res) =>
res.status(404).json({
success: false,
description: "Endpoint Not Found.",
})
);
export default router;
import { Router } from "express";
import adminRouter from "./admin/router";
import authRouter from "./auth/router";
import clientsRouter from "./clients/router";
import configRouter from "./config/router";
import gamesRouter from "./games/router";
import importRouter from "./import/router";
import importsRouter from "./imports/router";
import oauthRouter from "./oauth/router";
import scoresRouter from "./scores/router";
import searchRouter from "./search/router";
import sessionsRouter from "./sessions/router";
import statusRouter from "./status/router";
import usersRouter from "./users/router";
const router: Router = Router({ mergeParams: true });
router.use("/admin", adminRouter);
router.use("/auth", authRouter);
router.use("/status", statusRouter);
router.use("/import", importRouter);
router.use("/imports", importsRouter);
router.use("/users", usersRouter);
router.use("/games", gamesRouter);
router.use("/search", searchRouter);
router.use("/scores", scoresRouter);
router.use("/sessions", sessionsRouter);
router.use("/oauth", oauthRouter);
router.use("/clients", clientsRouter);
router.use("/config", configRouter);
/**
* Return a JSON 404 response if an endpoint is hit that does not exist.
*
* @name ALL /api/v1/*
*/
router.all("*", (req, res) =>
res.status(404).json({
success: false,
description: "Endpoint Not Found.",
})
);
export default router;
server/src/server/server.ts
+170 -170
View File
@@ -1,170 +1,170 @@
import express, { Express } from "express";
import "express-async-errors";
import expressSession from "express-session";
import { integer } from "tachi-common";
import { RedisClient } from "external/redis/redis";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import connectRedis from "connect-redis";
import helmet from "helmet";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
let store;
if (Environment.nodeEnv !== "test") {
logger.info("Connecting ExpressSession to Redis.");
const RedisStore = connectRedis(expressSession);
store = new RedisStore({
host: "localhost",
port: 6379,
client: RedisClient,
prefix: TachiConfig.NAME,
});
}
const userSessionMiddleware = expressSession({
// append node_env onto the end of the session name
// so we can separate tokens under the same URL.
// say for staging.kamaitachi.xyz
name: `${TachiConfig.NAME.replace(/ /gu, "_")}_SESSION`,
secret: ServerConfig.SESSION_SECRET,
store,
resave: true,
saveUninitialized: false,
cookie: {
secure: Environment.nodeEnv === "production" || ServerConfig.ENABLE_SERVER_HTTPS,
sameSite: "lax", // Very important. Without this, we're vulnerable to CSRF!
},
});
const app: Express = express();
if (Environment.nodeEnv !== "production" && ServerConfig.CLIENT_DEV_SERVER) {
logger.warn(`Enabling CORS requests from ${ServerConfig.CLIENT_DEV_SERVER}.`, {
bootInfo: true,
});
// Allow CORS requests from another server (since we have our dev server hosted separately).
app.use((req, res, next) => {
res.header("Access-Control-Allow-Origin", ServerConfig.CLIENT_DEV_SERVER!);
res.header(
"Access-Control-Allow-Headers",
"Origin, X-Requested-With, Content-Type, Accept, X-User-Intent"
);
res.header("Access-Control-Allow-Credentials", "true");
res.header("Access-Control-Allow-Methods", "GET,POST,PATCH,PUT,DELETE,OPTIONS");
next();
});
// hack to allow all OPTIONS requests. Remember that this setting should not be on in production!
if (ServerConfig.OPTIONS_ALWAYS_SUCCEEDS) {
app.options("*", (req, res) => res.send());
}
} else {
if (Environment.nodeEnv !== "test") {
logger.info("Enabling Helmet, as no CLIENT_DEV_SERVER was set, or we are in production.", {
bootInfo: true,
});
}
app.use(helmet());
}
app.use(userSessionMiddleware);
// Most of these options are leveraged from KTAPI
// Pass the IP of the user up our increasingly insane chain of nginx/docker nonsense
app.set("trust proxy", ["loopback", "linklocal", "uniquelocal"]);
// we don't allow nesting in query strings.
app.set("query parser", "simple");
// taken from https://nodejs.org/api/process.html#process_event_unhandledrejection
// to avoid future deprecation.
process.on("unhandledRejection", (reason, promise) => {
// @ts-expect-error reason is an error, and the logger can handle errors
// it just refuses.
logger.error(reason, { promise });
});
// enable reading json bodies
// limit them so as not to choke the api
app.use(express.json({ limit: "4mb" }));
app.use((req, res, next) => {
if (req.method !== "GET" && !req.body) {
req.body = {};
}
return next();
});
import mainRouter from "./router/router";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import { RequestLoggerMiddleware } from "./middleware/request-logger";
app.use(RequestLoggerMiddleware);
app.use("/", mainRouter);
// The SERVE_OWN_CDN option means that our /cdn path has to be hosted by us. In production,
// this is not the case (we have a dedicated nginx box for it running in a separate process).
// In dev, this is a pain to setup, so we can just run it locally.
if (
ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM" &&
ServerConfig.CDN_CONFIG.SAVE_LOCATION.SERVE_OWN_CDN
) {
if (Environment.nodeEnv === "production") {
logger.warn(
`Running LOCAL_FILESYSTEM OWN_CDN in production. Consider making a separate process handle your CDN for performance.`,
{ bootInfo: true }
);
}
logger.info(`Running own CDN at ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION}.`, {
bootInfo: true,
});
app.use("/cdn", express.static(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION));
app.get("/cdn/*", (req, res) => res.status(404).send("No content here."));
}
// completely stolen from ktapi error handler
interface ExpressJSONErr extends SyntaxError {
status: integer;
message: string;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const MAIN_ERR_HANDLER: express.ErrorRequestHandler = (err, req, res, next) => {
if (err instanceof SyntaxError) {
const expErr: ExpressJSONErr = err as ExpressJSONErr;
if (expErr.status === 400 && "body" in expErr) {
logger.info(`JSON Parsing Error?`, {
url: req.originalUrl,
userID: req[SYMBOL_TachiAPIAuth]?.userID,
});
return res.status(400).send({ success: false, description: err.message });
}
// else, this isn't a JSON parsing error
}
if (err.type === "entity.too.large") {
return res.status(413).json({
success: false,
description: "Your request body was too large. The limit is 4MB.",
});
}
logger.error(err, req.route);
return res.status(500).json({
success: false,
description: "A fatal internal server error has occured.",
});
};
app.use(MAIN_ERR_HANDLER);
export default app;
import express, { Express } from "express";
import "express-async-errors";
import expressSession from "express-session";
import { integer } from "tachi-common";
import { RedisClient } from "external/redis/redis";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import connectRedis from "connect-redis";
import helmet from "helmet";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
let store;
if (Environment.nodeEnv !== "test") {
logger.info("Connecting ExpressSession to Redis.");
const RedisStore = connectRedis(expressSession);
store = new RedisStore({
host: "localhost",
port: 6379,
client: RedisClient,
prefix: TachiConfig.NAME,
});
}
const userSessionMiddleware = expressSession({
// append node_env onto the end of the session name
// so we can separate tokens under the same URL.
// say for staging.kamaitachi.xyz
name: `${TachiConfig.NAME.replace(/ /gu, "_")}_SESSION`,
secret: ServerConfig.SESSION_SECRET,
store,
resave: true,
saveUninitialized: false,
cookie: {
secure: Environment.nodeEnv === "production" || ServerConfig.ENABLE_SERVER_HTTPS,
sameSite: "lax", // Very important. Without this, we're vulnerable to CSRF!
},
});
const app: Express = express();
if (Environment.nodeEnv !== "production" && ServerConfig.CLIENT_DEV_SERVER) {
logger.warn(`Enabling CORS requests from ${ServerConfig.CLIENT_DEV_SERVER}.`, {
bootInfo: true,
});
// Allow CORS requests from another server (since we have our dev server hosted separately).
app.use((req, res, next) => {
res.header("Access-Control-Allow-Origin", ServerConfig.CLIENT_DEV_SERVER!);
res.header(
"Access-Control-Allow-Headers",
"Origin, X-Requested-With, Content-Type, Accept, X-User-Intent"
);
res.header("Access-Control-Allow-Credentials", "true");
res.header("Access-Control-Allow-Methods", "GET,POST,PATCH,PUT,DELETE,OPTIONS");
next();
});
// hack to allow all OPTIONS requests. Remember that this setting should not be on in production!
if (ServerConfig.OPTIONS_ALWAYS_SUCCEEDS) {
app.options("*", (req, res) => res.send());
}
} else {
if (Environment.nodeEnv !== "test") {
logger.info("Enabling Helmet, as no CLIENT_DEV_SERVER was set, or we are in production.", {
bootInfo: true,
});
}
app.use(helmet());
}
app.use(userSessionMiddleware);
// Most of these options are leveraged from KTAPI
// Pass the IP of the user up our increasingly insane chain of nginx/docker nonsense
app.set("trust proxy", ["loopback", "linklocal", "uniquelocal"]);
// we don't allow nesting in query strings.
app.set("query parser", "simple");
// taken from https://nodejs.org/api/process.html#process_event_unhandledrejection
// to avoid future deprecation.
process.on("unhandledRejection", (reason, promise) => {
// @ts-expect-error reason is an error, and the logger can handle errors
// it just refuses.
logger.error(reason, { promise });
});
// enable reading json bodies
// limit them so as not to choke the api
app.use(express.json({ limit: "4mb" }));
app.use((req, res, next) => {
if (req.method !== "GET" && !req.body) {
req.body = {};
}
return next();
});
import mainRouter from "./router/router";
import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi";
import { RequestLoggerMiddleware } from "./middleware/request-logger";
app.use(RequestLoggerMiddleware);
app.use("/", mainRouter);
// The SERVE_OWN_CDN option means that our /cdn path has to be hosted by us. In production,
// this is not the case (we have a dedicated nginx box for it running in a separate process).
// In dev, this is a pain to setup, so we can just run it locally.
if (
ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM" &&
ServerConfig.CDN_CONFIG.SAVE_LOCATION.SERVE_OWN_CDN
) {
if (Environment.nodeEnv === "production") {
logger.warn(
`Running LOCAL_FILESYSTEM OWN_CDN in production. Consider making a separate process handle your CDN for performance.`,
{ bootInfo: true }
);
}
logger.info(`Running own CDN at ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION}.`, {
bootInfo: true,
});
app.use("/cdn", express.static(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION));
app.get("/cdn/*", (req, res) => res.status(404).send("No content here."));
}
// completely stolen from ktapi error handler
interface ExpressJSONErr extends SyntaxError {
status: integer;
message: string;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const MAIN_ERR_HANDLER: express.ErrorRequestHandler = (err, req, res, next) => {
if (err instanceof SyntaxError) {
const expErr: ExpressJSONErr = err as ExpressJSONErr;
if (expErr.status === 400 && "body" in expErr) {
logger.info(`JSON Parsing Error?`, {
url: req.originalUrl,
userID: req[SYMBOL_TachiAPIAuth]?.userID,
});
return res.status(400).send({ success: false, description: err.message });
}
// else, this isn't a JSON parsing error
}
if (err.type === "entity.too.large") {
return res.status(413).json({
success: false,
description: "Your request body was too large. The limit is 4MB.",
});
}
logger.error(err, req.route);
return res.status(500).json({
success: false,
description: "A fatal internal server error has occured.",
});
};
app.use(MAIN_ERR_HANDLER);
export default app;
server/src/test-utils/mock-api.ts
+18 -18
View File
@@ -1,18 +1,18 @@
import supertest from "supertest";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
import server from "../server/server";
logger.verbose("Creating Mock Server Connection...");
const connection = server.listen();
logger.verbose("Connecting to Supertest...");
const mockApi = supertest(connection);
export function CloseServerConnection() {
connection.close();
}
export default mockApi;
import supertest from "supertest";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
import server from "../server/server";
logger.verbose("Creating Mock Server Connection...");
const connection = server.listen();
logger.verbose("Connecting to Supertest...");
const mockApi = supertest(connection);
export function CloseServerConnection() {
connection.close();
}
export default mockApi;
server/src/test-utils/mock-fetch.ts
+32 -32
View File
@@ -1,32 +1,32 @@
// We know! these are mock functions.
/* eslint-disable require-await */
import { Response } from "node-fetch";
import { NodeFetch } from "utils/fetch";
/**
* Creates a basic Fetch function used for statusCode checks.
*/
export function MockBasicFetch(data: Partial<Response>) {
return (async () => data) as unknown as NodeFetch;
}
/**
* Creates a fake "fetch" function which takes a url and returns
* a fake JSON -> data function at that key.
*/
export function MockJSONFetch(urlDataMap: Record<string, unknown>) {
return (async (url: string) => {
if (urlDataMap[url]) {
return {
status: 200,
json: async () => urlDataMap[url],
};
}
throw new Error(
`Unexpected url ${url} - No Data Present? Valid urls are ${Object.keys(urlDataMap).join(
", "
)}`
);
}) as NodeFetch;
}
// We know! these are mock functions.
/* eslint-disable require-await */
import { Response } from "node-fetch";
import { NodeFetch } from "utils/fetch";
/**
* Creates a basic Fetch function used for statusCode checks.
*/
export function MockBasicFetch(data: Partial<Response>) {
return (async () => data) as unknown as NodeFetch;
}
/**
* Creates a fake "fetch" function which takes a url and returns
* a fake JSON -> data function at that key.
*/
export function MockJSONFetch(urlDataMap: Record<string, unknown>) {
return (async (url: string) => {
if (urlDataMap[url]) {
return {
status: 200,
json: async () => urlDataMap[url],
};
}
throw new Error(
`Unexpected url ${url} - No Data Present? Valid urls are ${Object.keys(urlDataMap).join(
", "
)}`
);
}) as NodeFetch;
}
server/src/test-utils/resets.ts
+113 -113
View File
@@ -1,113 +1,113 @@
/**
* Resets the state of the database.
*/
import db from "external/mongo/db";
import fs from "fs";
import path from "path";
import CreateLogCtx from "lib/logger/logger";
// im installing an entire library for rm rf...
import rimraf from "rimraf";
import { SetIndexes } from "external/mongo/indexes";
import { Environment, ServerConfig } from "lib/setup/config";
if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") {
throw new Error(
`Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)`
);
}
const logger = CreateLogCtx(__filename);
const DATA_DIR = path.join(__dirname, "./mock-db");
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const CACHE: Record<string, any[]> = {};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async function ResetState(data: any[], collection: any) {
await collection.remove({});
await collection.insert(data);
}
function GetAndCache(filename: string, fileLoc: string) {
let collection;
if (filename.startsWith("songs-")) {
// @ts-expect-error it's right, but we know what we're doing!
collection = db.songs[filename.split("-")[1]];
} else if (filename.startsWith("charts-")) {
// @ts-expect-error see above
collection = db.charts[filename.split("-")[1]];
} else {
// @ts-expect-error see above
collection = db[filename];
}
if (CACHE[filename]) {
return { data: CACHE[filename], collection };
}
const data = JSON.parse(fs.readFileSync(fileLoc, "utf-8"));
if (!Array.isArray(data)) {
throw new Error(`Panic, ${filename} not JSONArray?`);
}
CACHE[filename] = data;
return { data, collection };
}
let CACHE_FILENAMES: string[];
export default async function ResetDBState() {
let files;
if (CACHE_FILENAMES) {
files = CACHE_FILENAMES;
} else {
files = fs.readdirSync(DATA_DIR);
CACHE_FILENAMES = files;
}
const promises = [];
for (const file of files) {
const filename = path.basename(file, ".json");
const fileLoc = path.join(DATA_DIR, file);
const { data, collection } = GetAndCache(filename, fileLoc);
promises.push(ResetState(data, collection));
}
await Promise.all(promises);
}
export function ResetCDN() {
return new Promise<void>((resolve, reject) => {
if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") {
throw new Error(
`Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)`
);
}
rimraf(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION, (err) => {
if (err) {
reject(err);
}
resolve();
});
});
}
export async function SetIndexesForDB() {
await ResetDBState();
const url = `${Environment.mongoUrl}/testingdb`;
logger.info(`Setting indexes for ${url}`);
await SetIndexes(url, true);
logger.info(`Done.`);
return true;
}
/**
* Resets the state of the database.
*/
import db from "external/mongo/db";
import fs from "fs";
import path from "path";
import CreateLogCtx from "lib/logger/logger";
// im installing an entire library for rm rf...
import rimraf from "rimraf";
import { SetIndexes } from "external/mongo/indexes";
import { Environment, ServerConfig } from "lib/setup/config";
if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") {
throw new Error(
`Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)`
);
}
const logger = CreateLogCtx(__filename);
const DATA_DIR = path.join(__dirname, "./mock-db");
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const CACHE: Record<string, any[]> = {};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async function ResetState(data: any[], collection: any) {
await collection.remove({});
await collection.insert(data);
}
function GetAndCache(filename: string, fileLoc: string) {
let collection;
if (filename.startsWith("songs-")) {
// @ts-expect-error it's right, but we know what we're doing!
collection = db.songs[filename.split("-")[1]];
} else if (filename.startsWith("charts-")) {
// @ts-expect-error see above
collection = db.charts[filename.split("-")[1]];
} else {
// @ts-expect-error see above
collection = db[filename];
}
if (CACHE[filename]) {
return { data: CACHE[filename], collection };
}
const data = JSON.parse(fs.readFileSync(fileLoc, "utf-8"));
if (!Array.isArray(data)) {
throw new Error(`Panic, ${filename} not JSONArray?`);
}
CACHE[filename] = data;
return { data, collection };
}
let CACHE_FILENAMES: string[];
export default async function ResetDBState() {
let files;
if (CACHE_FILENAMES) {
files = CACHE_FILENAMES;
} else {
files = fs.readdirSync(DATA_DIR);
CACHE_FILENAMES = files;
}
const promises = [];
for (const file of files) {
const filename = path.basename(file, ".json");
const fileLoc = path.join(DATA_DIR, file);
const { data, collection } = GetAndCache(filename, fileLoc);
promises.push(ResetState(data, collection));
}
await Promise.all(promises);
}
export function ResetCDN() {
return new Promise<void>((resolve, reject) => {
if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") {
throw new Error(
`Cannot run tests when CDN_CONFIG.SAVE_LOCATION.TYPE is not LOCAL_FILESYSTEM! (Got ${ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE}.)`
);
}
rimraf(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION, (err) => {
if (err) {
reject(err);
}
resolve();
});
});
}
export async function SetIndexesForDB() {
await ResetDBState();
const url = `${Environment.mongoUrl}/testingdb`;
logger.info(`Setting indexes for ${url}`);
await SetIndexes(url, true);
logger.info(`Done.`);
return true;
}
server/src/test-utils/server.ts
+7 -7
View File
@@ -1,7 +1,7 @@
import server from "../server/server";
import supertest from "supertest";
const mockServer = supertest(server);
// mock the server for requests
export default mockServer;
import server from "../server/server";
import supertest from "supertest";
const mockServer = supertest(server);
// mock the server for requests
export default mockServer;
server/src/test-utils/test-data.ts
+241 -241
View File
@@ -1,241 +1,241 @@
import db from "external/mongo/db";
import fs from "fs";
import {
USC_DEFAULT_HOLD,
USC_DEFAULT_MISS,
USC_DEFAULT_NEAR,
USC_DEFAULT_PERFECT,
USC_DEFAULT_SLAM,
} from "lib/constants/usc-ir";
import { DryScore } from "lib/score-import/framework/common/types";
import { BarbatosScore } from "lib/score-import/import-types/ir/barbatos/types";
import { KsHookSV3CScore } from "lib/score-import/import-types/ir/kshook-sv3c/types";
import path from "path";
import { USCClientScore } from "server/router/ir/usc/_playtype/types";
import {
ChartDocument,
FolderDocument,
GoalDocument,
MilestoneDocument,
PBScoreDocument,
ScoreDocument,
SongDocument,
UserGoalDocument,
} from "tachi-common";
const file = (name: string) => path.join(__dirname, "/test-data", name);
export const GetKTDataJSON = (name: string) => JSON.parse(fs.readFileSync(file(name), "utf-8"));
export const GetKTDataBuffer = (name: string) => fs.readFileSync(file(name));
export const TestingIIDXSPDryScore = GetKTDataJSON(
"./tachi/iidx-dryscore.json"
) as DryScore<"iidx:SP">;
export const TestingIIDXSPScorePB = GetKTDataJSON(
"./tachi/iidx-scorepb.json"
) as PBScoreDocument<"iidx:SP">;
export const TestingIIDXSPScore = GetKTDataJSON(
"./tachi/iidx-score.json"
) as ScoreDocument<"iidx:SP">;
export const TestingKsHookSV3CScore = GetKTDataJSON("./kshook-sv3c/base.json") as KsHookSV3CScore;
export const Testing511SPA = GetKTDataJSON("./tachi/iidx-511spa.json") as ChartDocument<"iidx:SP">;
export const Testing511Song = GetKTDataJSON("./tachi/iidx-511-song.json") as SongDocument<"iidx">;
export const TestingAlbidaADV = GetKTDataJSON(
"./tachi/sdvx-albida-chart.json"
) as ChartDocument<"sdvx:Single">;
export const TestingSDVXAlbidaSong = GetKTDataJSON(
"./tachi/sdvx-albida-song.json"
) as SongDocument<"sdvx">;
export const TestingDoraChart = GetKTDataJSON(
"./tachi/gitadora-ithinkabtyou.json"
) as ChartDocument<"gitadora:Dora">;
export const TestingSDVXSingleDryScore = GetKTDataJSON(
"./tachi/sdvx-dryscore.json"
) as DryScore<"sdvx:Single">;
export const TestingGITADORADoraDryScore = GetKTDataJSON(
"./tachi/gitadora-dryscore.json"
) as DryScore<"gitadora:Dora">;
export const TestingIIDXEamusementCSV26 = GetKTDataBuffer(
"./eamusement-iidx-csv/pre-leggendaria.csv"
);
export const TestingIIDXEamusementCSV27 = GetKTDataBuffer(
"./eamusement-iidx-csv/post-leggendaria.csv"
);
export const TestingBarbatosScore: BarbatosScore = GetKTDataJSON("./barbatos/base.json");
export const HC511Goal: GoalDocument = {
charts: {
type: "single",
data: Testing511SPA.chartID,
},
createdBy: 1,
game: "iidx",
goalID: "mock_goalID",
playtype: "SP",
timeAdded: 0,
title: "HC 5.1.1. SPA",
criteria: {
mode: "single",
value: 5,
key: "scoreData.lampIndex",
},
};
export const HC511UserGoal: UserGoalDocument = {
achieved: false,
timeAchieved: null,
game: "iidx",
playtype: "SP",
goalID: "mock_goalID",
lastInteraction: null,
outOf: 5,
outOfHuman: "HARD CLEAR",
progress: null,
progressHuman: "NO DATA",
timeSet: 0,
userID: 1,
};
export const TestingIIDXFolderSP10: FolderDocument = {
title: "Level 10",
game: "iidx",
playtype: "SP",
type: "charts",
folderID: "ed9d8c734447ce67d7135c0067441a98cc81aeaf",
data: {
level: "10",
"flags¬IN BASE GAME": true,
},
searchTerms: [],
inactive: false,
};
export const TestingIIDXSPMilestone: MilestoneDocument = {
createdBy: 1,
criteria: {
type: "all",
},
desc: "testing milestone",
game: "iidx",
playtype: "SP",
group: "ExampleGroup",
groupIndex: 1,
milestoneID: "example_milestone_id",
name: "Example Milestone",
milestoneData: [
{
title: "Group1",
desc: "Foo",
goals: [
{
goalID: "eg_goal_1",
},
{
goalID: "eg_goal_2",
},
],
},
{
title: "Group2",
desc: "Bar",
goals: [
{
goalID: "eg_goal_3",
},
{
goalID: "eg_goal_4",
},
],
},
],
};
let KTDATA_CACHE: { songs: unknown[]; charts: unknown[] } | undefined;
export async function LoadTachiIIDXData() {
let songs;
let charts;
if (KTDATA_CACHE) {
songs = KTDATA_CACHE.songs;
charts = KTDATA_CACHE.charts;
} else {
songs = GetKTDataJSON("./tachi/tachi-songs-iidx.json");
charts = GetKTDataJSON("./tachi/tachi-charts-iidx.json");
KTDATA_CACHE = { songs, charts };
}
await db.songs.iidx.remove({});
await db.songs.iidx.insert(songs);
await db.charts.iidx.remove({});
await db.charts.iidx.insert(charts);
}
export const barbScore: BarbatosScore = {
clear_type: 2,
did_fail: false,
difficulty: 1,
critical: 100,
error: 5,
near_total: 50,
near_fast: 40,
near_slow: 10,
gauge_type: 1,
is_skill_analyser: false,
level: 10,
max_chain: 100,
percent: 90,
score: 9_000_000,
song_id: 1,
};
export const uscChart: ChartDocument<"usc:Controller" | "usc:Keyboard"> = {
rgcID: null,
chartID: "USC_CHART_ID",
difficulty: "NOV",
songID: 1,
playtype: "Keyboard",
levelNum: 1,
level: "1",
data: {
hashSHA1: "USC_CHART_HASH",
isOfficial: false,
},
tierlistInfo: {},
isPrimary: true,
versions: [],
};
export const uscScore: USCClientScore = {
crit: 100,
error: 5,
near: 50,
gauge: 0.8,
options: {
autoFlags: 0,
gaugeOpt: 0,
gaugeType: 1,
mirror: true,
random: false,
},
score: 9_000_000,
timestamp: 0,
windows: {
good: USC_DEFAULT_NEAR,
perfect: USC_DEFAULT_PERFECT,
hold: USC_DEFAULT_HOLD,
miss: USC_DEFAULT_MISS,
slam: USC_DEFAULT_SLAM,
},
};
import db from "external/mongo/db";
import fs from "fs";
import {
USC_DEFAULT_HOLD,
USC_DEFAULT_MISS,
USC_DEFAULT_NEAR,
USC_DEFAULT_PERFECT,
USC_DEFAULT_SLAM,
} from "lib/constants/usc-ir";
import { DryScore } from "lib/score-import/framework/common/types";
import { BarbatosScore } from "lib/score-import/import-types/ir/barbatos/types";
import { KsHookSV3CScore } from "lib/score-import/import-types/ir/kshook-sv3c/types";
import path from "path";
import { USCClientScore } from "server/router/ir/usc/_playtype/types";
import {
ChartDocument,
FolderDocument,
GoalDocument,
MilestoneDocument,
PBScoreDocument,
ScoreDocument,
SongDocument,
UserGoalDocument,
} from "tachi-common";
const file = (name: string) => path.join(__dirname, "/test-data", name);
export const GetKTDataJSON = (name: string) => JSON.parse(fs.readFileSync(file(name), "utf-8"));
export const GetKTDataBuffer = (name: string) => fs.readFileSync(file(name));
export const TestingIIDXSPDryScore = GetKTDataJSON(
"./tachi/iidx-dryscore.json"
) as DryScore<"iidx:SP">;
export const TestingIIDXSPScorePB = GetKTDataJSON(
"./tachi/iidx-scorepb.json"
) as PBScoreDocument<"iidx:SP">;
export const TestingIIDXSPScore = GetKTDataJSON(
"./tachi/iidx-score.json"
) as ScoreDocument<"iidx:SP">;
export const TestingKsHookSV3CScore = GetKTDataJSON("./kshook-sv3c/base.json") as KsHookSV3CScore;
export const Testing511SPA = GetKTDataJSON("./tachi/iidx-511spa.json") as ChartDocument<"iidx:SP">;
export const Testing511Song = GetKTDataJSON("./tachi/iidx-511-song.json") as SongDocument<"iidx">;
export const TestingAlbidaADV = GetKTDataJSON(
"./tachi/sdvx-albida-chart.json"
) as ChartDocument<"sdvx:Single">;
export const TestingSDVXAlbidaSong = GetKTDataJSON(
"./tachi/sdvx-albida-song.json"
) as SongDocument<"sdvx">;
export const TestingDoraChart = GetKTDataJSON(
"./tachi/gitadora-ithinkabtyou.json"
) as ChartDocument<"gitadora:Dora">;
export const TestingSDVXSingleDryScore = GetKTDataJSON(
"./tachi/sdvx-dryscore.json"
) as DryScore<"sdvx:Single">;
export const TestingGITADORADoraDryScore = GetKTDataJSON(
"./tachi/gitadora-dryscore.json"
) as DryScore<"gitadora:Dora">;
export const TestingIIDXEamusementCSV26 = GetKTDataBuffer(
"./eamusement-iidx-csv/pre-leggendaria.csv"
);
export const TestingIIDXEamusementCSV27 = GetKTDataBuffer(
"./eamusement-iidx-csv/post-leggendaria.csv"
);
export const TestingBarbatosScore: BarbatosScore = GetKTDataJSON("./barbatos/base.json");
export const HC511Goal: GoalDocument = {
charts: {
type: "single",
data: Testing511SPA.chartID,
},
createdBy: 1,
game: "iidx",
goalID: "mock_goalID",
playtype: "SP",
timeAdded: 0,
title: "HC 5.1.1. SPA",
criteria: {
mode: "single",
value: 5,
key: "scoreData.lampIndex",
},
};
export const HC511UserGoal: UserGoalDocument = {
achieved: false,
timeAchieved: null,
game: "iidx",
playtype: "SP",
goalID: "mock_goalID",
lastInteraction: null,
outOf: 5,
outOfHuman: "HARD CLEAR",
progress: null,
progressHuman: "NO DATA",
timeSet: 0,
userID: 1,
};
export const TestingIIDXFolderSP10: FolderDocument = {
title: "Level 10",
game: "iidx",
playtype: "SP",
type: "charts",
folderID: "ed9d8c734447ce67d7135c0067441a98cc81aeaf",
data: {
level: "10",
"flags¬IN BASE GAME": true,
},
searchTerms: [],
inactive: false,
};
export const TestingIIDXSPMilestone: MilestoneDocument = {
createdBy: 1,
criteria: {
type: "all",
},
desc: "testing milestone",
game: "iidx",
playtype: "SP",
group: "ExampleGroup",
groupIndex: 1,
milestoneID: "example_milestone_id",
name: "Example Milestone",
milestoneData: [
{
title: "Group1",
desc: "Foo",
goals: [
{
goalID: "eg_goal_1",
},
{
goalID: "eg_goal_2",
},
],
},
{
title: "Group2",
desc: "Bar",
goals: [
{
goalID: "eg_goal_3",
},
{
goalID: "eg_goal_4",
},
],
},
],
};
let KTDATA_CACHE: { songs: unknown[]; charts: unknown[] } | undefined;
export async function LoadTachiIIDXData() {
let songs;
let charts;
if (KTDATA_CACHE) {
songs = KTDATA_CACHE.songs;
charts = KTDATA_CACHE.charts;
} else {
songs = GetKTDataJSON("./tachi/tachi-songs-iidx.json");
charts = GetKTDataJSON("./tachi/tachi-charts-iidx.json");
KTDATA_CACHE = { songs, charts };
}
await db.songs.iidx.remove({});
await db.songs.iidx.insert(songs);
await db.charts.iidx.remove({});
await db.charts.iidx.insert(charts);
}
export const barbScore: BarbatosScore = {
clear_type: 2,
did_fail: false,
difficulty: 1,
critical: 100,
error: 5,
near_total: 50,
near_fast: 40,
near_slow: 10,
gauge_type: 1,
is_skill_analyser: false,
level: 10,
max_chain: 100,
percent: 90,
score: 9_000_000,
song_id: 1,
};
export const uscChart: ChartDocument<"usc:Controller" | "usc:Keyboard"> = {
rgcID: null,
chartID: "USC_CHART_ID",
difficulty: "NOV",
songID: 1,
playtype: "Keyboard",
levelNum: 1,
level: "1",
data: {
hashSHA1: "USC_CHART_HASH",
isOfficial: false,
},
tierlistInfo: {},
isPrimary: true,
versions: [],
};
export const uscScore: USCClientScore = {
crit: 100,
error: 5,
near: 50,
gauge: 0.8,
options: {
autoFlags: 0,
gaugeOpt: 0,
gaugeType: 1,
mirror: true,
random: false,
},
score: 9_000_000,
timestamp: 0,
windows: {
good: USC_DEFAULT_NEAR,
perfect: USC_DEFAULT_PERFECT,
hold: USC_DEFAULT_HOLD,
miss: USC_DEFAULT_MISS,
slam: USC_DEFAULT_SLAM,
},
};
server/src/utils/db.test.ts
+53 -53
View File
@@ -1,53 +1,53 @@
import db from "external/mongo/db";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { DecrementCounterValue, GetNextCounterValue } from "./db";
t.test("#GetNextCounterValue", (t) => {
t.beforeEach(ResetDBState);
t.test("Increments on valid counter hit", async (t) => {
const response = await GetNextCounterValue("real-counter");
// database starts with this at one
t.equal(response, 2, "Counter should return the current number stored");
const dbData = await db.counters.findOne({
counterName: "real-counter",
});
t.equal(dbData!.value, 3, "Counter should increment after being hit");
});
t.rejects(
() => GetNextCounterValue("fake-counter"),
"Could not find sequence document for fake-counter."
);
t.end();
});
t.test("#DecrementCounterValue", (t) => {
t.beforeEach(ResetDBState);
t.test("Should decrease a counter.", async (t) => {
await db.counters.insert({ counterName: "foo", value: 3 });
const res = await DecrementCounterValue("foo");
t.equal(res, 2);
const doc = await db.counters.findOne({ counterName: "foo" });
t.equal(doc?.value, 2);
t.end();
});
t.rejects(
() => GetNextCounterValue("fake-counter"),
"Could not find sequence document for fake-counter."
);
t.end();
});
import db from "external/mongo/db";
import t from "tap";
import ResetDBState from "test-utils/resets";
import { DecrementCounterValue, GetNextCounterValue } from "./db";
t.test("#GetNextCounterValue", (t) => {
t.beforeEach(ResetDBState);
t.test("Increments on valid counter hit", async (t) => {
const response = await GetNextCounterValue("real-counter");
// database starts with this at one
t.equal(response, 2, "Counter should return the current number stored");
const dbData = await db.counters.findOne({
counterName: "real-counter",
});
t.equal(dbData!.value, 3, "Counter should increment after being hit");
});
t.rejects(
() => GetNextCounterValue("fake-counter"),
"Could not find sequence document for fake-counter."
);
t.end();
});
t.test("#DecrementCounterValue", (t) => {
t.beforeEach(ResetDBState);
t.test("Should decrease a counter.", async (t) => {
await db.counters.insert({ counterName: "foo", value: 3 });
const res = await DecrementCounterValue("foo");
t.equal(res, 2);
const doc = await db.counters.findOne({ counterName: "foo" });
t.equal(doc?.value, 2);
t.end();
});
t.rejects(
() => GetNextCounterValue("fake-counter"),
"Could not find sequence document for fake-counter."
);
t.end();
});
server/src/utils/db.ts
+70 -70
View File
@@ -1,70 +1,70 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
import { integer, Game, ScoreDocument, PBScoreDocument } from "tachi-common";
export async function GetNextCounterValue(counterName: string): Promise<integer> {
const sequenceDoc = await db.counters.findOneAndUpdate(
{
counterName,
},
{
$inc: {
value: 1,
},
},
{
// this is marked as deprecated, but it shouldn't be, as returnDocument: "before"
// does nothing.
returnOriginal: true,
}
);
if (!sequenceDoc) {
logger.error(`Could not find sequence document for ${counterName}`);
throw new Error(`Could not find sequence document for ${counterName}.`);
}
return sequenceDoc.value;
}
export async function DecrementCounterValue(counterName: string): Promise<integer> {
logger.verbose(`Decrementing Counter Value ${counterName}.`);
const sequenceDoc = await db.counters.findOneAndUpdate(
{
counterName,
},
{
$inc: {
value: -1,
},
},
{
returnOriginal: false,
}
);
if (!sequenceDoc) {
logger.error(`Could not find sequence document for ${counterName}`);
throw new Error(`Could not find sequence document for ${counterName}.`);
}
return sequenceDoc.value;
}
export async function GetRelevantSongsAndCharts(
scores: (ScoreDocument | PBScoreDocument)[],
game: Game
) {
const [songs, charts] = await Promise.all([
db.songs[game].find({
id: { $in: scores.map((e) => e.songID) },
}),
db.charts[game].find({
chartID: { $in: scores.map((e) => e.chartID) },
}),
]);
return { songs, charts };
}
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
import { integer, Game, ScoreDocument, PBScoreDocument } from "tachi-common";
export async function GetNextCounterValue(counterName: string): Promise<integer> {
const sequenceDoc = await db.counters.findOneAndUpdate(
{
counterName,
},
{
$inc: {
value: 1,
},
},
{
// this is marked as deprecated, but it shouldn't be, as returnDocument: "before"
// does nothing.
returnOriginal: true,
}
);
if (!sequenceDoc) {
logger.error(`Could not find sequence document for ${counterName}`);
throw new Error(`Could not find sequence document for ${counterName}.`);
}
return sequenceDoc.value;
}
export async function DecrementCounterValue(counterName: string): Promise<integer> {
logger.verbose(`Decrementing Counter Value ${counterName}.`);
const sequenceDoc = await db.counters.findOneAndUpdate(
{
counterName,
},
{
$inc: {
value: -1,
},
},
{
returnOriginal: false,
}
);
if (!sequenceDoc) {
logger.error(`Could not find sequence document for ${counterName}`);
throw new Error(`Could not find sequence document for ${counterName}.`);
}
return sequenceDoc.value;
}
export async function GetRelevantSongsAndCharts(
scores: (ScoreDocument | PBScoreDocument)[],
game: Game
) {
const [songs, charts] = await Promise.all([
db.songs[game].find({
id: { $in: scores.map((e) => e.songID) },
}),
db.charts[game].find({
chartID: { $in: scores.map((e) => e.chartID) },
}),
]);
return { songs, charts };
}
server/src/utils/fetch.ts
+13 -13
View File
@@ -1,13 +1,13 @@
import { Environment } from "lib/setup/config";
import nodeFetch from "node-fetch";
const fetch =
Environment.nodeEnv === "test"
? () => {
throw new Error("Cannot use real fetch inside testing env!");
}
: nodeFetch;
export type NodeFetch = (url: RequestInfo, init?: RequestInit | undefined) => Promise<Response>;
export default fetch as unknown as NodeFetch;
import { Environment } from "lib/setup/config";
import nodeFetch from "node-fetch";
const fetch =
Environment.nodeEnv === "test"
? () => {
throw new Error("Cannot use real fetch inside testing env!");
}
: nodeFetch;
export type NodeFetch = (url: RequestInfo, init?: RequestInit | undefined) => Promise<Response>;
export default fetch as unknown as NodeFetch;
server/src/utils/queries/charts.ts
+319 -319
View File
@@ -1,319 +1,319 @@
import db from "external/mongo/db";
import { FilterQuery } from "mongodb";
import {
ChartDocument,
Difficulties,
Game,
GPTSupportedVersions,
IDStrings,
integer,
Playtypes,
} from "tachi-common";
export function FindChartWithChartID(game: Game, chartID: string) {
return db.charts[game].findOne({ chartID });
}
/**
* Find chart with PlaytypeDifficulty. This only finds charts that have `isPrimary` set to true.
* If you want to find charts that are not primary, you need to use PTDFVersion.
* @see FindChartWithPTDFVersion
*/
export function FindChartWithPTDF<
G extends Game = Game,
P extends Playtypes[G] = Playtypes[G],
I extends IDStrings = IDStrings
>(game: G, songID: integer, playtype: P, difficulty: Difficulties[I]) {
return db.charts[game].findOne({
songID: songID,
playtype: playtype,
difficulty: difficulty,
isPrimary: true,
});
}
/**
* Find chart with Playtype, Difficulty and a given version. This does not necessarily return a chart that has
* `isPrimary` set.
*/
export function FindChartWithPTDFVersion<
G extends Game = Game,
P extends Playtypes[G] = Playtypes[G],
I extends IDStrings = IDStrings
>(
game: G,
songID: integer,
playtype: P,
difficulty: Difficulties[I],
version: GPTSupportedVersions[I]
) {
return db.charts[game].findOne({
songID: songID,
playtype: playtype,
difficulty: difficulty,
versions: version,
});
}
/**
* Finds a DDR Chart based on its "song hash".
* Songs in DDR have a consistent checksum-like identifier used on the e-amusement website.
* We can use this to locate a chart by combining it with a playtype and difficulty.
*
* Despite the potentially confusing name of "songHash", songs are NOT meant to store lookup-like tokens.
* This is just for simplification reasons.
* @param songHash The identifier for the song.
* @param playtype The playtype for the chart.
* @param difficulty The difficulty for the chart.
*/
export function FindDDRChartOnSongHash(
songHash: string,
// Technically both of these should be "ddr" instead of Game, but it proves hard to work with.
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings]
) {
// note: this only works on accident because monk
// allows any strings like "foo.bar".
// We *should* normally cast this to ChartDocument<"ddr:SP" | "ddr:DP">
return db.charts.ddr.findOne({
"data.songHash": songHash,
playtype,
difficulty,
isPrimary: true,
});
}
/**
* Find a BMS chart on either its md5sum or its sha256sum.
* @param hash The md5 or sha256 hash to look for.
*/
export function FindBMSChartOnHash(hash: string) {
return db.charts.bms.findOne({
$or: [{ "data.hashMD5": hash }, { "data.hashSHA256": hash }],
});
}
/**
* Find a chart on its in-game-ID, playtype and difficulty.
*/
export function FindChartOnInGameID(
game: Game,
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings]
) {
if (game === "bms" || game === "usc") {
throw new Error(`Cannot call FindChartOnInGameID for game ${game}.`);
}
return db.charts[game].findOne({
"data.inGameID": inGameID,
playtype,
difficulty,
});
}
/**
* Finds a non-custom chart on its in-game-ID, playtype and difficulty.
* This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate.
*/
export function FindIIDXChartOnInGameID(
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings]
) {
return db.charts.iidx.findOne({
"data.inGameID": inGameID,
"data.2dxtraSet": null,
isPrimary: true,
playtype,
difficulty,
});
}
/**
* Finds a non-custom chart on its in-game-ID, playtype and difficulty.
* This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate.
*/
export function FindIIDXChartOnInGameIDVersion(
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings],
version: GPTSupportedVersions[IDStrings]
) {
return db.charts.iidx.findOne({
"data.inGameID": inGameID,
"data.2dxtraSet": null,
playtype,
difficulty,
versions: version,
});
}
/**
* Find a chart on its in-game-ID, playtype, difficulty and version.
*/
export function FindChartOnInGameIDVersion<I extends IDStrings = IDStrings>(
game: Game,
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[I],
version: GPTSupportedVersions[I]
) {
return db.charts[game].findOne({
"data.inGameID": inGameID,
versions: version,
playtype,
difficulty,
});
}
/**
* Finds an IIDX chart on its 2dxtra hash, which is the sha256 of the .1 buffer.
*/
export function FindIIDXChartWith2DXtraHash(hash: string) {
return db.charts.iidx.findOne({
"data.hashSHA256": hash,
});
}
/**
* Find an SDVX Chart on its in game ID. This exists to handle
* oddities with SDVX difficulties - If "ANY_INF" is sent, it actually
* refers to any of INF, GRV, HVN or VVD. This is because some services treat
* all of those as the same difficulty, but we do not.
*/
export function FindSDVXChartOnInGameID(
inGameID: number,
difficulty: Difficulties["sdvx:Single"] | "ANY_INF"
) {
const diffQuery =
difficulty === "ANY_INF"
? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] }
: difficulty;
return db.charts.sdvx.findOne({
"data.inGameID": inGameID,
difficulty: diffQuery,
isPrimary: true,
});
}
export function FindSDVXChartOnInGameIDVersion(
inGameID: number,
difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF",
version: GPTSupportedVersions["sdvx:Single"]
) {
const diffQuery =
difficulty === "ANY_INF"
? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] }
: difficulty;
return db.charts.sdvx.findOne({
"data.inGameID": inGameID,
difficulty: diffQuery,
versions: version,
});
}
export function FindSDVXChartOnDFVersion(
songID: integer,
difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF",
version: GPTSupportedVersions["sdvx:Single"]
) {
const diffQuery =
difficulty === "ANY_INF"
? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] }
: difficulty;
return db.charts.sdvx.findOne({
songID,
difficulty: diffQuery,
versions: version,
});
}
export function FindChartOnSHA256(game: Game, hash: string) {
if (game !== "bms" && game !== "usc" && game !== "iidx") {
throw new Error(`Cannot call FindChartOnSHA256 for game ${game}.`);
}
return db.charts[game].findOne({
"data.hashSHA256": hash,
});
}
export function FindChartOnARCID(game: "iidx" | "ddr" | "jubeat" | "sdvx", arcID: string) {
return db.charts[game].findOne({
"data.arcChartID": arcID,
});
}
/**
* Returns the N most popular charts for this game + playtype.
* Popularity is determined by how many scores match in the score
* collection.
*/
export async function FindChartsOnPopularity(
game: Game,
playtype: Playtypes[Game],
songIDs?: integer[],
skip = 0,
limit = 100,
scoreCollection: "personal-bests" | "scores" = "personal-bests"
): Promise<(ChartDocument & { __playcount: integer })[]> {
const matchQuery: FilterQuery<ChartDocument> = {
playtype,
};
if (songIDs) {
matchQuery.songID = { $in: songIDs };
}
// MongoDB is a hard beast to wield.
// This code might look very inefficient, but originally this *was*
// a single aggregate pipeline.
//
// We've split it up into multiple queries as this is an order of
// magnitude faster.
// Not entirely sure why, but $lookup is incredibly inefficient,
// and you should just avoid it.
const charts = (await db.charts[game].find(matchQuery)) as unknown as (ChartDocument & {
__playcount: integer;
})[];
const scoreCounts = (await db[scoreCollection].aggregate([
{
$match: { chartID: { $in: charts.map((e) => e.chartID) } },
},
{
$group: {
_id: "$chartID",
count: { $sum: 1 },
},
},
{
$sort: {
count: -1,
},
},
{
$skip: skip,
},
{
$limit: limit,
},
])) as { _id: string; count: integer }[];
const scoreCountMap = new Map();
for (const sc of scoreCounts) {
scoreCountMap.set(sc._id, sc.count);
}
for (const chart of charts) {
chart.__playcount = scoreCountMap.get(chart.chartID) ?? 0;
}
return charts.sort((a, b) => b.__playcount - a.__playcount).slice(skip, skip + limit);
}
import db from "external/mongo/db";
import { FilterQuery } from "mongodb";
import {
ChartDocument,
Difficulties,
Game,
GPTSupportedVersions,
IDStrings,
integer,
Playtypes,
} from "tachi-common";
export function FindChartWithChartID(game: Game, chartID: string) {
return db.charts[game].findOne({ chartID });
}
/**
* Find chart with PlaytypeDifficulty. This only finds charts that have `isPrimary` set to true.
* If you want to find charts that are not primary, you need to use PTDFVersion.
* @see FindChartWithPTDFVersion
*/
export function FindChartWithPTDF<
G extends Game = Game,
P extends Playtypes[G] = Playtypes[G],
I extends IDStrings = IDStrings
>(game: G, songID: integer, playtype: P, difficulty: Difficulties[I]) {
return db.charts[game].findOne({
songID: songID,
playtype: playtype,
difficulty: difficulty,
isPrimary: true,
});
}
/**
* Find chart with Playtype, Difficulty and a given version. This does not necessarily return a chart that has
* `isPrimary` set.
*/
export function FindChartWithPTDFVersion<
G extends Game = Game,
P extends Playtypes[G] = Playtypes[G],
I extends IDStrings = IDStrings
>(
game: G,
songID: integer,
playtype: P,
difficulty: Difficulties[I],
version: GPTSupportedVersions[I]
) {
return db.charts[game].findOne({
songID: songID,
playtype: playtype,
difficulty: difficulty,
versions: version,
});
}
/**
* Finds a DDR Chart based on its "song hash".
* Songs in DDR have a consistent checksum-like identifier used on the e-amusement website.
* We can use this to locate a chart by combining it with a playtype and difficulty.
*
* Despite the potentially confusing name of "songHash", songs are NOT meant to store lookup-like tokens.
* This is just for simplification reasons.
* @param songHash The identifier for the song.
* @param playtype The playtype for the chart.
* @param difficulty The difficulty for the chart.
*/
export function FindDDRChartOnSongHash(
songHash: string,
// Technically both of these should be "ddr" instead of Game, but it proves hard to work with.
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings]
) {
// note: this only works on accident because monk
// allows any strings like "foo.bar".
// We *should* normally cast this to ChartDocument<"ddr:SP" | "ddr:DP">
return db.charts.ddr.findOne({
"data.songHash": songHash,
playtype,
difficulty,
isPrimary: true,
});
}
/**
* Find a BMS chart on either its md5sum or its sha256sum.
* @param hash The md5 or sha256 hash to look for.
*/
export function FindBMSChartOnHash(hash: string) {
return db.charts.bms.findOne({
$or: [{ "data.hashMD5": hash }, { "data.hashSHA256": hash }],
});
}
/**
* Find a chart on its in-game-ID, playtype and difficulty.
*/
export function FindChartOnInGameID(
game: Game,
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings]
) {
if (game === "bms" || game === "usc") {
throw new Error(`Cannot call FindChartOnInGameID for game ${game}.`);
}
return db.charts[game].findOne({
"data.inGameID": inGameID,
playtype,
difficulty,
});
}
/**
* Finds a non-custom chart on its in-game-ID, playtype and difficulty.
* This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate.
*/
export function FindIIDXChartOnInGameID(
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings]
) {
return db.charts.iidx.findOne({
"data.inGameID": inGameID,
"data.2dxtraSet": null,
isPrimary: true,
playtype,
difficulty,
});
}
/**
* Finds a non-custom chart on its in-game-ID, playtype and difficulty.
* This explicitly ignores 2dxtra charts, and is necessary to use for iidx to disambiguate.
*/
export function FindIIDXChartOnInGameIDVersion(
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[IDStrings],
version: GPTSupportedVersions[IDStrings]
) {
return db.charts.iidx.findOne({
"data.inGameID": inGameID,
"data.2dxtraSet": null,
playtype,
difficulty,
versions: version,
});
}
/**
* Find a chart on its in-game-ID, playtype, difficulty and version.
*/
export function FindChartOnInGameIDVersion<I extends IDStrings = IDStrings>(
game: Game,
inGameID: number,
playtype: Playtypes[Game],
difficulty: Difficulties[I],
version: GPTSupportedVersions[I]
) {
return db.charts[game].findOne({
"data.inGameID": inGameID,
versions: version,
playtype,
difficulty,
});
}
/**
* Finds an IIDX chart on its 2dxtra hash, which is the sha256 of the .1 buffer.
*/
export function FindIIDXChartWith2DXtraHash(hash: string) {
return db.charts.iidx.findOne({
"data.hashSHA256": hash,
});
}
/**
* Find an SDVX Chart on its in game ID. This exists to handle
* oddities with SDVX difficulties - If "ANY_INF" is sent, it actually
* refers to any of INF, GRV, HVN or VVD. This is because some services treat
* all of those as the same difficulty, but we do not.
*/
export function FindSDVXChartOnInGameID(
inGameID: number,
difficulty: Difficulties["sdvx:Single"] | "ANY_INF"
) {
const diffQuery =
difficulty === "ANY_INF"
? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] }
: difficulty;
return db.charts.sdvx.findOne({
"data.inGameID": inGameID,
difficulty: diffQuery,
isPrimary: true,
});
}
export function FindSDVXChartOnInGameIDVersion(
inGameID: number,
difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF",
version: GPTSupportedVersions["sdvx:Single"]
) {
const diffQuery =
difficulty === "ANY_INF"
? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] }
: difficulty;
return db.charts.sdvx.findOne({
"data.inGameID": inGameID,
difficulty: diffQuery,
versions: version,
});
}
export function FindSDVXChartOnDFVersion(
songID: integer,
difficulty: "NOV" | "ADV" | "EXH" | "MXM" | "ANY_INF",
version: GPTSupportedVersions["sdvx:Single"]
) {
const diffQuery =
difficulty === "ANY_INF"
? { $in: ["INF", "GRV", "HVN", "VVD"] as Difficulties["sdvx:Single"][] }
: difficulty;
return db.charts.sdvx.findOne({
songID,
difficulty: diffQuery,
versions: version,
});
}
export function FindChartOnSHA256(game: Game, hash: string) {
if (game !== "bms" && game !== "usc" && game !== "iidx") {
throw new Error(`Cannot call FindChartOnSHA256 for game ${game}.`);
}
return db.charts[game].findOne({
"data.hashSHA256": hash,
});
}
export function FindChartOnARCID(game: "iidx" | "ddr" | "jubeat" | "sdvx", arcID: string) {
return db.charts[game].findOne({
"data.arcChartID": arcID,
});
}
/**
* Returns the N most popular charts for this game + playtype.
* Popularity is determined by how many scores match in the score
* collection.
*/
export async function FindChartsOnPopularity(
game: Game,
playtype: Playtypes[Game],
songIDs?: integer[],
skip = 0,
limit = 100,
scoreCollection: "personal-bests" | "scores" = "personal-bests"
): Promise<(ChartDocument & { __playcount: integer })[]> {
const matchQuery: FilterQuery<ChartDocument> = {
playtype,
};
if (songIDs) {
matchQuery.songID = { $in: songIDs };
}
// MongoDB is a hard beast to wield.
// This code might look very inefficient, but originally this *was*
// a single aggregate pipeline.
//
// We've split it up into multiple queries as this is an order of
// magnitude faster.
// Not entirely sure why, but $lookup is incredibly inefficient,
// and you should just avoid it.
const charts = (await db.charts[game].find(matchQuery)) as unknown as (ChartDocument & {
__playcount: integer;
})[];
const scoreCounts = (await db[scoreCollection].aggregate([
{
$match: { chartID: { $in: charts.map((e) => e.chartID) } },
},
{
$group: {
_id: "$chartID",
count: { $sum: 1 },
},
},
{
$sort: {
count: -1,
},
},
{
$skip: skip,
},
{
$limit: limit,
},
])) as { _id: string; count: integer }[];
const scoreCountMap = new Map();
for (const sc of scoreCounts) {
scoreCountMap.set(sc._id, sc.count);
}
for (const chart of charts) {
chart.__playcount = scoreCountMap.get(chart.chartID) ?? 0;
}
return charts.sort((a, b) => b.__playcount - a.__playcount).slice(skip, skip + limit);
}
server/src/utils/queries/songs.ts
+75 -75
View File
@@ -1,75 +1,75 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { InternalFailure } from "lib/score-import/framework/common/converter-failures";
import { FindOneResult } from "monk";
import { Game, integer, SongDocument } from "tachi-common";
import { EscapeStringRegexp } from "../misc";
/**
* Finds a song document for the given game with the given title (or alt-title).
* This is NOT the preferred way to find a song, as encodings, and typos, make this
* rather difficult. Prefer other functions!
* @param game - The game to search upon.
* @param title - The song title to match.
* @returns SongDocument
*/
export function FindSongOnTitle(game: Game, title: string): Promise<FindOneResult<SongDocument>> {
// @optimisable: Performance should be tested here by having a utility field for all-titles.
return db.songs[game].findOne({
$or: [
{
title: title,
},
{
altTitles: title,
},
],
});
}
/**
* Finds a song on a song title case-insensitively.
* This is needed for services that provide horrifically mutated string titles.
*/
export function FindSongOnTitleInsensitive(
game: Game,
title: string
): Promise<FindOneResult<SongDocument>> {
// @optimisable: Performance should be tested here by having a utility field for all-titles.
const regex = new RegExp(`^${EscapeStringRegexp(title)}$`, "iu");
return db.songs[game].findOne({
$or: [
{
title: { $regex: regex },
},
{
altTitles: { $regex: regex },
},
],
});
}
/**
* Finds a song document based on the Tachi songID. Depending on the database this might
* also be the in-game-ID.
* @param game - The game to search upon.
* @param songID - The song ID to match.
* @returns SongDocument
*/
export function FindSongOnID(game: Game, songID: integer): Promise<FindOneResult<SongDocument>> {
return db.songs[game].findOne({
id: songID,
});
}
export async function FindSongOnIDGuaranteed(game: Game, songID: integer, logger: KtLogger) {
const song = await FindSongOnID(game, songID);
if (!song) {
logger.severe(`Song-Chart desync for ${songID}. Has charts, but no song.`);
throw new InternalFailure(`Song-Chart desync for ${songID}. Has charts, but no song.`);
}
return song;
}
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { InternalFailure } from "lib/score-import/framework/common/converter-failures";
import { FindOneResult } from "monk";
import { Game, integer, SongDocument } from "tachi-common";
import { EscapeStringRegexp } from "../misc";
/**
* Finds a song document for the given game with the given title (or alt-title).
* This is NOT the preferred way to find a song, as encodings, and typos, make this
* rather difficult. Prefer other functions!
* @param game - The game to search upon.
* @param title - The song title to match.
* @returns SongDocument
*/
export function FindSongOnTitle(game: Game, title: string): Promise<FindOneResult<SongDocument>> {
// @optimisable: Performance should be tested here by having a utility field for all-titles.
return db.songs[game].findOne({
$or: [
{
title: title,
},
{
altTitles: title,
},
],
});
}
/**
* Finds a song on a song title case-insensitively.
* This is needed for services that provide horrifically mutated string titles.
*/
export function FindSongOnTitleInsensitive(
game: Game,
title: string
): Promise<FindOneResult<SongDocument>> {
// @optimisable: Performance should be tested here by having a utility field for all-titles.
const regex = new RegExp(`^${EscapeStringRegexp(title)}$`, "iu");
return db.songs[game].findOne({
$or: [
{
title: { $regex: regex },
},
{
altTitles: { $regex: regex },
},
],
});
}
/**
* Finds a song document based on the Tachi songID. Depending on the database this might
* also be the in-game-ID.
* @param game - The game to search upon.
* @param songID - The song ID to match.
* @returns SongDocument
*/
export function FindSongOnID(game: Game, songID: integer): Promise<FindOneResult<SongDocument>> {
return db.songs[game].findOne({
id: songID,
});
}
export async function FindSongOnIDGuaranteed(game: Game, songID: integer, logger: KtLogger) {
const song = await FindSongOnID(game, songID);
if (!song) {
logger.severe(`Song-Chart desync for ${songID}. Has charts, but no song.`);
throw new InternalFailure(`Song-Chart desync for ${songID}. Has charts, but no song.`);
}
return song;
}
server/src/utils/types.ts
+67 -67
View File
@@ -1,67 +1,67 @@
import {
FolderDocument,
TableDocument,
SessionDocument,
ScoreDocument,
ChartDocument,
PublicUserDocument,
UserGameStats,
Game,
Playtypes,
SongDocument,
UserSettings,
TachiAPIClientDocument,
} from "tachi-common";
declare module "express-session" {
// Inject additional properties on express-session
interface SessionData {
tachi: TachiSessionData;
}
}
export interface TachiSessionData {
user: PublicUserDocument;
settings: UserSettings;
}
export interface TachiAPIFailResponse {
success: false;
description: string;
}
export interface TachiAPISuccessResponse {
success: true;
description: string;
body: Record<string, unknown>;
}
export type TachiAPIReponse = TachiAPIFailResponse | TachiAPISuccessResponse;
/**
* Clarity type for empty objects - such as in context.
*/
export type EmptyObject = Record<string, never>;
/**
* Data that may be monkey-patched onto req.tachi. This holds things such as middleware results.
*/
export interface TachiRequestData {
uscChartDoc?: ChartDocument<"usc:Controller" | "usc:Keyboard">;
beatorajaChartDoc?: ChartDocument<"bms:7K" | "bms:14K">;
requestedUser?: PublicUserDocument;
requestedUserGameStats?: UserGameStats;
game?: Game;
playtype?: Playtypes[Game];
chartDoc?: ChartDocument;
songDoc?: SongDocument;
scoreDoc?: ScoreDocument;
sessionDoc?: SessionDocument;
tableDoc?: TableDocument;
folderDoc?: FolderDocument;
apiClientDoc: Omit<TachiAPIClientDocument, "clientSecret">;
}
import {
FolderDocument,
TableDocument,
SessionDocument,
ScoreDocument,
ChartDocument,
PublicUserDocument,
UserGameStats,
Game,
Playtypes,
SongDocument,
UserSettings,
TachiAPIClientDocument,
} from "tachi-common";
declare module "express-session" {
// Inject additional properties on express-session
interface SessionData {
tachi: TachiSessionData;
}
}
export interface TachiSessionData {
user: PublicUserDocument;
settings: UserSettings;
}
export interface TachiAPIFailResponse {
success: false;
description: string;
}
export interface TachiAPISuccessResponse {
success: true;
description: string;
body: Record<string, unknown>;
}
export type TachiAPIReponse = TachiAPIFailResponse | TachiAPISuccessResponse;
/**
* Clarity type for empty objects - such as in context.
*/
export type EmptyObject = Record<string, never>;
/**
* Data that may be monkey-patched onto req.tachi. This holds things such as middleware results.
*/
export interface TachiRequestData {
uscChartDoc?: ChartDocument<"usc:Controller" | "usc:Keyboard">;
beatorajaChartDoc?: ChartDocument<"bms:7K" | "bms:14K">;
requestedUser?: PublicUserDocument;
requestedUserGameStats?: UserGameStats;
game?: Game;
playtype?: Playtypes[Game];
chartDoc?: ChartDocument;
songDoc?: SongDocument;
scoreDoc?: ScoreDocument;
sessionDoc?: SessionDocument;
tableDoc?: TableDocument;
folderDoc?: FolderDocument;
apiClientDoc: Omit<TachiAPIClientDocument, "clientSecret">;
}
server/src/utils/user.test.ts
+59 -59
View File
@@ -1,59 +1,59 @@
import t from "tap";
import { DatabaseSchemas } from "external/mongo/schemas";
import { prAssert } from "test-utils/asserts";
import { GetUserCaseInsensitive, FormatUserDoc } from "./user";
import { PublicUserDocument } from "tachi-common";
import ResetDBState from "test-utils/resets";
t.test("#GetUserCaseInsensitive", (t) => {
t.beforeEach(ResetDBState);
t.test("Should return the user for an exact username", async (t) => {
const result = await GetUserCaseInsensitive("test_zkldi");
t.not(result, null, "Should not return null");
t.equal(result!.username, "test_zkldi", "Should return test_zkldi");
t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument");
// @ts-expect-error yeah
t.equal(result.password, undefined, "Should not return password");
// @ts-expect-error yeah
t.equal(result.email, undefined, "Should not return email");
});
t.test("Should return the user for an incorrectly cased username", async (t) => {
const result = await GetUserCaseInsensitive("tesT_ZkLdi");
t.not(result, null, "Should not return null");
t.equal(result!.username, "test_zkldi", "Should return test_zkldi");
t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument");
// @ts-expect-error yeah
t.equal(result.password, undefined, "Should not return password");
// @ts-expect-error yeah
t.equal(result.email, undefined, "Should not return email");
});
t.test("Should not return the user for a username that does not exist", async (t) => {
const result = await GetUserCaseInsensitive("foobar");
t.equal(result, null, "Should return null");
});
t.end();
});
t.test("#FormatUserDoc", (t) => {
t.equal(
FormatUserDoc({ username: "zkldi", id: 123 } as PublicUserDocument),
"zkldi (#123)",
"Should format a user document into username #id format."
);
t.end();
});
import t from "tap";
import { DatabaseSchemas } from "external/mongo/schemas";
import { prAssert } from "test-utils/asserts";
import { GetUserCaseInsensitive, FormatUserDoc } from "./user";
import { PublicUserDocument } from "tachi-common";
import ResetDBState from "test-utils/resets";
t.test("#GetUserCaseInsensitive", (t) => {
t.beforeEach(ResetDBState);
t.test("Should return the user for an exact username", async (t) => {
const result = await GetUserCaseInsensitive("test_zkldi");
t.not(result, null, "Should not return null");
t.equal(result!.username, "test_zkldi", "Should return test_zkldi");
t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument");
// @ts-expect-error yeah
t.equal(result.password, undefined, "Should not return password");
// @ts-expect-error yeah
t.equal(result.email, undefined, "Should not return email");
});
t.test("Should return the user for an incorrectly cased username", async (t) => {
const result = await GetUserCaseInsensitive("tesT_ZkLdi");
t.not(result, null, "Should not return null");
t.equal(result!.username, "test_zkldi", "Should return test_zkldi");
t.ok(DatabaseSchemas.users(result), "Should return a conforming PublicUserDocument");
// @ts-expect-error yeah
t.equal(result.password, undefined, "Should not return password");
// @ts-expect-error yeah
t.equal(result.email, undefined, "Should not return email");
});
t.test("Should not return the user for a username that does not exist", async (t) => {
const result = await GetUserCaseInsensitive("foobar");
t.equal(result, null, "Should return null");
});
t.end();
});
t.test("#FormatUserDoc", (t) => {
t.equal(
FormatUserDoc({ username: "zkldi", id: 123 } as PublicUserDocument),
"zkldi (#123)",
"Should format a user document into username #id format."
);
t.end();
});
server/src/utils/user.ts
+224 -224
View File
@@ -1,224 +1,224 @@
import {
integer,
PublicUserDocument,
UserGameStats,
GetGamePTConfig,
Game,
Playtypes,
UGSRatingsLookup,
IDStrings,
} from "tachi-common";
import { FindOneResult } from "monk";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
/**
* Returns a user's username from their ID. Throws if no user with that ID exists.
*/
export async function GetUsernameFromUserID(userID: integer): Promise<string> {
const partialDoc = await db.users.findOne(
{
id: userID,
},
{
projection: {
username: 1,
},
}
);
if (!partialDoc) {
throw new Error(`Could not find username for userID ${userID}.`);
}
return partialDoc.username;
}
/**
* Gets a user based on their username case-insensitively.
*/
export function GetUserCaseInsensitive(
username: string
): Promise<FindOneResult<PublicUserDocument>> {
return db.users.findOne({
usernameLowercase: username.toLowerCase(),
}) as Promise<FindOneResult<PublicUserDocument>>;
}
export async function CheckIfEmailInUse(email: string) {
const doc = await db["user-private-information"].findOne({ email });
return !!doc;
}
export function GetUserPrivateInfo(userID: integer) {
return db["user-private-information"].findOne({ userID });
}
/**
* Gets a user from their userID.
*/
export function GetUserWithID(userID: integer): Promise<FindOneResult<PublicUserDocument>> {
return db.users.findOne({
id: userID,
}) as Promise<FindOneResult<PublicUserDocument>>;
}
export function GetSettingsForUser(userID: integer) {
return db["user-settings"].findOne({
userID: userID,
});
}
/**
* Gets the users for these user IDs.
*/
export function GetUsersWithIDs(userIDs: integer[]) {
return db.users.find({
id: { $in: userIDs },
});
}
/**
* Retrieve a user document that is expected to exist.
* If the user document is not found, a severe error is logged, and this
* function throws.
*/
export async function GetUserWithIDGuaranteed(userID: integer): Promise<PublicUserDocument> {
const userDoc = await GetUserWithID(userID);
if (!userDoc) {
logger.severe(
`User ${userID} does not have an associated user document, but one was expected.`
);
throw new Error(
`User ${userID} does not have an associated user document, but one was expected.`
);
}
return userDoc;
}
/**
* Gets a user based on either their username case-insensitively, or a direct lookup of their ID.
* This is used in URLs to resolve the passed user.
*/
export function ResolveUser(usernameOrID: string) {
// user ID passed
if (usernameOrID.match(/^[0-9]+$/u)) {
const intID = Number(usernameOrID);
return GetUserWithID(intID);
}
return GetUserCaseInsensitive(usernameOrID);
}
/**
* Returns a formatted string indicating the user. This is used for logging.
*/
export function FormatUserDoc(userdoc: PublicUserDocument) {
return `${userdoc.username} (#${userdoc.id})`;
}
export async function GetUsersRanking(stats: UserGameStats) {
const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
const aggRes = await db["game-stats"].aggregate([
{
$match: {
game: stats.game,
playtype: stats.playtype,
},
},
{
$group: {
_id: null,
ranking: {
$sum: {
$cond: {
if: {
$gt: [
`$ratings.${gptConfig.defaultProfileRatingAlg}`,
stats.ratings[gptConfig.defaultProfileRatingAlg],
],
},
then: 1,
else: 0,
},
},
},
},
},
]);
return (aggRes[0].ranking + 1) as integer;
}
export function GetUGPTPlaycount(userID: integer, game: Game, playtype: Playtypes[Game]) {
return db.scores.count({ userID, game, playtype });
}
export async function GetAllRankings(stats: UserGameStats) {
const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
const entries = await Promise.all(
gptConfig.profileRatingAlgs.map((k) =>
GetUsersRankingAndOutOf(stats, k).then((r) => [k, r])
)
);
return Object.fromEntries(entries);
}
export async function GetUsersRankingAndOutOf(
stats: UserGameStats,
alg?: UGSRatingsLookup[IDStrings]
) {
const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
const ratingAlg = alg ?? gptConfig.defaultProfileRatingAlg;
const aggRes = await db["game-stats"].aggregate([
{
$match: {
game: stats.game,
playtype: stats.playtype,
},
},
{
$group: {
_id: null,
outOf: { $sum: 1 },
ranking: {
$sum: {
$cond: {
if: {
$gt: [`$ratings.${ratingAlg}`, stats.ratings[ratingAlg]],
},
then: 1,
else: 0,
},
},
},
},
},
]);
return {
ranking: (aggRes[0].ranking + 1) as integer,
outOf: aggRes[0].outOf as integer,
};
}
const FIVE_MINUTES = 1000 * 60 * 5;
/**
* Returns the cutoff point for "being online" in tachi. This means the user
* has made any page request in the past 5 minutes.
*/
export function GetOnlineCutoff() {
return Date.now() - FIVE_MINUTES;
}
import {
integer,
PublicUserDocument,
UserGameStats,
GetGamePTConfig,
Game,
Playtypes,
UGSRatingsLookup,
IDStrings,
} from "tachi-common";
import { FindOneResult } from "monk";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
/**
* Returns a user's username from their ID. Throws if no user with that ID exists.
*/
export async function GetUsernameFromUserID(userID: integer): Promise<string> {
const partialDoc = await db.users.findOne(
{
id: userID,
},
{
projection: {
username: 1,
},
}
);
if (!partialDoc) {
throw new Error(`Could not find username for userID ${userID}.`);
}
return partialDoc.username;
}
/**
* Gets a user based on their username case-insensitively.
*/
export function GetUserCaseInsensitive(
username: string
): Promise<FindOneResult<PublicUserDocument>> {
return db.users.findOne({
usernameLowercase: username.toLowerCase(),
}) as Promise<FindOneResult<PublicUserDocument>>;
}
export async function CheckIfEmailInUse(email: string) {
const doc = await db["user-private-information"].findOne({ email });
return !!doc;
}
export function GetUserPrivateInfo(userID: integer) {
return db["user-private-information"].findOne({ userID });
}
/**
* Gets a user from their userID.
*/
export function GetUserWithID(userID: integer): Promise<FindOneResult<PublicUserDocument>> {
return db.users.findOne({
id: userID,
}) as Promise<FindOneResult<PublicUserDocument>>;
}
export function GetSettingsForUser(userID: integer) {
return db["user-settings"].findOne({
userID: userID,
});
}
/**
* Gets the users for these user IDs.
*/
export function GetUsersWithIDs(userIDs: integer[]) {
return db.users.find({
id: { $in: userIDs },
});
}
/**
* Retrieve a user document that is expected to exist.
* If the user document is not found, a severe error is logged, and this
* function throws.
*/
export async function GetUserWithIDGuaranteed(userID: integer): Promise<PublicUserDocument> {
const userDoc = await GetUserWithID(userID);
if (!userDoc) {
logger.severe(
`User ${userID} does not have an associated user document, but one was expected.`
);
throw new Error(
`User ${userID} does not have an associated user document, but one was expected.`
);
}
return userDoc;
}
/**
* Gets a user based on either their username case-insensitively, or a direct lookup of their ID.
* This is used in URLs to resolve the passed user.
*/
export function ResolveUser(usernameOrID: string) {
// user ID passed
if (usernameOrID.match(/^[0-9]+$/u)) {
const intID = Number(usernameOrID);
return GetUserWithID(intID);
}
return GetUserCaseInsensitive(usernameOrID);
}
/**
* Returns a formatted string indicating the user. This is used for logging.
*/
export function FormatUserDoc(userdoc: PublicUserDocument) {
return `${userdoc.username} (#${userdoc.id})`;
}
export async function GetUsersRanking(stats: UserGameStats) {
const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
const aggRes = await db["game-stats"].aggregate([
{
$match: {
game: stats.game,
playtype: stats.playtype,
},
},
{
$group: {
_id: null,
ranking: {
$sum: {
$cond: {
if: {
$gt: [
`$ratings.${gptConfig.defaultProfileRatingAlg}`,
stats.ratings[gptConfig.defaultProfileRatingAlg],
],
},
then: 1,
else: 0,
},
},
},
},
},
]);
return (aggRes[0].ranking + 1) as integer;
}
export function GetUGPTPlaycount(userID: integer, game: Game, playtype: Playtypes[Game]) {
return db.scores.count({ userID, game, playtype });
}
export async function GetAllRankings(stats: UserGameStats) {
const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
const entries = await Promise.all(
gptConfig.profileRatingAlgs.map((k) =>
GetUsersRankingAndOutOf(stats, k).then((r) => [k, r])
)
);
return Object.fromEntries(entries);
}
export async function GetUsersRankingAndOutOf(
stats: UserGameStats,
alg?: UGSRatingsLookup[IDStrings]
) {
const gptConfig = GetGamePTConfig(stats.game, stats.playtype);
const ratingAlg = alg ?? gptConfig.defaultProfileRatingAlg;
const aggRes = await db["game-stats"].aggregate([
{
$match: {
game: stats.game,
playtype: stats.playtype,
},
},
{
$group: {
_id: null,
outOf: { $sum: 1 },
ranking: {
$sum: {
$cond: {
if: {
$gt: [`$ratings.${ratingAlg}`, stats.ratings[ratingAlg]],
},
then: 1,
else: 0,
},
},
},
},
},
]);
return {
ranking: (aggRes[0].ranking + 1) as integer,
outOf: aggRes[0].outOf as integer,
};
}
const FIVE_MINUTES = 1000 * 60 * 5;
/**
* Returns the cutoff point for "being online" in tachi. This means the user
* has made any page request in the past 5 minutes.
*/
export function GetOnlineCutoff() {
return Date.now() - FIVE_MINUTES;
}