experimental: initial autofix blast

This commit is contained in:
zkldi
2022-05-06 03:39:03 +01:00
parent 61ea7a4bf7
commit 0adc39b5cb
275 changed files with 2207 additions and 1877 deletions
+9 -9
View File
@@ -1,12 +1,11 @@
import { Session, SessionData } from "express-session";
import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "../../src/lib/constants/tachi";
import { TachiRequestData, TachiSessionData } from "../../src/utils/types";
import { APITokenDocument } from "tachi-common";
export {};
import type { SYMBOL_TACHI_API_AUTH, SYMBOL_TACHI_DATA } from "../../src/lib/constants/tachi";
import type { TachiRequestData, TachiSessionData } from "../../src/utils/types";
import type { Session, SessionData } from "express-session";
import type { APITokenDocument } from "tachi-common";
// this only exists for testing context - it doesn't seem to work properly otherwise.
declare module "express-session" {
// Inject additional properties on express-session
interface SessionData {
tachi: TachiSessionData;
@@ -16,12 +15,13 @@ declare module "express-session" {
declare global {
namespace Express {
interface Request {
session: Session & Partial<SessionData>;
session: Partial<SessionData> & Session;
[SYMBOL_TACHI_DATA]?: Partial<TachiRequestData>;
[SYMBOL_TachiData]?: Partial<TachiRequestData>;
// even though this is technically *not* present on every request
// it's always assigned in the main router, so its functionally equivalent.
[SYMBOL_TachiAPIAuth]: APITokenDocument;
[SYMBOL_TACHI_API_AUTH]: APITokenDocument;
}
}
}
+10
View File
@@ -106,6 +106,7 @@ export const adjectives = [
"Biodegradable",
"Bite-sized",
"Bitter",
// "Black", - Could result in racism.
"Black-and-white",
"Bland",
@@ -135,6 +136,7 @@ export const adjectives = [
"Brisk",
"Broken",
"Bronze",
// "Brown", - Could result in racism
"Bruised",
"Bubbly",
@@ -702,6 +704,7 @@ export const adjectives = [
"Magnificent",
"Majestic",
"Major",
// "Male", - try and avoid sexism
"Mammoth",
"Married",
@@ -1308,6 +1311,7 @@ export const adjectives = [
"Whimsical",
"Whirlwind",
"Whispered",
// "White", - Could result in racism
"Whole",
"Whopping",
@@ -2027,6 +2031,7 @@ export const nouns = [
"Share",
"Balance",
"Bit",
// "Black", I have visions of this one resulting in racism
"Bottom", // This one gets to stay because it's hilarious
"Choice",
@@ -2136,6 +2141,7 @@ export const nouns = [
"Park",
"Plenty",
"Quarter",
// "Skin", - Could possibly be racist.
"Sort",
"Weight",
@@ -2236,6 +2242,7 @@ export const nouns = [
"Pleasure",
"Score",
"Screw",
// "Sex", - Absolutely not, Thanks for this being in a SFW wordlist...
"Shop",
"Shower",
@@ -2333,6 +2340,7 @@ export const nouns = [
"Aside",
"Associate",
"Bet",
// "Blow", - maybe not.
"Border",
"Branch",
@@ -2609,6 +2617,7 @@ export const nouns = [
"Report",
"Rise",
"Walk",
// "White", - Could result in racism.
"Beyond",
"Junior",
@@ -2830,6 +2839,7 @@ export const nouns = [
"Dear",
"Delay",
"Drunk",
// "Female", try and avoid potential sexism
"Hurry",
"Inevitable",
+9 -7
View File
@@ -1,9 +1,11 @@
import { ONE_MINUTE } from "lib/constants/time";
import CreateLogCtx from "lib/logger/logger";
import { OrphanScoreDocument } from "lib/score-import/import-types/common/types";
import { Environment, ServerConfig } from "lib/setup/config";
import monk, { TMiddleware } from "monk";
import {
import monk from "monk";
import { GetMillisecondsSince } from "utils/misc";
import type { OrphanScoreDocument } from "lib/score-import/import-types/common/types";
import type { TMiddleware } from "monk";
import type {
APITokenDocument,
ARCSavedProfileDocument,
BMSCourseDocument,
@@ -40,8 +42,8 @@ import {
UserGameStatsSnapshot,
UserSettings,
} from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
import { MigrationDocument, PrivateUserInfoDocument } from "utils/types";
import type { MigrationDocument, PrivateUserInfoDocument } from "utils/types";
const logger = CreateLogCtx(__filename);
let dbName = ServerConfig.MONGO_DATABASE_NAME;
@@ -197,8 +199,8 @@ const db = {
notifications: monkDB.get<NotificationDocument>("notifications"),
};
export type StaticDatabases = Exclude<keyof typeof db, "songs" | "charts">;
export type StaticDatabases = Exclude<keyof typeof db, "charts" | "songs">;
export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`;
export type Databases = StaticDatabases | `charts-${Game}` | `songs-${Game}`;
export default db;
+9 -5
View File
@@ -1,11 +1,13 @@
/* eslint-disable no-await-in-loop */
import db, { monkDB } from "./db";
import { ONE_DAY } from "lib/constants/time";
import CreateLogCtx from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
import { IndexOptions } from "mongodb";
import monk, { IMonkManager } from "monk";
import monk from "monk";
import { Random20Hex } from "utils/misc";
import db, { Databases, monkDB } from "./db";
import type { Databases } from "./db";
import type { IndexOptions } from "mongodb";
import type { IMonkManager } from "monk";
const logger = CreateLogCtx(__filename);
@@ -20,7 +22,7 @@ function index(fields: Record<string, unknown>, options?: IndexOptions) {
const UNIQUE = { unique: true };
const staticIndexes: Partial<Record<Databases, Index[]>> = {
const staticIndexes: Partial<Record<Databases, Array<Index>>> = {
scores: [
index({ scoreID: 1 }, UNIQUE),
index({ chartID: 1, userID: 1 }),
@@ -33,6 +35,7 @@ const staticIndexes: Partial<Record<Databases, Index[]>> = {
],
sessions: [
index({ userID: 1, game: 1, playtype: 1, timeStarted: 1, timeEnded: 1 }),
// Optimises score modification, since sessions need to be repointed.
// also, just generally useful.
index({ "scoreInfo.scoreID": 1 }),
@@ -122,7 +125,7 @@ const staticIndexes: Partial<Record<Databases, Index[]>> = {
notifications: [index({ notifID: 1 }, UNIQUE), index({ sentTo: 1, sentAt: 1 })],
};
const indexes: Partial<Record<Databases, Index[]>> = staticIndexes;
const indexes: Partial<Record<Databases, Array<Index>>> = staticIndexes;
for (const game of TachiConfig.GAMES) {
if (indexes[`charts-${game}` as Databases]) {
@@ -166,6 +169,7 @@ export async function SetIndexesWithDB(db: IMonkManager, reset: boolean) {
// this creates a collection, i cant find the createCollection
// call.
const tmp = Random20Hex();
await db.get(collection).insert({ __tmp: tmp });
await db.get(collection).remove({ __tmp: tmp });
}
+6 -3
View File
@@ -1,7 +1,9 @@
import { TachiConfig } from "lib/setup/config";
import p, { PrudenceSchema } from "prudence";
import { SCHEMAS, SchemaValidatorFunction } from "tachi-common/js/lib/schemas";
import { Databases } from "./db";
import p from "prudence";
import { SCHEMAS } from "tachi-common/js/lib/schemas";
import type { Databases } from "./db";
import type { PrudenceSchema } from "prudence";
import type { SchemaValidatorFunction } from "tachi-common/js/lib/schemas";
// The idea of this file is to export a function for every database
// that will validate that collection.
@@ -67,6 +69,7 @@ export const DatabaseSchemas: Record<Databases, SchemaValidatorFunction> = {
orphanID: "string",
timeInserted: p.isPositive,
errMsg: "?string",
// @todo Type these properly? This will take a lot of effort, as it will require
// formal types for all parser data:context duos.
data: p.any,
+1 -1
View File
@@ -1,5 +1,5 @@
import CreateLogCtx from "lib/logger/logger";
import db from "./db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
+3 -3
View File
@@ -15,14 +15,14 @@ const logger = CreateLogCtx(__filename);
// Stub types. In the future, we may actually use this.
export type RedisIPCChannels = "";
export type RedisIPCData = {
export interface RedisIPCData {
"": unknown;
};
}
type RedisSubCallback<T extends RedisIPCChannels> = (data: RedisIPCData[T]) => void;
type SubCallbacks = {
[K in RedisIPCChannels]: RedisSubCallback<K>[];
[K in RedisIPCChannels]: Array<RedisSubCallback<K>>;
};
const SubCallbacks: Partial<SubCallbacks> = {};
@@ -4,14 +4,15 @@ import { DatabaseSchemas } from "external/mongo/schemas";
import fjsh from "fast-json-stable-hash";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig, TachiConfig } from "lib/setup/config";
import { PrudenceError } from "prudence";
import { TachiAPIClientDocument, UserAuthLevels } from "tachi-common";
import { UserAuthLevels } from "tachi-common";
import { Random20Hex } from "utils/misc";
import { FormatPrError } from "utils/prudence";
import type { PrudenceError } from "prudence";
import type { TachiAPIClientDocument } from "tachi-common";
const logger = CreateLogCtx(__filename);
type DefaultClients = Omit<TachiAPIClientDocument, "clientSecret" | "author">[];
type DefaultClients = Array<Omit<TachiAPIClientDocument, "author" | "clientSecret">>;
// Defines some Tachi API Clients that should come default with a Tachi
// environment.
+5 -5
View File
@@ -1,11 +1,11 @@
import { Response } from "express";
import fs from "fs";
import { DeleteFromS3, PushToS3 } from "./s3";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import mkdirp from "mkdirp";
import fs from "fs";
import path from "path";
import { promisify } from "util";
import { DeleteFromS3, PushToS3 } from "./s3";
import type { Response } from "express";
const readFilePromise = promisify(fs.readFile);
const writeFilePromise = promisify(fs.writeFile);
@@ -50,13 +50,13 @@ export function CDNRetrieve(fileLoc: string) {
* Redirects the response to the CDN server at the given path.
*/
export function CDNRedirect(res: Response, fileLoc: string) {
if (fileLoc[0] !== "/") {
if (!fileLoc.startsWith("/")) {
throw new Error(`Invalid fileLoc - did not start with /.`);
}
logger.debug(`CDN Redirecting to ${ServerConfig.CDN_CONFIG.WEB_LOCATION}${fileLoc}.`);
return res.redirect(`${ServerConfig.CDN_CONFIG.WEB_LOCATION}${fileLoc}`);
res.redirect(`${ServerConfig.CDN_CONFIG.WEB_LOCATION}${fileLoc}`);
}
/**
+2 -1
View File
@@ -4,10 +4,11 @@ import { ServerConfig } from "lib/setup/config";
const logger = CreateLogCtx("S3 Client");
let s3: null | S3Client = null;
let s3: S3Client | null = null;
if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "S3_BUCKET") {
const data = ServerConfig.CDN_CONFIG.SAVE_LOCATION;
logger.info(`Using S3_BUCKET as CDN location.`, { bootInfo: true });
s3 = new S3Client({
endpoint: data.ENDPOINT,
+1 -1
View File
@@ -1,4 +1,4 @@
import { integer } from "tachi-common";
import type { integer } from "tachi-common";
export function GetUSCIRReplayURL(scoreID: string) {
return `/uscir/replays/${scoreID}`;
+2 -2
View File
@@ -1,3 +1,3 @@
export const SYMBOL_TachiData = Symbol("tachi-request-patch");
export const SYMBOL_TachiAPIAuth = Symbol("tachi-api-auth-info");
export const SYMBOL_TACHI_DATA = Symbol("tachi-request-patch");
export const SYMBOL_TACHI_API_AUTH = Symbol("tachi-api-auth-info");
export const JOB_RETRY_COUNT = 7;
+16 -13
View File
@@ -1,26 +1,26 @@
import fs from "fs/promises";
import CreateLogCtx from "lib/logger/logger";
import { Environment, ServerConfig } from "lib/setup/config";
import { asyncExec } from "utils/misc";
import fs from "fs/promises";
import os from "os";
import path from "path";
import { Game } from "tachi-common";
import { asyncExec } from "utils/misc";
import type { Game } from "tachi-common";
const logger = CreateLogCtx(__filename);
export type SeedsCollections =
| `charts-${Game}`
| `songs-${Game}`
| "bms-course-lookup"
| "folders"
| "tables";
| "tables"
| `charts-${Game}`
| `songs-${Game}`;
/**
* Class that encapsulates the behaviour of a seeds repo.
*/
export class DatabaseSeedsRepo {
private baseDir: string;
private logger;
private readonly baseDir: string;
private readonly logger;
constructor(baseDir: string) {
this.baseDir = baseDir;
@@ -36,12 +36,12 @@ export class DatabaseSeedsRepo {
*
* @returns The data in the requested collection.
*/
async ReadCollection<D>(collectionName: SeedsCollections): Promise<D[]> {
async ReadCollection<D>(collectionName: SeedsCollections): Promise<Array<D>> {
const data = await fs.readFile(this.CollectionNameToPath(collectionName), {
encoding: "utf-8",
});
const parsedData = JSON.parse(data) as D[];
const parsedData = JSON.parse(data) as Array<D>;
return parsedData;
}
@@ -52,7 +52,7 @@ export class DatabaseSeedsRepo {
* @param collectionName - The collection to write to.
* @param content - A new array of objects to write.
*/
async WriteCollection(collectionName: SeedsCollections, content: unknown[]) {
async WriteCollection(collectionName: SeedsCollections, content: Array<unknown>) {
await fs.writeFile(this.CollectionNameToPath(collectionName), JSON.stringify(content));
// Deterministically sort whatever content we just wrote.
@@ -64,7 +64,7 @@ export class DatabaseSeedsRepo {
async *IterateCollections() {
const collectionNames = (await fs.readdir(path.join(this.baseDir, "collections"))).map(
(e) => path.parse(e).name
) as SeedsCollections[];
) as Array<SeedsCollections>;
for (const collectionName of collectionNames) {
// eslint-disable-next-line no-await-in-loop
@@ -78,7 +78,10 @@ export class DatabaseSeedsRepo {
* @param collectionName - The collection to mutate.
* @param mutator - A function that takes the entire collection as an array, then returns a new array.
*/
async MutateCollection<D>(collectionName: SeedsCollections, mutator: (dataset: D[]) => D[]) {
async MutateCollection<D>(
collectionName: SeedsCollections,
mutator: (dataset: Array<D>) => Array<D>
) {
const dataset = await this.ReadCollection<D>(collectionName);
const newData = mutator(dataset);
+9 -12
View File
@@ -1,7 +1,8 @@
import bunyan from "bunyan";
import CreateLogCtx from "lib/logger/logger";
import { Environment, ServerConfig } from "lib/setup/config";
import nodemailer, { SentMessageInfo, Transporter } from "nodemailer";
import nodemailer from "nodemailer";
import type { SentMessageInfo, Transporter } from "nodemailer";
const logger = CreateLogCtx(__filename);
@@ -12,17 +13,13 @@ if (ServerConfig.EMAIL_CONFIG) {
const conf = ServerConfig.EMAIL_CONFIG;
try {
transporter = nodemailer.createTransport(
Object.assign(
{
newline: "unix",
logger: conf.TRANSPORT_OPS?.debug
? bunyan.createLogger({ name: "Email Logger" })
: undefined,
},
conf.TRANSPORT_OPS ?? {}
)
);
transporter = nodemailer.createTransport({
newline: "unix",
logger: conf.TRANSPORT_OPS?.debug
? bunyan.createLogger({ name: "Email Logger" })
: undefined,
...(conf.TRANSPORT_OPS ?? {}),
});
transporter.verify((err) => {
if (err) {
@@ -1,6 +1,6 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { Game, integer, Playtype } from "tachi-common";
import type { Game, integer, Playtype } from "tachi-common";
const logger = CreateLogCtx(__filename);
+2 -2
View File
@@ -1,9 +1,9 @@
import { monkDB } from "external/mongo/db";
import { CloseRedisConnection } from "external/redis/redis";
import http from "http";
import https from "https";
import { rootLogger } from "lib/logger/logger";
import { CloseScoreImportQueue } from "lib/score-import/worker/queue";
import type http from "http";
import type https from "https";
const logger = rootLogger;
+1 -1
View File
@@ -1,7 +1,7 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { DeleteMultipleScores } from "lib/score-mutation/delete-scores";
import { ImportDocument } from "tachi-common";
import type { ImportDocument } from "tachi-common";
const logger = CreateLogCtx(__filename);
+2 -2
View File
@@ -1,7 +1,7 @@
import { ONE_MONTH } from "lib/constants/time";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import { PublicUserDocument } from "tachi-common";
import type { PublicUserDocument } from "tachi-common";
const logger = CreateLogCtx(__filename);
@@ -27,7 +27,7 @@ export function GetTotalAllowedInvites(user: PublicUserDocument) {
let invites = monthsSinceJoin * ServerConfig.INVITE_CODE_CONFIG.BATCH_SIZE;
if (user.badges.includes("alpha") || user.badges.includes("beta")) {
invites += ServerConfig.INVITE_CODE_CONFIG.BETA_USER_BONUS;
invites = invites + ServerConfig.INVITE_CODE_CONFIG.BETA_USER_BONUS;
}
if (invites > ServerConfig.INVITE_CODE_CONFIG.INVITE_CAP) {
@@ -14,6 +14,7 @@ export async function BacksyncBMSPMSSongsAndCharts() {
for (const game of ["bms", "pms"] as const) {
logger.info(`Fetching BMS songs from DB.`);
// did you know, this code is liable to blow up in my face and OOM one day?
let songs = await db.songs[game].find({});
+14 -11
View File
@@ -1,9 +1,17 @@
import db from "external/mongo/db";
import { BMS_TABLES } from "lib/constants/bms-tables";
/* eslint-disable no-await-in-loop */
import CreateLogCtx from "lib/logger/logger";
import fetch from "node-fetch";
import { CreateFolderID } from "utils/folder";
import { FormatBMSTables } from "utils/misc";
import type { ChartDocument } from "tachi-common";
// I have no confidence in half of these links surviving.
// Eventually, some of these are going to disappear, and we'll have to find
// something else. probably.
const registeredTables: BMSTablesDataset[] = [
const registeredTables: Array<BMSTablesDataset> = [
{
name: "Insane",
humanisedPrefix: "insane",
@@ -102,14 +110,6 @@ const registeredTables: BMSTablesDataset[] = [
},
];
/* eslint-disable no-await-in-loop */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import fetch from "node-fetch";
import { ChartDocument } from "tachi-common";
import { CreateFolderID } from "utils/folder";
import { FormatBMSTables } from "utils/misc";
// this seems to be all we care about
interface TableJSONDoc {
md5: string;
@@ -119,7 +119,7 @@ interface TableJSONDoc {
const logger = CreateLogCtx(__filename);
async function ImportTableLevels(tableJSON: TableJSONDoc[], prefix: string) {
async function ImportTableLevels(tableJSON: Array<TableJSONDoc>, prefix: string) {
let failures = 0;
let success = 0;
const total = tableJSON.length;
@@ -201,7 +201,7 @@ export async function UpdateTable(table: BMSTablesDataset) {
// instead of using res.json() we have to use res.text() here
// so we have proper logging when some of these bms urls
// spontaneously return HTML.
let tableJSON: BMSTableChart[] | undefined;
let tableJSON: Array<BMSTableChart> | undefined;
// We need to have a retry counter because some of the bms tables
// randomly return useless html for no reason.
@@ -209,6 +209,7 @@ export async function UpdateTable(table: BMSTablesDataset) {
// i hate bms.
while (!tableJSON) {
let text;
try {
text = await res.text();
tableJSON = JSON.parse(text);
@@ -220,6 +221,7 @@ export async function UpdateTable(table: BMSTablesDataset) {
logger.error(`SKIPPING TABLE!`);
return;
}
logger.info(`Retrying ${table.url}.`);
}
}
@@ -234,6 +236,7 @@ export async function UpdateTable(table: BMSTablesDataset) {
"data¬tableFolders": {
"~elemMatch": {
table: table.prefix,
// just incase some dude tries numbers
level: level.toString(),
},
@@ -1,12 +1,12 @@
import { Queue, Worker } from "bullmq";
import CreateLogCtx from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
import { DedupeArr } from "utils/misc";
import { BacksyncBMSPMSSongsAndCharts } from "../backsync-bms-pms-data";
import { DeoprhanScores } from "../deorphan-scores";
import { UGSSnapshot } from "../ugs-snapshot";
import { UpdatePoyashiData } from "../update-bpi-data";
import { UpdateDPTiers } from "../update-dp-tiers";
import { Queue, Worker } from "bullmq";
import CreateLogCtx from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
import { DedupeArr } from "utils/misc";
interface Job {
name: string;
@@ -14,7 +14,7 @@ interface Job {
run: () => Promise<void>;
}
const jobs: Job[] = [
const jobs: Array<Job> = [
{
name: "Snapshot User Game Stats",
cronFormat: "0 0 * * *",
@@ -22,6 +22,7 @@ const jobs: Job[] = [
},
{
name: "De-Orphan Scores",
// We run an hour after snapshotting UGS
// just to spread load out a bit.
cronFormat: "1 0 * * *",
@@ -79,6 +80,7 @@ export function InitialiseJobRunner() {
new Worker("Job Runner", async (j) => {
const name = j.data.jobName;
logger.info(`Running job ${name}.`);
const jobInfo = jobNameMap.get(name);
+3 -2
View File
@@ -1,8 +1,8 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { UserGameStats, UserGameStatsSnapshot } from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
import { GetAllRankings } from "utils/user";
import type { UserGameStats, UserGameStatsSnapshot } from "tachi-common";
const logger = CreateLogCtx(__filename);
@@ -10,7 +10,7 @@ const logger = CreateLogCtx(__filename);
// nonsense happens. we'll have to see.
const currentTime = new Date().setUTCHours(0, 0, 0, 0);
let batchWrite: UserGameStatsSnapshot[] = [];
let batchWrite: Array<UserGameStatsSnapshot> = [];
// This code is intentionally *very* robust, and handles a lot of unanticipated failures
// because if it breaks, we brick the database.
@@ -34,6 +34,7 @@ export async function UGSSnapshot() {
try {
await db["game-stats"]
.find({})
// @ts-expect-error faulty TS types
.each(async (ugs: UserGameStats, { pause, resume }) => {
pause();
+9 -11
View File
@@ -1,6 +1,8 @@
import { PullDatabaseSeeds } from "lib/database-seeds/repo";
import CreateLogCtx from "lib/logger/logger";
import {
import { RecalcAllScores } from "utils/calculations/recalc-scores";
import fetch from "utils/fetch";
import type {
ChartDocument,
Difficulties,
GPTSupportedVersions,
@@ -8,8 +10,6 @@ import {
Playtypes,
SongDocument,
} from "tachi-common";
import { RecalcAllScores } from "utils/calculations/recalc-scores";
import fetch from "utils/fetch";
const logger = CreateLogCtx(__filename);
@@ -20,7 +20,7 @@ const difficultyResolve = {
9: ["DP", "ANOTHER"],
10: ["SP", "LEGGENDARIA"],
11: ["DP", "LEGGENDARIA"],
} as Record<string, [Playtypes["iidx"], Difficulties["iidx:SP" | "iidx:DP"]]>;
} as Record<string, [Playtypes["iidx"], Difficulties["iidx:DP" | "iidx:SP"]]>;
interface PoyashiProxyBPIInfo {
title: string;
@@ -39,7 +39,7 @@ interface PoyashiProxyBPIInfo {
interface PoyashiProxyData {
version: integer;
requireVersion: string;
body: PoyashiProxyBPIInfo[];
body: Array<PoyashiProxyBPIInfo>;
}
/**
@@ -59,10 +59,8 @@ export async function UpdatePoyashiData() {
logger.info("Fetched data.");
const iidxSongs = (await repo.ReadCollection("songs-iidx")) as SongDocument<"iidx">[];
const iidxCharts = (await repo.ReadCollection("charts-iidx")) as ChartDocument<
"iidx:SP" | "iidx:DP"
>[];
const iidxSongs = await repo.ReadCollection("songs-iidx");
const iidxCharts = await repo.ReadCollection("charts-iidx");
// Utility functions for finding matching charts.
function FindSongOnTitle(title: string) {
@@ -78,8 +76,8 @@ export async function UpdatePoyashiData() {
function FindChartWithPTDFVersion(
songID: integer,
playtype: Playtypes["iidx"],
diff: Difficulties["iidx:SP" | "iidx:DP"],
version: GPTSupportedVersions["iidx:SP" | "iidx:DP"]
diff: Difficulties["iidx:DP" | "iidx:SP"],
version: GPTSupportedVersions["iidx:DP" | "iidx:SP"]
) {
for (const chart of iidxCharts) {
if (
+1 -1
View File
@@ -117,7 +117,7 @@ function ParseTierStr(tierStr: string) {
return null;
}
const result = tierStr.match(/\((.*)\)$/u);
const result = /\((.*)\)$/u.exec(tierStr);
if (result && result[1]) {
return Number(result[1]);
+8 -6
View File
@@ -1,10 +1,11 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { DiscordColours } from "./colours";
import { ONE_MINUTE } from "lib/constants/time";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import SafeJSONStringify from "safe-json-stringify";
import fetch from "utils/fetch";
import Transport, { TransportStreamOptions } from "winston-transport";
import { DiscordColours } from "./colours";
import Transport from "winston-transport";
import type { TransportStreamOptions } from "winston-transport";
interface DiscordTransportOptions extends TransportStreamOptions {
/** Webhook obtained from Discord */
@@ -12,8 +13,8 @@ interface DiscordTransportOptions extends TransportStreamOptions {
}
interface LogLevelCountState {
warn: string[];
error: string[];
warn: Array<string>;
error: Array<string>;
}
/**
@@ -27,7 +28,7 @@ export default class DiscordTransport extends Transport {
private webhookUrl = "";
/** Initialization promise resolved after retrieving discord id and token */
private initalised: Promise<void>;
private readonly initalised: Promise<void>;
private bucketData: LogLevelCountState = {
warn: [],
@@ -90,7 +91,7 @@ export default class DiscordTransport extends Transport {
return;
}
this.bucketData[info.level as "warn" | "error"].push(info.message);
this.bucketData[info.level as "error" | "warn"].push(info.message);
if (!this.isBucketing) {
this.isBucketing = true;
@@ -158,6 +159,7 @@ export default class DiscordTransport extends Transport {
embeds: [
{
description: `[${info.level}] ${info.message}`,
// it's Colour!!
color: DiscordColours[info.level as keyof typeof DiscordColours],
timestamp: new Date().toISOString(),
+12 -10
View File
@@ -1,9 +1,10 @@
import { Transport as SeqTransport } from "@valuabletouch/winston-seq";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import SafeJSONStringify from "safe-json-stringify";
import { SeqLogLevel } from "seq-logging";
import { EscapeStringRegexp } from "utils/misc";
import winston, { format, LeveledLogMethod, Logger, transports } from "winston";
import winston, { format, transports } from "winston";
import type { SeqLogLevel } from "seq-logging";
import type { LeveledLogMethod, Logger } from "winston";
import "winston-daily-rotate-file";
import DiscordWinstonTransport from "./discord-transport";
@@ -13,12 +14,14 @@ const level = process.env.LOG_LEVEL ?? ServerConfig.LOGGER_CONFIG.LOG_LEVEL;
const formatExcessProperties = (meta: Record<string, unknown>, limit = false) => {
let i = 0;
for (const key in meta) {
const val = meta[key];
if (val instanceof Error) {
meta[key] = { message: val.message, stack: val.stack };
}
i++;
}
@@ -41,7 +44,7 @@ function StrCap(string: string) {
const formatExcessPropertiesNoStack = (
meta: Record<string, unknown>,
omitKeys: string[] = [],
omitKeys: Array<string> = [],
limit = false
) => {
const realMeta: Record<string, unknown> = {};
@@ -116,7 +119,7 @@ const consoleFormatRoute = format.combine(
})
);
const tports: winston.transport[] = [];
const tports: Array<winston.transport> = [];
if (ServerConfig.LOGGER_CONFIG.FILE) {
tports.push(
@@ -159,6 +162,7 @@ if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) {
error: "Error",
warn: "Warning",
info: "Information",
// Note that Seq interprets these in reverse,
// however, it's easier to read this code if I just
// use the same levels, instead of the right ones.
@@ -196,12 +200,12 @@ export const rootLogger = winston.createLogger({
transports: tports,
defaultMeta: {
__ServerName: TachiConfig.NAME,
__Worker: !!process.env.IS_WORKER,
__Worker: Boolean(process.env.IS_WORKER),
__ReplicaID: Environment.replicaIdentity,
},
}) as KtLogger;
if (!!ServerConfig.LOGGER_CONFIG.SEQ_API_KEY !== !!Environment.seqUrl) {
if (Boolean(ServerConfig.LOGGER_CONFIG.SEQ_API_KEY) !== Boolean(Environment.seqUrl)) {
rootLogger.warn(
`Only one of SEQ_API_KEY (conf.json5) and SEQ_URL (Environment) were set. Not sending logs to Seq, as both must be provided.`
);
@@ -224,9 +228,7 @@ function CreateLogCtx(filename: string, lg = rootLogger): KtLogger {
context: [replacedFilename],
}) as KtLogger;
logger.defaultMeta = Object.assign({}, logger.defaultMeta ?? {}, {
context: [replacedFilename],
});
logger.defaultMeta = { ...(logger.defaultMeta ?? {}), context: [replacedFilename] };
return logger;
}
@@ -237,7 +239,7 @@ export function AppendLogCtx(context: string, lg: KtLogger): KtLogger {
}
export function ChangeRootLogLevel(
level: "crit" | "severe" | "error" | "warn" | "info" | "verbose" | "debug"
level: "crit" | "debug" | "error" | "info" | "severe" | "verbose" | "warn"
) {
rootLogger.info(`Changing log level to ${level}.`);
+4 -4
View File
@@ -1,8 +1,8 @@
import UGPTRivalsMigration from "./migrations/add-rivals-to-ugpt";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { Environment } from "lib/setup/config";
import { Migration } from "utils/types";
import UGPTRivalsMigration from "./migrations/add-rivals-to-ugpt";
import type { Migration } from "utils/types";
const logger = CreateLogCtx(__filename);
@@ -18,12 +18,12 @@ export const FAKE_MIGRATION: Migration = {
// Migrations are stored in an array because they have some concept of order
// That is, migrations should ideally be applied in a fixed order just to avoid
// any potential unsavoury interactions.
const REGISTERED_MIGRATIONS: Migration[] =
const REGISTERED_MIGRATIONS: Array<Migration> =
// If we're testing, we should pull fake migrations instead to ensure the tests
// stay consistent
Environment.nodeEnv === "test" ? [FAKE_MIGRATION] : [UGPTRivalsMigration];
function CreateMigrationLookupMap(migrations: Migration[]) {
function CreateMigrationLookupMap(migrations: Array<Migration>) {
const map = new Map<string, Migration>();
for (const mig of migrations) {
@@ -1,5 +1,5 @@
import db from "external/mongo/db";
import { Migration } from "utils/types";
import type { Migration } from "utils/types";
const migration: Migration = {
id: "add-rivals-to-ugpt",
@@ -1,6 +1,7 @@
import db from "external/mongo/db";
import { FormatGame, Game, integer, Playtype, PublicUserDocument } from "tachi-common";
import { SendNotification } from "./notifications";
import db from "external/mongo/db";
import { FormatGame } from "tachi-common";
import type { Game, integer, Playtype, PublicUserDocument } from "tachi-common";
/**
* A utility wrapper for sending the SET_RIVAL notification.
@@ -1,6 +1,6 @@
import db from "external/mongo/db";
import { integer, NotificationBody, NotificationDocument } from "tachi-common";
import { Random20Hex } from "utils/misc";
import type { integer, NotificationBody, NotificationDocument } from "tachi-common";
function ConstructNotificationDoc(
title: string,
@@ -34,7 +34,11 @@ export function SendNotification(title: string, toUserID: integer, body: Notific
* Send notifications to multiple users at once. This is more efficient than calling
* send notification in parallel.
*/
export function BulkSendNotification(title: string, toUserIDs: integer[], body: NotificationBody) {
export function BulkSendNotification(
title: string,
toUserIDs: Array<integer>,
body: NotificationBody
) {
const notifications = toUserIDs.map((e) => ConstructNotificationDoc(title, e, body));
return db.notifications.insert(notifications);
+7 -5
View File
@@ -1,7 +1,9 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { FilterQuery } from "mongodb";
import {
import { GetNextCounterValue } from "utils/db";
import { DedupeArr } from "utils/misc";
import type { FilterQuery } from "mongodb";
import type {
ChartDocument,
IDStrings,
IDStringToGame,
@@ -9,8 +11,6 @@ import {
OrphanChart,
SongDocument,
} from "tachi-common";
import { GetNextCounterValue } from "utils/db";
import { DedupeArr } from "utils/misc";
const logger = CreateLogCtx(__filename);
@@ -39,7 +39,7 @@ export async function HandleOrphanQueue<I extends IDStrings>(
logger.debug(`Received orphanqueue request for ${chartName}.`);
const orphanChart = await db["orphan-chart-queue"].findOne(
Object.assign({ idString }, orphanMatchCriteria),
{ idString, ...orphanMatchCriteria },
{
projectID: true,
}
@@ -63,8 +63,10 @@ export async function HandleOrphanQueue<I extends IDStrings>(
const uniqueUsersArr = DedupeArr(orphanChart.userIDs);
const playcount = uniqueUsersArr.length;
// If N or more people have played this chart while orphaned, unorphan
// it.
if (playcount >= queueSize) {
logger.info(`Song ${chartName} was unorphaned by userIDs ${uniqueUsersArr.join(", ")}.`);
const songID = await GetNextCounterValue(`${game}-song-id`);
+3 -2
View File
@@ -2,9 +2,10 @@ import db from "external/mongo/db";
import { SetRivalsFailReasons } from "lib/constants/err-codes";
import CreateLogCtx from "lib/logger/logger";
import { SendSetRivalNotification } from "lib/notifications/notification-wrappers";
import { FormatGame, Game, integer, Playtype } from "tachi-common";
import { FormatGame } from "tachi-common";
import { ArrayDiff } from "utils/misc";
import { GetUsersWithIDs, GetUserWithIDGuaranteed } from "utils/user";
import type { Game, integer, Playtype } from "tachi-common";
const logger = CreateLogCtx(__filename);
@@ -56,7 +57,7 @@ export async function SetRivals(
userID: integer,
game: Game,
playtype: Playtype,
newRivals: integer[]
newRivals: Array<integer>
) {
if (newRivals.length > 5) {
return SetRivalsFailReasons.TOO_MANY;
@@ -1,4 +1,10 @@
import { KtLogger } from "lib/logger/logger";
import {
CalculateKTLampRatingIIDXDP,
CalculateKTLampRatingIIDXSP,
CalculateKTRating,
CalculateMFCP,
CalculateSieglinde,
} from "./stats";
import {
CHUNITHMRating,
GITADORASkill,
@@ -8,23 +14,10 @@ import {
Volforce,
WACCARate,
} from "rg-stats";
import {
ChartDocument,
Game,
GetGamePTConfig,
IDStrings,
Lamps,
Playtypes,
ScoreDocument,
} from "tachi-common";
import { DryScore } from "../common/types";
import {
CalculateKTLampRatingIIDXDP,
CalculateKTLampRatingIIDXSP,
CalculateKTRating,
CalculateMFCP,
CalculateSieglinde,
} from "./stats";
import { GetGamePTConfig } from "tachi-common";
import type { DryScore } from "../common/types";
import type { KtLogger } from "lib/logger/logger";
import type { ChartDocument, Game, IDStrings, Lamps, Playtypes, ScoreDocument } from "tachi-common";
export async function CreateCalculatedData(
dryScore: DryScore,
@@ -84,6 +77,7 @@ export async function CalculateDataForGamePT<G extends Game>(
playtype: Playtypes[G],
chart: ChartDocument,
dryScore: DryScore,
// ESD gets specially passed through because it's not part of the DryScore, but
// can be used for statistics anyway.
esd: number | null,
@@ -97,7 +91,7 @@ type CalculatedData<I extends IDStrings> = Required<ScoreDocument<I>["calculated
function CalculateDataIIDX(
dryScore: DryScore,
chart: ChartDocument<"iidx:SP" | "iidx:DP">,
chart: ChartDocument<"iidx:DP" | "iidx:SP">,
logger: KtLogger
): CalculatedData<"iidx:SP"> {
let bpi;
@@ -131,7 +125,7 @@ function CalculateDataSDVXorUSC(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): CalculatedData<"sdvx:Single" | "usc:Keyboard" | "usc:Controller"> {
): CalculatedData<"sdvx:Single" | "usc:Controller" | "usc:Keyboard"> {
// for usc, unofficial charts currently have no VF6 value.
if (
dryScore.game === "usc" &&
@@ -185,7 +179,7 @@ function CalculateDataCHUNITHM(
function CalculateDataGitadora(
dryScore: DryScore,
chart: ChartDocument
): CalculatedData<"gitadora:Gita" | "gitadora:Dora"> {
): CalculatedData<"gitadora:Dora" | "gitadora:Gita"> {
return {
skill: GITADORASkill.calculate(dryScore.scoreData.percent, chart.levelNum),
};
@@ -195,7 +189,7 @@ function CalculateDataDDR(
dryScore: DryScore,
chart: ChartDocument,
logger: KtLogger
): CalculatedData<"ddr:SP" | "ddr:DP"> {
): CalculatedData<"ddr:DP" | "ddr:SP"> {
return {
MFCP: CalculateMFCP(dryScore, chart, logger),
ktRating: CalculateKTRating(dryScore, "ddr", chart.playtype, chart, logger),
@@ -1,6 +1,7 @@
import { KtLogger } from "lib/logger/logger";
import { ChartDocument, Game, GetGamePTConfig, IIDX_LAMPS, integer, Playtype } from "tachi-common";
import { DryScore } from "../common/types";
import { GetGamePTConfig, IIDX_LAMPS } from "tachi-common";
import type { DryScore } from "../common/types";
import type { KtLogger } from "lib/logger/logger";
import type { ChartDocument, Game, integer, Playtype } from "tachi-common";
/**
* Calculate Marvelous Full Combo Points. This algorithm
@@ -1,7 +1,7 @@
// Various Errors that can occur during processing.
import { ImportTypes } from "tachi-common";
import { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types";
import type { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types";
import type { ImportTypes } from "tachi-common";
export class ConverterFailure {
message: string | null;
@@ -1,12 +1,12 @@
import { KtLogger } from "lib/logger/logger";
import {
import { Parsers } from "lib/score-import/import-types/parsers";
import type { KtLogger } from "lib/logger/logger";
import type {
ImportTypeContextMap,
ImportTypeDataMap,
ParserFunctionReturns,
} from "lib/score-import/import-types/common/types";
import { Parsers } from "lib/score-import/import-types/parsers";
import { ScoreImportJobData } from "lib/score-import/worker/types";
import { ImportTypes } from "tachi-common";
import type { ScoreImportJobData } from "lib/score-import/worker/types";
import type { ImportTypes } from "tachi-common";
export function GetInputParser<I extends ImportTypes>(jobData: ScoreImportJobData<I>) {
// Retrieve the set parser function for this import type.
@@ -1,6 +1,7 @@
import { KtLogger, rootLogger } from "lib/logger/logger";
import { ImportTypes, PublicUserDocument } from "tachi-common";
import { rootLogger } from "lib/logger/logger";
import { FormatUserDoc } from "utils/user";
import type { KtLogger } from "lib/logger/logger";
import type { ImportTypes, PublicUserDocument } from "tachi-common";
export function CreateScoreLogger(
user: PublicUserDocument,
@@ -1,15 +1,7 @@
import CreateLogCtx from "lib/logger/logger";
import {
ChartDocument,
ESDCore,
Game,
GameToIDStrings,
GetGamePTConfig,
Grades,
IDStrings,
Playtype,
} from "tachi-common";
import { InternalFailure, InvalidScoreFailure } from "./converter-failures";
import CreateLogCtx from "lib/logger/logger";
import { ESDCore, GameToIDStrings, GetGamePTConfig } from "tachi-common";
import type { ChartDocument, Game, Grades, IDStrings, Playtype } from "tachi-common";
const logger = CreateLogCtx(__filename);
@@ -64,7 +56,7 @@ export function GenericCalculatePercent(game: Game, score: number, chart?: Chart
return score;
case "bms":
case "pms":
case "iidx":
case "iidx": {
if (!chart) {
logger.severe("No Chart passed to GenericCalcPercent but game was iidx/bms/pms.");
throw new InternalFailure(
@@ -77,19 +69,22 @@ export function GenericCalculatePercent(game: Game, score: number, chart?: Chart
const MAX =
(
chart as ChartDocument<
| "iidx:SP"
| "iidx:DP"
| "bms:7K"
| "bms:14K"
| "iidx:DP"
| "iidx:SP"
| "pms:Controller"
| "pms:Keyboard"
>
).data.notecount * 2;
return (100 * score) / MAX;
default:
}
default: {
logger.severe(`Invalid game passed of ${game} to GenericCalcPercent.`);
throw new InternalFailure(`Invalid game passed of ${game} to GenericCalcPercent.`);
}
}
}
@@ -108,6 +103,7 @@ export function ValidatePercent(
// i love needing a helper function for *ONE* game.
if (game === "maimai") {
const mmChart = chart as ChartDocument<"maimai:Single">;
if (percent > mmChart.data.maxPercent) {
throw new InvalidScoreFailure(
`Invalid percent - expected a number less than ${mmChart.data.maxPercent}.`
@@ -144,7 +140,7 @@ export function GenericGetGradeAndPercent<G extends Game>(
ValidatePercent(game, chart.playtype, percent, chart);
const grade = GetGradeFromPercent(game, chart.playtype, percent) as Grades[GameToIDStrings[G]];
const grade = GetGradeFromPercent(game, chart.playtype, percent);
return { percent, grade };
}
@@ -193,7 +189,7 @@ export function CalculateESDForGame(
* Parses and validates a date from a string.
* @returns Milliseconds from the unix epoch, or null if the initial argument was null or undefined.
*/
export function ParseDateFromString(str: string | undefined | null): number | null {
export function ParseDateFromString(str: string | null | undefined): number | null {
if (!str) {
return null;
}
@@ -1,5 +1,7 @@
import { Difficulties, Game, GetGamePTConfig, IDStrings, Playtype } from "tachi-common";
import { InvalidScoreFailure } from "./converter-failures";
import { GetGamePTConfig } from "tachi-common";
import type { Difficulties, Game, IDStrings, Playtype } from "tachi-common";
export function AssertStrAsDifficulty(
strVal: string,
game: Game,
@@ -1,4 +1,5 @@
import { IDStrings, ScoreDocument, Playtype, Game } from "tachi-common";
import { Game } from "tachi-common";
import type { IDStrings, ScoreDocument, Playtype } from "tachi-common";
/**
* An intermediate score format that will be filled out by
@@ -6,9 +7,9 @@ import { IDStrings, ScoreDocument, Playtype, Game } from "tachi-common";
*/
export type DryScore<I extends IDStrings = IDStrings> = Pick<
ScoreDocument<I>,
"service" | "game" | "scoreMeta" | "timeAchieved" | "comment" | "importType"
"comment" | "game" | "importType" | "scoreMeta" | "service" | "timeAchieved"
> & {
scoreData: Omit<ScoreDocument<I>["scoreData"], "gradeIndex" | "lampIndex" | "esd">;
scoreData: Omit<ScoreDocument<I>["scoreData"], "esd" | "gradeIndex" | "lampIndex">;
};
export type ScorePlaytypeMap = Partial<Record<Playtype, ScoreDocument[]>>;
export type ScorePlaytypeMap = Partial<Record<Playtype, Array<ScoreDocument>>>;
@@ -1,15 +1,15 @@
import { MakeScoreImport } from "./score-import";
import ScoreImportFatalError from "./score-importing/score-import-error";
import CreateLogCtx from "lib/logger/logger";
import {
import { Random20Hex } from "utils/misc";
import type { ParserArguments } from "../worker/types";
import type {
ImportDocument,
ImportTypes,
integer,
SuccessfulAPIResponse,
UnsuccessfulAPIResponse,
} from "tachi-common";
import { Random20Hex } from "utils/misc";
import { ParserArguments } from "../worker/types";
import { MakeScoreImport } from "./score-import";
import ScoreImportFatalError from "./score-importing/score-import-error";
export interface WrappedAPIResponse {
statusCode: number;
@@ -1,8 +1,8 @@
import { integer, Game, GoalDocument, GoalSubscriptionDocument } from "tachi-common";
import { EvaluateGoalForUser } from "lib/targets/goals";
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { EvaluateGoalForUser } from "lib/targets/goals";
import { EmitWebhookEvent } from "lib/webhooks/webhooks";
import type { KtLogger } from "lib/logger/logger";
import type { integer, Game, GoalDocument, GoalSubscriptionDocument } from "tachi-common";
/**
* Update a user's progress on all of their set goals.
@@ -26,7 +26,7 @@ export async function GetAndUpdateUsersGoals(
}
export async function UpdateGoalsForUser(
goals: GoalDocument[],
goals: Array<GoalDocument>,
goalSubsMap: Map<string, GoalSubscriptionDocument>,
userID: integer,
logger: KtLogger
@@ -122,7 +122,9 @@ export async function ProcessGoal(
};
let webhookEvent = null;
// if this is a newly-achieved goal
if (res.achieved && !goalSub.achieved) {
webhookEvent = {
goalID: goal.goalID,
@@ -135,6 +137,7 @@ export async function ProcessGoal(
const setData = {
...newData,
timeAchieved: newData.achieved ? Date.now() : null,
// we're guaranteed that this works, because things
// that haven't changed return nothing instead of
// getting to this point.
@@ -187,7 +190,7 @@ export async function GetRelevantGoals(
userID: integer,
chartIDs: Set<string>,
logger: KtLogger
): Promise<{ goals: GoalDocument[]; goalSubsMap: Map<string, GoalSubscriptionDocument> }> {
): Promise<{ goals: Array<GoalDocument>; goalSubsMap: Map<string, GoalSubscriptionDocument> }> {
const goalSubs = await db["goal-subs"].find({ game, userID }, { projectID: true });
logger.verbose(`Found user has ${goalSubs.length} goals.`);
@@ -198,7 +201,8 @@ export async function GetRelevantGoals(
const goalIDs = goalSubs.map((e) => e.goalID);
const chartIDsArr: string[] = [];
const chartIDsArr: Array<string> = [];
for (const c of chartIDs) {
chartIDsArr.push(c);
}
@@ -225,6 +229,7 @@ export async function GetRelevantGoals(
if (!goalSet.has(goalSub.goalID)) {
continue;
}
// since these are guaranteed to be unique, lets make a hot map of goalID -> goalSubDocument, so we can
// pull them in for post-processing and filter out the goalSubDocuments that aren't relevant.
goalSubsMap.set(goalSub.goalID, goalSub);
@@ -240,7 +245,7 @@ export async function GetRelevantGoals(
* Returns the set of goals where its folder contains any member
* of chartIDsArr.
*/
export function GetRelevantFolderGoals(goalIDs: string[], chartIDsArr: string[]) {
export function GetRelevantFolderGoals(goalIDs: Array<string>, chartIDsArr: Array<string>) {
// Slightly black magic - this is kind of like doing an SQL join.
// it's weird to do this in mongodb, but this seems like the right
// way to actually handle this.
@@ -1,5 +1,5 @@
import db from "external/mongo/db";
import { integer } from "tachi-common";
import type { integer } from "tachi-common";
/**
* If a user has no ongoing import, enable the import lock and return true.
@@ -1,8 +1,9 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { CalculateMilestoneOutOf, GetGoalIDsFromMilestone } from "lib/targets/milestones";
import { EmitWebhookEvent } from "lib/webhooks/webhooks";
import { BulkWriteUpdateOneOperation } from "mongodb";
import {
import type { KtLogger } from "lib/logger/logger";
import type { BulkWriteUpdateOneOperation } from "mongodb";
import type {
Game,
GoalImportInfo,
integer,
@@ -11,7 +12,6 @@ import {
Playtype,
MilestoneSubscriptionDocument,
} from "tachi-common";
import { CalculateMilestoneOutOf, GetGoalIDsFromMilestone } from "lib/targets/milestones";
/**
* Processes and updates a user's milestones from their Goal Import Info (i.e. what is returned
@@ -50,15 +50,16 @@ export function ProcessMilestoneFromGII(
}
export async function UpdateUsersMilestones(
importGoalInfo: GoalImportInfo[],
importGoalInfo: Array<GoalImportInfo>,
game: Game,
playtypes: Playtype[],
playtypes: Array<Playtype>,
userID: integer,
logger: KtLogger
) {
const goalSubInfoMap: Map<string, GoalImportInfo["new"]> = new Map();
const goalIDs = [];
for (const e of importGoalInfo) {
goalSubInfoMap.set(e.goalID, e.new);
goalIDs.push(e.goalID);
@@ -75,6 +76,7 @@ export async function UpdateUsersMilestones(
// create a map here to avoid linear searching when
// co-iterating
const milestoneSubMap = new Map();
for (const um of milestoneSubs) {
milestoneSubMap.set(um.milestoneID, um);
}
@@ -85,9 +87,9 @@ export async function UpdateUsersMilestones(
importGoalMap.set(ig.goalID, ig.new);
}
const bwrite: BulkWriteUpdateOneOperation<MilestoneSubscriptionDocument>[] = [];
const bwrite: Array<BulkWriteUpdateOneOperation<MilestoneSubscriptionDocument>> = [];
const importMilestoneInfo: MilestoneImportInfo[] = [];
const importMilestoneInfo: Array<MilestoneImportInfo> = [];
for (const milestone of milestones) {
const { achieved, progress } = ProcessMilestoneFromGII(milestone, importGoalMap);
@@ -128,6 +130,7 @@ export async function UpdateUsersMilestones(
if (progress !== milestoneSub.progress) {
importMilestoneInfo.push(milestoneInfo);
// @ts-expect-error This property isn't read only, because I said so.
bwriteOp.updateOne.update.$set!.lastInteraction = Date.now();
}
@@ -155,9 +158,9 @@ export async function UpdateUsersMilestones(
}
async function GetRelevantMilestones(
goalIDs: string[],
goalIDs: Array<string>,
game: Game,
playtypes: Playtype[],
playtypes: Array<Playtype>,
userID: integer,
logger: KtLogger
) {
@@ -1,23 +1,23 @@
import db from "external/mongo/db";
import {
ConverterFunction,
ConverterFnReturnOrFailure,
ImportTypeContextMap,
ImportTypeDataMap,
OrphanScoreDocument,
} from "../../import-types/common/types";
import { Game, ImportTypes, integer } from "tachi-common";
import fjsh from "fast-json-stable-hash";
import { KtLogger } from "lib/logger/logger";
import { Converters } from "../../import-types/converters";
import {
ConverterFailure,
InternalFailure,
KTDataNotFoundFailure,
} from "../common/converter-failures";
import { ProcessSuccessfulConverterReturn } from "../score-importing/score-importing";
import { HandlePostImportSteps } from "../score-importing/score-import-main";
import { ProcessSuccessfulConverterReturn } from "../score-importing/score-importing";
import db from "external/mongo/db";
import fjsh from "fast-json-stable-hash";
import { GetUserWithID } from "utils/user";
import type {
ConverterFunction,
ConverterFnReturnOrFailure,
ImportTypeContextMap,
ImportTypeDataMap,
OrphanScoreDocument,
} from "../../import-types/common/types";
import type { KtLogger } from "lib/logger/logger";
import type { Game, ImportTypes, integer } from "tachi-common";
/**
* Creates an OrphanedScore document from the data and context,
@@ -34,7 +34,7 @@ export async function OrphanScore<T extends ImportTypes = ImportTypes>(
game: Game,
logger: KtLogger
) {
const orphan: Pick<OrphanScoreDocument, "importType" | "data" | "context" | "userID"> = {
const orphan: Pick<OrphanScoreDocument, "context" | "data" | "importType" | "userID"> = {
importType,
data,
context,
@@ -44,6 +44,7 @@ export async function OrphanScore<T extends ImportTypes = ImportTypes>(
logger.debug("Orphaning document", orphan);
let orphanID;
try {
orphanID = `O${fjsh.hash(orphan, "sha256")}`;
} catch (err) {
@@ -83,7 +84,7 @@ export async function OrphanScore<T extends ImportTypes = ImportTypes>(
*/
export async function ReprocessOrphan(
orphan: OrphanScoreDocument,
blacklist: string[],
blacklist: Array<string>,
logger: KtLogger
) {
const ConverterFunction = Converters[orphan.importType] as ConverterFunction<
@@ -1,7 +1,3 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { BulkWriteUpdateOneOperation } from "mongodb";
import { integer, PBScoreDocument, ScoreDocument } from "tachi-common";
import {
BMSMergeFn,
IIDXMergeFn,
@@ -9,6 +5,10 @@ import {
SDVXMergeFn,
USCMergeFn,
} from "./game-specific-merge";
import db from "external/mongo/db";
import type { KtLogger } from "lib/logger/logger";
import type { BulkWriteUpdateOneOperation } from "mongodb";
import type { integer, PBScoreDocument, ScoreDocument } from "tachi-common";
export type PBScoreDocumentNoRank = Omit<PBScoreDocument, "rankingData">;
@@ -75,12 +75,13 @@ export async function UpdateChartRanking(chartID: string) {
}
);
const bwrite: BulkWriteUpdateOneOperation<PBScoreDocument>[] = [];
const bwrite: Array<BulkWriteUpdateOneOperation<PBScoreDocument>> = [];
let rank = 0;
for (let i = 0; i < scores.length; i++) {
const score = scores[i];
rank++;
bwrite.push({
@@ -165,6 +166,7 @@ async function MergeScoreLampIntoPB(
};
const GameSpecificMergeFn = GAME_SPECIFIC_MERGE_FNS[scorePB.game];
if (GameSpecificMergeFn) {
const success = await GameSpecificMergeFn(pbDoc, scorePB, lampPB, logger);
@@ -1,14 +1,14 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { Volforce } from "rg-stats";
import { PBScoreDocument, ScoreDocument } from "tachi-common";
import { FindChartWithChartID } from "utils/queries/charts";
import { InternalFailure } from "../common/converter-failures";
import db from "external/mongo/db";
import { Volforce } from "rg-stats";
import { FindChartWithChartID } from "utils/queries/charts";
import type { KtLogger } from "lib/logger/logger";
import type { PBScoreDocument, ScoreDocument } from "tachi-common";
export async function IIDXMergeFn(
pbDoc: PBScoreDocument<"iidx:SP" | "iidx:DP">,
scorePB: ScoreDocument<"iidx:SP" | "iidx:DP">,
lampPB: ScoreDocument<"iidx:SP" | "iidx:DP">,
pbDoc: PBScoreDocument<"iidx:DP" | "iidx:SP">,
scorePB: ScoreDocument<"iidx:DP" | "iidx:SP">,
lampPB: ScoreDocument<"iidx:DP" | "iidx:SP">,
logger: KtLogger
): Promise<boolean> {
// lampRating needs to be updated.
@@ -26,13 +26,14 @@ export async function IIDXMergeFn(
"scoreData.hitMeta.bp": 1, // bp 0 is the best BP, bp 1 is worse, so on
},
}
)) as ScoreDocument<"iidx:SP" | "iidx:DP">;
)) as ScoreDocument<"iidx:DP" | "iidx:SP">;
if (!bpPB) {
logger.verbose(
`Could not find BP PB for ${scorePB.userID} ${scorePB.chartID} in PB joining. User likely has no scores with BP defined.`,
{ pbDoc }
);
// this isn't actually an error! we just don't have to do anything.
return true;
}
@@ -87,9 +88,9 @@ export function BMSMergeFn(
* This is near-identical to the SDVXMergeFn. See that.
*/
export async function USCMergeFn(
pbDoc: PBScoreDocument<"usc:Keyboard" | "usc:Controller">,
scorePB: ScoreDocument<"usc:Keyboard" | "usc:Controller">,
lampPB: ScoreDocument<"usc:Keyboard" | "usc:Controller">,
pbDoc: PBScoreDocument<"usc:Controller" | "usc:Keyboard">,
scorePB: ScoreDocument<"usc:Controller" | "usc:Keyboard">,
lampPB: ScoreDocument<"usc:Controller" | "usc:Keyboard">,
logger: KtLogger
) {
// @optimisable - see SDVXMergeFn
@@ -1,7 +1,8 @@
import { CreatePBDoc, UpdateChartRanking } from "./create-pb-doc";
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { integer } from "tachi-common";
import { CreatePBDoc, PBScoreDocumentNoRank, UpdateChartRanking } from "./create-pb-doc";
import type { PBScoreDocumentNoRank } from "./create-pb-doc";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
/**
* Process, recalculate and update a users PBs for this set of chartIDs.
@@ -23,7 +24,7 @@ export async function ProcessPBs(
const pbDocsReturn = await Promise.all(promises);
const pbDocs: PBScoreDocumentNoRank[] = [];
const pbDocs: Array<PBScoreDocumentNoRank> = [];
for (const doc of pbDocsReturn) {
if (!doc) {
@@ -1,14 +1,14 @@
/* eslint-disable no-await-in-loop */
import { JOB_RETRY_COUNT } from "lib/constants/tachi";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import { ImportDocument, ImportTypes, integer } from "tachi-common";
import { Sleep } from "utils/misc";
import ScoreImportQueue, { ScoreImportQueueEvents } from "../worker/queue";
import { ScoreImportJobData } from "../worker/types";
import { GetInputParser } from "./common/get-input-parser";
import ScoreImportFatalError from "./score-importing/score-import-error";
import ScoreImportMain from "./score-importing/score-import-main";
import ScoreImportQueue, { ScoreImportQueueEvents } from "../worker/queue";
import { JOB_RETRY_COUNT } from "lib/constants/tachi";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import { Sleep } from "utils/misc";
import type { ScoreImportJobData } from "../worker/types";
import type { ImportDocument, ImportTypes, integer } from "tachi-common";
const logger = CreateLogCtx(__filename);
@@ -1,8 +1,9 @@
import { ChartDocument, integer, ScoreDocument, SongDocument, GetGamePTConfig } from "tachi-common";
import { KtLogger } from "lib/logger/logger";
import { CreateCalculatedData } from "../calculated-data/calculated-data";
import { CalculateESDForGame } from "../common/score-utils";
import { DryScore } from "../common/types";
import { GetGamePTConfig } from "tachi-common";
import type { DryScore } from "../common/types";
import type { KtLogger } from "lib/logger/logger";
import type { ChartDocument, integer, ScoreDocument, SongDocument } from "tachi-common";
/**
* Takes an "intermediate" score and appends the rest of the data it needs.
@@ -30,20 +31,20 @@ export async function HydrateScore(
const gptConfig = GetGamePTConfig(dryScore.game, chart.playtype);
// Fill out the rest of the fields we want for scoreData
const scoreData = Object.assign(
{
lampIndex: gptConfig.lamps.indexOf(dryScore.scoreData.lamp),
gradeIndex: gptConfig.grades.indexOf(dryScore.scoreData.grade),
esd,
},
dryScoreData
);
const scoreData = {
lampIndex: gptConfig.lamps.indexOf(dryScore.scoreData.lamp),
gradeIndex: gptConfig.grades.indexOf(dryScore.scoreData.grade),
esd,
...dryScoreData,
};
const score: ScoreDocument = {
// extract all of the non-scoreData props from a dry score and push them here
...rest,
// then push our score data.
scoreData,
// everything below this point is sane
highlight: false,
timeAdded: Date.now(),
@@ -1,12 +1,12 @@
import { ScoreDocument, integer } from "tachi-common";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import type { ScoreDocument, integer } from "tachi-common";
const logger = CreateLogCtx(__filename);
const MAX_PIPELINE_LENGTH = 500;
interface ScoreQueue {
queue: ScoreDocument[];
queue: Array<ScoreDocument>;
scoreIDSet: Set<string>;
}
@@ -1,7 +1,7 @@
import crypto from "crypto";
import db from "external/mongo/db";
import { integer } from "tachi-common";
import { DryScore } from "../common/types";
import crypto from "crypto";
import type { DryScore } from "../common/types";
import type { integer } from "tachi-common";
/**
* Performs sha256 hashing on the input data.
@@ -1,9 +1,25 @@
import ScoreImportFatalError from "./score-import-error";
import { ImportAllIterableData } from "./score-importing";
import { Converters } from "../../import-types/converters";
import { InternalFailure } from "../common/converter-failures";
import { CreateScoreLogger } from "../common/import-logger";
import { GetAndUpdateUsersGoals } from "../goals/goals";
import { CheckAndSetOngoingImportLock, UnsetOngoingImportLock } from "../import-locks/lock";
import { UpdateUsersMilestones } from "../milestones/milestones";
import { ProcessPBs } from "../pb/process-pbs";
import { CreateSessions } from "../sessions/sessions";
import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs";
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { ScoreImportJob } from "lib/score-import/worker/types";
import {
import { GetGameConfig } from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
import { GetUserWithID } from "utils/user";
import type { ConverterFunction, ImportInputParser } from "../../import-types/common/types";
import type { ScorePlaytypeMap } from "../common/types";
import type { ClassHandler } from "../user-game-stats/types";
import type { KtLogger } from "lib/logger/logger";
import type { ScoreImportJob } from "lib/score-import/worker/types";
import type {
Game,
GetGameConfig,
IDStrings,
ImportDocument,
ImportProcessingInfo,
@@ -12,22 +28,6 @@ import {
Playtype,
PublicUserDocument,
} from "tachi-common";
import { GetMillisecondsSince } from "utils/misc";
import { GetUserWithID } from "utils/user";
import { ConverterFunction, ImportInputParser } from "../../import-types/common/types";
import { Converters } from "../../import-types/converters";
import { InternalFailure } from "../common/converter-failures";
import { CreateScoreLogger } from "../common/import-logger";
import { ScorePlaytypeMap } from "../common/types";
import { GetAndUpdateUsersGoals } from "../goals/goals";
import { CheckAndSetOngoingImportLock, UnsetOngoingImportLock } from "../import-locks/lock";
import { UpdateUsersMilestones } from "../milestones/milestones";
import { ProcessPBs } from "../pb/process-pbs";
import { CreateSessions } from "../sessions/sessions";
import { ClassHandler } from "../user-game-stats/types";
import { UpdateUsersGamePlaytypeStats } from "../user-game-stats/update-ugs";
import ScoreImportFatalError from "./score-import-error";
import { ImportAllIterableData } from "./score-importing";
/**
* Performs a Score Import.
@@ -68,6 +68,7 @@ export default async function ScoreImportMain<D, C>(
if (hasNoOngoingImport) {
logger.info(`User ${userID} made an import while they had one ongoing.`);
// @danger
// Throwing away an import if the user already has one outgoing is *bad*, as in the case
// of degraded performance we might just start throwing scores away.
@@ -186,7 +187,7 @@ export default async function ScoreImportMain<D, C>(
// Create and Save an import document to the database, and finish everything up!
const ImportDocument: ImportDocument = {
importType,
idStrings: playtypes.map((e) => `${game}:${e}`) as IDStrings[],
idStrings: playtypes.map((e) => `${game}:${e}`) as Array<IDStrings>,
scoreIDs,
playtypes,
game,
@@ -255,7 +256,7 @@ export default async function ScoreImportMain<D, C>(
* and updating a users game stats.
*/
export async function HandlePostImportSteps(
importInfo: ImportProcessingInfo[],
importInfo: Array<ImportProcessingInfo>,
user: PublicUserDocument,
importType: ImportTypes,
game: Game,
@@ -296,6 +297,7 @@ export async function HandlePostImportSteps(
// This function also handles conjoining different scores together (such as unioning best lamp and
// best score).
const pbTimeStart = process.hrtime.bigint();
await ProcessPBs(user.id, chartIDs, logger);
const pbTime = GetMillisecondsSince(pbTimeStart);
@@ -303,7 +305,7 @@ export async function HandlePostImportSteps(
logger.debug(`PB Processing took ${pbTime} milliseconds (${pbTimeRel}ms/doc)`);
const playtypes = Object.keys(scorePlaytypeMap) as Playtype[];
const playtypes = Object.keys(scorePlaytypeMap) as Array<Playtype>;
SetJobProgress(job, "Updating profile statistics.");
@@ -368,7 +370,7 @@ export async function HandlePostImportSteps(
*/
async function UpdateUsersGameStats(
game: Game,
modifiedPlaytypes: Playtype[],
modifiedPlaytypes: Array<Playtype>,
userID: integer,
classHandler: ClassHandler | null,
logger: KtLogger
@@ -387,6 +389,7 @@ async function UpdateUsersGameStats(
}
const r = await Promise.all(promises);
return r.flat(1);
}
@@ -397,7 +400,7 @@ async function UpdateUsersGameStats(
* A set of unique chartIDs involved in the import and the scores mapped
* on their playtype.
*/
function ParseImportInfo(importInfo: ImportProcessingInfo[]) {
function ParseImportInfo(importInfo: Array<ImportProcessingInfo>) {
const scorePlaytypeMap: ScorePlaytypeMap = Object.create(null);
const scoreIDs = [];
@@ -1,21 +1,6 @@
import db from "external/mongo/db";
import { AppendLogCtx, KtLogger } from "lib/logger/logger";
import { ScoreImportJob } from "lib/score-import/worker/types";
import {
ChartDocument,
Game,
IDStrings,
ImportProcessingInfo,
ImportTypes,
integer,
ScoreDocument,
SongDocument,
} from "tachi-common";
import {
ConverterFnReturnOrFailure,
ConverterFnSuccessReturn,
ConverterFunction,
} from "../../import-types/common/types";
import { HydrateScore } from "./hydrate-score";
import { GetScoreQueueMaybe, InsertQueue, QueueScoreInsert } from "./insert-score";
import { CreateScoreID } from "./score-id";
import {
ConverterFailure,
InternalFailure,
@@ -23,11 +8,27 @@ import {
KTDataNotFoundFailure,
SkipScoreFailure,
} from "../common/converter-failures";
import { DryScore } from "../common/types";
import { OrphanScore } from "../orphans/orphans";
import { HydrateScore } from "./hydrate-score";
import { GetScoreQueueMaybe, InsertQueue, QueueScoreInsert } from "./insert-score";
import { CreateScoreID } from "./score-id";
import db from "external/mongo/db";
import { AppendLogCtx } from "lib/logger/logger";
import { IDStrings } from "tachi-common";
import type {
ConverterFnReturnOrFailure,
ConverterFnSuccessReturn,
ConverterFunction,
} from "../../import-types/common/types";
import type { DryScore } from "../common/types";
import type { KtLogger } from "lib/logger/logger";
import type { ScoreImportJob } from "lib/score-import/worker/types";
import type {
ChartDocument,
Game,
ImportProcessingInfo,
ImportTypes,
integer,
ScoreDocument,
SongDocument,
} from "tachi-common";
/**
* Processes the iterable data into the Tachi database.
@@ -40,13 +41,13 @@ import { CreateScoreID } from "./score-id";
export async function ImportAllIterableData<D, C>(
userID: integer,
importType: ImportTypes,
iterableData: Iterable<D> | AsyncIterable<D>,
iterableData: AsyncIterable<D> | Iterable<D>,
ConverterFunction: ConverterFunction<D, C>,
context: C,
game: Game,
logger: KtLogger,
job: ScoreImportJob | undefined
): Promise<ImportProcessingInfo[]> {
): Promise<Array<ImportProcessingInfo>> {
logger.verbose("Getting Blacklist...");
// @optimisable: could filter harder with score.game and score.playtype
@@ -92,9 +93,7 @@ export async function ImportAllIterableData<D, C>(
logger.verbose(`Finished Importing Data (${processedResults.length} datapoints).`);
logger.debug(`Removing null returns...`);
const datapoints = processedResults.filter(
(e) => e !== null
) as ImportProcessingInfo<IDStrings>[];
const datapoints = processedResults.filter((e) => e !== null) as Array<ImportProcessingInfo>;
logger.debug(`Removed null from results.`);
@@ -126,7 +125,7 @@ export async function ImportIterableDatapoint<D, C>(
ConverterFunction: ConverterFunction<D, C>,
context: C,
game: Game,
blacklist: string[],
blacklist: Array<string>,
logger: KtLogger
): Promise<ImportProcessingInfo | null> {
// Converter Function Return
@@ -198,23 +197,24 @@ export async function ImportIterableDatapoint<D, C>(
return {
success: false,
type: "InternalError",
// could return cfnReturn.message here, but we might want to hide the details of the crash.
message: "An internal error has occured.",
content: {},
};
} else if (cfnReturn instanceof SkipScoreFailure) {
return null;
} else {
logger.warn(`Unknown error returned as ConverterFailure, Ignoring.`, {
err: cfnReturn,
});
return {
success: false,
type: "InternalError",
message: "An internal service error has occured.",
content: {},
};
}
logger.warn(`Unknown error returned as ConverterFailure, Ignoring.`, {
err: cfnReturn,
});
return {
success: false,
type: "InternalError",
message: "An internal service error has occured.",
content: {},
};
}
if (cfnReturn instanceof Error) {
@@ -229,18 +229,13 @@ export async function ImportIterableDatapoint<D, C>(
};
}
return ProcessSuccessfulConverterReturn(
userID,
cfnReturn as ConverterFnSuccessReturn,
blacklist,
logger
);
return ProcessSuccessfulConverterReturn(userID, cfnReturn, blacklist, logger);
}
export async function ProcessSuccessfulConverterReturn(
userID: integer,
cfnReturn: ConverterFnSuccessReturn,
blacklist: string[],
blacklist: Array<string>,
logger: KtLogger,
forceImmediateImport = false
): Promise<ImportProcessingInfo | null> {
@@ -289,7 +284,7 @@ async function HydrateAndInsertScore(
dryScore: DryScore,
chart: ChartDocument,
song: SongDocument,
blacklist: string[],
blacklist: Array<string>,
importLogger: KtLogger,
force = false
): Promise<ScoreDocument | null> {
@@ -333,6 +328,7 @@ async function HydrateAndInsertScore(
const score = await HydrateScore(userID, dryScore, chart, song, scoreID, logger);
let res;
if (force) {
res = await db.scores.insert(score);
} else {
@@ -1,11 +1,11 @@
import { Game, Playtype, Playtypes, ScoreDocument, SessionDocument } from "tachi-common";
import type { Game, Playtype, Playtypes, ScoreDocument, SessionDocument } from "tachi-common";
type ScoreCalculatedDataOnly = Pick<ScoreDocument, "calculatedData">;
const RELEVANT_SCORES = 10;
function AverageBest10(vals: (number | null | undefined)[]) {
const numbers = vals.filter((e) => typeof e === "number") as number[];
function AverageBest10(vals: Array<number | null | undefined>) {
const numbers = vals.filter((e) => typeof e === "number") as Array<number>;
if (numbers.length < RELEVANT_SCORES) {
return null;
@@ -19,24 +19,27 @@ function AverageBest10(vals: (number | null | undefined)[]) {
);
}
function SumAll(arr: ScoreCalculatedDataOnly[], prop: keyof ScoreDocument["calculatedData"]) {
function SumAll(arr: Array<ScoreCalculatedDataOnly>, prop: keyof ScoreDocument["calculatedData"]) {
return arr.reduce((a, e) => {
if (typeof e.calculatedData[prop] === "number") {
return a + (e.calculatedData[prop] as number);
return a + e.calculatedData[prop]!;
}
return a;
}, 0);
}
function AvgBest10Map(arr: ScoreCalculatedDataOnly[], prop: keyof ScoreDocument["calculatedData"]) {
function AvgBest10Map(
arr: Array<ScoreCalculatedDataOnly>,
prop: keyof ScoreDocument["calculatedData"]
) {
return AverageBest10(arr.map((e) => e.calculatedData[prop]));
}
type CalculatedDataFunctions = {
[G in Game]: {
[P in Playtypes[G]]: (
scoreCalcData: ScoreCalculatedDataOnly[]
scoreCalcData: Array<ScoreCalculatedDataOnly>
) => SessionDocument["calculatedData"];
};
};
@@ -147,7 +150,7 @@ const CalculatedDataFunctions: CalculatedDataFunctions = {
export function CreateSessionCalcData(
game: Game,
playtype: Playtype,
scoreCalcData: ScoreCalculatedDataOnly[]
scoreCalcData: Array<ScoreCalculatedDataOnly>
): SessionDocument["calculatedData"] {
// @ts-expect-error standard game->pt stuff.
return CalculatedDataFunctions[game][playtype](scoreCalcData);
@@ -1,7 +1,12 @@
import crypto from "crypto";
import { CreateSessionCalcData } from "./calculated-data";
import { GenerateRandomSessionName } from "./name-generation";
import db from "external/mongo/db";
import { AppendLogCtx, KtLogger } from "lib/logger/logger";
import {
import { AppendLogCtx } from "lib/logger/logger";
import { GetScoresFromSession } from "utils/session";
import crypto from "crypto";
import type { ScorePlaytypeMap } from "../common/types";
import type { KtLogger } from "lib/logger/logger";
import type {
Game,
ImportTypes,
integer,
@@ -12,10 +17,6 @@ import {
SessionInfoReturn,
SessionScoreInfo,
} from "tachi-common";
import { GetScoresFromSession } from "utils/session";
import { ScorePlaytypeMap } from "../common/types";
import { CreateSessionCalcData } from "./calculated-data";
import { GenerateRandomSessionName } from "./name-generation";
const TWO_HOURS = 1000 * 60 * 60 * 2;
@@ -31,7 +32,7 @@ export async function CreateSessions(
/* eslint-disable no-await-in-loop */
for (const playtype in scorePtMap) {
// @ts-expect-error This is my least favourite thing about ts.
const scores = scorePtMap[playtype] as ScoreDocument[];
const scores = scorePtMap[playtype] as Array<ScoreDocument>;
const sessionInfo = await LoadScoresIntoSessions(
userID,
@@ -80,9 +81,9 @@ export function CreateSessionID() {
function UpdateExistingSession(
existingSession: SessionDocument,
newInfo: SessionScoreInfo[],
oldScores: ScoreDocument[],
newScores: ScoreDocument[]
newInfo: Array<SessionScoreInfo>,
oldScores: Array<ScoreDocument>,
newScores: Array<ScoreDocument>
) {
const allScores = [...oldScores, ...newScores];
@@ -109,8 +110,8 @@ function UpdateExistingSession(
function CreateSession(
userID: integer,
importType: ImportTypes,
groupInfo: SessionScoreInfo[],
groupScores: ScoreDocument[],
groupInfo: Array<SessionScoreInfo>,
groupScores: Array<ScoreDocument>,
game: Game,
playtype: Playtype
): SessionDocument {
@@ -139,11 +140,11 @@ function CreateSession(
export async function LoadScoresIntoSessions(
userID: integer,
importType: ImportTypes,
importScores: ScoreDocument[],
importScores: Array<ScoreDocument>,
game: Game,
playtype: Playtype,
baseLogger: KtLogger
): Promise<SessionInfoReturn[]> {
): Promise<Array<SessionInfoReturn>> {
const logger = AppendLogCtx("Session Generation", baseLogger);
const timestampedScores = [];
@@ -151,6 +152,7 @@ export async function LoadScoresIntoSessions(
for (const score of importScores) {
if (!score.timeAchieved) {
logger.verbose(`Ignored score ${score.scoreID}, as it had no timeAchieved.`);
// ignore scores without timestamps. We can't use these for sessions.
continue;
}
@@ -170,8 +172,8 @@ export async function LoadScoresIntoSessions(
// The "Score Groups" for the array of scores provided.
// This contains scores split on 2hr margins, which allows for more optimised
// session db requests.
const sessionScoreGroups: ScoreDocument[][] = [];
let curGroup: ScoreDocument[] = [];
const sessionScoreGroups: Array<Array<ScoreDocument>> = [];
let curGroup: Array<ScoreDocument> = [];
let lastTimestamp = 0;
for (const score of timestampedScores) {
@@ -181,6 +183,7 @@ export async function LoadScoresIntoSessions(
sessionScoreGroups.push(curGroup);
curGroup = [score];
}
lastTimestamp = score.timeAchieved!;
}
@@ -190,7 +193,7 @@ export async function LoadScoresIntoSessions(
logger.verbose(`Created ${sessionScoreGroups.length} groups from timestamped scores.`);
const sessionInfoReturns: SessionInfoReturn[] = [];
const sessionInfoReturns: Array<SessionInfoReturn> = [];
// All async operations inside here *need* to be done in lockstep to avoid colliding sessions.
// realistically, that shouldn't be possible, but hey.
@@ -213,6 +216,7 @@ export async function LoadScoresIntoSessions(
});
const pbMap: Map<string, PBScoreDocument> = new Map();
for (const pb of pbs) {
pbMap.set(pb.chartID, pb);
}
@@ -6,9 +6,9 @@ import {
SDVXVFClasses,
WACCA_COLOURS,
} from "lib/constants/classes";
import { KtLogger } from "lib/logger/logger";
import { Game, integer, Playtype, ScoreCalculatedDataLookup } from "tachi-common";
import { GameClasses } from "tachi-common/js/game-classes";
import type { KtLogger } from "lib/logger/logger";
import type { Game, integer, Playtype, ScoreCalculatedDataLookup } from "tachi-common";
import type { GameClasses } from "tachi-common/js/game-classes";
export function CalculateSDVXClass(
game: Game,
@@ -1,18 +1,3 @@
import deepmerge from "deepmerge";
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import { EmitWebhookEvent } from "lib/webhooks/webhooks";
import {
ClassDelta,
Game,
IDStrings,
integer,
Playtype,
Playtypes,
UserGameStats,
} from "tachi-common";
import { GameClasses } from "tachi-common/js/game-classes";
import { ReturnClassIfGreater } from "utils/class";
import {
CalculateChunithmColour,
CalculateGitadoraColour,
@@ -21,7 +6,22 @@ import {
CalculateSDVXClass,
CalculateWACCAColour,
} from "./builtin-class-handlers";
import { ClassHandler, ScoreClasses } from "./types";
import deepmerge from "deepmerge";
import db from "external/mongo/db";
import { EmitWebhookEvent } from "lib/webhooks/webhooks";
import { ReturnClassIfGreater } from "utils/class";
import type { ClassHandler, ScoreClasses } from "./types";
import type { KtLogger } from "lib/logger/logger";
import type {
ClassDelta,
Game,
IDStrings,
integer,
Playtype,
Playtypes,
UserGameStats,
} from "tachi-common";
import type { GameClasses } from "tachi-common/js/game-classes";
type ClassHandlerMap = {
[G in Game]:
@@ -136,8 +136,8 @@ export async function ProcessClassDeltas(
userGameStats: UserGameStats | null,
userID: integer,
logger: KtLogger
): Promise<ClassDelta[]> {
const deltas: ClassDelta[] = [];
): Promise<Array<ClassDelta>> {
const deltas: Array<ClassDelta> = [];
const achievementOps = [];
@@ -157,6 +157,7 @@ export async function ProcessClassDeltas(
continue;
} else {
let delta: ClassDelta;
if (isGreater === null) {
delta = {
game,
@@ -1,6 +1,6 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import {
import type { KtLogger } from "lib/logger/logger";
import type {
Game,
IDStrings,
integer,
@@ -16,9 +16,9 @@ type CustomCalcNames = ScoreCalculatedDataLookup[IDStrings];
function LazySumAll(key: CustomCalcNames) {
return async (game: Game, playtype: Playtype, userID: integer) => {
const sc = await db["personal-bests"].find({
game: game,
playtype: playtype,
userID: userID,
game,
playtype,
userID,
isPrimary: true,
[`calculatedData.${key}`]: { $type: "number" },
});
@@ -53,9 +53,9 @@ function LazyCalcN(
return async (game: Game, playtype: Playtype, userID: integer) => {
const sc = await db["personal-bests"].find(
{
game: game,
playtype: playtype,
userID: userID,
game,
playtype,
userID,
isPrimary: true,
[`calculatedData.${key}`]: { $type: "number" },
},
@@ -291,20 +291,21 @@ async function CalculateGitadoraSkill(
}
let skill = 0;
skill += bestHotScores.reduce((a, e) => a + e.calculatedData.skill!, 0);
skill += bestScores.reduce((a, e) => a + e.calculatedData.skill!, 0);
skill = skill + bestHotScores.reduce((a, e) => a + e.calculatedData.skill!, 0);
skill = skill + bestScores.reduce((a, e) => a + e.calculatedData.skill!, 0);
return skill;
}
export async function GetBestRatingOnSongs(
songIDs: integer[],
songIDs: Array<integer>,
userID: integer,
game: Game,
playtype: Playtype,
ratingProp: "skill",
limit = 25
): Promise<PBScoreDocument[]> {
): Promise<Array<PBScoreDocument>> {
const r = await db["personal-bests"].aggregate([
{
$match: {
@@ -373,8 +374,9 @@ async function CalculateJubility(
]);
let skill = 0;
skill += bestHotScores.reduce((a, e) => a + e.calculatedData.jubility!, 0);
skill += bestScores.reduce((a, e) => a + e.calculatedData.jubility!, 0);
skill = skill + bestHotScores.reduce((a, e) => a + e.calculatedData.jubility!, 0);
skill = skill + bestScores.reduce((a, e) => a + e.calculatedData.jubility!, 0);
return skill;
}
@@ -1,15 +1,13 @@
import { KtLogger } from "lib/logger/logger";
import { Game, IDStrings, integer, Playtype } from "tachi-common";
import { GameClasses } from "tachi-common/js/game-classes";
import type { KtLogger } from "lib/logger/logger";
import type { Game, IDStrings, integer, Playtype } from "tachi-common";
import type { GameClasses } from "tachi-common/js/game-classes";
export type ScoreClasses = Partial<GameClasses<IDStrings>>;
export interface ClassHandler {
(
game: Game,
playtype: Playtype,
userID: integer,
ratings: Record<string, number>,
logger: KtLogger
): Promise<ScoreClasses> | ScoreClasses | undefined;
}
export type ClassHandler = (
game: Game,
playtype: Playtype,
userID: integer,
ratings: Record<string, number>,
logger: KtLogger
) => Promise<ScoreClasses> | ScoreClasses | undefined;
@@ -2,13 +2,13 @@
// Some games have dedicated methods to calculate statistics like these, other games do not.
// That's about all there is to it!
import db from "external/mongo/db";
import { CreateGameSettings } from "lib/game-settings/create-game-settings";
import { KtLogger } from "lib/logger/logger";
import { ClassDelta, Game, integer, Playtype, UserGameStats } from "tachi-common";
import { ProcessClassDeltas, UpdateUGSClasses } from "./classes";
import { CalculateRatings } from "./rating";
import { ClassHandler } from "./types";
import db from "external/mongo/db";
import { CreateGameSettings } from "lib/game-settings/create-game-settings";
import type { ClassHandler } from "./types";
import type { KtLogger } from "lib/logger/logger";
import type { ClassDelta, Game, integer, Playtype, UserGameStats } from "tachi-common";
export async function UpdateUsersGamePlaytypeStats(
game: Game,
@@ -16,7 +16,7 @@ export async function UpdateUsersGamePlaytypeStats(
userID: integer,
classHandler: ClassHandler | null,
logger: KtLogger
): Promise<ClassDelta[]> {
): Promise<Array<ClassDelta>> {
const ratings = await CalculateRatings(game, playtype, userID, logger);
// Attempt to find a users game stats if one already exists. If one doesn't exist,
@@ -3,7 +3,7 @@ import { ServerConfig } from "lib/setup/config";
import nodeFetch from "utils/fetch";
import { HasOwnProperty } from "utils/misc";
import { CreateURLWithParams } from "utils/url";
import { ClassHandler } from "../../../framework/user-game-stats/types";
import type { ClassHandler } from "../../../framework/user-game-stats/types";
export async function CreateArcIIDXClassHandler(
profileID: string,
@@ -44,7 +44,7 @@ export async function CreateArcIIDXClassHandler(
// we're just going to lazily path directly towards the rank.
// if ARC sends us an unexpected JSON structure or whatever
// we wont crash.
let arcClass: string | undefined | null;
let arcClass: string | null | undefined;
if (playtype === "SP") {
arcClass = json?._items?.[0]?.sp?.rank;
@@ -1,10 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import p, { PrudenceSchema } from "prudence";
import { Lamps } from "tachi-common";
import { FormatPrError } from "utils/prudence";
import { FindChartOnARCID } from "utils/queries/charts";
import { FindSongOnIDGuaranteed } from "utils/queries/songs";
import { EmptyObject } from "utils/types";
import {
InvalidScoreFailure,
KTDataNotFoundFailure,
@@ -13,9 +7,16 @@ import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../framework/common/score-utils";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../../common/types";
import { ARCIIDXScore } from "./types";
import p from "prudence";
import { FormatPrError } from "utils/prudence";
import { FindChartOnARCID } from "utils/queries/charts";
import { FindSongOnIDGuaranteed } from "utils/queries/songs";
import type { DryScore } from "../../../framework/common/types";
import type { ConverterFunction } from "../../common/types";
import type { ARCIIDXScore } from "./types";
import type { PrudenceSchema } from "prudence";
import type { Lamps } from "tachi-common";
import type { EmptyObject } from "utils/types";
// There's a bunch of other useless fields but we don't care
const PR_ArcIIDXScore: PrudenceSchema = {
@@ -50,6 +51,7 @@ export const ConvertAPIArcIIDX: ConverterFunction<unknown, EmptyObject> = async
// Remove internal properties.
delete (data as any)._links;
delete (data as any)._etag;
// Most importantly, this one will cause crashes.
delete (data as any)._id;
@@ -75,7 +77,7 @@ export const ConvertAPIArcIIDX: ConverterFunction<unknown, EmptyObject> = async
const lamp = ResolveARCIIDXLamp(score.lamp);
const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = {
const dryScore: DryScore<"iidx:DP" | "iidx:SP"> = {
comment: null,
game: "iidx",
importType,
@@ -97,7 +99,7 @@ export const ConvertAPIArcIIDX: ConverterFunction<unknown, EmptyObject> = async
return { song, chart, dryScore };
};
export function ResolveARCIIDXLamp(lamp: ARCIIDXScore["lamp"]): Lamps["iidx:SP" | "iidx:DP"] {
export function ResolveARCIIDXLamp(lamp: ARCIIDXScore["lamp"]): Lamps["iidx:DP" | "iidx:SP"] {
switch (lamp) {
case "NO_PLAY":
return "NO PLAY";
@@ -1,12 +1,12 @@
import { KtLogger } from "lib/logger/logger";
import { CreateArcIIDXClassHandler } from "./class-handler";
import { TraverseKaiAPI } from "../../common/api-kai/traverse-api";
import { ServerConfig } from "lib/setup/config";
import { integer } from "tachi-common";
import nodeFetch from "utils/fetch";
import { GetArcAuthGuaranteed } from "utils/queries/auth";
import { EmptyObject } from "utils/types";
import { TraverseKaiAPI } from "../../common/api-kai/traverse-api";
import { ParserFunctionReturns } from "../../common/types";
import { CreateArcIIDXClassHandler } from "./class-handler";
import type { ParserFunctionReturns } from "../../common/types";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
import type { EmptyObject } from "utils/types";
export async function ParseArcIIDX(
userID: integer,
@@ -24,6 +24,7 @@ export async function ParseArcIIDX(
return {
iterable: TraverseKaiAPI(
ServerConfig.ARC_API_URL,
// BISTROVER
// The trailing slash is critical! ARC only allows /? queries.
`/api/v1/iidx/28/player_bests/?profile_id=${authDoc.accountID}`,
@@ -1,16 +1,16 @@
import { integer } from "tachi-common";
import type { integer } from "tachi-common";
export interface ARCIIDXScore {
chart_id: string;
lamp:
| "NO_PLAY"
| "FAILED"
| "ASSIST_CLEAR"
| "EASY_CLEAR"
| "CLEAR"
| "HARD_CLEAR"
| "EASY_CLEAR"
| "EX_HARD_CLEAR"
| "FULL_COMBO";
| "FAILED"
| "FULL_COMBO"
| "HARD_CLEAR"
| "NO_PLAY";
ex_score: integer;
miss_count: integer | null;
timestamp: string;
@@ -1,9 +1,3 @@
import p, { PrudenceSchema } from "prudence";
import { Lamps } from "tachi-common";
import { FormatPrError } from "utils/prudence";
import { FindChartOnARCID } from "utils/queries/charts";
import { FindSongOnIDGuaranteed } from "utils/queries/songs";
import { EmptyObject } from "utils/types";
import {
InvalidScoreFailure,
KTDataNotFoundFailure,
@@ -12,9 +6,16 @@ import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../framework/common/score-utils";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../../common/types";
import { ARCSDVXScore } from "./types";
import p from "prudence";
import { FormatPrError } from "utils/prudence";
import { FindChartOnARCID } from "utils/queries/charts";
import { FindSongOnIDGuaranteed } from "utils/queries/songs";
import type { DryScore } from "../../../framework/common/types";
import type { ConverterFunction } from "../../common/types";
import type { ARCSDVXScore } from "./types";
import type { PrudenceSchema } from "prudence";
import type { Lamps } from "tachi-common";
import type { EmptyObject } from "utils/types";
// There's a bunch of other useless fields but we don't care
const PR_ArcSDVXScore: PrudenceSchema = {
@@ -1,11 +1,11 @@
import { KtLogger } from "lib/logger/logger";
import { TraverseKaiAPI } from "../../common/api-kai/traverse-api";
import { ServerConfig } from "lib/setup/config";
import { integer } from "tachi-common";
import nodeFetch from "utils/fetch";
import { GetArcAuthGuaranteed } from "utils/queries/auth";
import { EmptyObject } from "utils/types";
import { TraverseKaiAPI } from "../../common/api-kai/traverse-api";
import { ParserFunctionReturns } from "../../common/types";
import type { ParserFunctionReturns } from "../../common/types";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
import type { EmptyObject } from "utils/types";
export async function ParseArcSDVX(
userID: integer,
@@ -23,6 +23,7 @@ export async function ParseArcSDVX(
return {
iterable: TraverseKaiAPI(
ServerConfig.ARC_API_URL,
// VIVID WAVE.
// The trailing slash is critical! ARC only allows /? queries.
`/api/v1/sdvx/5/player_bests/?profile_id=${authDoc.accountID}`,
@@ -1,9 +1,9 @@
import { integer } from "tachi-common";
import type { integer } from "tachi-common";
export interface ARCSDVXScore {
chart_id: string;
score: integer;
lamp: "CLEAR" | "UC" | "PUC" | "PLAY" | "HC";
lamp: "CLEAR" | "HC" | "PLAY" | "PUC" | "UC";
btn_rate: number;
long_rate: number;
vol_rate: number;
@@ -1,7 +1,7 @@
import { KtLogger } from "lib/logger/logger";
import { integer } from "tachi-common";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import { ParseKaiIIDX } from "../../common/api-kai/iidx/parser";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
export async function ParseEagIIDX(userID: integer, logger: KtLogger) {
const authDoc = await GetKaiAuthGuaranteed(userID, "EAG", logger);
@@ -1,7 +1,7 @@
import { KtLogger } from "lib/logger/logger";
import { integer } from "tachi-common";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import { ParseKaiSDVX } from "../../common/api-kai/sdvx/parser";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
export async function ParseEagSDVX(userID: integer, logger: KtLogger) {
const authDoc = await GetKaiAuthGuaranteed(userID, "EAG", logger);
@@ -1,7 +1,7 @@
import { KtLogger } from "lib/logger/logger";
import { ParseKaiIIDX } from "../../common/api-kai/iidx/parser";
import { integer } from "tachi-common";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
export async function ParseFloIIDX(userID: integer, logger: KtLogger) {
const authDoc = await GetKaiAuthGuaranteed(userID, "FLO", logger);
@@ -1,7 +1,7 @@
import { KtLogger } from "lib/logger/logger";
import { integer } from "tachi-common";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import { ParseKaiSDVX } from "../../common/api-kai/sdvx/parser";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
export async function ParseFloSDVX(userID: integer, logger: KtLogger) {
const authDoc = await GetKaiAuthGuaranteed(userID, "FLO", logger);
@@ -1,7 +1,7 @@
import { KtLogger } from "lib/logger/logger";
import { integer } from "tachi-common";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import { ParseKaiSDVX } from "../../common/api-kai/sdvx/parser";
import { GetKaiAuthGuaranteed } from "utils/queries/auth";
import type { KtLogger } from "lib/logger/logger";
import type { integer } from "tachi-common";
export async function ParseMinSDVX(userID: integer, logger: KtLogger) {
const authDoc = await GetKaiAuthGuaranteed(userID, "MIN", logger);
@@ -1,10 +1,10 @@
import { IIDXDans } from "lib/constants/classes";
import { ClassHandler } from "lib/score-import/framework/user-game-stats/types";
import nodeFetch from "utils/fetch";
import { KaiTypeToBaseURL } from "../utils";
import { IIDXDans } from "lib/constants/classes";
import nodeFetch from "utils/fetch";
import type { ClassHandler } from "lib/score-import/framework/user-game-stats/types";
export async function CreateKaiIIDXClassHandler(
kaiType: "FLO" | "EAG",
kaiType: "EAG" | "FLO",
token: string,
fetch = nodeFetch
): Promise<ClassHandler> {
@@ -1,8 +1,3 @@
import p from "prudence";
import { integer, Lamps } from "tachi-common";
import { FormatPrError } from "utils/prudence";
import { FindIIDXChartOnInGameIDVersion } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import {
InternalFailure,
InvalidScoreFailure,
@@ -12,9 +7,14 @@ import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../../framework/common/score-utils";
import { DryScore } from "../../../../framework/common/types";
import { ConverterFunction } from "../../types";
import { KaiContext, KaiIIDXScore } from "../types";
import p from "prudence";
import { FormatPrError } from "utils/prudence";
import { FindIIDXChartOnInGameIDVersion } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import type { DryScore } from "../../../../framework/common/types";
import type { ConverterFunction } from "../../types";
import type { KaiContext, KaiIIDXScore } from "../types";
import type { integer, Lamps } from "tachi-common";
const PR_KaiIIDXScore = {
music_id: p.isPositiveInteger,
@@ -29,7 +29,7 @@ const PR_KaiIIDXScore = {
timestamp: "string",
};
function ResolveKaiLamp(lamp: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7): Lamps["iidx:SP" | "iidx:DP"] {
function ResolveKaiLamp(lamp: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7): Lamps["iidx:DP" | "iidx:SP"] {
switch (lamp) {
case 0:
return "NO PLAY";
@@ -56,94 +56,139 @@ function ResolveKaiLamp(lamp: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7): Lamps["iidx:SP" |
const OldLeggendariaConversionTable: Record<integer, integer> = {
// RUGGED ASH† -> RUGGED ASH
1100: 1017,
// Clione† -> Clione
4100: 4005,
// ABSOLUTE† -> ABSOLUTE
4101: 4001,
// RIDE ON THE LIGHT (HI GREAT MIX) † -> RIDE ON THE LIGHT (HI GREAT MIX)
5100: 5014,
// RED ZONE† -> RED ZONE
11100: 11032,
// spiral galaxy† -> spiral galaxy
11101: 11012,
// Little Little Princess† -> Little Little Princess
12100: 12002,
// CONTRACT† -> CONTRACT
13100: 13010,
// VANESSA† -> VANESSA
14100: 14009,
// KAMAITACHI† -> KAMAITACHI
14101: 14046,
// ICARUS† -> ICARUS
15101: 15023,
// THE DEEP STRIKER† -> THE DEEP STRIKER
15102: 15007,
// Blue Rain† -> Blue Rain
15104: 15004,
// Wanna Party?† -> Wanna Party?
15105: 15045,
// 凛として咲く花の如く† -> 凛として咲く花の如く
16101: 16050,
// THANK YOU FOR PLAYING† -> THANK YOU FOR PLAYING
16102: 16045,
// naughty girl@Queen's Palace† -> naughty girl@Queen's Palace
16103: 16031,
// Kung-fu Empire† -> Kung-fu Empire
16104: 16015,
// SOLID STATE SQUAD† -> SOLID STATE SQUAD
17101: 17060,
// Golden Palms† -> Golden Palms
18100: 18025,
// おおきなこえで† -> おおきなこえで
18103: 18011,
// QUANTUM TELEPORTATION† -> QUANTUM TELEPORTATION
19100: 19063,
// 朧† -> 朧
20103: 20100,
// 仮想空間の旅人たち† -> 仮想空間の旅人たち
20104: 20039,
// LUV CAN SAVE U† -> LUV CAN SAVE U
20105: 20068,
// Howling† -> Howling
20106: 20024,
// 龍と少女とデコヒーレンス† -> 龍と少女とデコヒーレンス
20107: 20019,
// Close the World feat.a☆ru†LEGGENDARIA -> Close the World feat. a☆ru
21100: 21012,
// Sigmund†LEGGENDARIA -> Sigmund
21101: 21059,
// Ancient Scapes†LEGGENDARIA -> Ancient Scapes
21102: 21069,
// invoker†LEGGENDARIA -> invoker
21103: 21073,
// Feel The Beat†LEGGENDARIA -> Feel The Beat
21104: 21052,
// 疾風迅雷†LEGGENDARIA -> 疾風迅雷
21105: 21048,
// Verflucht†LEGGENDARIA -> Verflucht
21106: 21050,
// 廿† -> 廿
21107: 21029,
// CHRONO DIVER -NORNIR-† -> CHRONO DIVER -NORNIR-
22101: 22008,
// chrono diver -fragment-† -> chrono diver -fragment-
22102: 22013,
// 恋は白帯、サンシロー† -> 恋は白帯、サンシロー
22103: 22024,
// Beat Radiance† -> Beat Radiance
22104: 22027,
// 超青少年ノ為ノ超多幸ナ超古典的超舞曲† -> 超青少年ノ為ノ超多幸ナ超古典的超舞曲
22105: 22031,
// EBONY & IVORY† -> EBONY & IVORY
22106: 22089,
// Cosmic Cat† -> Cosmic Cat
22107: 22006,
// Damage Per Second† -> Damage Per Second
23100: 23054,
// STARLIGHT DANCEHALL† -> STARLIGHT DANCEHALL
23101: 23031,
// Amazing Mirage† -> Amazing Mirage
24100: 24041,
// 冬椿 ft. Kanae Asaba† -> 冬椿 ft. Kanae Asaba
24101: 24011,
};
@@ -184,6 +229,7 @@ export const ConvertAPIKaiIIDX: ConverterFunction<unknown, KaiContext> = async (
musicID,
playtype,
score.difficulty,
// they send integers like 25, 27 - this will convert to our versions.
version as "20" | "21" | "22" | "23" | "24" | "25" | "26" | "27" | "28"
);
@@ -210,7 +256,7 @@ export const ConvertAPIKaiIIDX: ConverterFunction<unknown, KaiContext> = async (
const timeAchieved = ParseDateFromString(score.timestamp);
const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = {
const dryScore: DryScore<"iidx:DP" | "iidx:SP"> = {
comment: null,
game: "iidx",
importType,
@@ -1,15 +1,15 @@
import { KtLogger } from "lib/logger/logger";
import { KaiAuthDocument } from "tachi-common";
import nodeFetch from "utils/fetch";
import { ParserFunctionReturns } from "../../types";
import { CreateKaiIIDXClassHandler } from "./class-handler";
import { CreateKaiReauthFunction } from "../reauth";
import { TraverseKaiAPI } from "../traverse-api";
import { KaiContext } from "../types";
import { KaiTypeToBaseURL } from "../utils";
import { CreateKaiIIDXClassHandler } from "./class-handler";
import nodeFetch from "utils/fetch";
import type { ParserFunctionReturns } from "../../types";
import type { KaiContext } from "../types";
import type { KtLogger } from "lib/logger/logger";
import type { KaiAuthDocument } from "tachi-common";
export async function ParseKaiIIDX(
service: "FLO" | "EAG",
service: "EAG" | "FLO",
authDoc: KaiAuthDocument,
logger: KtLogger,
fetch = nodeFetch
@@ -1,11 +1,11 @@
import { GetKaiTypeClientCredentials, KaiTypeToBaseURL } from "./utils";
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error";
import p from "prudence";
import { KaiAuthDocument } from "tachi-common";
import nodeFetch from "utils/fetch";
import { CreateURLWithParams } from "utils/url";
import { GetKaiTypeClientCredentials, KaiTypeToBaseURL } from "./utils";
import type { KtLogger } from "lib/logger/logger";
import type { KaiAuthDocument } from "tachi-common";
const REAUTH_SCHEMA = {
access_token: "string",
@@ -13,7 +13,7 @@ const REAUTH_SCHEMA = {
};
export function CreateKaiReauthFunction(
kaiType: "FLO" | "EAG" | "MIN",
kaiType: "EAG" | "FLO" | "MIN",
authDoc: KaiAuthDocument,
logger: KtLogger,
fetch = nodeFetch
@@ -35,6 +35,7 @@ export function CreateKaiReauthFunction(
return async () => {
let res;
try {
const url = CreateURLWithParams(`${KaiTypeToBaseURL(kaiType)}/oauth/token`, {
refresh_token: authDoc.refreshToken,
@@ -63,6 +64,7 @@ export function CreateKaiReauthFunction(
let json;
/* istanbul ignore next */
try {
json = await res.json();
} catch (err) {
@@ -1,10 +1,10 @@
import { SDVXDans } from "lib/constants/classes";
import { ClassHandler } from "lib/score-import/framework/user-game-stats/types";
import nodeFetch from "utils/fetch";
import { KaiTypeToBaseURL } from "../utils";
import { SDVXDans } from "lib/constants/classes";
import nodeFetch from "utils/fetch";
import type { ClassHandler } from "lib/score-import/framework/user-game-stats/types";
export async function CreateKaiSDVXClassHandler(
kaiType: "FLO" | "EAG" | "MIN",
kaiType: "EAG" | "FLO" | "MIN",
token: string,
fetch = nodeFetch
): Promise<ClassHandler> {
@@ -1,8 +1,3 @@
import p from "prudence";
import { Lamps } from "tachi-common";
import { FormatPrError } from "utils/prudence";
import { FindSDVXChartOnInGameIDVersion } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import {
InternalFailure,
InvalidScoreFailure,
@@ -12,9 +7,14 @@ import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../../framework/common/score-utils";
import { DryScore } from "../../../../framework/common/types";
import { ConverterFunction } from "../../types";
import { KaiContext, KaiSDVXScore } from "../types";
import p from "prudence";
import { FormatPrError } from "utils/prudence";
import { FindSDVXChartOnInGameIDVersion } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import type { DryScore } from "../../../../framework/common/types";
import type { ConverterFunction } from "../../types";
import type { KaiContext, KaiSDVXScore } from "../types";
import type { Lamps } from "tachi-common";
const PR_KaiSDVXScore = {
music_id: p.isPositiveInteger,
@@ -1,15 +1,15 @@
import { KtLogger } from "lib/logger/logger";
import { KaiAuthDocument } from "tachi-common";
import nodeFetch from "utils/fetch";
import { ParserFunctionReturns } from "../../types";
import { CreateKaiSDVXClassHandler } from "./class-handler";
import { CreateKaiReauthFunction } from "../reauth";
import { TraverseKaiAPI } from "../traverse-api";
import { KaiContext } from "../types";
import { KaiTypeToBaseURL } from "../utils";
import { CreateKaiSDVXClassHandler } from "./class-handler";
import nodeFetch from "utils/fetch";
import type { ParserFunctionReturns } from "../../types";
import type { KaiContext } from "../types";
import type { KtLogger } from "lib/logger/logger";
import type { KaiAuthDocument } from "tachi-common";
export async function ParseKaiSDVX(
service: "FLO" | "EAG" | "MIN",
service: "EAG" | "FLO" | "MIN",
authDoc: KaiAuthDocument,
logger: KtLogger,
fetch = nodeFetch
@@ -1,9 +1,9 @@
import { VERSION_STR } from "lib/constants/version";
import { KtLogger } from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
import { URL } from "url";
import nodeFetch from "utils/fetch";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { VERSION_STR } from "lib/constants/version";
import { TachiConfig } from "lib/setup/config";
import nodeFetch from "utils/fetch";
import { URL } from "url";
import type { KtLogger } from "lib/logger/logger";
/**
* A Kai Reauth function is an async function that returns a string
@@ -64,8 +64,10 @@ export async function* TraverseKaiAPI(
let json;
let res;
// wrap all this in a try catch just incase the fetch or the res
// json call fails.
try {
// eslint-disable-next-line no-await-in-loop
res = await fetch(url, {
@@ -1,9 +1,9 @@
import { integer } from "tachi-common";
import type { integer } from "tachi-common";
export interface KaiIIDXScore {
music_id: integer;
play_style: "SINGLE" | "DOUBLE";
difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
play_style: "DOUBLE" | "SINGLE";
difficulty: "ANOTHER" | "BEGINNER" | "HYPER" | "LEGGENDARIA" | "NORMAL";
version_played: integer;
lamp: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7;
ex_score: integer;
@@ -31,5 +31,5 @@ export interface KaiSDVXScore {
}
export interface KaiContext {
service: "FLO" | "EAG" | "MIN";
service: "EAG" | "FLO" | "MIN";
}
@@ -1,27 +1,30 @@
import { ServerConfig } from "lib/setup/config";
export function KaiTypeToBaseURL(kaiType: "FLO" | "EAG" | "MIN") {
export function KaiTypeToBaseURL(kaiType: "EAG" | "FLO" | "MIN") {
if (kaiType === "FLO") {
if (!ServerConfig.FLO_API_URL) {
throw new Error(`Got kaiType FLO, but no API_URL was defined?`);
}
return ServerConfig.FLO_API_URL;
} else if (kaiType === "EAG") {
if (!ServerConfig.EAG_API_URL) {
throw new Error(`Got kaiType EAG, but no API_URL was defined?`);
}
return ServerConfig.EAG_API_URL;
} else if (kaiType === "MIN") {
if (!ServerConfig.MIN_API_URL) {
throw new Error(`Got kaiType MIN, but no API_URL was defined?`);
}
return ServerConfig.MIN_API_URL;
}
throw new Error(`Invalid Kai Type ${kaiType} provided.`);
}
export function GetKaiTypeClientCredentials(kaiType: "FLO" | "EAG" | "MIN") {
export function GetKaiTypeClientCredentials(kaiType: "EAG" | "FLO" | "MIN") {
if (kaiType === "FLO") {
return ServerConfig.FLO_OAUTH2_INFO;
} else if (kaiType === "EAG") {
@@ -1,7 +1,8 @@
import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error";
import { ClassHandler } from "lib/score-import/framework/user-game-stats/types";
import { GamePTConfig, GetGamePTConfig, IDStrings } from "tachi-common";
import { GameClasses, GameClassSets } from "tachi-common/js/game-classes";
import { GetGamePTConfig } from "tachi-common";
import type { ClassHandler } from "lib/score-import/framework/user-game-stats/types";
import type { GamePTConfig, IDStrings } from "tachi-common";
import type { GameClasses, GameClassSets } from "tachi-common/js/game-classes";
// Note: This is tested by batch-manuals parser.test.ts.
export function CreateBatchManualClassHandler(
@@ -1,20 +1,3 @@
import db from "external/mongo/db";
import { KtLogger } from "lib/logger/logger";
import {
BatchManualScore,
ChartDocument,
Grades,
IDStrings,
ImportTypes,
SongDocument,
} from "tachi-common";
import {
FindBMSChartOnHash,
FindChartWithPTDF,
FindChartWithPTDFVersion,
FindDDRChartOnSongHash,
} from "utils/queries/charts";
import { FindSongOnID, FindSongOnTitleInsensitive } from "utils/queries/songs";
import {
InternalFailure,
InvalidScoreFailure,
@@ -25,9 +8,27 @@ import {
AssertStrAsDifficulty,
AssertStrAsPositiveInt,
} from "../../../framework/common/string-asserts";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../types";
import { BatchManualContext } from "./types";
import db from "external/mongo/db";
import {
FindBMSChartOnHash,
FindChartWithPTDF,
FindChartWithPTDFVersion,
FindDDRChartOnSongHash,
} from "utils/queries/charts";
import { FindSongOnID, FindSongOnTitleInsensitive } from "utils/queries/songs";
import type { DryScore } from "../../../framework/common/types";
import type { ConverterFunction } from "../types";
import type { BatchManualContext } from "./types";
import type { KtLogger } from "lib/logger/logger";
import type {
BatchManualScore,
ChartDocument,
Grades,
IDStrings,
ImportTypes,
SongDocument,
} from "tachi-common";
/**
* Creates a ConverterFn for the BatchManualScore format. This curries
* the importType into the function, so the right failures can be
@@ -58,10 +59,7 @@ export const ConverterBatchManual: ConverterFunction<BatchManualScore, BatchManu
);
}
if (
data.percent > 100 &&
(chart as ChartDocument<"jubeat:Single">).data.isHardMode === false
) {
if (data.percent > 100 && !(chart as ChartDocument<"jubeat:Single">).data.isHardMode) {
throw new InvalidScoreFailure(`The percent field must be <= 100 for normal mode.`);
}
@@ -89,13 +87,13 @@ export const ConverterBatchManual: ConverterFunction<BatchManualScore, BatchManu
let service = context.service;
if (importType === "ir/direct-manual") {
service += " (DIRECT-MANUAL)";
service = `${service} (DIRECT-MANUAL)`;
} else if (importType === "file/batch-manual") {
service += " (BATCH-MANUAL)";
service = `${service} (BATCH-MANUAL)`;
}
const dryScore: DryScore = {
game: game,
game,
service,
comment: data.comment ?? null,
importType,
@@ -272,7 +270,9 @@ export async function ResolveMatchTypeToKTData(
}
let identifier: number | string = data.identifier;
// ddr uses weird strings as IDs instead of numbers
if (game !== "ddr") {
identifier = Number(data.identifier);
}
@@ -334,6 +334,7 @@ export async function ResolveMatchTypeToKTData(
logger.error(
`Invalid matchType ${data.matchType} ended up in conversion - should have been rejected by prudence?`
);
// really, this could be a larger error. - it's an internal failure because prudence should reject this.
throw new InvalidScoreFailure(`Invalid matchType ${data.matchType}.`);
}
@@ -1,21 +1,15 @@
import deepmerge from "deepmerge";
import { KtLogger } from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
import p, { PrudenceSchema, ValidSchemaValue } from "prudence";
import {
BatchManual,
BatchManualScore,
Game,
GetGameConfig,
GetGamePTConfig,
ImportTypes,
Playtype,
} from "tachi-common";
import { FormatPrError } from "utils/prudence";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../types";
import { CreateBatchManualClassHandler } from "./class-handler";
import { BatchManualContext } from "./types";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import deepmerge from "deepmerge";
import { TachiConfig } from "lib/setup/config";
import p from "prudence";
import { GetGameConfig, GetGamePTConfig } from "tachi-common";
import { FormatPrError } from "utils/prudence";
import type { ParserFunctionReturns } from "../types";
import type { BatchManualContext } from "./types";
import type { KtLogger } from "lib/logger/logger";
import type { PrudenceSchema, ValidSchemaValue } from "prudence";
import type { BatchManual, BatchManualScore, Game, ImportTypes, Playtype } from "tachi-common";
const optNull = (v: ValidSchemaValue) => p.optional(p.nullable(v));
@@ -150,6 +144,7 @@ const PR_HitMeta = (game: Game): PrudenceSchema => {
const PR_BatchManualScore = (game: Game, playtype: Playtype): PrudenceSchema => {
const gptConfig = GetGamePTConfig(game, playtype);
return {
score: p.isPositiveInteger,
lamp: p.isIn(gptConfig.lamps),
@@ -187,6 +182,7 @@ const PR_BatchManualScore = (game: Game, playtype: Playtype): PrudenceSchema =>
// @ts-expect-error shush
const v = self[key];
if ((!Number.isSafeInteger(v) || v < 0) && v !== null) {
return `Key ${key} had an invalid value of ${v} [${typeof v}]`;
}
@@ -264,9 +260,10 @@ export function ParseBatchManualFromObject(
// attempt to retrieve game
// @ts-expect-error man.
const possiblyGame = object?.meta?.game;
const possiblyGame = object.meta?.game;
// @ts-expect-error man.
const possiblyPlaytype = object?.meta?.playtype;
const possiblyPlaytype = object.meta?.playtype;
if (!possiblyGame) {
throw new ScoreImportFatalError(
@@ -326,10 +323,11 @@ export function ParseBatchManualFromObject(
version: batchManual.meta.version ?? null,
},
iterable: batchManual.scores,
// if classes are provided, use those as a class handler. Otherwise, we
// don't care.
classHandler: batchManual.classes
? CreateBatchManualClassHandler(batchManual.classes!)
? CreateBatchManualClassHandler(batchManual.classes)
: null,
};
}
@@ -1,4 +1,4 @@
import { Game, GPTSupportedVersions, IDStrings, Playtype } from "tachi-common";
import type { Game, GPTSupportedVersions, IDStrings, Playtype } from "tachi-common";
export interface BatchManualContext {
game: Game;
@@ -1,6 +1,3 @@
import { ChartDocument, Lamps } from "tachi-common";
import { FindChartWithPTDFVersion } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
import {
InvalidScoreFailure,
KTDataNotFoundFailure,
@@ -11,11 +8,14 @@ import {
ParseDateFromString,
} from "../../../framework/common/score-utils";
import { AssertStrAsPositiveInt } from "../../../framework/common/string-asserts";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../types";
import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
import { FindChartWithPTDFVersion } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
import type { DryScore } from "../../../framework/common/types";
import type { ConverterFunction } from "../types";
import type { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
import type { ChartDocument, Lamps } from "tachi-common";
const EAMUSEMENT_LAMP_RESOLVER: Map<string, Lamps["iidx:SP" | "iidx:DP"]> = new Map([
const EAMUSEMENT_LAMP_RESOLVER: Map<string, Lamps["iidx:DP" | "iidx:SP"]> = new Map([
["NO PLAY", "NO PLAY"],
["FAILED", "FAILED"],
["FULLCOMBO CLEAR", "FULL COMBO"],
@@ -50,7 +50,7 @@ const ConvertEamIIDXCSV: ConverterFunction<
// if pre-HV, leggendarias were stored in a wacky form.
if (!context.hasBeginnerAndLegg) {
// hack fix for legacy LEGGENDARIA titles
if (data.title.match(/(†|†LEGGENDARIA)$/u)) {
if (/(†|†LEGGENDARIA)$/u.exec(data.title)) {
data.title = data.title.replace(/(†|†LEGGENDARIA)$/u, "").trimEnd();
isLegacyLeggendaria = true;
}
@@ -82,7 +82,7 @@ const ConvertEamIIDXCSV: ConverterFunction<
context.playtype,
eamScore.difficulty,
context.importVersion
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
)) as ChartDocument<"iidx:DP" | "iidx:SP">;
if (!tachiChart) {
throw new KTDataNotFoundFailure(
@@ -135,7 +135,7 @@ const ConvertEamIIDXCSV: ConverterFunction<
const timestamp = ParseDateFromString(data.timestamp);
const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = {
const dryScore: DryScore<"iidx:DP" | "iidx:SP"> = {
service: context.service,
comment: null,
game: "iidx",
@@ -152,6 +152,7 @@ const ConvertEamIIDXCSV: ConverterFunction<
grade,
},
scoreMeta: {},
// japan is gmt+9
timeAchieved: timestamp ? timestamp - NINE_HOURS : null,
};
@@ -164,6 +165,7 @@ const ConvertEamIIDXCSV: ConverterFunction<
`${HUMANISED_CHART_TITLE} - Invalid BP of ${eamScore.bp}.`
);
}
dryScore.scoreData.hitMeta.bp = numBP;
} else if (eamScore.bp === "---") {
logger.debug(
@@ -1,8 +1,8 @@
import { KtLogger } from "lib/logger/logger";
import { CSVParseError, NaiveCSVParse } from "utils/naive-csv-parser";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../types";
import { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
import { CSVParseError, NaiveCSVParse } from "utils/naive-csv-parser";
import type { ParserFunctionReturns } from "../types";
import type { EamusementScoreData, IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./types";
import type { KtLogger } from "lib/logger/logger";
enum EAM_VERSION_NAMES {
"1st&substream" = 1,
@@ -72,7 +72,7 @@ const HV_HEADER_COUNT = 41;
// "timestamp",
// ];
export function ResolveHeaders(headers: string[], logger: KtLogger) {
export function ResolveHeaders(headers: Array<string>, logger: KtLogger) {
if (headers.length === PRE_HV_HEADER_COUNT) {
logger.verbose("PRE_HV csv received.");
return {
@@ -93,14 +93,16 @@ export function ResolveHeaders(headers: string[], logger: KtLogger) {
}
export function IIDXCSVParse(csvBuffer: Buffer, logger: KtLogger) {
let rawHeaders: string[];
let rawRows: string[][];
let rawHeaders: Array<string>;
let rawRows: Array<Array<string>>;
try {
({ rawHeaders, rawRows } = NaiveCSVParse(csvBuffer, logger));
} catch (e) {
if (e instanceof CSVParseError) {
throw new ScoreImportFatalError(400, e.message);
}
throw e;
}
@@ -134,7 +136,7 @@ export function IIDXCSVParse(csvBuffer: Buffer, logger: KtLogger) {
gameVersion = versionNum;
}
const scores: EamusementScoreData[] = [];
const scores: Array<EamusementScoreData> = [];
for (let d = 0; d < diffs.length; d++) {
const diff = diffs[d];
@@ -182,7 +184,7 @@ function GenericParseEamIIDXCSV(
service: string,
logger: KtLogger
): ParserFunctionReturns<IIDXEamusementCSVData, IIDXEamusementCSVContext> {
let playtype: "SP" | "DP";
let playtype: "DP" | "SP";
if (body.playtype === "SP") {
playtype = "SP";
@@ -1,7 +1,7 @@
import { GPTSupportedVersions } from "tachi-common";
import type { GPTSupportedVersions } from "tachi-common";
export interface EamusementScoreData {
difficulty: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
difficulty: "ANOTHER" | "BEGINNER" | "HYPER" | "LEGGENDARIA" | "NORMAL";
lamp: string;
exscore: string;
pgreat: string;
@@ -10,26 +10,27 @@ export interface EamusementScoreData {
level: string;
}
type BaseProps = {
interface BaseProps {
title: string;
timestamp: string;
}
export type IIDXEamusementCSVData = BaseProps & {
score: EamusementScoreData;
};
export type IIDXEamusementCSVData = {
score: EamusementScoreData;
} & BaseProps;
type Props = "bp" | "exscore" | "great" | "lamp" | "level" | "pgreat";
type Props = "exscore" | "lamp" | "pgreat" | "great" | "bp" | "level";
type RawPropKeys = `${"another" | "beginner" | "hyper" | "leggendaria" | "normal"}-${Props}`;
type RawPropKeys = `${"beginner" | "normal" | "hyper" | "another" | "leggendaria"}-${Props}`;
export type RawIIDXEamusementCSVData = {
[K in RawPropKeys]: unknown;
} & BaseProps & { [index: string]: unknown };
export type RawIIDXEamusementCSVData = BaseProps &
Record<string, unknown> & {
[K in RawPropKeys]: unknown;
};
export interface IIDXEamusementCSVContext {
playtype: "SP" | "DP";
importVersion: GPTSupportedVersions["iidx:SP" | "iidx:DP"];
playtype: "DP" | "SP";
importVersion: GPTSupportedVersions["iidx:DP" | "iidx:SP"];
hasBeginnerAndLegg: boolean;
service: string;
}
@@ -1,6 +1,22 @@
import { KtLogger } from "lib/logger/logger";
import { USCClientScore } from "server/router/ir/usc/_playtype/types";
import {
import type { ConverterFailure } from "../../framework/common/converter-failures";
import type { DryScore } from "../../framework/common/types";
import type { ClassHandler } from "../../framework/user-game-stats/types";
import type { SDVXEamusementCSVData } from "../file/eamusement-sdvx-csv/types";
import type { MerScore } from "../file/mer-iidx/types";
import type { S3Score } from "../file/solid-state-squad/types";
import type { BarbatosContext, BarbatosScore } from "../ir/barbatos/types";
import type { BeatorajaContext, BeatorajaScore } from "../ir/beatoraja/types";
import type { FervidexStaticContext, FervidexStaticScore } from "../ir/fervidex-static/types";
import type { FervidexContext, FervidexScore } from "../ir/fervidex/types";
import type { KsHookSV6CContext, KsHookSV6CScore } from "../ir/kshook-sv6c/types";
import type { LR2HookContext, LR2HookScore } from "../ir/lr2hook/types";
import type { IRUSCContext } from "../ir/usc/types";
import type { KaiContext } from "./api-kai/types";
import type { BatchManualContext } from "./batch-manual/types";
import type { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./eamusement-iidx-csv/types";
import type { KtLogger } from "lib/logger/logger";
import type { USCClientScore } from "server/router/ir/usc/_playtype/types";
import type {
BatchManualScore,
ChartDocument,
Game,
@@ -9,23 +25,7 @@ import {
MongoDBDocument,
SongDocument,
} from "tachi-common";
import { EmptyObject } from "utils/types";
import { ConverterFailure } from "../../framework/common/converter-failures";
import { DryScore } from "../../framework/common/types";
import { ClassHandler } from "../../framework/user-game-stats/types";
import { SDVXEamusementCSVData } from "../file/eamusement-sdvx-csv/types";
import { MerScore } from "../file/mer-iidx/types";
import { S3Score } from "../file/solid-state-squad/types";
import { BarbatosContext, BarbatosScore } from "../ir/barbatos/types";
import { BeatorajaContext, BeatorajaScore } from "../ir/beatoraja/types";
import { FervidexStaticContext, FervidexStaticScore } from "../ir/fervidex-static/types";
import { FervidexContext, FervidexScore } from "../ir/fervidex/types";
import { KsHookSV6CContext, KsHookSV6CScore } from "../ir/kshook-sv6c/types";
import { LR2HookContext, LR2HookScore } from "../ir/lr2hook/types";
import { IRUSCContext } from "../ir/usc/types";
import { KaiContext } from "./api-kai/types";
import { BatchManualContext } from "./batch-manual/types";
import { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./eamusement-iidx-csv/types";
import type { EmptyObject } from "utils/types";
export interface ImportTypeDataMap {
"file/eamusement-iidx-csv": IIDXEamusementCSVData;
@@ -100,24 +100,24 @@ export interface ConverterFnSuccessReturn {
export type ConverterFnReturnOrFailure = ConverterFailure | ConverterFnSuccessReturn;
export interface ConverterFunction<D, C> {
(
data: D,
processContext: C,
importType: ImportTypes,
logger: KtLogger
): Promise<ConverterFnSuccessReturn>;
}
export type ConverterFunction<D, C> = (
data: D,
processContext: C,
importType: ImportTypes,
logger: KtLogger
) => Promise<ConverterFnSuccessReturn>;
export interface ImportInputParser<D, C> {
(logger: KtLogger): ParserFunctionReturns<D, C> | Promise<ParserFunctionReturns<D, C>>;
}
export type ImportInputParser<D, C> = (
logger: KtLogger
) => ParserFunctionReturns<D, C> | Promise<ParserFunctionReturns<D, C>>;
export interface ParserFunctionReturns<D, C> {
iterable: Iterable<D> | AsyncIterable<D>;
iterable: AsyncIterable<D> | Iterable<D>;
context: C;
game: Game;
classHandler: ClassHandler | null;
}
export type ParserFunction<D, C, A extends unknown[]> = (...args: A) => ParserFunctionReturns<D, C>;
export type ParserFunction<D, C, A extends Array<unknown>> = (
...args: A
) => ParserFunctionReturns<D, C>;
@@ -1,11 +1,9 @@
import { ImportTypes } from "tachi-common";
import { ConvertAPIArcIIDX } from "./api/arc-iidx/converter";
import { ConvertAPIArcSDVX } from "./api/arc-sdvx/converter";
import { ConvertAPIKaiIIDX } from "./common/api-kai/iidx/converter";
import { ConvertAPIKaiSDVX } from "./common/api-kai/sdvx/converter";
import { ConverterBatchManual } from "./common/batch-manual/converter";
import ConvertEamIIDXCSV from "./common/eamusement-iidx-csv/converter";
import { ConverterFunction, ImportTypeContextMap, ImportTypeDataMap } from "./common/types";
import ConvertEamSDVXCSV from "./file/eamusement-sdvx-csv/converter";
import { ConvertFileMerIIDX } from "./file/mer-iidx/converter";
import { ConvertFileS3 } from "./file/solid-state-squad/converter";
@@ -16,6 +14,8 @@ import { ConverterIRFervidex } from "./ir/fervidex/converter";
import { ConverterIRKsHookSV6C } from "./ir/kshook-sv6c/converter";
import { ConverterLR2Hook } from "./ir/lr2hook/converter";
import { ConverterIRUSC } from "./ir/usc/converter";
import type { ConverterFunction, ImportTypeContextMap, ImportTypeDataMap } from "./common/types";
import type { ImportTypes } from "tachi-common";
export type ConverterMap = {
[I in ImportTypes]: ConverterFunction<ImportTypeDataMap[I], ImportTypeContextMap[I]>;
@@ -1,9 +1,9 @@
import { KtLogger } from "lib/logger/logger";
import { BatchManualScore } from "tachi-common";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParseBatchManualFromObject } from "../../common/batch-manual/parser";
import { BatchManualContext } from "../../common/batch-manual/types";
import { ParserFunctionReturns } from "../../common/types";
import type { BatchManualContext } from "../../common/batch-manual/types";
import type { ParserFunctionReturns } from "../../common/types";
import type { KtLogger } from "lib/logger/logger";
import type { BatchManualScore } from "tachi-common";
/**
* Parses a buffer of BATCH-MANUAL data.
@@ -22,7 +22,7 @@ function ParseBatchManual(
} catch (err) {
throw new ScoreImportFatalError(
400,
`Invalid JSON. (${(err as Error)?.message ?? "No Error Message Available."})`
`Invalid JSON. (${(err as Error).message ?? "No Error Message Available."})`
);
}
@@ -1,10 +1,10 @@
import { KtLogger } from "lib/logger/logger";
import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser";
import {
import type {
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
} from "../../common/eamusement-iidx-csv/types";
import { ParserFunctionReturns } from "../../common/types";
import type { ParserFunctionReturns } from "../../common/types";
import type { KtLogger } from "lib/logger/logger";
function ParseEamusementIIDXCSV(
fileData: Express.Multer.File,
@@ -4,13 +4,13 @@ import {
} from "lib/score-import/framework/common/converter-failures";
import { GenericGetGradeAndPercent } from "lib/score-import/framework/common/score-utils";
import { AssertStrAsPositiveInt } from "lib/score-import/framework/common/string-asserts";
import { DryScore } from "lib/score-import/framework/common/types";
import { Difficulties, Lamps } from "tachi-common";
import { FindChartWithPTDF } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
import { EmptyObject } from "utils/types";
import { ConverterFunction } from "../../common/types";
import { SDVXEamusementCSVData } from "./types";
import type { ConverterFunction } from "../../common/types";
import type { SDVXEamusementCSVData } from "./types";
import type { DryScore } from "lib/score-import/framework/common/types";
import type { Difficulties, Lamps } from "tachi-common";
import type { EmptyObject } from "utils/types";
const DIFFICULTY_MAP: Map<string, Difficulties["sdvx:Single"]> = new Map([
["NOVICE", "NOV"],
@@ -98,6 +98,7 @@ const ConvertEamSDVXCSV: ConverterFunction<SDVXEamusementCSVData, EmptyObject> =
service: "e-amusement",
game: "sdvx",
scoreMeta: {},
// No timestamp data :(
timeAchieved: null,
comment: null,
@@ -1,9 +1,9 @@
import { KtLogger } from "lib/logger/logger";
import { CSVParseError, NaiveCSVParse } from "utils/naive-csv-parser";
import { EmptyObject } from "utils/types";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../../common/types";
import { SDVXEamusementCSVData } from "./types";
import { CSVParseError, NaiveCSVParse } from "utils/naive-csv-parser";
import type { ParserFunctionReturns } from "../../common/types";
import type { SDVXEamusementCSVData } from "./types";
import type { KtLogger } from "lib/logger/logger";
import type { EmptyObject } from "utils/types";
const HEADER_COUNT = 11;
@@ -12,14 +12,16 @@ export default function ParseEamusementSDVXCSV(
_body: Record<string, unknown>,
logger: KtLogger
): ParserFunctionReturns<SDVXEamusementCSVData, EmptyObject> {
let rawHeaders: string[];
let rawRows: string[][];
let rawHeaders: Array<string>;
let rawRows: Array<Array<string>>;
try {
({ rawHeaders, rawRows } = NaiveCSVParse(fileData.buffer, logger));
} catch (e) {
if (e instanceof CSVParseError) {
throw new ScoreImportFatalError(400, e.message);
}
throw e;
}
@@ -37,9 +39,11 @@ export default function ParseEamusementSDVXCSV(
level: cells[2],
lamp: cells[3],
score: cells[5],
// @todo exscore is currently unused, but we should eventually store it.
// It is 0 if the score was played without S-criticals.
exscore: cells[6],
// The other columns (grade, # of different clears) are essentially useless.
// There is no timestamp :(
}));
@@ -1,7 +1,3 @@
import { Lamps } from "tachi-common";
import { FindIIDXChartOnInGameID } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import { EmptyObject } from "utils/types";
import {
InternalFailure,
KTDataNotFoundFailure,
@@ -10,9 +6,13 @@ import {
GenericGetGradeAndPercent,
ParseDateFromString,
} from "../../../framework/common/score-utils";
import { DryScore } from "../../../framework/common/types";
import { ConverterFunction } from "../../common/types";
import { MerScore } from "./types";
import { FindIIDXChartOnInGameID } from "utils/queries/charts";
import { FindSongOnID } from "utils/queries/songs";
import type { DryScore } from "../../../framework/common/types";
import type { ConverterFunction } from "../../common/types";
import type { MerScore } from "./types";
import type { Lamps } from "tachi-common";
import type { EmptyObject } from "utils/types";
function ConvertMERLamp(lamp: MerScore["clear_type"]): Lamps["iidx:DP" | "iidx:SP"] {
if (lamp === "FULLCOMBO CLEAR") {
@@ -54,7 +54,7 @@ export const ConvertFileMerIIDX: ConverterFunction<MerScore, EmptyObject> = asyn
const timeAchieved = ParseDateFromString(ConvertDateToJST(data.update_time));
const dryScore: DryScore<"iidx:SP" | "iidx:DP"> = {
const dryScore: DryScore<"iidx:DP" | "iidx:SP"> = {
game: "iidx",
comment: null,
importType: "file/mer-iidx",
@@ -1,10 +1,10 @@
import { KtLogger } from "lib/logger/logger";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import p from "prudence";
import { FormatPrError } from "utils/prudence";
import { EmptyObject } from "utils/types";
import ScoreImportFatalError from "../../../framework/score-importing/score-import-error";
import { ParserFunctionReturns } from "../../common/types";
import { MerScore } from "./types";
import type { ParserFunctionReturns } from "../../common/types";
import type { MerScore } from "./types";
import type { KtLogger } from "lib/logger/logger";
import type { EmptyObject } from "utils/types";
const PR_MerIIDX = {
music_id: p.isPositiveInteger,
@@ -55,7 +55,7 @@ export function ParseMerIIDX(
return {
classHandler: null,
context: {},
iterable: jsonData as MerScore[],
iterable: jsonData as Array<MerScore>,
game: "iidx",
};
}
@@ -1,19 +1,19 @@
import { integer } from "tachi-common";
import type { integer } from "tachi-common";
export interface MerScore {
music_id: integer;
play_type: "SINGLE" | "DOUBLE";
diff_type: "BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA";
play_type: "DOUBLE" | "SINGLE";
diff_type: "ANOTHER" | "BEGINNER" | "HYPER" | "LEGGENDARIA" | "NORMAL";
score: integer;
miss_count: integer;
clear_type:
| "NO PLAY"
| "FAILED"
| "ASSIST CLEAR"
| "EASY CLEAR"
| "CLEAR"
| "HARD CLEAR"
| "EASY CLEAR"
| "EX HARD CLEAR"
| "FULLCOMBO CLEAR";
| "FAILED"
| "FULLCOMBO CLEAR"
| "HARD CLEAR"
| "NO PLAY";
update_time: string;
}
@@ -1,10 +1,10 @@
import { KtLogger } from "lib/logger/logger";
import GenericParseEamIIDXCSV from "../../common/eamusement-iidx-csv/parser";
import {
import type {
IIDXEamusementCSVContext,
IIDXEamusementCSVData,
} from "../../common/eamusement-iidx-csv/types";
import { ParserFunctionReturns } from "../../common/types";
import type { ParserFunctionReturns } from "../../common/types";
import type { KtLogger } from "lib/logger/logger";
function ParsePLIIIDXCSV(
fileData: Express.Multer.File,

Some files were not shown because too many files have changed in this diff Show More