From ef162464dc2205bbfafa1ea77a5edc2571b7c581 Mon Sep 17 00:00:00 2001 From: zkldi Date: Sat, 31 Jul 2021 01:53:06 +0100 Subject: [PATCH 1/9] Fixes #259 --- server/src/main.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main.ts b/server/src/main.ts index 01ff7f216..bb6b98668 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -1,6 +1,6 @@ import CreateLogCtx from "lib/logger/logger"; import server from "server/server"; -import { CONF_INFO, LOG_LEVEL, PORT, ENABLE_SERVER_HTTPS } from "lib/setup/config"; +import { ServerTypeInfo, ServerConfig } from "lib/setup/config"; import https from "https"; import fs from "fs"; @@ -8,10 +8,10 @@ import { FormatVersion } from "./lib/constants/version"; const logger = CreateLogCtx(__filename); -logger.info(`Booting ${CONF_INFO.name} - ${FormatVersion()} [ENV: ${process.env.NODE_ENV}]`); -logger.info(`Log level is set to ${LOG_LEVEL}.`); +logger.info(`Booting ${ServerTypeInfo.name} - ${FormatVersion()} [ENV: ${process.env.NODE_ENV}]`); +logger.info(`Log level is set to ${ServerConfig.LOG_LEVEL}.`); -if (ENABLE_SERVER_HTTPS) { +if (ServerConfig.ENABLE_SERVER_HTTPS) { logger.warn( "HTTPS Mode is enabled. This should not be used in production, and you should instead run behind a reverse proxy." ); @@ -20,9 +20,9 @@ if (ENABLE_SERVER_HTTPS) { const httpsServer = https.createServer({ key: privateKey, cert: certificate }, server); - httpsServer.listen(PORT); - logger.info(`HTTPS Listening on port ${PORT}`); + httpsServer.listen(ServerConfig.PORT); + logger.info(`HTTPS Listening on port ${ServerConfig.PORT}`); } else { - server.listen(PORT); - logger.info(`HTTP Listening on port ${PORT}`); + server.listen(ServerConfig.PORT); + logger.info(`HTTP Listening on port ${ServerConfig.PORT}`); } From 5a6a334e69e0c184c7df9e2a31c688217c6f06eb Mon Sep 17 00:00:00 2001 From: zkldi Date: Sat, 31 Jul 2021 01:54:33 +0100 Subject: [PATCH 2/9] Config Code refactor --- server/src/external/mongo/db.ts | 8 ++-- server/src/external/mongo/indexes.ts | 4 +- server/src/external/mongo/schemas.ts | 6 +-- server/src/external/redis/redis-IPC.ts | 4 +- server/src/lib/cdn/cdn.ts | 4 +- server/src/lib/logger/logger.ts | 4 +- .../import-types/api/arc-ddr/parser.ts | 6 +-- .../api/arc-iidx/class-handler.ts | 20 ++++++--- .../import-types/api/arc-iidx/parser.ts | 8 ++-- .../import-types/api/arc-sdvx/parser.ts | 6 +-- .../common/api-kai/iidx/parser.ts | 4 +- .../common/api-kai/sdvx/parser.ts | 4 +- .../common/api-kai/traverse-api.ts | 6 +-- .../common/batch-manual/parser.ts | 10 +++-- server/src/lib/search/search.ts | 4 +- server/src/lib/setup/config.ts | 37 +++++----------- server/src/server/middleware/rate-limiter.ts | 4 +- server/src/server/middleware/type-require.ts | 6 +-- .../src/server/router/api/v1/admin/router.ts | 4 +- server/src/server/router/api/v1/auth/auth.ts | 4 +- .../router/api/v1/games/_game/middleware.ts | 4 +- .../server/router/api/v1/games/router.test.ts | 6 +-- .../src/server/router/api/v1/games/router.ts | 10 +++-- .../src/server/router/api/v1/import/router.ts | 4 +- server/src/server/server.ts | 44 ++++--------------- server/src/utils/misc.ts | 4 +- 26 files changed, 97 insertions(+), 128 deletions(-) diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index d678227ab..775dc39f5 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -29,14 +29,14 @@ import { SessionViewDocument, } from "tachi-common"; import monk, { TMiddleware } from "monk"; -import { MONGO_CONNECTION_URL, MONGO_DATABASE_NAME } from "lib/setup/config"; import CreateLogCtx from "lib/logger/logger"; import { OrphanScoreDocument } from "lib/score-import/import-types/common/types"; import { GetMilisecondsSince } from "utils/misc"; +import { ServerConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); -let dbName = MONGO_DATABASE_NAME; +let dbName = ServerConfig.MONGO_DATABASE_NAME; /* istanbul ignore next */ if (process.env.NODE_ENV === "test") { @@ -46,11 +46,11 @@ if (process.env.NODE_ENV === "test") { let dbtime: bigint; /* istanbul ignore next */ if (process.env.NODE_ENV !== "test") { - logger.info(`Connecting to database ${MONGO_CONNECTION_URL}/${dbName}...`); + logger.info(`Connecting to database ${ServerConfig.MONGO_CONNECTION_URL}/${dbName}...`); dbtime = process.hrtime.bigint(); } -export const monkDB = monk(`${MONGO_CONNECTION_URL}/${dbName}`); +export const monkDB = monk(`${ServerConfig.MONGO_CONNECTION_URL}/${dbName}`); /* istanbul ignore next */ monkDB diff --git a/server/src/external/mongo/indexes.ts b/server/src/external/mongo/indexes.ts index 5dad5ea6f..132fbf251 100644 --- a/server/src/external/mongo/indexes.ts +++ b/server/src/external/mongo/indexes.ts @@ -3,7 +3,7 @@ import monk from "monk"; import { IndexOptions } from "mongodb"; import { ValidDatabases } from "tachi-common"; import CreateLogCtx from "lib/logger/logger"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; import { ONE_DAY } from "lib/constants/time"; const logger = CreateLogCtx(__filename); @@ -89,7 +89,7 @@ const staticIndexes: Partial> = { const indexes: Partial> = staticIndexes; -for (const game of CONF_INFO.supportedGames) { +for (const game of ServerTypeInfo.supportedGames) { if (indexes[`charts-${game}` as ValidDatabases]) { indexes[`charts-${game}` as ValidDatabases]!.push( index({ chartID: 1 }, UNIQUE), diff --git a/server/src/external/mongo/schemas.ts b/server/src/external/mongo/schemas.ts index c05764395..61e3c9cee 100644 --- a/server/src/external/mongo/schemas.ts +++ b/server/src/external/mongo/schemas.ts @@ -1,7 +1,7 @@ import deepmerge from "deepmerge"; import { Game, Playtypes, GetGameConfig, GetGamePTConfig } from "tachi-common"; import p, { PrudenceSchema, ValidSchemaValue } from "prudence"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; const LAZY_EMAIL_REGEX = /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/u; @@ -87,7 +87,7 @@ export const PR_SCORE_GENERIC = { comment: p.nullable(p.isBoundedString(1, 240)), timeAdded: p.isPositive, scoreID: "string", // temp - importType: p.nullable(p.isIn(CONF_INFO.supportedImportTypes)), + importType: p.nullable(p.isIn(ServerTypeInfo.supportedImportTypes)), }; type ScoreSchemas = { @@ -409,7 +409,7 @@ const PRUDENCE_FOLDER_CHART_LOOKUP: PrudenceSchema = { const PRUDENCE_FOLDER: PrudenceSchema = { title: "string", - game: p.isIn(CONF_INFO.supportedGames), + game: p.isIn(ServerTypeInfo.supportedGames), playtype: (self, parent) => p.isIn(GetGameConfig(parent.game as Game).validPlaytypes)(self), folderID: "string", table: "string", diff --git a/server/src/external/redis/redis-IPC.ts b/server/src/external/redis/redis-IPC.ts index 31f0f57b5..6e37859b3 100644 --- a/server/src/external/redis/redis-IPC.ts +++ b/server/src/external/redis/redis-IPC.ts @@ -1,7 +1,7 @@ import redis from "redis"; import { RedisIPCChannels, RedisIPCData } from "tachi-common"; import CreateLogCtx from "lib/logger/logger"; -import { CONFIG } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); @@ -19,7 +19,7 @@ const SubCallbacks: Partial = {}; const SubClient = redis.createClient(); const PubClient = redis.createClient(); -const PREFIX = CONFIG.TYPE.toUpperCase(); // KTCHI or BTCHI +const PREFIX = ServerConfig.TYPE.toUpperCase(); // KTCHI or BTCHI export function RedisPub(channel: T, data: RedisIPCData[T]) { PubClient.publish(`${PREFIX}-${channel}`, JSON.stringify(data)); diff --git a/server/src/lib/cdn/cdn.ts b/server/src/lib/cdn/cdn.ts index a8575bd15..f36cc595a 100644 --- a/server/src/lib/cdn/cdn.ts +++ b/server/src/lib/cdn/cdn.ts @@ -1,10 +1,10 @@ import fs from "fs"; import path from "path"; -import { CDN_FILE_ROOT } from "lib/setup/config"; import CreateLogCtx from "lib/logger/logger"; import { promisify } from "util"; import mkdirp from "mkdirp"; import { Response } from "express"; +import { ServerConfig } from "lib/setup/config"; const readFilePromise = promisify(fs.readFile); const writeFilePromise = promisify(fs.writeFile); @@ -19,7 +19,7 @@ const logger = CreateLogCtx(__filename); * Path directory traversal *is* possible, and *will* ruin your day. */ function CDNRoot(fileLoc: string) { - return path.join(CDN_FILE_ROOT, fileLoc); + return path.join(ServerConfig.CDN_FILE_ROOT, fileLoc); } /** diff --git a/server/src/lib/logger/logger.ts b/server/src/lib/logger/logger.ts index f227afe55..74ac25f00 100644 --- a/server/src/lib/logger/logger.ts +++ b/server/src/lib/logger/logger.ts @@ -1,11 +1,11 @@ import winston, { format, transports, Logger, LeveledLogMethod } from "winston"; import { EscapeStringRegexp } from "utils/misc"; -import { LOG_LEVEL } from "lib/setup/config"; import SafeJSONStringify from "safe-json-stringify"; +import { ServerConfig } from "lib/setup/config"; export type KtLogger = Logger & { severe: LeveledLogMethod }; -const level = LOG_LEVEL ?? "info"; +const level = ServerConfig.LOG_LEVEL; const IN_TESTING = process.env.NODE_ENV === "test"; diff --git a/server/src/lib/score-import/import-types/api/arc-ddr/parser.ts b/server/src/lib/score-import/import-types/api/arc-ddr/parser.ts index df8d24640..3efa31c1c 100644 --- a/server/src/lib/score-import/import-types/api/arc-ddr/parser.ts +++ b/server/src/lib/score-import/import-types/api/arc-ddr/parser.ts @@ -3,7 +3,7 @@ import nodeFetch from "utils/fetch"; import { TraverseKaiAPI } from "../../common/api-kai/traverse-api"; import { ParserFunctionReturns } from "../../common/types"; import { EmptyObject } from "utils/types"; -import { ARC_API_URL, ARC_AUTH_TOKEN } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; export function ParseArcDDR( arcProfileID: string, @@ -12,10 +12,10 @@ export function ParseArcDDR( ): ParserFunctionReturns { return { iterable: TraverseKaiAPI( - ARC_API_URL, + ServerConfig.ARC_API_URL, // DDR Ace. `/api/v1/ddr/16/player_bests?profile_id=${arcProfileID}`, - ARC_AUTH_TOKEN, + ServerConfig.ARC_AUTH_TOKEN, logger, fetch ), diff --git a/server/src/lib/score-import/import-types/api/arc-iidx/class-handler.ts b/server/src/lib/score-import/import-types/api/arc-iidx/class-handler.ts index 04cf88b5f..1f21d67ea 100644 --- a/server/src/lib/score-import/import-types/api/arc-iidx/class-handler.ts +++ b/server/src/lib/score-import/import-types/api/arc-iidx/class-handler.ts @@ -1,8 +1,8 @@ import nodeFetch from "utils/fetch"; import { HasOwnProperty } from "utils/misc"; import { IIDXClasses } from "lib/constants/classes"; -import { ARC_API_URL } from "lib/setup/config"; import { ClassHandler } from "../../../framework/user-game-stats/types"; +import { ServerConfig } from "lib/setup/config"; export async function CreateArcIIDXClassHandler( profileID: string, @@ -16,11 +16,14 @@ export async function CreateArcIIDXClassHandler( // SP and DP dans are located in the same place, // fetch once, then return a function that traverses this data. try { - const res = await fetch(`${ARC_API_URL}/api/v1/iidx/27/profiles?_id=${profileID}`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); + const res = await fetch( + `${ServerConfig.ARC_API_URL}/api/v1/iidx/27/profiles?_id=${profileID}`, + { + headers: { + Authorization: `Bearer ${token}`, + }, + } + ); json = await res.json(); } catch (e) { @@ -29,7 +32,10 @@ export async function CreateArcIIDXClassHandler( return (game, playtype, userID, ratings, logger) => { if (err) { - logger.error(`An error occured while updating classes for ${ARC_API_URL}.`, { err }); + logger.error( + `An error occured while updating classes for ${ServerConfig.ARC_API_URL}.`, + { err } + ); return {}; } diff --git a/server/src/lib/score-import/import-types/api/arc-iidx/parser.ts b/server/src/lib/score-import/import-types/api/arc-iidx/parser.ts index ae73eb2b5..7c5218109 100644 --- a/server/src/lib/score-import/import-types/api/arc-iidx/parser.ts +++ b/server/src/lib/score-import/import-types/api/arc-iidx/parser.ts @@ -3,7 +3,7 @@ import nodeFetch from "utils/fetch"; import { TraverseKaiAPI } from "../../common/api-kai/traverse-api"; import { ParserFunctionReturns } from "../../common/types"; import { EmptyObject } from "utils/types"; -import { ARC_API_URL, ARC_AUTH_TOKEN } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; import { CreateArcIIDXClassHandler } from "./class-handler"; export async function ParseArcIIDX( @@ -13,15 +13,15 @@ export async function ParseArcIIDX( ): Promise> { return { iterable: TraverseKaiAPI( - ARC_API_URL, + ServerConfig.ARC_API_URL, // HEROIC VERSE `/api/v1/iidx/27/player_bests?profile_id=${arcProfileID}`, - ARC_AUTH_TOKEN, + ServerConfig.ARC_AUTH_TOKEN, logger, fetch ), context: {}, - classHandler: await CreateArcIIDXClassHandler(arcProfileID, ARC_AUTH_TOKEN), + classHandler: await CreateArcIIDXClassHandler(arcProfileID, ServerConfig.ARC_AUTH_TOKEN), game: "iidx", }; } diff --git a/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts b/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts index fbd98b68e..1bf0e6f68 100644 --- a/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts +++ b/server/src/lib/score-import/import-types/api/arc-sdvx/parser.ts @@ -3,7 +3,7 @@ import nodeFetch from "utils/fetch"; import { TraverseKaiAPI } from "../../common/api-kai/traverse-api"; import { ParserFunctionReturns } from "../../common/types"; import { EmptyObject } from "utils/types"; -import { ARC_API_URL, ARC_AUTH_TOKEN } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; export function ParseArcSDVX( arcProfileID: string, @@ -12,10 +12,10 @@ export function ParseArcSDVX( ): ParserFunctionReturns { return { iterable: TraverseKaiAPI( - ARC_API_URL, + ServerConfig.ARC_API_URL, // VIVID WAVE. `/api/v1/sdvx/5/player_bests?profile_id=${arcProfileID}`, - ARC_AUTH_TOKEN, + ServerConfig.ARC_AUTH_TOKEN, logger, fetch ), diff --git a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts index 65a7b6dfb..d042c1009 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts @@ -4,7 +4,7 @@ import { KaiAuthDocument } from "tachi-common"; import { KaiContext } from "../types"; import { TraverseKaiAPI } from "../traverse-api"; import { ParserFunctionReturns } from "../../types"; -import { FLO_API_URL, EAG_API_URL } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; export function ParseKaiIIDX( service: "FLO" | "EAG", @@ -12,7 +12,7 @@ export function ParseKaiIIDX( logger: KtLogger, fetch = nodeFetch ): ParserFunctionReturns { - const baseUrl = service === "FLO" ? FLO_API_URL : EAG_API_URL; + const baseUrl = service === "FLO" ? ServerConfig.FLO_API_URL : ServerConfig.EAG_API_URL; return { iterable: TraverseKaiAPI( diff --git a/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts b/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts index 836d44cca..a1d3fec6e 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts @@ -4,7 +4,7 @@ import { KaiAuthDocument } from "tachi-common"; import { KaiContext } from "../types"; import { TraverseKaiAPI } from "../traverse-api"; import { ParserFunctionReturns } from "../../types"; -import { EAG_API_URL, FLO_API_URL } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; export function ParseKaiSDVX( service: "FLO" | "EAG", @@ -12,7 +12,7 @@ export function ParseKaiSDVX( logger: KtLogger, fetch = nodeFetch ): ParserFunctionReturns { - const baseUrl = service === "FLO" ? FLO_API_URL : EAG_API_URL; + const baseUrl = service === "FLO" ? ServerConfig.FLO_API_URL : ServerConfig.EAG_API_URL; return { iterable: TraverseKaiAPI( diff --git a/server/src/lib/score-import/import-types/common/api-kai/traverse-api.ts b/server/src/lib/score-import/import-types/common/api-kai/traverse-api.ts index 45ac5595b..f7cd29876 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/traverse-api.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/traverse-api.ts @@ -2,7 +2,7 @@ import { KtLogger } from "lib/logger/logger"; import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; import nodeFetch from "utils/fetch"; import { VERSION_STR } from "lib/constants/version"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; /** * Traverses a Kai-like personal_bests api. @@ -42,7 +42,7 @@ export async function* TraverseKaiAPI( const res = await fetch(url, { headers: { Authorization: `Bearer ${token}`, - "User-Agent": `${CONF_INFO.name}/${VERSION_STR}`, + "User-Agent": `${ServerTypeInfo.name}/${VERSION_STR}`, }, }); @@ -61,7 +61,7 @@ export async function* TraverseKaiAPI( throw new ScoreImportFatalError( 500, - `Recieved no _links prop from ${url}. This is not an error with ${CONF_INFO.name}.` + `Recieved no _links prop from ${url}. This is not an error with ${ServerTypeInfo.name}.` ); } diff --git a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts index 948be8fc0..cc1d315cc 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts @@ -6,7 +6,7 @@ import { Game, ImportTypes, GetGamePTConfig, Playtypes, GetGameConfig } from "ta import deepmerge from "deepmerge"; import { FormatPrError } from "utils/prudence"; import { ParserFunctionReturns } from "../types"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; const optNull = (v: ValidSchemaValue) => p.optional(p.nullable(v)); @@ -95,7 +95,7 @@ const PR_BatchManualScore = (game: Game, playtype: Playtypes[Game]): PrudenceSch const PR_BatchManual = (game: Game, playtype: Playtypes[Game]): PrudenceSchema => ({ meta: { service: p.isBoundedString(3, 15), - game: p.isIn(CONF_INFO.supportedGames), + game: p.isIn(ServerTypeInfo.supportedGames), playtype: p.is(playtype), version: "*?string", }, @@ -144,10 +144,12 @@ export function ParseBatchManualFromObject( ); } - if (!CONF_INFO.supportedGames.includes(possiblyGame)) { + if (!ServerTypeInfo.supportedGames.includes(possiblyGame)) { throw new ScoreImportFatalError( 400, - `Invalid game ${possiblyGame} - expected any of ${CONF_INFO.supportedGames.join(", ")}.` + `Invalid game ${possiblyGame} - expected any of ${ServerTypeInfo.supportedGames.join( + ", " + )}.` ); } diff --git a/server/src/lib/search/search.ts b/server/src/lib/search/search.ts index ffa29612d..98cb50082 100644 --- a/server/src/lib/search/search.ts +++ b/server/src/lib/search/search.ts @@ -14,7 +14,7 @@ import { } from "tachi-common"; import { EscapeStringRegexp } from "utils/misc"; import { GetOnlineCutoff, OMIT_PRIVATE_USER_RETURNS } from "utils/user"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; const logger = CreateLogCtx(__filename); @@ -165,7 +165,7 @@ async function SearchAllGamesSingleGame(game: Game, search: string) { export async function SearchAllGamesSongs(search: string) { const promises = []; - for (const game of CONF_INFO.supportedGames) { + for (const game of ServerTypeInfo.supportedGames) { promises.push(SearchAllGamesSingleGame(game, search)); } diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index be91b367a..29a5e55e5 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -50,11 +50,10 @@ export interface TachiConfig { CDN_FILE_ROOT: string; TYPE: "ktchi" | "btchi" | "omni"; PORT: integer; - CLIENT_INDEX_HTML_PATH: string; - ENABLE_SERVER_HTTPS: boolean; - RUN_OWN_CDN: boolean; + ENABLE_SERVER_HTTPS?: boolean; + RUN_OWN_CDN?: boolean; CLIENT_DEV_SERVER?: string | null; - TYPE_INFO: StaticConfig.ServerConfig; + SERVER_TYPE_INFO: StaticConfig.ServerConfig; } const err = p(config, { @@ -69,9 +68,8 @@ const err = p(config, { ARC_AUTH_TOKEN: "string", CDN_FILE_ROOT: "string", PORT: p.isPositiveInteger, - CLIENT_INDEX_HTML_PATH: "string", - ENABLE_SERVER_HTTPS: "boolean", - RUN_OWN_CDN: "boolean", + ENABLE_SERVER_HTTPS: "*boolean", + RUN_OWN_CDN: "*boolean", CLIENT_DEV_SERVER: "*?string", TYPE: p.isIn("ktchi", "btchi", "omni"), }); @@ -81,11 +79,11 @@ if (err) { } if (config.TYPE === "ktchi") { - config.TYPE_INFO = StaticConfig.KTCHI_CONFIG; + config.SERVER_TYPE_INFO = StaticConfig.KTCHI_CONFIG; } else if (config.TYPE === "btchi") { - config.TYPE_INFO = StaticConfig.BTCHI_CONFIG; + config.SERVER_TYPE_INFO = StaticConfig.BTCHI_CONFIG; } else if (config.TYPE === "omni") { - config.TYPE_INFO = StaticConfig.OMNI_CONFIG; + config.SERVER_TYPE_INFO = StaticConfig.OMNI_CONFIG; } const tachiConfig = config as TachiConfig; @@ -98,20 +96,5 @@ try { process.exit(1); } -export const MONGO_CONNECTION_URL = tachiConfig.MONGO_CONNECTION_URL; -export const MONGO_DATABASE_NAME = tachiConfig.MONGO_DATABASE_NAME; -export const LOG_LEVEL = tachiConfig.LOG_LEVEL; -export const SESSION_SECRET = tachiConfig.SESSION_SECRET; -export const CAPTCHA_SECRET_KEY = tachiConfig.CAPTCHA_SECRET_KEY; -export const FLO_API_URL = tachiConfig.FLO_API_URL; -export const EAG_API_URL = tachiConfig.EAG_API_URL; -export const ARC_API_URL = tachiConfig.ARC_API_URL; -export const ARC_AUTH_TOKEN = tachiConfig.ARC_AUTH_TOKEN; -export const CDN_FILE_ROOT = tachiConfig.CDN_FILE_ROOT; -export const CONF_INFO = tachiConfig.TYPE_INFO; -export const PORT = tachiConfig.PORT; -export const CONFIG = tachiConfig; -export const CLIENT_INDEX_HTML_PATH = tachiConfig.CLIENT_INDEX_HTML_PATH; -export const ENABLE_SERVER_HTTPS = tachiConfig.ENABLE_SERVER_HTTPS; -export const RUN_OWN_CDN = tachiConfig.RUN_OWN_CDN; -export const CLIENT_DEV_SERVER = tachiConfig.CLIENT_DEV_SERVER; +export const ServerTypeInfo = tachiConfig.SERVER_TYPE_INFO; +export const ServerConfig = tachiConfig; diff --git a/server/src/server/middleware/rate-limiter.ts b/server/src/server/middleware/rate-limiter.ts index 5af644109..9de6072bc 100644 --- a/server/src/server/middleware/rate-limiter.ts +++ b/server/src/server/middleware/rate-limiter.ts @@ -1,5 +1,5 @@ import RateLimitRedis from "rate-limit-redis"; -import { CONFIG } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; import rateLimit from "express-rate-limit"; import CreateLogCtx from "lib/logger/logger"; import { RedisClient } from "external/redis/redis"; @@ -8,7 +8,7 @@ const logger = CreateLogCtx(__filename); const store = process.env.NODE_ENV === "production" - ? new RateLimitRedis({ prefix: `${CONFIG.TYPE}-RL:`, client: RedisClient }) + ? new RateLimitRedis({ prefix: `${ServerConfig.TYPE}-RL:`, client: RedisClient }) : undefined; // undefined forces a default to an in-memory store export function ClearTestingRateLimitCache() { diff --git a/server/src/server/middleware/type-require.ts b/server/src/server/middleware/type-require.ts index 753d699a8..721a760cd 100644 --- a/server/src/server/middleware/type-require.ts +++ b/server/src/server/middleware/type-require.ts @@ -1,4 +1,4 @@ -import { CONFIG } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; import { RequestHandler } from "express"; /** @@ -7,7 +7,7 @@ import { RequestHandler } from "express"; * this restriction is bypassed. */ export const RequireBokutachi: RequestHandler = (req, res, next) => { - if (CONFIG.TYPE === "btchi" || CONFIG.TYPE === "omni") { + if (ServerConfig.TYPE === "btchi" || ServerConfig.TYPE === "omni") { return next(); } @@ -23,7 +23,7 @@ export const RequireBokutachi: RequestHandler = (req, res, next) => { * this restriction is bypassed. */ export const RequireKamaitachi: RequestHandler = (req, res, next) => { - if (CONFIG.TYPE === "ktchi" || CONFIG.TYPE === "omni") { + if (ServerConfig.TYPE === "ktchi" || ServerConfig.TYPE === "omni") { return next(); } diff --git a/server/src/server/router/api/v1/admin/router.ts b/server/src/server/router/api/v1/admin/router.ts index 3940ccfaf..ca0f28957 100644 --- a/server/src/server/router/api/v1/admin/router.ts +++ b/server/src/server/router/api/v1/admin/router.ts @@ -5,7 +5,7 @@ import CreateLogCtx, { ChangeRootLogLevel, GetLogLevel } from "lib/logger/logger import prValidate from "server/middleware/prudence-validate"; import { GetUserWithID } from "utils/user"; import { ONE_MINUTE } from "lib/constants/time"; -import { LOG_LEVEL } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); @@ -42,6 +42,8 @@ const RequireAdminLevel: RequestHandler = async (req, res, next) => { return next(); }; +const LOG_LEVEL = ServerConfig.LOG_LEVEL; + router.use(RequireAdminLevel); let currentLogLevelTimer: NodeJS.Timeout | null = null; diff --git a/server/src/server/router/api/v1/auth/auth.ts b/server/src/server/router/api/v1/auth/auth.ts index 33e167f7f..be994f027 100644 --- a/server/src/server/router/api/v1/auth/auth.ts +++ b/server/src/server/router/api/v1/auth/auth.ts @@ -6,8 +6,8 @@ import { GetNextCounterValue } from "utils/db"; import CreateLogCtx from "lib/logger/logger"; import { FormatUserDoc } from "utils/user"; import nodeFetch from "utils/fetch"; -import { CAPTCHA_SECRET_KEY } from "lib/setup/config"; import { Random20Hex } from "utils/misc"; +import { ServerConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); @@ -104,7 +104,7 @@ export async function ValidateCaptcha( fetch = nodeFetch ) { const r = await fetch( - `https://www.google.com/recaptcha/api/siteverify?secret=${CAPTCHA_SECRET_KEY}&response=${recaptcha}&remoteip=${remoteAddr}` + `https://www.google.com/recaptcha/api/siteverify?secret=${ServerConfig.CAPTCHA_SECRET_KEY}&response=${recaptcha}&remoteip=${remoteAddr}` ); if (r.status !== 200) { diff --git a/server/src/server/router/api/v1/games/_game/middleware.ts b/server/src/server/router/api/v1/games/_game/middleware.ts index 7db2ad012..eaa50a1ce 100644 --- a/server/src/server/router/api/v1/games/_game/middleware.ts +++ b/server/src/server/router/api/v1/games/_game/middleware.ts @@ -1,5 +1,5 @@ import { RequestHandler } from "express"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; import { IsValidGame } from "utils/misc"; import { AssignToReqTachiData } from "utils/req-tachi-data"; @@ -9,7 +9,7 @@ export const ValidateGameFromParam: RequestHandler = (req, res, next) => { success: false, description: `Invalid/unsupported game ${ req.params.game - } - Expected any of ${CONF_INFO.supportedGames.join(", ")}`, + } - Expected any of ${ServerTypeInfo.supportedGames.join(", ")}`, }); } diff --git a/server/src/server/router/api/v1/games/router.test.ts b/server/src/server/router/api/v1/games/router.test.ts index 35e2bd788..5e4b7959f 100644 --- a/server/src/server/router/api/v1/games/router.test.ts +++ b/server/src/server/router/api/v1/games/router.test.ts @@ -1,6 +1,6 @@ import { GetGameConfig } from "tachi-common"; import t from "tap"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; import { CloseAllConnections } from "test-utils/close-connections"; import mockApi from "test-utils/mock-api"; @@ -9,10 +9,10 @@ t.test("GET /api/v1/games", async (t) => { // and also returns configs properly. const res = await mockApi.get("/api/v1/games"); - t.strictSame(res.body.body.supportedGames, CONF_INFO.supportedGames); + t.strictSame(res.body.body.supportedGames, ServerTypeInfo.supportedGames); t.strictSame(res.body.body.configs.iidx, GetGameConfig("iidx")); - t.equal(Object.keys(res.body.body.configs).length, CONF_INFO.supportedGames.length); + t.equal(Object.keys(res.body.body.configs).length, ServerTypeInfo.supportedGames.length); t.end(); }); diff --git a/server/src/server/router/api/v1/games/router.ts b/server/src/server/router/api/v1/games/router.ts index 0a9009c91..b85e13205 100644 --- a/server/src/server/router/api/v1/games/router.ts +++ b/server/src/server/router/api/v1/games/router.ts @@ -1,5 +1,5 @@ import { Router } from "express"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; import { GetGameConfig } from "tachi-common"; import gameRouter from "./_game/router"; @@ -13,13 +13,15 @@ const router: Router = Router({ mergeParams: true }); */ router.get("/", (req, res) => { // this line is a bit too 'smart' for its own good, but whatever. - const configs = Object.fromEntries(CONF_INFO.supportedGames.map((e) => [e, GetGameConfig(e)])); + const configs = Object.fromEntries( + ServerTypeInfo.supportedGames.map((e) => [e, GetGameConfig(e)]) + ); return res.status(200).json({ success: true, - description: `Returned support information for ${CONF_INFO.supportedGames.length} game(s).`, + description: `Returned support information for ${ServerTypeInfo.supportedGames.length} game(s).`, body: { - supportedGames: CONF_INFO.supportedGames, + supportedGames: ServerTypeInfo.supportedGames, configs, }, }); diff --git a/server/src/server/router/api/v1/import/router.ts b/server/src/server/router/api/v1/import/router.ts index 1e955f230..558384cf4 100644 --- a/server/src/server/router/api/v1/import/router.ts +++ b/server/src/server/router/api/v1/import/router.ts @@ -14,7 +14,7 @@ import ParseBatchManual from "lib/score-import/import-types/file/batch-manual/pa import { ParseSolidStateXML } from "lib/score-import/import-types/file/solid-state-squad/parser"; import { ParseMerIIDX } from "lib/score-import/import-types/file/mer-iidx/parser"; import ParsePLIIIDXCSV from "lib/score-import/import-types/file/pli-iidx-csv/parser"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; import { RequirePermissions } from "server/middleware/auth"; const logger = CreateLogCtx(__filename); @@ -37,7 +37,7 @@ router.post( ParseMultipartScoredata, prValidate( { - importType: Prudence.isIn(CONF_INFO.supportedImportTypes), + importType: Prudence.isIn(ServerTypeInfo.supportedImportTypes), }, {}, { allowExcessKeys: true } diff --git a/server/src/server/server.ts b/server/src/server/server.ts index c2a98528e..d4004670b 100644 --- a/server/src/server/server.ts +++ b/server/src/server/server.ts @@ -3,15 +3,7 @@ import "express-async-errors"; import expressSession from "express-session"; import { integer } from "tachi-common"; import { RedisClient } from "external/redis/redis"; -import { - CDN_FILE_ROOT, - CLIENT_DEV_SERVER, - CLIENT_INDEX_HTML_PATH, - CONFIG, - ENABLE_SERVER_HTTPS, - RUN_OWN_CDN, - SESSION_SECRET, -} from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; import connectRedis from "connect-redis"; import helmet from "helmet"; import fs from "fs"; @@ -35,24 +27,24 @@ const userSessionMiddleware = expressSession({ // append node_env onto the end of the session name // so we can separate tokens under the same URL. // say for staging.kamaitachi.xyz - name: `${CONFIG.TYPE}_${process.env.NODE_ENV}_session`, - secret: SESSION_SECRET, + name: `${ServerConfig.TYPE}_${process.env.NODE_ENV}_session`, + secret: ServerConfig.SESSION_SECRET, store, resave: true, saveUninitialized: false, cookie: { - secure: process.env.NODE_ENV === "production" || ENABLE_SERVER_HTTPS, + secure: process.env.NODE_ENV === "production" || ServerConfig.ENABLE_SERVER_HTTPS, }, }); const app: Express = express(); -if (process.env.NODE_ENV !== "production" && CLIENT_DEV_SERVER) { - logger.warn(`Enabling CORS requests from ${CLIENT_DEV_SERVER}.`); +if (process.env.NODE_ENV !== "production" && ServerConfig.CLIENT_DEV_SERVER) { + logger.warn(`Enabling CORS requests from ${ServerConfig.CLIENT_DEV_SERVER}.`); // Allow CORS requests from another server (since we have our dev server hosted separately). app.use((req, res, next) => { - res.header("Access-Control-Allow-Origin", CLIENT_DEV_SERVER!); + res.header("Access-Control-Allow-Origin", ServerConfig.CLIENT_DEV_SERVER!); res.header( "Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept" @@ -112,7 +104,7 @@ app.use("/", mainRouter); // The RUN_OWN_CDN option means that our /cdn path has to be hosted by us. In production, // this is not the case (we have a dedicated nginx box for it running in a separate process). // In dev, this is a pain to setup, so we can just run it locally. -if (RUN_OWN_CDN) { +if (ServerConfig.RUN_OWN_CDN) { if (process.env.NODE_ENV === "production") { logger.warn( `Running OWN_CDN in production. Consider making a separate process handle your CDN for performance.` @@ -121,28 +113,10 @@ if (RUN_OWN_CDN) { logger.info("Running own CDN."); } - app.use("/cdn", express.static(CDN_FILE_ROOT)); + app.use("/cdn", express.static(ServerConfig.CDN_FILE_ROOT)); app.get("/cdn/*", (req, res) => res.status(404).send("No content here.")); } -const indexHTMLContent = fs.readFileSync(CLIENT_INDEX_HTML_PATH); - -/** - * If any user gets to this point, we send them index.html and yield routing to - * the ktchi-client react router. - * @name GET * - */ -app.get("*", (req, res) => res.setHeader("Content-Type", "text/html").send(indexHTMLContent)); - -/** - * If any user gets to this point, and are not making a GET request, they are probably making an API request. - * And want JSON back. - * @name ALL (except GET) * - */ -app.all("*", (req, res) => - res.status(404).json({ success: false, description: "404: This URL does not exist." }) -); - // completely stolen from ktapi error handler interface ExpressJSONErr extends SyntaxError { status: integer; diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 3e41da231..3fbe19e81 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -1,5 +1,5 @@ import crypto from "crypto"; -import { CONF_INFO } from "lib/setup/config"; +import { ServerTypeInfo } from "lib/setup/config"; import { Game, Playtypes, GetGameConfig, AnySongDocument, AnyChartDocument } from "tachi-common"; // https://github.com/sindresorhus/escape-string-regexp/blob/main/index.js @@ -62,7 +62,7 @@ export function HasOwnProperty(obj: T, key: string | number | symbol): key is } export function IsValidGame(str: string): str is Game { - return !!CONF_INFO.supportedGames.includes(str as Game); + return !!ServerTypeInfo.supportedGames.includes(str as Game); } export function IsValidPlaytype(game: Game, str: string): str is Playtypes[Game] { From 8993a27ff4772ffa16b98531465af2547eb7bb8a Mon Sep 17 00:00:00 2001 From: zkldi Date: Sat, 31 Jul 2021 01:59:44 +0100 Subject: [PATCH 3/9] remove index html tests and update github conf --- server/.github/test.conf.json5 | 3 +-- server/src/server/server.test.ts | 17 ----------------- server/src/test-utils/resets.ts | 6 +++--- 3 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 server/src/server/server.test.ts diff --git a/server/.github/test.conf.json5 b/server/.github/test.conf.json5 index 5e6d9afab..9c07fc144 100644 --- a/server/.github/test.conf.json5 +++ b/server/.github/test.conf.json5 @@ -12,8 +12,7 @@ ARC_AUTH_TOKEN: "unused", CDN_FILE_ROOT: "./local-cdn", PORT: 8080, - TYPE: "omni", // It really doesn't matter whether we're in bokutachi or kamaitachi mode. If we had the performance we could run these in parallel? - CLIENT_INDEX_HTML_PATH: "./src/test-utils/mock-index.html", + TYPE: "omni", ENABLE_SERVER_HTTPS: false, RUN_OWN_CDN: true } diff --git a/server/src/server/server.test.ts b/server/src/server/server.test.ts deleted file mode 100644 index ee563c68e..000000000 --- a/server/src/server/server.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import t from "tap"; -import { CloseAllConnections } from "test-utils/close-connections"; -import mockApi from "test-utils/mock-api"; - -t.test("GET /", (t) => { - t.test("Should return our index.html", async (t) => { - const res = await mockApi.get("/"); - - t.equal(res.text, "
Hello World.
"); - - t.end(); - }); - - t.end(); -}); - -t.teardown(CloseAllConnections); diff --git a/server/src/test-utils/resets.ts b/server/src/test-utils/resets.ts index 2121ce1fc..62cf090e2 100644 --- a/server/src/test-utils/resets.ts +++ b/server/src/test-utils/resets.ts @@ -7,8 +7,8 @@ import path from "path"; import CreateLogCtx from "lib/logger/logger"; // im installing an entire library for rm rf... import rimraf from "rimraf"; -import { CDN_FILE_ROOT, MONGO_CONNECTION_URL } from "lib/setup/config"; import { SetIndexes } from "external/mongo/indexes"; +import { ServerConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); @@ -79,7 +79,7 @@ export default async function ResetDBState() { export function ResetCDN() { return new Promise((resolve, reject) => - rimraf(CDN_FILE_ROOT, (err) => { + rimraf(ServerConfig.CDN_FILE_ROOT, (err) => { if (err) { reject(err); } @@ -89,7 +89,7 @@ export function ResetCDN() { } export async function SetIndexesForDB() { - const url = `${MONGO_CONNECTION_URL}/testingdb`; + const url = `${ServerConfig.MONGO_CONNECTION_URL}/testingdb`; logger.info(`Setting indexes for ${url}`); From e145ae60d2a07b48ab5184403ecc42e31a906f8d Mon Sep 17 00:00:00 2001 From: zkldi Date: Sat, 31 Jul 2021 20:42:45 +0100 Subject: [PATCH 4/9] remove client_index_static_html binding --- server/src/lib/setup/config.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index 29a5e55e5..e3cb6eebc 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -88,13 +88,5 @@ if (config.TYPE === "ktchi") { const tachiConfig = config as TachiConfig; -// note: we dont use fs.exists here because the file may be moved underfoot. -try { - fs.readFileSync(config.CLIENT_INDEX_HTML_PATH); -} catch (err) { - logger.error(`Error while opening file at CLIENT_INDEX_HTML_PATH. Is one here?`); - process.exit(1); -} - export const ServerTypeInfo = tachiConfig.SERVER_TYPE_INFO; export const ServerConfig = tachiConfig; From f8f8d58defeec8fd4e53349bf0111f6b0f54cc0f Mon Sep 17 00:00:00 2001 From: zkldi Date: Wed, 4 Aug 2021 19:21:25 +0100 Subject: [PATCH 5/9] Remove ability for server to serve index.html Fixes #259 --- server/src/lib/constants/classes.ts | 13 +++++++ .../user-game-stats/builtin-class-handlers.ts | 37 ++++++++++++++++++- .../framework/user-game-stats/classes.ts | 10 ++++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/server/src/lib/constants/classes.ts b/server/src/lib/constants/classes.ts index 1c9f3161a..8f36f2074 100644 --- a/server/src/lib/constants/classes.ts +++ b/server/src/lib/constants/classes.ts @@ -159,3 +159,16 @@ export enum SDVXVFClasses { IMPERIAL_III, IMPERIAL_IV, } + +export enum CHUNITHM_COLOURS { + BLUE, + GREEN, + ORANGE, + RED, + PURPLE, + BRONZE, + SILVER, + GOLD, + PLATINUM, + RAINBOW, +} diff --git a/server/src/lib/score-import/framework/user-game-stats/builtin-class-handlers.ts b/server/src/lib/score-import/framework/user-game-stats/builtin-class-handlers.ts index 97baf1cb3..a7b58cd70 100644 --- a/server/src/lib/score-import/framework/user-game-stats/builtin-class-handlers.ts +++ b/server/src/lib/score-import/framework/user-game-stats/builtin-class-handlers.ts @@ -1,6 +1,6 @@ import { Game, Playtypes, integer, ScoreCalculatedDataLookup } from "tachi-common"; import { GameClasses } from "tachi-common/js/game-classes"; -import { GitadoraColours, SDVXVFClasses } from "lib/constants/classes"; +import { CHUNITHM_COLOURS, GitadoraColours, SDVXVFClasses } from "lib/constants/classes"; import { KtLogger } from "lib/logger/logger"; export function CalculateSDVXClass( @@ -94,6 +94,41 @@ export function GitadoraSkillToColour(sk: number) { return GitadoraColours.WHITE; } +export function CalculateChunithmColour( + game: Game, + playtype: Playtypes[Game], + userID: integer, + ratings: Record +) { + const colour = ChuniRatingToColour(ratings.naiveRating); + + return { colour }; +} + +function ChuniRatingToColour(rating: number) { + if (rating >= 15) { + return CHUNITHM_COLOURS.RAINBOW; + } else if (rating >= 14.5) { + return CHUNITHM_COLOURS.PLATINUM; + } else if (rating >= 14) { + return CHUNITHM_COLOURS.GOLD; + } else if (rating >= 13) { + return CHUNITHM_COLOURS.SILVER; + } else if (rating >= 12) { + return CHUNITHM_COLOURS.BRONZE; + } else if (rating >= 10) { + return CHUNITHM_COLOURS.PURPLE; + } else if (rating >= 7) { + return CHUNITHM_COLOURS.RED; + } else if (rating >= 4) { + return CHUNITHM_COLOURS.ORANGE; + } else if (rating >= 2) { + return CHUNITHM_COLOURS.GREEN; + } + + return CHUNITHM_COLOURS.BLUE; +} + // export function CalculateJubeatColour( // game: Game, // playtype: Playtypes[Game], diff --git a/server/src/lib/score-import/framework/user-game-stats/classes.ts b/server/src/lib/score-import/framework/user-game-stats/classes.ts index 95cca8fca..882d350e8 100644 --- a/server/src/lib/score-import/framework/user-game-stats/classes.ts +++ b/server/src/lib/score-import/framework/user-game-stats/classes.ts @@ -2,7 +2,11 @@ import { Game, Playtypes, integer, UserGameStats, ClassDelta, IDStrings } from " import { GameClasses } from "tachi-common/js/game-classes"; import deepmerge from "deepmerge"; import { KtLogger } from "lib/logger/logger"; -import { CalculateGitadoraColour, CalculateSDVXClass } from "./builtin-class-handlers"; +import { + CalculateChunithmColour, + CalculateGitadoraColour, + CalculateSDVXClass, +} from "./builtin-class-handlers"; import { ReturnClassIfGreater } from "utils/class"; import { RedisPub } from "external/redis/redis-IPC"; import { ClassHandler, ScoreClasses } from "./types"; @@ -27,7 +31,9 @@ type ClassHandlerMap = { const STATIC_CLASS_HANDLERS: ClassHandlerMap = { iidx: null, bms: null, - chunithm: null, + chunithm: { + colour: CalculateChunithmColour, + }, ddr: null, gitadora: { Gita: CalculateGitadoraColour, From d962de5735144db86e915a88382ba4964ee8aa61 Mon Sep 17 00:00:00 2001 From: zkldi Date: Sat, 7 Aug 2021 00:04:08 +0100 Subject: [PATCH 6/9] Remove ability for server to serve index.html Fixes #259 --- server/pnpm-lock.yaml | 8 ++++---- .../lib/score-import/framework/user-game-stats/classes.ts | 2 +- .../import-types/common/batch-manual/parser.ts | 1 + .../v1/users/_userID/games/_game/_playtype/pbs/router.ts | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 554968c97..d5cf9cde0 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -84,7 +84,7 @@ dependencies: redis: 3.1.2 rimraf: 3.0.2 safe-json-stringify: 1.2.0 - tachi-common: github.com/zkldi/tachi-common/42a4da4ced2934d73b862f31a38814a0cc0a16f0_ts-node@10.0.0+typescript@4.3.4 + tachi-common: github.com/zkldi/tachi-common/960879126ca93099d412866ca0f3a8327979034f_ts-node@10.0.0+typescript@4.3.4 typescript: 4.3.4 winston: 3.3.3 @@ -4226,9 +4226,9 @@ packages: engines: {node: '>=6'} dev: true - github.com/zkldi/tachi-common/42a4da4ced2934d73b862f31a38814a0cc0a16f0_ts-node@10.0.0+typescript@4.3.4: - resolution: {tarball: https://codeload.github.com/zkldi/tachi-common/tar.gz/42a4da4ced2934d73b862f31a38814a0cc0a16f0} - id: github.com/zkldi/tachi-common/42a4da4ced2934d73b862f31a38814a0cc0a16f0 + github.com/zkldi/tachi-common/960879126ca93099d412866ca0f3a8327979034f_ts-node@10.0.0+typescript@4.3.4: + resolution: {tarball: https://codeload.github.com/zkldi/tachi-common/tar.gz/960879126ca93099d412866ca0f3a8327979034f} + id: github.com/zkldi/tachi-common/960879126ca93099d412866ca0f3a8327979034f name: tachi-common version: 0.1.0 dependencies: diff --git a/server/src/lib/score-import/framework/user-game-stats/classes.ts b/server/src/lib/score-import/framework/user-game-stats/classes.ts index 882d350e8..dd50e7cdb 100644 --- a/server/src/lib/score-import/framework/user-game-stats/classes.ts +++ b/server/src/lib/score-import/framework/user-game-stats/classes.ts @@ -32,7 +32,7 @@ const STATIC_CLASS_HANDLERS: ClassHandlerMap = { iidx: null, bms: null, chunithm: { - colour: CalculateChunithmColour, + Single: CalculateChunithmColour, }, ddr: null, gitadora: { diff --git a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts index cc1d315cc..d8be91320 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts @@ -71,6 +71,7 @@ const PR_BatchManualScore = (game: Game, playtype: Playtypes[Game]): PrudenceSch } for (const key in self) { + // @ts-expect-error check if (!gptConfig.judgements.includes(key)) { return `Invalid Key ${key}. Expected any of ${gptConfig.judgements.join(", ")}`; } diff --git a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.ts b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.ts index d29b46483..94aef3e1a 100644 --- a/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.ts +++ b/server/src/server/router/api/v1/users/_userID/games/_game/_playtype/pbs/router.ts @@ -144,7 +144,7 @@ router.get("/:chartID", async (req, res) => { return res.status(200).json({ success: true, - description: `Successfull retrieved PB for user.`, + description: `Successfully retrieved PB for user.`, body: { scores, chart, From a57245f3737645fc73d1fdf98ea7802c4aa3ea99 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 10 Aug 2021 04:28:43 +0100 Subject: [PATCH 7/9] i accidentally the eslint --- .../beatoraja/import-courses.ts | 4 +- .../__JSON-TO-KTDB/beatoraja/import.ts | 14 +- .../scripts/__JSON-TO-KTDB/iidx/diff-sync.ts | 132 +++++----- server/scripts/__JSON-TO-KTDB/iidx/types.ts | 16 +- server/scripts/__JSON-TO-KTDB/usc/dedupe.ts | 30 +-- server/scripts/__JSON-TO-KTDB/usc/insert.ts | 8 +- .../charts-chunithm.ts | 70 ++--- .../__KT_DATABASE_MIGRATION/charts-ddr.ts | 78 +++--- .../charts-gitadora.ts | 76 +++--- .../__KT_DATABASE_MIGRATION/charts-iidx.ts | 76 +++--- .../__KT_DATABASE_MIGRATION/charts-museca.ts | 66 ++--- .../__KT_DATABASE_MIGRATION/charts-sdvx.ts | 72 +++--- .../__KT_DATABASE_MIGRATION/folders.ts | 48 ++-- .../scripts/__KT_DATABASE_MIGRATION/goals.ts | 108 ++++---- .../__KT_DATABASE_MIGRATION/migrate.ts | 60 ++--- .../other/museca-tierlist.ts | 54 ++-- .../__KT_DATABASE_MIGRATION/songs-chunithm.ts | 32 +-- .../__KT_DATABASE_MIGRATION/songs-ddr.ts | 30 +-- .../__KT_DATABASE_MIGRATION/songs-gitadora.ts | 30 +-- .../__KT_DATABASE_MIGRATION/songs-iidx.ts | 32 +-- .../__KT_DATABASE_MIGRATION/songs-museca.ts | 30 +-- .../__KT_DATABASE_MIGRATION/songs-sdvx.ts | 32 +-- .../__KT_DATABASE_MIGRATION/tierlists.ts | 56 ++-- .../__KT_DATABASE_MIGRATION/user-goals.ts | 16 +- .../scripts/__KT_DATABASE_MIGRATION/users.ts | 58 ++--- server/scripts/arc-db-sync/arc-db-fetch.ts | 98 +++---- server/scripts/arc-db-sync/arc-db-merge.ts | 122 ++++----- server/scripts/iidx-tierlist/get-sp12-data.ts | 242 +++++++++--------- .../iidx-tierlist/parse-iidxkr-data.ts | 236 ++++++++--------- server/scripts/initialise-counters.ts | 24 +- server/scripts/single-use/add-2dxtra-flag.ts | 28 +- server/scripts/single-use/add-hcnver.ts | 18 +- server/scripts/single-use/fix-chuni-port.ts | 26 +- .../fix-iidx-versions/fix-iidx-versions.ts | 38 +-- .../single-use/fix-iidx-versions/stat.ts | 20 +- .../fix-iidx-versions/update-versions.ts | 132 +++++----- .../single-use/fix-legacy-folder-format.ts | 40 +-- .../scripts/single-use/numify-tableindex.ts | 36 +-- server/scripts/single-use/s3check.ts | 8 +- server/scripts/single-use/sample-iidx-db.ts | 6 +- server/scripts/update-bpi-poyashi.ts | 140 +++++----- .../framework/calculated-data/stats.test.ts | 6 +- .../common/api-kai/iidx/converter.test.ts | 6 +- .../common/api-kai/sdvx/converter.test.ts | 6 +- .../common/eamusement-iidx-csv/parser.test.ts | 5 +- 45 files changed, 1225 insertions(+), 1240 deletions(-) diff --git a/server/scripts/__JSON-TO-KTDB/beatoraja/import-courses.ts b/server/scripts/__JSON-TO-KTDB/beatoraja/import-courses.ts index fcf0ee58d..b2e81c65f 100644 --- a/server/scripts/__JSON-TO-KTDB/beatoraja/import-courses.ts +++ b/server/scripts/__JSON-TO-KTDB/beatoraja/import-courses.ts @@ -4,7 +4,7 @@ import path from "path"; import db from "../../../src/external/mongo/db"; (async () => { - const data = JSON.parse(fs.readFileSync(path.join(__dirname, "courses.json"), "utf-8")); + const data = JSON.parse(fs.readFileSync(path.join(__dirname, "courses.json"), "utf-8")); - await db["bms-course-lookup"].insert(data); + await db["bms-course-lookup"].insert(data); })(); diff --git a/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts b/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts index e23de619b..319e0cbdd 100644 --- a/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts +++ b/server/scripts/__JSON-TO-KTDB/beatoraja/import.ts @@ -8,13 +8,13 @@ const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8")); (async () => { - logger.info("starting chart import."); - await db.charts.bms.remove({}); - await db.charts.bms.insert(charts); - logger.info("imported charts."); + logger.info("starting chart import."); + await db.charts.bms.remove({}); + await db.charts.bms.insert(charts); + logger.info("imported charts."); - await db.songs.bms.remove({}); - await db.songs.bms.insert(songs); + await db.songs.bms.remove({}); + await db.songs.bms.insert(songs); - logger.info("done."); + logger.info("done."); })(); diff --git a/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts b/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts index 2361b6f50..ea7ce62f7 100644 --- a/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts +++ b/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts @@ -18,83 +18,83 @@ program.parse(process.argv); const options = program.opts(); const dataset = JSON.parse( - fs.readFileSync(path.join(__dirname, "./out.json"), "utf-8") + fs.readFileSync(path.join(__dirname, "./out.json"), "utf-8") ) as MDBJson[]; const logger = CreateLogCtx(__filename); (async () => { - for (const data of dataset) { - for (const d in data.difficulties) { - const diff = d as keyof typeof data.difficulties; + for (const data of dataset) { + for (const d in data.difficulties) { + const diff = d as keyof typeof data.difficulties; - if (data.difficulties[diff] === 0) { - logger.verbose(`Skipping chart ${data.title} ${diff}`); - continue; - } + if (data.difficulties[diff] === 0) { + logger.verbose(`Skipping chart ${data.title} ${diff}`); + continue; + } - const spl = diff.split("-"); - const chart = (await db.charts.iidx.findOne({ - "data.inGameID": data.songID, - playtype: spl[0] as Playtypes["iidx"], - difficulty: spl[1] as Difficulties["iidx:SP" | "iidx:DP"], - isPrimary: true, - })) as ChartDocument<"iidx:SP" | "iidx:DP">; + const spl = diff.split("-"); + const chart = (await db.charts.iidx.findOne({ + "data.inGameID": data.songID, + playtype: spl[0] as Playtypes["iidx"], + difficulty: spl[1] as Difficulties["iidx:SP" | "iidx:DP"], + isPrimary: true, + })) as ChartDocument<"iidx:SP" | "iidx:DP">; - if (!chart) { - logger.warn(`No chart exists for ${data.title} (${diff}).`); - continue; - } + if (!chart) { + logger.warn(`No chart exists for ${data.title} (${diff}).`); + continue; + } - if (chart.levelNum !== data.difficulties[diff]) { - logger.warn( - `Level Mismatch with ${data.title} (${diff}) [${chart.levelNum} -> ${data.difficulties[diff]}]` - ); + if (chart.levelNum !== data.difficulties[diff]) { + logger.warn( + `Level Mismatch with ${data.title} (${diff}) [${chart.levelNum} -> ${data.difficulties[diff]}]` + ); - if (options.sync) { - await db.charts.iidx.update( - { - _id: chart._id, - }, - { - $set: { - levelNum: data.difficulties[diff], - level: data.difficulties[diff].toString(), - }, - } - ); - } - } + if (options.sync) { + await db.charts.iidx.update( + { + _id: chart._id, + }, + { + $set: { + levelNum: data.difficulties[diff], + level: data.difficulties[diff].toString(), + }, + } + ); + } + } - if (chart.data.notecount !== data.notecounts[diff]) { - if ( - data.notecounts[diff] === undefined || - Math.abs(chart.data.notecount - data.notecounts[diff]!) > 2 - ) { - logger.error( - `Significant Notecount Mismatch with ${data.title} (${diff}) [${chart.data.notecount} -> ${data.notecounts[diff]}] - skipping.` - ); - // continue; - } + if (chart.data.notecount !== data.notecounts[diff]) { + if ( + data.notecounts[diff] === undefined || + Math.abs(chart.data.notecount - data.notecounts[diff]!) > 2 + ) { + logger.error( + `Significant Notecount Mismatch with ${data.title} (${diff}) [${chart.data.notecount} -> ${data.notecounts[diff]}] - skipping.` + ); + // continue; + } - logger.warn( - `Notecount Mismatch with ${data.title} (${diff}) [${chart.data.notecount} -> ${data.notecounts[diff]}]` - ); + logger.warn( + `Notecount Mismatch with ${data.title} (${diff}) [${chart.data.notecount} -> ${data.notecounts[diff]}]` + ); - if (options.sync) { - await db.charts.iidx.update( - { - _id: chart._id, - }, - { - $set: { - "data.notecount": data.notecounts[diff], - }, - } - ); - } - } - } - } - logger.info("done"); + if (options.sync) { + await db.charts.iidx.update( + { + _id: chart._id, + }, + { + $set: { + "data.notecount": data.notecounts[diff], + }, + } + ); + } + } + } + } + logger.info("done"); })(); diff --git a/server/scripts/__JSON-TO-KTDB/iidx/types.ts b/server/scripts/__JSON-TO-KTDB/iidx/types.ts index 68a522a32..ed187efd7 100644 --- a/server/scripts/__JSON-TO-KTDB/iidx/types.ts +++ b/server/scripts/__JSON-TO-KTDB/iidx/types.ts @@ -1,12 +1,12 @@ export interface MDBJson { - title: string; - artist: string; - marquee: string; - folder: number; - genre: string; - difficulties: Record; - notecounts: Partial>; - songID: number; + title: string; + artist: string; + marquee: string; + folder: number; + genre: string; + difficulties: Record; + notecounts: Partial>; + songID: number; } type DiffNames = `${"SP" | "DP"}-${"BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"}`; diff --git a/server/scripts/__JSON-TO-KTDB/usc/dedupe.ts b/server/scripts/__JSON-TO-KTDB/usc/dedupe.ts index 781f5e997..0d28a3c66 100644 --- a/server/scripts/__JSON-TO-KTDB/usc/dedupe.ts +++ b/server/scripts/__JSON-TO-KTDB/usc/dedupe.ts @@ -7,25 +7,25 @@ const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "charts.json"), " const map: Map = new Map(); for (const chart of charts) { - const id = `${chart.songID}-${chart.difficulty}`; - const doc = map.get(id); + const id = `${chart.songID}-${chart.difficulty}`; + const doc = map.get(id); - if (doc) { - if (Array.isArray(doc.data.hashSHA1)) { - doc.data.hashSHA1.push(chart.data.hashSHA1); - } else { - doc.data.hashSHA1 = [doc.data.hashSHA1, chart.data.hashSHA1]; - } + if (doc) { + if (Array.isArray(doc.data.hashSHA1)) { + doc.data.hashSHA1.push(chart.data.hashSHA1); + } else { + doc.data.hashSHA1 = [doc.data.hashSHA1, chart.data.hashSHA1]; + } - // insane hack - chart.erasethis = true; - } else { - map.set(id, chart); - } + // insane hack + chart.erasethis = true; + } else { + map.set(id, chart); + } } // works because byref fs.writeFileSync( - path.join(__dirname, "deduped-charts.json"), - JSON.stringify(charts.filter((e: any) => e.erasethis !== true)) + path.join(__dirname, "deduped-charts.json"), + JSON.stringify(charts.filter((e: any) => e.erasethis !== true)) ); diff --git a/server/scripts/__JSON-TO-KTDB/usc/insert.ts b/server/scripts/__JSON-TO-KTDB/usc/insert.ts index f28e7f3bc..cdeed0b89 100644 --- a/server/scripts/__JSON-TO-KTDB/usc/insert.ts +++ b/server/scripts/__JSON-TO-KTDB/usc/insert.ts @@ -6,8 +6,8 @@ const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "songs.json"), "ut const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "deduped-charts.json"), "utf-8")); (async () => { - await db.songs.usc.remove({}); - await db.charts.usc.remove({}); - await db.songs.usc.insert(songs); - await db.charts.usc.insert(charts); + await db.songs.usc.remove({}); + await db.charts.usc.remove({}); + await db.songs.usc.insert(songs); + await db.charts.usc.insert(charts); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts index 54e5cf407..21e880a18 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts @@ -9,48 +9,48 @@ import { gameOrders } from "tachi-common/js/config"; const logger = CreateLogCtx(__filename); async function ConvertFn(c: any): Promise> { - const song = await db.songs.chunithm.findOne({ - id: c.id, - }); + const song = await db.songs.chunithm.findOne({ + id: c.id, + }); - if (!song) { - logger.severe(`Cannot find song with ID ${c.id}?`); - throw new Error(`Cannot find song with ID ${c.id}?`); - } + if (!song) { + logger.severe(`Cannot find song with ID ${c.id}?`); + throw new Error(`Cannot find song with ID ${c.id}?`); + } - const newChartDoc: ChartDocument<"chunithm:Single"> = { - rgcID: null, - chartID: c.chartID, - difficulty: c.difficulty, - songID: c.id, - playtype: c.playtype, - levelNum: c.levelNum, - level: c.level.toString(), - flags: { - "IN BASE GAME": !!c.flags["IN BASE GAME"], - OMNIMIX: !!c.flags.OMNIMIX, - }, - data: { - inGameID: c.internals.inGameID, - }, - isPrimary: true, - versions: [], // sentinel - }; + const newChartDoc: ChartDocument<"chunithm:Single"> = { + rgcID: null, + chartID: c.chartID, + difficulty: c.difficulty, + songID: c.id, + playtype: c.playtype, + levelNum: c.levelNum, + level: c.level.toString(), + flags: { + "IN BASE GAME": !!c.flags["IN BASE GAME"], + OMNIMIX: !!c.flags.OMNIMIX, + }, + data: { + inGameID: c.internals.inGameID, + }, + isPrimary: true, + versions: [], // sentinel + }; - const idx = gameOrders.chunithm.indexOf(song.firstVersion!); + const idx = gameOrders.chunithm.indexOf(song.firstVersion!); - if (idx === -1) { - logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); - newChartDoc.versions = [song.firstVersion!]; - } else { - newChartDoc.versions = gameOrders.chunithm.slice(idx); - } + if (idx === -1) { + logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); + newChartDoc.versions = [song.firstVersion!]; + } else { + newChartDoc.versions = gameOrders.chunithm.slice(idx); + } - return newChartDoc; + return newChartDoc; } (async () => { - await MigrateRecords(db.charts.chunithm, "charts-chunithm", ConvertFn); + await MigrateRecords(db.charts.chunithm, "charts-chunithm", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts index c28ed6366..c3351ed5c 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts @@ -10,53 +10,53 @@ import { oldKTDB } from "./old-db"; const logger = CreateLogCtx(__filename); async function ConvertFn(c: any): Promise> { - const song = (await db.songs.ddr.findOne({ - id: c.id, - })) as SongDocument<"ddr">; + const song = (await db.songs.ddr.findOne({ + id: c.id, + })) as SongDocument<"ddr">; - const oldSong = await oldKTDB.get("songs-ddr").findOne({ - id: c.id, - }); + const oldSong = await oldKTDB.get("songs-ddr").findOne({ + id: c.id, + }); - if (!song) { - logger.severe(`Cannot find song with ID ${c.id}?`); - throw new Error(`Cannot find song with ID ${c.id}?`); - } + if (!song) { + logger.severe(`Cannot find song with ID ${c.id}?`); + throw new Error(`Cannot find song with ID ${c.id}?`); + } - const newChartDoc: ChartDocument<"ddr:SP" | "ddr:DP"> = { - rgcID: null, - chartID: c.chartID, - difficulty: c.difficulty, - songID: c.id, - playtype: c.playtype, - levelNum: c.levelNum, - level: c.level.toString(), - flags: { - "IN BASE GAME": true, - "N-1": true, - }, - data: { - inGameID: c.internals.inGameID, - songHash: oldSong.internals.songHash, - }, - isPrimary: true, - versions: [], // sentinel - }; + const newChartDoc: ChartDocument<"ddr:SP" | "ddr:DP"> = { + rgcID: null, + chartID: c.chartID, + difficulty: c.difficulty, + songID: c.id, + playtype: c.playtype, + levelNum: c.levelNum, + level: c.level.toString(), + flags: { + "IN BASE GAME": true, + "N-1": true, + }, + data: { + inGameID: c.internals.inGameID, + songHash: oldSong.internals.songHash, + }, + isPrimary: true, + versions: [], // sentinel + }; - const idx = gameOrders.ddr.indexOf(song.firstVersion!); + const idx = gameOrders.ddr.indexOf(song.firstVersion!); - if (idx === -1) { - logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); - newChartDoc.versions = [song.firstVersion!]; - } else { - newChartDoc.versions = gameOrders.ddr.slice(idx); - } + if (idx === -1) { + logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); + newChartDoc.versions = [song.firstVersion!]; + } else { + newChartDoc.versions = gameOrders.ddr.slice(idx); + } - return newChartDoc; + return newChartDoc; } (async () => { - await MigrateRecords(db.charts.ddr, "charts-ddr", ConvertFn); + await MigrateRecords(db.charts.ddr, "charts-ddr", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts index 0b041a2f1..7ccbae4ea 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts @@ -9,52 +9,52 @@ import { oldKTDB } from "./old-db"; const logger = CreateLogCtx(__filename); async function ConvertFn(c: any): Promise> { - const song = (await db.songs.gitadora.findOne({ - id: c.id, - })) as SongDocument<"gitadora">; + const song = (await db.songs.gitadora.findOne({ + id: c.id, + })) as SongDocument<"gitadora">; - const oldSong = await oldKTDB.get("songs-gitadora").findOne({ - id: c.id, - }); + const oldSong = await oldKTDB.get("songs-gitadora").findOne({ + id: c.id, + }); - if (!song) { - logger.severe(`Cannot find song with ID ${c.id}?`); - throw new Error(`Cannot find song with ID ${c.id}?`); - } + if (!song) { + logger.severe(`Cannot find song with ID ${c.id}?`); + throw new Error(`Cannot find song with ID ${c.id}?`); + } - const newChartDoc: ChartDocument<"gitadora:Gita" | "gitadora:Dora"> = { - rgcID: null, - chartID: c.chartID, - difficulty: c.difficulty, - songID: c.id, - playtype: c.playtype, - levelNum: c.levelNum, - level: c.level.toString(), - flags: { - "IN BASE GAME": true, - OMNIMIX: true, - }, - data: { - inGameID: c.internals.inGameINTID, - }, - isPrimary: true, - versions: [], // sentinel - }; + const newChartDoc: ChartDocument<"gitadora:Gita" | "gitadora:Dora"> = { + rgcID: null, + chartID: c.chartID, + difficulty: c.difficulty, + songID: c.id, + playtype: c.playtype, + levelNum: c.levelNum, + level: c.level.toString(), + flags: { + "IN BASE GAME": true, + OMNIMIX: true, + }, + data: { + inGameID: c.internals.inGameINTID, + }, + isPrimary: true, + versions: [], // sentinel + }; - const idx = gameOrders.gitadora.indexOf(song.firstVersion!); + const idx = gameOrders.gitadora.indexOf(song.firstVersion!); - if (idx === -1) { - logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); - newChartDoc.versions = [song.firstVersion!]; - } else { - newChartDoc.versions = gameOrders.gitadora.slice(idx); - } + if (idx === -1) { + logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); + newChartDoc.versions = [song.firstVersion!]; + } else { + newChartDoc.versions = gameOrders.gitadora.slice(idx); + } - return newChartDoc; + return newChartDoc; } (async () => { - await MigrateRecords(db.charts.gitadora, "charts-gitadora", ConvertFn); + await MigrateRecords(db.charts.gitadora, "charts-gitadora", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts index f190a6e5a..e14884cf0 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts @@ -9,51 +9,51 @@ import { gameOrders } from "tachi-common/js/config"; const logger = CreateLogCtx(__filename); async function ConvertFn(c: any): Promise> { - const song = await db.songs.iidx.findOne({ - id: c.id, - }); + const song = await db.songs.iidx.findOne({ + id: c.id, + }); - if (!song) { - logger.severe(`Cannot find song with ID ${c.id}?`); - throw new Error(`Cannot find song with ID ${c.id}?`); - } + if (!song) { + logger.severe(`Cannot find song with ID ${c.id}?`); + throw new Error(`Cannot find song with ID ${c.id}?`); + } - const newChartDoc: ChartDocument<"iidx:SP" | "iidx:DP"> = { - rgcID: null, - chartID: c.chartID, - difficulty: c.difficulty, - songID: c.id, - playtype: c.playtype, - levelNum: c.levelNum, - level: c.level.toString(), - flags: { - "IN BASE GAME": !!c.flags["IN BASE GAME"], - OMNIMIX: !!c.flags.OMNIMIX, - "N-1": !!c.flags["N-1"], - }, - data: { - inGameID: c.internals.inGameINTID, - notecount: c.notedata.notecount, - arcChartID: null, - }, - isPrimary: true, - versions: [], // sentinel - }; + const newChartDoc: ChartDocument<"iidx:SP" | "iidx:DP"> = { + rgcID: null, + chartID: c.chartID, + difficulty: c.difficulty, + songID: c.id, + playtype: c.playtype, + levelNum: c.levelNum, + level: c.level.toString(), + flags: { + "IN BASE GAME": !!c.flags["IN BASE GAME"], + OMNIMIX: !!c.flags.OMNIMIX, + "N-1": !!c.flags["N-1"], + }, + data: { + inGameID: c.internals.inGameINTID, + notecount: c.notedata.notecount, + arcChartID: null, + }, + isPrimary: true, + versions: [], // sentinel + }; - const idx = gameOrders.iidx.indexOf(song.firstVersion!); + const idx = gameOrders.iidx.indexOf(song.firstVersion!); - if (idx === -1) { - logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); - newChartDoc.versions = [song.firstVersion!]; - } else { - newChartDoc.versions = gameOrders.iidx.slice(idx); - } + if (idx === -1) { + logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); + newChartDoc.versions = [song.firstVersion!]; + } else { + newChartDoc.versions = gameOrders.iidx.slice(idx); + } - return newChartDoc; + return newChartDoc; } (async () => { - await MigrateRecords(db.charts.iidx, "charts-iidx", ConvertFn); + await MigrateRecords(db.charts.iidx, "charts-iidx", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts index 65d2b4337..eeab079a6 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts @@ -8,46 +8,46 @@ import { gameOrders } from "tachi-common/js/config"; const logger = CreateLogCtx(__filename); async function ConvertFn(c: any): Promise> { - const song = await db.songs.museca.findOne({ - id: c.id, - }); + const song = await db.songs.museca.findOne({ + id: c.id, + }); - if (!song) { - logger.severe(`Cannot find song with ID ${c.id}?`); - throw new Error(`Cannot find song with ID ${c.id}?`); - } + if (!song) { + logger.severe(`Cannot find song with ID ${c.id}?`); + throw new Error(`Cannot find song with ID ${c.id}?`); + } - const newChartDoc: ChartDocument<"museca:Single"> = { - rgcID: null, - chartID: c.chartID, - difficulty: c.difficulty, - songID: c.id, - playtype: c.playtype, - levelNum: c.levelNum, - level: c.level.toString(), - flags: { - "IN BASE GAME": true, - OMNIMIX: true, // false, needs to be overrode for 3 or so charts - }, - data: {}, - isPrimary: true, - versions: [], // sentinel - }; + const newChartDoc: ChartDocument<"museca:Single"> = { + rgcID: null, + chartID: c.chartID, + difficulty: c.difficulty, + songID: c.id, + playtype: c.playtype, + levelNum: c.levelNum, + level: c.level.toString(), + flags: { + "IN BASE GAME": true, + OMNIMIX: true, // false, needs to be overrode for 3 or so charts + }, + data: {}, + isPrimary: true, + versions: [], // sentinel + }; - const idx = gameOrders.museca.indexOf(song.firstVersion!); + const idx = gameOrders.museca.indexOf(song.firstVersion!); - if (idx === -1) { - logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); - newChartDoc.versions = [song.firstVersion!]; - } else { - newChartDoc.versions = gameOrders.museca.slice(idx); - } + if (idx === -1) { + logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); + newChartDoc.versions = [song.firstVersion!]; + } else { + newChartDoc.versions = gameOrders.museca.slice(idx); + } - return newChartDoc; + return newChartDoc; } (async () => { - await MigrateRecords(db.charts.museca, "charts-museca", ConvertFn); + await MigrateRecords(db.charts.museca, "charts-museca", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts index 271f23608..f9c061ab7 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts @@ -8,49 +8,49 @@ import { gameOrders } from "tachi-common/js/config"; const logger = CreateLogCtx(__filename); async function ConvertFn(c: any): Promise> { - const song = await db.songs.sdvx.findOne({ - id: c.id, - }); + const song = await db.songs.sdvx.findOne({ + id: c.id, + }); - if (!song) { - logger.severe(`Cannot find song with ID ${c.id}?`); - throw new Error(`Cannot find song with ID ${c.id}?`); - } + if (!song) { + logger.severe(`Cannot find song with ID ${c.id}?`); + throw new Error(`Cannot find song with ID ${c.id}?`); + } - const newChartDoc: ChartDocument<"sdvx:Single"> = { - rgcID: null, - chartID: c.chartID, - difficulty: c.difficulty, - songID: c.id, - playtype: c.playtype, - levelNum: c.levelNum, - level: c.level.toString(), - flags: { - "IN BASE GAME": !!c.flags["IN BASE GAME"], - OMNIMIX: !!c.flags.OMNIMIX, - "N-1": !!c.flags["N-1"], - }, - data: { - inGameID: c.internals.inGameINTID, - }, - isPrimary: true, - versions: [], // sentinel - }; + const newChartDoc: ChartDocument<"sdvx:Single"> = { + rgcID: null, + chartID: c.chartID, + difficulty: c.difficulty, + songID: c.id, + playtype: c.playtype, + levelNum: c.levelNum, + level: c.level.toString(), + flags: { + "IN BASE GAME": !!c.flags["IN BASE GAME"], + OMNIMIX: !!c.flags.OMNIMIX, + "N-1": !!c.flags["N-1"], + }, + data: { + inGameID: c.internals.inGameINTID, + }, + isPrimary: true, + versions: [], // sentinel + }; - const idx = gameOrders.sdvx.indexOf(song.firstVersion!); + const idx = gameOrders.sdvx.indexOf(song.firstVersion!); - if (idx === -1) { - logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); - newChartDoc.versions = [song.firstVersion!]; - } else { - newChartDoc.versions = gameOrders.sdvx.slice(idx); - } + if (idx === -1) { + logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); + newChartDoc.versions = [song.firstVersion!]; + } else { + newChartDoc.versions = gameOrders.sdvx.slice(idx); + } - return newChartDoc; + return newChartDoc; } (async () => { - await MigrateRecords(db.charts.sdvx, "charts-sdvx", ConvertFn); + await MigrateRecords(db.charts.sdvx, "charts-sdvx", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/folders.ts b/server/scripts/__KT_DATABASE_MIGRATION/folders.ts index 8afd0f766..5ee11c210 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/folders.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/folders.ts @@ -10,39 +10,39 @@ import crypto from "crypto"; const logger = CreateLogCtx(__filename); function ConvertFn(c: any): FolderDocument[] { - throw new Error("This doesn't work, It was migrated elsewhere."); + throw new Error("This doesn't work, It was migrated elsewhere."); - // let docs = []; + // let docs = []; - // for (const playtype of validPlaytypes[c.game as Game]) { - // const fd: FolderDocument = { - // title: c.title, - // game: c.game, - // playtype, - // type: "query" as const, - // query: { - // collection: c.query.collection, - // body: c.query.query, - // }, - // folderID: crypto.randomBytes(20).toString("hex"), - // table: c.table, - // tableIndex: 0, // ??? - // }; + // for (const playtype of validPlaytypes[c.game as Game]) { + // const fd: FolderDocument = { + // title: c.title, + // game: c.game, + // playtype, + // type: "query" as const, + // query: { + // collection: c.query.collection, + // body: c.query.query, + // }, + // folderID: crypto.randomBytes(20).toString("hex"), + // table: c.table, + // tableIndex: 0, // ??? + // }; - // let gh = gameHuman[c.game as Game]; + // let gh = gameHuman[c.game as Game]; - // fd.title = fd.title.replace(new RegExp(`${gh} +`, "giu"), ""); + // fd.title = fd.title.replace(new RegExp(`${gh} +`, "giu"), ""); - // logger.info(`Porting folder ${fd.title} (${playtype})`); + // logger.info(`Porting folder ${fd.title} (${playtype})`); - // docs.push(fd); - // } + // docs.push(fd); + // } - // return docs; + // return docs; } (async () => { - await MigrateRecords(db.folders, "folders", ConvertFn); + await MigrateRecords(db.folders, "folders", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/goals.ts b/server/scripts/__KT_DATABASE_MIGRATION/goals.ts index a13248575..f6eb1fc9a 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/goals.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/goals.ts @@ -9,71 +9,71 @@ import MigrateRecords from "./migrate"; const logger = CreateLogCtx(__filename); function ConvertFn(c: any): GoalDocument | null { - let charts: GoalDocument["charts"]; + let charts: GoalDocument["charts"]; - if (c.directChartID) { - charts = { - type: "single", - data: c.directChartID, - }; - } else if (c.directChartIDs) { - charts = { - type: "multi", - data: c.directChartIDs, - }; - } else if (c.chartQuery) { - logger.warn(`Skipped goal ${c.title}, as it used cQuery.`); - return null; - } else { - logger.warn(`Any Goal ${c.goalID}, ${c.title}. Skipping.`); - return null; - } + if (c.directChartID) { + charts = { + type: "single", + data: c.directChartID, + }; + } else if (c.directChartIDs) { + charts = { + type: "multi", + data: c.directChartIDs, + }; + } else if (c.chartQuery) { + logger.warn(`Skipped goal ${c.title}, as it used cQuery.`); + return null; + } else { + logger.warn(`Any Goal ${c.goalID}, ${c.title}. Skipping.`); + return null; + } - const partialCriteria: Partial = { - mode: "single", - }; + const partialCriteria: Partial = { + mode: "single", + }; - if (c.scoreQuery["scoreData¬lampIndex"]) { - partialCriteria.key = "scoreData.lampIndex"; - } else if (c.scoreQuery["scoreData¬gradeIndex"]) { - partialCriteria.key = "scoreData.gradeIndex"; - } else if (c.scoreQuery["scoreData¬score"]) { - partialCriteria.key = "scoreData.score"; - } else if (c.scoreQuery["scoreData¬percent"]) { - partialCriteria.key = "scoreData.percent"; - } else { - throw new Error(`Invalid goal ${c.goalID} ${c.title}.`); - } + if (c.scoreQuery["scoreData¬lampIndex"]) { + partialCriteria.key = "scoreData.lampIndex"; + } else if (c.scoreQuery["scoreData¬gradeIndex"]) { + partialCriteria.key = "scoreData.gradeIndex"; + } else if (c.scoreQuery["scoreData¬score"]) { + partialCriteria.key = "scoreData.score"; + } else if (c.scoreQuery["scoreData¬percent"]) { + partialCriteria.key = "scoreData.percent"; + } else { + throw new Error(`Invalid goal ${c.goalID} ${c.title}.`); + } - const v = c.scoreQuery[partialCriteria.key.replace(/\./gu, "¬")]; + const v = c.scoreQuery[partialCriteria.key.replace(/\./gu, "¬")]; - const value = v ? v["~gte"] : undefined; + const value = v ? v["~gte"] : undefined; - // would break for value == 0, but a goal would never be - // zero. - if (!value) { - throw new Error(`Invalid goal ${c.goalID} ${c.title}.`); - } + // would break for value == 0, but a goal would never be + // zero. + if (!value) { + throw new Error(`Invalid goal ${c.goalID} ${c.title}.`); + } - partialCriteria.value = value; + partialCriteria.value = value; - // @ts-expect-error ts cant resolve this union properly - const newGoalDoc: GoalDocument = { - game: c.game, - playtype: c.playtype, - title: c.title, - timeAdded: c.timeAdded, - goalID: c.goalID, // maybe - createdBy: c.createdBy, - charts, - criteria: partialCriteria as GoalDocument["criteria"], - }; + // @ts-expect-error ts cant resolve this union properly + const newGoalDoc: GoalDocument = { + game: c.game, + playtype: c.playtype, + title: c.title, + timeAdded: c.timeAdded, + goalID: c.goalID, // maybe + createdBy: c.createdBy, + charts, + criteria: partialCriteria as GoalDocument["criteria"], + }; - return newGoalDoc; + return newGoalDoc; } (async () => { - await MigrateRecords(db.goals, "goals", ConvertFn); + await MigrateRecords(db.goals, "goals", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts index 592a0d073..1a7c1c5a5 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts @@ -7,43 +7,43 @@ const logger = CreateLogCtx(__filename); const newDocuments: unknown[] = []; export default async function MigrateRecords( - existingCollection: ICollection, - collectionName: string, - HandlerFN: (c: unknown) => unknown + existingCollection: ICollection, + collectionName: string, + HandlerFN: (c: unknown) => unknown ) { - const existingRecords = await existingCollection.count({}); - logger.info(`Starting migration for ${collectionName}...`); + const existingRecords = await existingCollection.count({}); + logger.info(`Starting migration for ${collectionName}...`); - if (existingRecords > 0) { - logger.error(`${existingRecords} documents already exist in db, terminating.`); - process.exit(1); - } + if (existingRecords > 0) { + logger.error(`${existingRecords} documents already exist in db, terminating.`); + process.exit(1); + } - let i = 0; + let i = 0; - await oldKTDB - .get(collectionName) - .find({}) - // @ts-expect-error it exists. - .each(async (c, { pause, resume }) => { - i++; - pause(); + await oldKTDB + .get(collectionName) + .find({}) + // @ts-expect-error it exists. + .each(async (c, { pause, resume }) => { + i++; + pause(); - if (i % 10000 === 0) { - logger.info(`Processed ${i} documents.`); - } - const newDoc = await HandlerFN(c); + if (i % 10000 === 0) { + logger.info(`Processed ${i} documents.`); + } + const newDoc = await HandlerFN(c); - if (newDoc !== null) { - newDocuments.push(newDoc); - } + if (newDoc !== null) { + newDocuments.push(newDoc); + } - resume(); - }); + resume(); + }); - logger.info(`Inserting ${newDocuments.length} documents.`); - await existingCollection.insert(newDocuments.flat()); - logger.info(`Done!`); + logger.info(`Inserting ${newDocuments.length} documents.`); + await existingCollection.insert(newDocuments.flat()); + logger.info(`Done!`); - process.exit(0); + process.exit(0); } diff --git a/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts b/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts index c076c37b6..1810e213e 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts @@ -8,39 +8,39 @@ const logger = CreateLogCtx(__filename); const TIERLIST_ID = "ed45403c2cd93922d8a45b3c5bc352a962a8f115"; function ConvertFn(c: any) { - const tdd: TierlistDataDocument<"Individual Difference"> = { - chartID: c.chartID, - type: "score", - key: null, - tierlistDataID: CalculateTierlistDataID(c.chartID, "score", null, TIERLIST_ID), - tierlistID: TIERLIST_ID, - data: { - flags: { - "Individual Difference": c.idvDifference === "FALSE", - }, - value: c.tiers["score-tier"], - humanised: `☆${c.tiers["score-tier"]}`, - }, - }; + const tdd: TierlistDataDocument<"Individual Difference"> = { + chartID: c.chartID, + type: "score", + key: null, + tierlistDataID: CalculateTierlistDataID(c.chartID, "score", null, TIERLIST_ID), + tierlistID: TIERLIST_ID, + data: { + flags: { + "Individual Difference": c.idvDifference === "FALSE", + }, + value: c.tiers["score-tier"], + humanised: `☆${c.tiers["score-tier"]}`, + }, + }; - return tdd; + return tdd; } (async () => { - const exists = await db["tierlist-data"].count({ - tierlistID: TIERLIST_ID, - }); + const exists = await db["tierlist-data"].count({ + tierlistID: TIERLIST_ID, + }); - if (exists > 0) { - logger.error(`Cancelling migration request as ${exists} documents already exist.`); - return process.exit(1); - } + if (exists > 0) { + logger.error(`Cancelling migration request as ${exists} documents already exist.`); + return process.exit(1); + } - const docs = await oldKTDB.get("tierlistdata").find({ - tierlistID: TIERLIST_ID, - }); + const docs = await oldKTDB.get("tierlistdata").find({ + tierlistID: TIERLIST_ID, + }); - const newDocs = await Promise.all(docs.map(ConvertFn)); + const newDocs = await Promise.all(docs.map(ConvertFn)); - await db["tierlist-data"].insert(newDocs); + await db["tierlist-data"].insert(newDocs); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts index f56576c9c..771928e04 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts @@ -5,25 +5,25 @@ import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): SongDocument<"chunithm"> { - const newSongDoc: SongDocument<"chunithm"> = { - title: c.title, - artist: c.artist, - id: c.id, - firstVersion: c.firstAppearance, - "alt-titles": c["alt-titles"].filter((e: string) => e !== c.title), - "search-titles": c["search-titles"] - .map((e: string) => e.toString()) - .filter((e: string) => e !== c.title), - data: { - genre: c.genre, - }, - }; + const newSongDoc: SongDocument<"chunithm"> = { + title: c.title, + artist: c.artist, + id: c.id, + firstVersion: c.firstAppearance, + "alt-titles": c["alt-titles"].filter((e: string) => e !== c.title), + "search-titles": c["search-titles"] + .map((e: string) => e.toString()) + .filter((e: string) => e !== c.title), + data: { + genre: c.genre, + }, + }; - return newSongDoc; + return newSongDoc; } (async () => { - await MigrateRecords(db.songs.chunithm, "songs-chunithm", ConvertFn); + await MigrateRecords(db.songs.chunithm, "songs-chunithm", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts index d9a4c7c2a..f4b112aaa 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts @@ -5,24 +5,24 @@ import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): SongDocument<"ddr"> { - const newSongDoc: SongDocument<"ddr"> = { - title: c.title, - artist: c.artist, - id: c.id, - firstVersion: c.firstAppearance, - "alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [], - "search-titles": c["search-title"] - .filter((e: string) => !!e) - .map((e: string) => e.toString()) - .filter((e: string) => e !== c.title), - data: {}, - }; + const newSongDoc: SongDocument<"ddr"> = { + title: c.title, + artist: c.artist, + id: c.id, + firstVersion: c.firstAppearance, + "alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [], + "search-titles": c["search-title"] + .filter((e: string) => !!e) + .map((e: string) => e.toString()) + .filter((e: string) => e !== c.title), + data: {}, + }; - return newSongDoc; + return newSongDoc; } (async () => { - await MigrateRecords(db.songs.ddr, "songs-ddr", ConvertFn); + await MigrateRecords(db.songs.ddr, "songs-ddr", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts index 16734c8a5..c24fb8d44 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts @@ -4,24 +4,24 @@ import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): SongDocument<"gitadora"> { - const newSongDoc: SongDocument<"gitadora"> = { - title: c.title, - artist: c.artist, - id: c.id, - firstVersion: c.firstAppearance, - "alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [], - "search-titles": c["search-titles"] - .filter((e: string) => !!e) - .map((e: string) => e.toString()) - .filter((e: string) => e !== c.title), - data: {}, - }; + const newSongDoc: SongDocument<"gitadora"> = { + title: c.title, + artist: c.artist, + id: c.id, + firstVersion: c.firstAppearance, + "alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [], + "search-titles": c["search-titles"] + .filter((e: string) => !!e) + .map((e: string) => e.toString()) + .filter((e: string) => e !== c.title), + data: {}, + }; - return newSongDoc; + return newSongDoc; } (async () => { - await MigrateRecords(db.songs.gitadora, "songs-gitadora", ConvertFn); + await MigrateRecords(db.songs.gitadora, "songs-gitadora", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts index 3f5b1a6c1..ca09f47b9 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts @@ -5,25 +5,25 @@ import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): SongDocument<"iidx"> { - const newSongDoc: SongDocument<"iidx"> = { - title: c.title, - artist: c.artist, - id: c.id, - firstVersion: c.firstAppearance, - "alt-titles": c["alt-titles"].filter((e: string) => e !== c.title), - "search-titles": c["search-titles"] - .map((e: string) => e.toString()) - .filter((e: string) => e !== c.title), - data: { - genre: c.genre, - }, - }; + const newSongDoc: SongDocument<"iidx"> = { + title: c.title, + artist: c.artist, + id: c.id, + firstVersion: c.firstAppearance, + "alt-titles": c["alt-titles"].filter((e: string) => e !== c.title), + "search-titles": c["search-titles"] + .map((e: string) => e.toString()) + .filter((e: string) => e !== c.title), + data: { + genre: c.genre, + }, + }; - return newSongDoc; + return newSongDoc; } (async () => { - await MigrateRecords(db.songs.iidx, "songs-iidx", ConvertFn); + await MigrateRecords(db.songs.iidx, "songs-iidx", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts index ee33f0a2e..2b69bddb2 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts @@ -4,24 +4,24 @@ import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): SongDocument<"museca"> { - const newSongDoc: SongDocument<"museca"> = { - title: c.title, - artist: c.artist, - id: c.id, - firstVersion: c.firstAppearance, - "alt-titles": [], - "search-titles": [], - data: { - titleJP: c.title_jp, - artistJP: c.artist_jp, - }, - }; + const newSongDoc: SongDocument<"museca"> = { + title: c.title, + artist: c.artist, + id: c.id, + firstVersion: c.firstAppearance, + "alt-titles": [], + "search-titles": [], + data: { + titleJP: c.title_jp, + artistJP: c.artist_jp, + }, + }; - return newSongDoc; + return newSongDoc; } (async () => { - await MigrateRecords(db.songs.museca, "songs-museca", ConvertFn); + await MigrateRecords(db.songs.museca, "songs-museca", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts index 165fb3820..aac8faf03 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts @@ -4,25 +4,25 @@ import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): SongDocument<"sdvx"> { - const newSongDoc: SongDocument<"sdvx"> = { - title: c.title, - artist: c.artist, - id: c.id, - firstVersion: c.firstAppearance, - "alt-titles": c["alt-titles"].filter((e: string) => e !== c.title), - "search-titles": c["search-titles"] - .map((e: string) => e.toString()) - .filter((e: string) => e !== c.title), - data: { - uscEquiv: null, - }, - }; + const newSongDoc: SongDocument<"sdvx"> = { + title: c.title, + artist: c.artist, + id: c.id, + firstVersion: c.firstAppearance, + "alt-titles": c["alt-titles"].filter((e: string) => e !== c.title), + "search-titles": c["search-titles"] + .map((e: string) => e.toString()) + .filter((e: string) => e !== c.title), + data: { + uscEquiv: null, + }, + }; - return newSongDoc; + return newSongDoc; } (async () => { - await MigrateRecords(db.songs.sdvx, "songs-sdvx", ConvertFn); + await MigrateRecords(db.songs.sdvx, "songs-sdvx", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/tierlists.ts b/server/scripts/__KT_DATABASE_MIGRATION/tierlists.ts index abb19de28..be16e2927 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/tierlists.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/tierlists.ts @@ -5,37 +5,37 @@ import db from "../../src/external/mongo/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): TierlistParent { - const tierlistParent: TierlistParent = { - createdAt: Date.now(), - tierlistID: c.tierlistID, - createdBy: 1, - description: `The official Kamaitachi Tierlist for ${gameHuman[c.game as Game]}${ - validPlaytypes[c.game as Game].length > 1 ? ` (${c.playtype})` : "" - }.`, - game: c.game, - playtype: c.playtype, - isDefault: true, - name: c.title, - lastUpdated: Date.now(), - permissions: { - anyPlayer: { - edit: 0, - submit: 1, - vote: 1, - }, - }, - config: { - autoHumanise: false, - flags: ["Individual Difference"], - requireState: "clear", - }, - }; + const tierlistParent: TierlistParent = { + createdAt: Date.now(), + tierlistID: c.tierlistID, + createdBy: 1, + description: `The official Kamaitachi Tierlist for ${gameHuman[c.game as Game]}${ + validPlaytypes[c.game as Game].length > 1 ? ` (${c.playtype})` : "" + }.`, + game: c.game, + playtype: c.playtype, + isDefault: true, + name: c.title, + lastUpdated: Date.now(), + permissions: { + anyPlayer: { + edit: 0, + submit: 1, + vote: 1, + }, + }, + config: { + autoHumanise: false, + flags: ["Individual Difference"], + requireState: "clear", + }, + }; - return tierlistParent; + return tierlistParent; } (async () => { - await MigrateRecords(db.tierlists, "tierlist", ConvertFn); + await MigrateRecords(db.tierlists, "tierlist", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts b/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts index b818e9e29..2e0b41953 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts @@ -8,19 +8,19 @@ import MigrateRecords from "./migrate"; const logger = CreateLogCtx(__filename); async function ConvertFn(c: any): Promise { - // const newUG: UserGoalDocument = {}; + // const newUG: UserGoalDocument = {}; - let relatedGoal = await db.goals.findOne({ goalID: c.goalID }); + const relatedGoal = await db.goals.findOne({ goalID: c.goalID }); - if (!relatedGoal) { - return null; - } + if (!relatedGoal) { + return null; + } - return c; + return c; } (async () => { - await MigrateRecords(db["user-goals"], "user-goals", ConvertFn); + await MigrateRecords(db["user-goals"], "user-goals", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/users.ts b/server/scripts/__KT_DATABASE_MIGRATION/users.ts index 87d83dc2c..48fa8d8d2 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/users.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/users.ts @@ -3,38 +3,38 @@ import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): PrivateUserDocument { - const newUserDoc: PrivateUserDocument = { - username: c.username, - usernameLowercase: c.username.toLowerCase(), - about: c.about, - clan: c.clan, - customBanner: c.custombanner, - customPfp: c.custompfp, - email: c.email, - friends: c.friends, - id: c.id, - lastSeen: c.lastSeen, - password: c.password, - permissions: {}, - settings: { - invisible: !!c.settings.invisible, - nsfwSplashes: !!c.settings.nsfwsplashes, - }, - socialMedia: { - discord: c.socialmedia.discord || null, - twitter: c.socialmedia.twitter || null, - youtube: c.socialmedia.youtube || null, - twitch: c.socialmedia.twitch || null, - github: c.socialmedia.github || null, - steam: c.socialmedia.steam || null, - }, - }; + const newUserDoc: PrivateUserDocument = { + username: c.username, + usernameLowercase: c.username.toLowerCase(), + about: c.about, + clan: c.clan, + customBanner: c.custombanner, + customPfp: c.custompfp, + email: c.email, + friends: c.friends, + id: c.id, + lastSeen: c.lastSeen, + password: c.password, + permissions: {}, + settings: { + invisible: !!c.settings.invisible, + nsfwSplashes: !!c.settings.nsfwsplashes, + }, + socialMedia: { + discord: c.socialmedia.discord || null, + twitter: c.socialmedia.twitter || null, + youtube: c.socialmedia.youtube || null, + twitch: c.socialmedia.twitch || null, + github: c.socialmedia.github || null, + steam: c.socialmedia.steam || null, + }, + }; - return newUserDoc; + return newUserDoc; } (async () => { - await MigrateRecords(db.users, "users", ConvertFn); + await MigrateRecords(db.users, "users", ConvertFn); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/arc-db-sync/arc-db-fetch.ts b/server/scripts/arc-db-sync/arc-db-fetch.ts index 94ebc3056..c64608505 100644 --- a/server/scripts/arc-db-sync/arc-db-fetch.ts +++ b/server/scripts/arc-db-sync/arc-db-fetch.ts @@ -18,74 +18,74 @@ program.parse(process.argv); const options = program.opts(); async function GetIIDXSongs() { - const allSongs = []; + const allSongs = []; - let url = `https://${options.url}/api/v1/iidx/27/music/`; + let url = `https://${options.url}/api/v1/iidx/27/music/`; - let moreData = true; - while (moreData) { - // eslint-disable-next-line no-await-in-loop - const songs = await fetch(url, { - headers: { - Authorization: `Bearer ${options.bearer}`, - }, - }).then((r) => r.json()); + let moreData = true; + while (moreData) { + // eslint-disable-next-line no-await-in-loop + const songs = await fetch(url, { + headers: { + Authorization: `Bearer ${options.bearer}`, + }, + }).then((r) => r.json()); - if (songs._links._next) { - url = songs._links._next; - } else { - moreData = false; - } + if (songs._links._next) { + url = songs._links._next; + } else { + moreData = false; + } - allSongs.push(...songs._items); - } + allSongs.push(...songs._items); + } - fs.writeFileSync(path.join(__dirname, "./songs.json"), JSON.stringify(allSongs)); + fs.writeFileSync(path.join(__dirname, "./songs.json"), JSON.stringify(allSongs)); } async function GetCharts() { - const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8")); + const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8")); - logger.info("Parsed songs.json"); + logger.info("Parsed songs.json"); - const allCharts: any[] = []; - const failed: any[] = []; + const allCharts: any[] = []; + const failed: any[] = []; - async function DoStuff(song: any) { - try { - logger.info( - `Starting Request https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true.` - ); + async function DoStuff(song: any) { + try { + logger.info( + `Starting Request https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true.` + ); - // eslint-disable-next-line no-await-in-loop - const charts = await fetch( - `https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true`, - { - headers: { - Authorization: `Bearer ${options.bearer}`, - }, - } - ).then((r) => r.json()); + // eslint-disable-next-line no-await-in-loop + const charts = await fetch( + `https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true`, + { + headers: { + Authorization: `Bearer ${options.bearer}`, + }, + } + ).then((r) => r.json()); - logger.info(`Parsed: ${charts._related.music[0].title}`); + logger.info(`Parsed: ${charts._related.music[0].title}`); - allCharts.push(charts); - } catch (err) { - logger.error(song); - logger.error(err); - failed.push(err); - } - } + allCharts.push(charts); + } catch (err) { + logger.error(song); + logger.error(err); + failed.push(err); + } + } - const promises = songs.map((s: any) => DoStuff(s)); + const promises = songs.map((s: any) => DoStuff(s)); - await Promise.all(promises); + await Promise.all(promises); - fs.writeFileSync(path.join(__dirname, "./charts.json"), JSON.stringify(allCharts)); + fs.writeFileSync(path.join(__dirname, "./charts.json"), JSON.stringify(allCharts)); - logger.info(failed); + logger.info(failed); - logger.info("done?"); + logger.info("done?"); } GetCharts(); diff --git a/server/scripts/arc-db-sync/arc-db-merge.ts b/server/scripts/arc-db-sync/arc-db-merge.ts index 276b32c41..01641440a 100644 --- a/server/scripts/arc-db-sync/arc-db-merge.ts +++ b/server/scripts/arc-db-sync/arc-db-merge.ts @@ -10,77 +10,77 @@ import { FindChartWithPTDFVersion } from "../../src/common/database-lookup/chart const logger = CreateLogCtx(__filename); async function MergeIDs() { - const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8")); + const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8")); - for (const chartData of charts) { - const songTitle = chartData._related.music[0].title; + for (const chartData of charts) { + const songTitle = chartData._related.music[0].title; - // eslint-disable-next-line no-await-in-loop - const song = await FindSongOnTitle("iidx", songTitle); + // eslint-disable-next-line no-await-in-loop + const song = await FindSongOnTitle("iidx", songTitle); - if (!song) { - logger.error(`Could not resolve ${songTitle}?`); - continue; - } + if (!song) { + logger.error(`Could not resolve ${songTitle}?`); + continue; + } - for (const chart of chartData._items) { - const playtype: "SP" | "DP" = chart.play_style === "DOUBLE" ? "DP" : "SP"; - let difficulty = chart.difficulty === "BLACK" ? "LEGGENDARIA" : chart.difficulty; - const version = "27-omni"; + for (const chart of chartData._items) { + const playtype: "SP" | "DP" = chart.play_style === "DOUBLE" ? "DP" : "SP"; + let difficulty = chart.difficulty === "BLACK" ? "LEGGENDARIA" : chart.difficulty; + const version = "27-omni"; - // yeah this fails for wacky playtype mismatches -- i don't care. - if ( - ["ミッドナイト堕天使", "Y&Co. is dead or alive", "State Of The Art"].includes( - songTitle - ) && - difficulty === "LEGGENDARIA" - ) { - difficulty = "ANOTHER"; - } + // yeah this fails for wacky playtype mismatches -- i don't care. + if ( + ["ミッドナイト堕天使", "Y&Co. is dead or alive", "State Of The Art"].includes( + songTitle + ) && + difficulty === "LEGGENDARIA" + ) { + difficulty = "ANOTHER"; + } - // eslint-disable-next-line no-await-in-loop - const tachiChart = await FindChartWithPTDFVersion( - "iidx", - song.id, - playtype, - difficulty, - version - ); + // eslint-disable-next-line no-await-in-loop + const tachiChart = await FindChartWithPTDFVersion( + "iidx", + song.id, + playtype, + difficulty, + version + ); - if (!tachiChart) { - logger.error( - `Could not find chart ${songTitle} ${playtype} ${difficulty} ${version}.` - ); - continue; - } + if (!tachiChart) { + logger.error( + `Could not find chart ${songTitle} ${playtype} ${difficulty} ${version}.` + ); + continue; + } - // eslint-disable-next-line no-await-in-loop - await ( - db.charts.iidx as unknown as ICollection> - ).update( - { - _id: tachiChart._id, - }, - { - $set: { - "data.arcChartID": chart._id, - }, - } - ); - } - } + // eslint-disable-next-line no-await-in-loop + await ( + db.charts.iidx as unknown as ICollection> + ).update( + { + _id: tachiChart._id, + }, + { + $set: { + "data.arcChartID": chart._id, + }, + } + ); + } + } - const r = await db.charts.iidx.update( - { - "data.arcChartID": { $exists: false }, - }, - { $set: { "data.arcChartID": null } }, - { - multi: true, - } - ); + const r = await db.charts.iidx.update( + { + "data.arcChartID": { $exists: false }, + }, + { $set: { "data.arcChartID": null } }, + { + multi: true, + } + ); - logger.info("done.", { r }); + logger.info("done.", { r }); } MergeIDs(); diff --git a/server/scripts/iidx-tierlist/get-sp12-data.ts b/server/scripts/iidx-tierlist/get-sp12-data.ts index f51a89e4d..1bb74f5f1 100644 --- a/server/scripts/iidx-tierlist/get-sp12-data.ts +++ b/server/scripts/iidx-tierlist/get-sp12-data.ts @@ -14,161 +14,161 @@ const logger = CreateLogCtx(__filename); const TIERLIST_ID = "ee9b756e50cff8282091102257b01f423ef855f2"; async function FetchSP12Data() { - const rj = await fetch("https://sp12.iidx.app/api/v1/sheets").then((r) => r.json()); // will throw if somethings wrong, anyway + const rj = await fetch("https://sp12.iidx.app/api/v1/sheets").then((r) => r.json()); // will throw if somethings wrong, anyway - if (!rj.sheets) { - throw new Error(`No sheets in return from sp12?`); - } + if (!rj.sheets) { + throw new Error(`No sheets in return from sp12?`); + } - const tdd: TierlistDataDocument<"Individual Difference">[] = []; + const tdd: TierlistDataDocument<"Individual Difference">[] = []; - for (const sh of rj.sheets) { - let chart; - try { - chart = await HumanisedTitleLookup(sh.title); - } catch (err) { - logger.error((err as Error).message); - continue; - } + for (const sh of rj.sheets) { + let chart; + try { + chart = await HumanisedTitleLookup(sh.title); + } catch (err) { + logger.error((err as Error).message); + continue; + } - for (const key of ["NORMAL CLEAR", "HARD CLEAR", "EX HARD CLEAR"]) { - let fakeKey; - let val; + for (const key of ["NORMAL CLEAR", "HARD CLEAR", "EX HARD CLEAR"]) { + let fakeKey; + let val; - switch (key) { - case "NORMAL CLEAR": - fakeKey = "n_clear"; - const v = Math.floor(sh[fakeKey] / 2); + switch (key) { + case "NORMAL CLEAR": + fakeKey = "n_clear"; + const v = Math.floor(sh[fakeKey] / 2); - if (v === 9) { - val = 11.8; - } else if (v === 8) { - val = 12.0; - } else if (v === 7) { - val = 12.2; - } else if (v === 6) { - val = 12.4; - } else if (v === 5) { - val = 12.6; - } else if (v < 0) { - continue; - } else { - val = 12.6 + (5 - v) * 0.1; - } + if (v === 9) { + val = 11.8; + } else if (v === 8) { + val = 12.0; + } else if (v === 7) { + val = 12.2; + } else if (v === 6) { + val = 12.4; + } else if (v === 5) { + val = 12.6; + } else if (v < 0) { + continue; + } else { + val = 12.6 + (5 - v) * 0.1; + } - break; - case "HARD CLEAR": - fakeKey = "hard"; - const v2 = Math.floor(sh[fakeKey] / 2); + break; + case "HARD CLEAR": + fakeKey = "hard"; + const v2 = Math.floor(sh[fakeKey] / 2); - if (v2 === 9) { - val = 11.9; - } else if (v2 === 8) { - val = 12.1; - } else if (v2 === 7) { - val = 12.3; - } else if (v2 === 6) { - val = 12.5; - } else if (v2 === 5) { - val = 12.7; - } else if (v2 < 0) { - continue; - } else { - val = 12.7 + (5 - v2) * 0.1; - } + if (v2 === 9) { + val = 11.9; + } else if (v2 === 8) { + val = 12.1; + } else if (v2 === 7) { + val = 12.3; + } else if (v2 === 6) { + val = 12.5; + } else if (v2 === 5) { + val = 12.7; + } else if (v2 < 0) { + continue; + } else { + val = 12.7 + (5 - v2) * 0.1; + } - break; - case "EX HARD CLEAR": - fakeKey = "exh"; + break; + case "EX HARD CLEAR": + fakeKey = "exh"; - const v3 = sh[fakeKey]; - if (v3 < 0) { - continue; - } + const v3 = sh[fakeKey]; + if (v3 < 0) { + continue; + } - val = 12.4 + (12 - v3) * 0.1; + val = 12.4 + (12 - v3) * 0.1; - break; - default: - throw new Error("??"); - } + break; + default: + throw new Error("??"); + } - const humanised = sh[`${fakeKey}_string`]; + const humanised = sh[`${fakeKey}_string`]; - if (humanised === "難易度未定") { - continue; - } + if (humanised === "難易度未定") { + continue; + } - if (val < 11.8) { - logger.info(`Skipping ${sh.title} ${key}, ${humanised}.`); - continue; - } + if (val < 11.8) { + logger.info(`Skipping ${sh.title} ${key}, ${humanised}.`); + continue; + } - val = parseFloat(val.toFixed(2)); + val = parseFloat(val.toFixed(2)); - tdd.push({ - chartID: chart.chartID, - type: "lamp", - tierlistID: TIERLIST_ID, - tierlistDataID: CalculateTierlistDataID(chart.chartID, "lamp", key, TIERLIST_ID), - key, - data: { - humanised, - value: val, - flags: { - "Individual Difference": !!humanised.match(/個人差/u), - }, - }, - }); - } - } + tdd.push({ + chartID: chart.chartID, + type: "lamp", + tierlistID: TIERLIST_ID, + tierlistDataID: CalculateTierlistDataID(chart.chartID, "lamp", key, TIERLIST_ID), + key, + data: { + humanised, + value: val, + flags: { + "Individual Difference": !!humanised.match(/個人差/u), + }, + }, + }); + } + } - logger.info(`Inserting ${tdd.length} documents...`); + logger.info(`Inserting ${tdd.length} documents...`); - await db["tierlist-data"].insert(tdd); + await db["tierlist-data"].insert(tdd); - logger.info("Done!"); + logger.info("Done!"); - process.exit(0); + process.exit(0); } function isOdd(num: number) { - return num % 2 !== 0; + return num % 2 !== 0; } async function HumanisedTitleLookup(originalTitle: string) { - let difficulty: Difficulties["iidx:SP"] = "ANOTHER"; + let difficulty: Difficulties["iidx:SP"] = "ANOTHER"; - let title = originalTitle; + let title = originalTitle; - if (title.match(/(†|†LEGGENDARIA)$/u)) { - difficulty = "LEGGENDARIA"; - title = title.split("†")[0]; // this will break if there is ever a ptp leggendaria - } else if (title.match(/\[H\]$/u)) { - difficulty = "HYPER"; - title = title.split("[")[0]; - } else if (title.match(/\[A\]$/u)) { - difficulty = "ANOTHER"; // lmao - title = title.split("[")[0]; - } + if (title.match(/(†|†LEGGENDARIA)$/u)) { + difficulty = "LEGGENDARIA"; + title = title.split("†")[0]; // this will break if there is ever a ptp leggendaria + } else if (title.match(/\[H\]$/u)) { + difficulty = "HYPER"; + title = title.split("[")[0]; + } else if (title.match(/\[A\]$/u)) { + difficulty = "ANOTHER"; // lmao + title = title.split("[")[0]; + } - const song = await FindSongOnTitle("iidx", title); + const song = await FindSongOnTitle("iidx", title); - if (!song) { - throw new Error( - `Could not resolve song ${title} (${difficulty}) (Original ${originalTitle}).` - ); - } + if (!song) { + throw new Error( + `Could not resolve song ${title} (${difficulty}) (Original ${originalTitle}).` + ); + } - const chart = await FindChartWithPTDF("iidx", song.id, "SP", difficulty); + const chart = await FindChartWithPTDF("iidx", song.id, "SP", difficulty); - if (!chart) { - throw new Error( - `Could not resolve chart ${title} ${difficulty} (Original ${originalTitle}).` - ); - } + if (!chart) { + throw new Error( + `Could not resolve chart ${title} ${difficulty} (Original ${originalTitle}).` + ); + } - return chart; + return chart; } FetchSP12Data(); diff --git a/server/scripts/iidx-tierlist/parse-iidxkr-data.ts b/server/scripts/iidx-tierlist/parse-iidxkr-data.ts index 28ffa057b..b16d01d7e 100644 --- a/server/scripts/iidx-tierlist/parse-iidxkr-data.ts +++ b/server/scripts/iidx-tierlist/parse-iidxkr-data.ts @@ -12,153 +12,153 @@ const logger = CreateLogCtx(__filename); const TIERLIST_ID = "ee9b756e50cff8282091102257b01f423ef855f2"; async function parseKr( - dir: string, - key: "NORMAL CLEAR" | "HARD CLEAR", - catVals: { h: string; v: number }[] + dir: string, + key: "NORMAL CLEAR" | "HARD CLEAR", + catVals: { h: string; v: number }[] ) { - const krdata = JSON.parse(fs.readFileSync(path.join(__dirname, "./iidx-pe-kr", dir), "utf-8")); + const krdata = JSON.parse(fs.readFileSync(path.join(__dirname, "./iidx-pe-kr", dir), "utf-8")); - const tdd = []; + const tdd = []; - for (let i = 0; i < krdata.categories.length; i++) { - const data = krdata.categories[i]; - if (data.sortindex < 0) { - continue; - } + for (let i = 0; i < krdata.categories.length; i++) { + const data = krdata.categories[i]; + if (data.sortindex < 0) { + continue; + } - const cv = catVals[i]; + const cv = catVals[i]; - if (!cv) { - throw new Error(`cv krdata mismatch ${krdata.categories.length} ${catVals.length}?`); - } + if (!cv) { + throw new Error(`cv krdata mismatch ${krdata.categories.length} ${catVals.length}?`); + } - for (const item of data.items) { - const song = await FindSongOnTitle("iidx", item.data.title.replace(/†$/u, "").trim()); + for (const item of data.items) { + const song = await FindSongOnTitle("iidx", item.data.title.replace(/†$/u, "").trim()); - if (!song) { - logger.warn(`Could not find song with title ${item.data.title}`); - continue; - } + if (!song) { + logger.warn(`Could not find song with title ${item.data.title}`); + continue; + } - const t = item.data.type; + const t = item.data.type; - let diff: Difficulties["iidx:SP"]; - if (t === "A") { - diff = "ANOTHER"; - } else if (t === "L") { - diff = "LEGGENDARIA"; - } else if (t === "H") { - diff = "HYPER"; - } else if (t === "N") { - diff = "NORMAL"; - } else { - logger.warn(`${song.title} Unknown difficulty ${t}.`); - continue; - } + let diff: Difficulties["iidx:SP"]; + if (t === "A") { + diff = "ANOTHER"; + } else if (t === "L") { + diff = "LEGGENDARIA"; + } else if (t === "H") { + diff = "HYPER"; + } else if (t === "N") { + diff = "NORMAL"; + } else { + logger.warn(`${song.title} Unknown difficulty ${t}.`); + continue; + } - const chart = await FindChartWithPTDF("iidx", song.id, "SP", diff); + const chart = await FindChartWithPTDF("iidx", song.id, "SP", diff); - if (!chart) { - logger.warn(`${song.title} ${diff} - Couldn't find chart?`); - continue; - } + if (!chart) { + logger.warn(`${song.title} ${diff} - Couldn't find chart?`); + continue; + } - tdd.push({ - chartID: chart.chartID, - type: "lamp" as const, - tierlistID: TIERLIST_ID, - tierlistDataID: CalculateTierlistDataID(chart.chartID, "lamp", key, TIERLIST_ID), - key, - data: { - humanised: cv.h, - value: cv.v, - flags: { - "Individual Difference": !!data.category.match(/個人差/u), - }, - }, - }); - } - } + tdd.push({ + chartID: chart.chartID, + type: "lamp" as const, + tierlistID: TIERLIST_ID, + tierlistDataID: CalculateTierlistDataID(chart.chartID, "lamp", key, TIERLIST_ID), + key, + data: { + humanised: cv.h, + value: cv.v, + flags: { + "Individual Difference": !!data.category.match(/個人差/u), + }, + }, + }); + } + } - logger.info(`Inserting ${tdd.length} documents...`); + logger.info(`Inserting ${tdd.length} documents...`); - const x = await db["tierlist-data"].bulkWrite( - tdd.map((e) => ({ - updateOne: { - filter: { - tierlistDataID: e.tierlistDataID, - }, - update: { - $set: e, - }, - upsert: true, - }, - })) - ); + const x = await db["tierlist-data"].bulkWrite( + tdd.map((e) => ({ + updateOne: { + filter: { + tierlistDataID: e.tierlistDataID, + }, + update: { + $set: e, + }, + upsert: true, + }, + })) + ); - logger.info(x); + logger.info(x); - logger.info("done..."); + logger.info("done..."); } function h(h: string, v: number) { - return { h, v }; + return { h, v }; } parseKr("sp9H.json", "HARD CLEAR", [ - h("S+", 10.2), - h("S", 10), - h("S 個人差", 10), - h("A+", 9.9), - h("A+ 個人差", 9.9), - h("A", 9.8), - h("B+", 9.7), - h("B", 9.6), - h("C+", 9.5), - h("C", 9.4), - h("D", 9.2), - h("E", 9.0), - h("F", 8.8), + h("S+", 10.2), + h("S", 10), + h("S 個人差", 10), + h("A+", 9.9), + h("A+ 個人差", 9.9), + h("A", 9.8), + h("B+", 9.7), + h("B", 9.6), + h("C+", 9.5), + h("C", 9.4), + h("D", 9.2), + h("E", 9.0), + h("F", 8.8), ]); parseKr("sp10H.json", "HARD CLEAR", [ - h("S", 11), - h("A+", 10.9), - h("A", 10.8), - h("B+", 10.7), - h("B", 10.6), - h("C+", 10.5), - h("C", 10.4), - h("D", 10.2), - h("E", 10.0), - h("F", 9.8), + h("S", 11), + h("A+", 10.9), + h("A", 10.8), + h("B+", 10.7), + h("B", 10.6), + h("C+", 10.5), + h("C", 10.4), + h("D", 10.2), + h("E", 10.0), + h("F", 9.8), ]); parseKr("sp11N.json", "NORMAL CLEAR", [ - h("S+", 12.2), - h("S", 12), - h("A+", 11.9), - h("A", 11.8), - h("B", 11.6), - h("C", 11.4), - h("D", 11.2), - h("E", 11.0), - h("F", 10.8), + h("S+", 12.2), + h("S", 12), + h("A+", 11.9), + h("A", 11.8), + h("B", 11.6), + h("C", 11.4), + h("D", 11.2), + h("E", 11.0), + h("F", 10.8), ]); parseKr("sp11H.json", "HARD CLEAR", [ - h("S+", 12.2), - h("S+ 個人差", 12.2), - h("S", 12), - h("S 個人差", 12), - h("A+", 11.9), - h("A+ 個人差", 11.9), - h("A", 11.8), - h("A 個人差", 11.0), - h("B+", 11.7), - h("B", 11.6), - h("C", 11.4), - h("D", 11.2), - h("E", 11.0), - h("F", 10.8), + h("S+", 12.2), + h("S+ 個人差", 12.2), + h("S", 12), + h("S 個人差", 12), + h("A+", 11.9), + h("A+ 個人差", 11.9), + h("A", 11.8), + h("A 個人差", 11.0), + h("B+", 11.7), + h("B", 11.6), + h("C", 11.4), + h("D", 11.2), + h("E", 11.0), + h("F", 10.8), ]); diff --git a/server/scripts/initialise-counters.ts b/server/scripts/initialise-counters.ts index 9826f0e2b..54c0775d1 100644 --- a/server/scripts/initialise-counters.ts +++ b/server/scripts/initialise-counters.ts @@ -4,19 +4,19 @@ import CreateLogCtx from "../common/logger"; const logger = CreateLogCtx(__filename); (async () => { - let users = await db.counters.findOne({ - counterName: "users", - }); + const users = await db.counters.findOne({ + counterName: "users", + }); - if (users) { - throw new Error(`"users" document already exists, exiting.`); - } + if (users) { + throw new Error(`"users" document already exists, exiting.`); + } - await db.counters.insert({ - counterName: "users", - value: 1, - }); + await db.counters.insert({ + counterName: "users", + value: 1, + }); - logger.info("Successfully initialised counter documents. Exiting."); - process.exit(0); + logger.info("Successfully initialised counter documents. Exiting."); + process.exit(0); })(); diff --git a/server/scripts/single-use/add-2dxtra-flag.ts b/server/scripts/single-use/add-2dxtra-flag.ts index ce7debc20..0f7421787 100644 --- a/server/scripts/single-use/add-2dxtra-flag.ts +++ b/server/scripts/single-use/add-2dxtra-flag.ts @@ -3,19 +3,19 @@ import db from "../../src/db/db"; db.charts.iidx - .find({}) - .each(async (c, { pause, resume }) => { - pause(); + .find({}) + .each(async (c, { pause, resume }) => { + pause(); - await db.charts.iidx.update( - { _id: c._id }, - { - $set: { - "flags.2dxtra": false, - }, - } - ); + await db.charts.iidx.update( + { _id: c._id }, + { + $set: { + "flags.2dxtra": false, + }, + } + ); - resume(); - }) - .then(() => console.log("done")); + resume(); + }) + .then(() => console.log("done")); diff --git a/server/scripts/single-use/add-hcnver.ts b/server/scripts/single-use/add-hcnver.ts index 016101bd3..c945a32de 100644 --- a/server/scripts/single-use/add-hcnver.ts +++ b/server/scripts/single-use/add-hcnver.ts @@ -7,18 +7,18 @@ import CreateLogCtx from "../../src/common/logger"; const logger = CreateLogCtx(__filename); -let data = JSON.parse(fs.readFileSync(path.join(__dirname, "./hcnverdata.json"), "utf-8")); +const data = JSON.parse(fs.readFileSync(path.join(__dirname, "./hcnverdata.json"), "utf-8")); for (const d of data) { - let r = p(d, PRUDENCE_CHART_SCHEMAS.iidx); - if (r) { - logger.error(r.userVal); - logger.error(r); - throw r; - } + const r = p(d, PRUDENCE_CHART_SCHEMAS.iidx); + if (r) { + logger.error(r.userVal); + logger.error(r); + throw r; + } } db.charts.iidx.insert(data).then(() => { - logger.info("Done."); - process.exit(0); + logger.info("Done."); + process.exit(0); }); diff --git a/server/scripts/single-use/fix-chuni-port.ts b/server/scripts/single-use/fix-chuni-port.ts index 96d6697c5..f180f9152 100644 --- a/server/scripts/single-use/fix-chuni-port.ts +++ b/server/scripts/single-use/fix-chuni-port.ts @@ -1,18 +1,18 @@ import db from "../../src/db/db"; (async () => { - let charts = await db.charts.chunithm.find({}); + const charts = await db.charts.chunithm.find({}); - for (const c of charts) { - await db.charts.chunithm.update( - { - _id: c._id, - }, - { - $set: { - "data.inGameID": c.songID, - }, - } - ); - } + for (const c of charts) { + await db.charts.chunithm.update( + { + _id: c._id, + }, + { + $set: { + "data.inGameID": c.songID, + }, + } + ); + } })(); diff --git a/server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts b/server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts index cb3fe5847..cdf2498f0 100644 --- a/server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts +++ b/server/scripts/single-use/fix-iidx-versions/fix-iidx-versions.ts @@ -2,27 +2,27 @@ import db from "../../../src/db/db"; import { ChartDocument } from "tachi-common"; db.charts.iidx - .find({}) - // @ts-expect-error it exists - .each(async (c: ChartDocument<"iidx:SP" | "iidx:DP">, { pause, resume }: any) => { - pause(); + .find({}) + // @ts-expect-error it exists + .each(async (c: ChartDocument<"iidx:SP" | "iidx:DP">, { pause, resume }: any) => { + pause(); - const song = (await db.songs.iidx.findOne({ id: c.songID }))!; + const song = (await db.songs.iidx.findOne({ id: c.songID }))!; - let versions = []; + let versions = []; - if (song.firstVersion === "27") { - versions = ["27", "28", "27-omni", "28-omni"]; - } else if (song.firstVersion === "28") { - versions = ["28", "28-omni"]; - } else if (song.firstVersion === "inf") { - versions = ["inf"]; - } else { - versions = ["26", "27", "28", "26-omni", "27-omni", "28-omni"]; - } + if (song.firstVersion === "27") { + versions = ["27", "28", "27-omni", "28-omni"]; + } else if (song.firstVersion === "28") { + versions = ["28", "28-omni"]; + } else if (song.firstVersion === "inf") { + versions = ["inf"]; + } else { + versions = ["26", "27", "28", "26-omni", "27-omni", "28-omni"]; + } - await db.charts.iidx.update({ _id: c._id }, { $set: { versions } }); + await db.charts.iidx.update({ _id: c._id }, { $set: { versions } }); - resume(); - }) - .then(() => console.log("done")); + resume(); + }) + .then(() => console.log("done")); diff --git a/server/scripts/single-use/fix-iidx-versions/stat.ts b/server/scripts/single-use/fix-iidx-versions/stat.ts index 7652275ea..5644d31b5 100644 --- a/server/scripts/single-use/fix-iidx-versions/stat.ts +++ b/server/scripts/single-use/fix-iidx-versions/stat.ts @@ -4,17 +4,17 @@ import db from "../../../src/db/db"; const logger = CreateLogCtx(__filename); (async () => { - const charts = await db.charts.iidx.find({ versions: { $size: 0 } }); + const charts = await db.charts.iidx.find({ versions: { $size: 0 } }); - for (const chart of charts) { - // eslint-disable-next-line no-await-in-loop - const song = await db.songs.iidx.findOne({ id: chart.songID }); + for (const chart of charts) { + // eslint-disable-next-line no-await-in-loop + const song = await db.songs.iidx.findOne({ id: chart.songID }); - if (!song) { - logger.error(`cannot find parent song ${chart.songID}`); - continue; - } + if (!song) { + logger.error(`cannot find parent song ${chart.songID}`); + continue; + } - logger.info(`${song.title} ${chart.playtype} ${chart.difficulty}`); - } + logger.info(`${song.title} ${chart.playtype} ${chart.difficulty}`); + } })(); diff --git a/server/scripts/single-use/fix-iidx-versions/update-versions.ts b/server/scripts/single-use/fix-iidx-versions/update-versions.ts index 57ef239f2..0872622a3 100644 --- a/server/scripts/single-use/fix-iidx-versions/update-versions.ts +++ b/server/scripts/single-use/fix-iidx-versions/update-versions.ts @@ -7,90 +7,90 @@ import { FindSongOnTitleInsensitive } from "../../../src/common/database-lookup/ const logger = CreateLogCtx(__filename); const comicallyLargeJSONData = JSON.parse( - fs.readFileSync(path.join(__dirname, "./versions.json"), "utf-8") + fs.readFileSync(path.join(__dirname, "./versions.json"), "utf-8") ); const DIFF_MAP = { - A: "ANOTHER", - H: "HYPER", - N: "NORMAL", - B: "BEGINNER", - X: "LEGGENDARIA", + A: "ANOTHER", + H: "HYPER", + N: "NORMAL", + B: "BEGINNER", + X: "LEGGENDARIA", } as any; function ConvertVersion(c: string) { - if (c.startsWith("AC")) { - return c.split("AC")[1]; - } else if (c.startsWith("OM")) { - return `${c.split("OM")[1]}-omni`; - } else if (c.startsWith("CS")) { - return `${c.split("CS")[1]}-cs`; - } else if (c === "INF") { - return "inf"; - } else if (c === "BMUS") { - return "bmus"; - } else { - logger.error(`Invalid Chart Version ${c}.`); - throw new Error("Invalid Chart Version"); - } + if (c.startsWith("AC")) { + return c.split("AC")[1]; + } else if (c.startsWith("OM")) { + return `${c.split("OM")[1]}-omni`; + } else if (c.startsWith("CS")) { + return `${c.split("CS")[1]}-cs`; + } else if (c === "INF") { + return "inf"; + } else if (c === "BMUS") { + return "bmus"; + } else { + logger.error(`Invalid Chart Version ${c}.`); + throw new Error("Invalid Chart Version"); + } } (async () => { - for (const data of comicallyLargeJSONData) { - const song = await FindSongOnTitleInsensitive("iidx", data["MUSIC NAME"]); + for (const data of comicallyLargeJSONData) { + const song = await FindSongOnTitleInsensitive("iidx", data["MUSIC NAME"]); - if (!song) { - logger.error(`Can't find song with title ${data["MUSIC NAME"]}.`); - continue; - } + if (!song) { + logger.error(`Can't find song with title ${data["MUSIC NAME"]}.`); + continue; + } - const playtype = data.STYLE === "S" ? "SP" : "DP"; + const playtype = data.STYLE === "S" ? "SP" : "DP"; - const difficulty = DIFF_MAP[data.DIFF]; + const difficulty = DIFF_MAP[data.DIFF]; - const relevantVersions = [ - "OM27", - "OM26", - "AC27", - "AC26", - "INF", - "CS16", - "CS15", - "CS14", - "CS13", - "CS12", - "CS11", - "CS10", - "CS9", - "CS8", - "CS7", - "BMUS", - "CS6", - "CS5", - "CS4", - "CS3", - ] - .filter((e) => data[e] === "1") - .map(ConvertVersion); + const relevantVersions = [ + "OM27", + "OM26", + "AC27", + "AC26", + "INF", + "CS16", + "CS15", + "CS14", + "CS13", + "CS12", + "CS11", + "CS10", + "CS9", + "CS8", + "CS7", + "BMUS", + "CS6", + "CS5", + "CS4", + "CS3", + ] + .filter((e) => data[e] === "1") + .map(ConvertVersion); - if (!relevantVersions.includes("27-omni") && !relevantVersions.includes("inf")) { - logger.warn(`Song ${song.title} (${playtype} ${difficulty}) has no relevant versions.`); - continue; - } + if (!relevantVersions.includes("27-omni") && !relevantVersions.includes("inf")) { + logger.warn(`Song ${song.title} (${playtype} ${difficulty}) has no relevant versions.`); + continue; + } - // logger.info(relevantVersions);q + // logger.info(relevantVersions);q - const chart = await db.charts.iidx.findOne({ songID: song.id, difficulty, playtype }); + const chart = await db.charts.iidx.findOne({ songID: song.id, difficulty, playtype }); - if (!chart) { - logger.error(`Can't find chart ${song.title} (${playtype} ${difficulty})`); - continue; - } + if (!chart) { + logger.error(`Can't find chart ${song.title} (${playtype} ${difficulty})`); + continue; + } - await db.charts.iidx.update({ _id: chart._id }, { $set: { versions: relevantVersions } }); - } + await db.charts.iidx.update({ _id: chart._id }, { $set: { versions: relevantVersions } }); + } - logger.info("done"); + logger.info("done"); - process.exit(0); + process.exit(0); })(); diff --git a/server/scripts/single-use/fix-legacy-folder-format.ts b/server/scripts/single-use/fix-legacy-folder-format.ts index a135bdbb4..c40c5f4f8 100644 --- a/server/scripts/single-use/fix-legacy-folder-format.ts +++ b/server/scripts/single-use/fix-legacy-folder-format.ts @@ -2,27 +2,27 @@ import db from "../../src/db/db"; import { FolderDocument } from "tachi-common"; (async () => { - const folders = (await db.folders.find({})) as any[]; + const folders = (await db.folders.find({})) as any[]; - for (const fl of folders) { - const f: FolderDocument = { - folderID: fl.folderID, - game: fl.game, - playtype: fl.playtype, - table: fl.table, - tableIndex: fl.tableIndex, - title: fl.title, - type: fl.query.collection === "charts" ? "charts" : "songs", - data: fl.query.body, - }; + for (const fl of folders) { + const f: FolderDocument = { + folderID: fl.folderID, + game: fl.game, + playtype: fl.playtype, + table: fl.table, + tableIndex: fl.tableIndex, + title: fl.title, + type: fl.query.collection === "charts" ? "charts" : "songs", + data: fl.query.body, + }; - await db.folders.update( - { - _id: fl._id, - }, - { $set: f } - ); - } + await db.folders.update( + { + _id: fl._id, + }, + { $set: f } + ); + } - console.log("done"); + console.log("done"); })(); diff --git a/server/scripts/single-use/numify-tableindex.ts b/server/scripts/single-use/numify-tableindex.ts index e54cfd29d..8782be945 100644 --- a/server/scripts/single-use/numify-tableindex.ts +++ b/server/scripts/single-use/numify-tableindex.ts @@ -1,24 +1,24 @@ import db from "../../src/db/db"; (async () => { - db.folders - .find({}) - // @ts-expect-error ??? monk - .each(async (c, { pause, resume }) => { - pause(); + db.folders + .find({}) + // @ts-expect-error ??? monk + .each(async (c, { pause, resume }) => { + pause(); - await db.folders.update( - { - _id: c._id, - }, - { - $set: { - tableIndex: Number(c.tableIndex), - }, - } - ); + await db.folders.update( + { + _id: c._id, + }, + { + $set: { + tableIndex: Number(c.tableIndex), + }, + } + ); - resume(); - }) - .then(() => console.log("done")); + resume(); + }) + .then(() => console.log("done")); })(); diff --git a/server/scripts/single-use/s3check.ts b/server/scripts/single-use/s3check.ts index 381fafd92..0520429b8 100644 --- a/server/scripts/single-use/s3check.ts +++ b/server/scripts/single-use/s3check.ts @@ -8,14 +8,14 @@ const logger = CreateLogCtx(__filename); let parsedXML; const fileData = fs.readFileSync( - path.join(__dirname, "../../src/test-utils/test-data/s3/valid.xml"), - "utf-8" + path.join(__dirname, "../../src/test-utils/test-data/s3/valid.xml"), + "utf-8" ); try { - parsedXML = parser.parse(fileData); + parsedXML = parser.parse(fileData); } catch (err) { - logger.error(err); + logger.error(err); } fs.writeFileSync(path.join(__dirname, "s3data.json"), JSON.stringify(parsedXML, null, 4)); diff --git a/server/scripts/single-use/sample-iidx-db.ts b/server/scripts/single-use/sample-iidx-db.ts index 43ce9a8ca..61f856df5 100644 --- a/server/scripts/single-use/sample-iidx-db.ts +++ b/server/scripts/single-use/sample-iidx-db.ts @@ -3,7 +3,7 @@ import db from "../../db/db"; import path from "path"; (async () => { - let data = await db.scores.find({ game: "iidx", playtype: "SP" }, { limit: 1000 }); - fs.writeFileSync(path.join(__dirname, "data.json"), JSON.stringify(data)); - process.exit(0); + const data = await db.scores.find({ game: "iidx", playtype: "SP" }, { limit: 1000 }); + fs.writeFileSync(path.join(__dirname, "data.json"), JSON.stringify(data)); + process.exit(0); })(); diff --git a/server/scripts/update-bpi-poyashi.ts b/server/scripts/update-bpi-poyashi.ts index 1fc69458e..24af9a128 100644 --- a/server/scripts/update-bpi-poyashi.ts +++ b/server/scripts/update-bpi-poyashi.ts @@ -6,8 +6,8 @@ import path from "path"; import { config, Difficulties, ESDCore, IIDXBPIData, ChartDocument } from "tachi-common"; import { FindSongOnTitle } from "../src/score-import/database-lookup/song-title"; import { - FindChartWithPTDF, - FindChartWithPTDFVersion, + FindChartWithPTDF, + FindChartWithPTDFVersion, } from "../src/score-import/database-lookup/chart-ptdf"; import db from "../src/db/db"; import CreateLogCtx from "../src/common/logger"; @@ -23,92 +23,92 @@ const options = program.opts(); const dataLoc = path.join(__dirname, "./bpi-poyashi-data/output.json"); const difficultyResolve: Record = { - 3: ["SP", "HYPER"], - 4: ["SP", "ANOTHER"], - 8: ["DP", "HYPER"], - 9: ["DP", "ANOTHER"], - 10: ["SP", "LEGGENDARIA"], - 11: ["DP", "LEGGENDARIA"], + 3: ["SP", "HYPER"], + 4: ["SP", "ANOTHER"], + 8: ["DP", "HYPER"], + 9: ["DP", "ANOTHER"], + 10: ["SP", "LEGGENDARIA"], + 11: ["DP", "LEGGENDARIA"], }; async function UpdatePoyashiData() { - let data; - if (options.fetch) { - // lol! - logger.info("Fetching data from github..."); - const rj = await fetch( - "https://raw.githubusercontent.com/potakusan/bpim-score-repo/master/output/release.json" - ).then((r) => r.json()); + let data; + if (options.fetch) { + // lol! + logger.info("Fetching data from github..."); + const rj = await fetch( + "https://raw.githubusercontent.com/potakusan/bpim-score-repo/master/output/release.json" + ).then((r) => r.json()); - logger.info("Fetched data."); - fs.writeFileSync(dataLoc, JSON.stringify(rj)); - logger.info("Saved data to output.json."); + logger.info("Fetched data."); + fs.writeFileSync(dataLoc, JSON.stringify(rj)); + logger.info("Saved data to output.json."); - data = rj; - } else { - logger.info("Reading data from output.json..."); - data = JSON.parse(fs.readFileSync(dataLoc, "utf-8")); - } + data = rj; + } else { + logger.info("Reading data from output.json..."); + data = JSON.parse(fs.readFileSync(dataLoc, "utf-8")); + } - const realData: IIDXBPIData[] = []; - for (const d of data.body) { - const res = difficultyResolve[d.difficulty]; + const realData: IIDXBPIData[] = []; + for (const d of data.body) { + const res = difficultyResolve[d.difficulty]; - if (!res) { - throw new Error(`Unknown difficulty ${d.difficulty}`); - } + if (!res) { + throw new Error(`Unknown difficulty ${d.difficulty}`); + } - const [playtype, diff] = res; + const [playtype, diff] = res; - const tachiSong = await FindSongOnTitle("iidx", d.title); + const tachiSong = await FindSongOnTitle("iidx", d.title); - if (!tachiSong) { - logger.warn(`Cannot find song ${d.title}?`); - continue; - } + if (!tachiSong) { + logger.warn(`Cannot find song ${d.title}?`); + continue; + } - const tachiChart = (await FindChartWithPTDFVersion( - "iidx", - tachiSong.id, - playtype as "SP" | "DP", - diff as Difficulties["iidx:DP" | "iidx:SP"], - "28" - )) as ChartDocument<"iidx:SP" | "iidx:DP">; + const tachiChart = (await FindChartWithPTDFVersion( + "iidx", + tachiSong.id, + playtype as "SP" | "DP", + diff as Difficulties["iidx:DP" | "iidx:SP"], + "28" + )) as ChartDocument<"iidx:SP" | "iidx:DP">; - if (!tachiChart) { - logger.warn( - `Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?` - ); - continue; - } + if (!tachiChart) { + logger.warn( + `Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?` + ); + continue; + } - const kavg = Number(d.avg); + const kavg = Number(d.avg); - if (kavg < 0) { - logger.warn( - `${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.` - ); - continue; - } + if (kavg < 0) { + logger.warn( + `${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.` + ); + continue; + } - const kesd = ESDCore.CalculateESD( - config.judgementWindows.iidx.SP, - kavg / (tachiChart.data.notecount * 2) - ); + const kesd = ESDCore.CalculateESD( + config.judgementWindows.iidx.SP, + kavg / (tachiChart.data.notecount * 2) + ); - realData.push({ - coef: d.coef === -1 ? null : d.coef, - kavg: Number(d.avg), - wr: Number(d.wr), - chartID: tachiChart.chartID, - kesd, - }); - } + realData.push({ + coef: d.coef === -1 ? null : d.coef, + kavg: Number(d.avg), + wr: Number(d.wr), + chartID: tachiChart.chartID, + kesd, + }); + } - await db["iidx-bpi-data"].remove({}); - await db["iidx-bpi-data"].insert(realData); + await db["iidx-bpi-data"].remove({}); + await db["iidx-bpi-data"].insert(realData); - logger.info(`Removed all, and inserted ${realData.length} documents.`); + logger.info(`Removed all, and inserted ${realData.length} documents.`); } UpdatePoyashiData(); diff --git a/server/src/lib/score-import/framework/calculated-data/stats.test.ts b/server/src/lib/score-import/framework/calculated-data/stats.test.ts index 3b140e9c0..659f787eb 100644 --- a/server/src/lib/score-import/framework/calculated-data/stats.test.ts +++ b/server/src/lib/score-import/framework/calculated-data/stats.test.ts @@ -13,11 +13,7 @@ import { import CreateLogCtx from "lib/logger/logger"; import { isApproximately } from "test-utils/asserts"; import { DryScore } from "../common/types"; -import { - GetKTDataJSON, - Testing511SPA, - TestingIIDXSPDryScore, -} from "test-utils/test-data"; +import { GetKTDataJSON, Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data"; import { CloseAllConnections } from "test-utils/close-connections"; import deepmerge from "deepmerge"; import ResetDBState from "test-utils/resets"; diff --git a/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.test.ts b/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.test.ts index 0a0379e64..6c73706b5 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.test.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/iidx/converter.test.ts @@ -1,11 +1,7 @@ import t from "tap"; import { CloseAllConnections } from "test-utils/close-connections"; import ResetDBState from "test-utils/resets"; -import { - GetKTDataJSON, - Testing511Song, - Testing511SPA, -} from "test-utils/test-data"; +import { GetKTDataJSON, Testing511Song, Testing511SPA } from "test-utils/test-data"; import CreateLogCtx from "lib/logger/logger"; import { ConvertAPIKaiIIDX } from "./converter"; import deepmerge from "deepmerge"; diff --git a/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts b/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts index e0518eea3..992befc44 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts @@ -1,11 +1,7 @@ import t from "tap"; import { CloseAllConnections } from "test-utils/close-connections"; import ResetDBState from "test-utils/resets"; -import { - GetKTDataJSON, - TestingAlbidaADV, - TestingSDVXAlbidaSong, -} from "test-utils/test-data"; +import { GetKTDataJSON, TestingAlbidaADV, TestingSDVXAlbidaSong } from "test-utils/test-data"; import CreateLogCtx from "lib/logger/logger"; import { ConvertAPIKaiSDVX, ConvertDifficulty, ConvertVersion, ResolveKaiLamp } from "./converter"; import deepmerge from "deepmerge"; diff --git a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts index 34390fe83..4ef3aab25 100644 --- a/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts +++ b/server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts @@ -2,10 +2,7 @@ import t from "tap"; import CreateLogCtx from "lib/logger/logger"; import ScoreImportFatalError from "../../../framework/score-importing/score-import-error"; import GenericParseEamIIDXCSV, { NaiveCSVParse, ResolveHeaders } from "./parser"; -import { - TestingIIDXEamusementCSV26, - TestingIIDXEamusementCSV27, -} from "test-utils/test-data"; +import { TestingIIDXEamusementCSV26, TestingIIDXEamusementCSV27 } from "test-utils/test-data"; import { MockMulterFile } from "test-utils/mock-multer"; import { CloseAllConnections } from "test-utils/close-connections"; From f28698f965a59452daad0d5059c718510cf227b0 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 10 Aug 2021 22:11:39 +0100 Subject: [PATCH 8/9] This branch has accidentally been used to run way more code than it should've -- whoops. --- .../__KT_DATABASE_MIGRATION/migrate.ts | 2 +- server/scripts/single-use/create-tables.ts | 44 +++++++++++++++++++ server/src/external/mongo/indexes.ts | 2 +- server/src/utils/folder.ts | 3 ++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 server/scripts/single-use/create-tables.ts diff --git a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts index 1a7c1c5a5..f15bd5640 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts @@ -1,5 +1,5 @@ +import CreateLogCtx from "lib/logger/logger"; import { ICollection } from "monk"; -import CreateLogCtx from "../../src/common/logger"; import { oldKTDB } from "./old-db"; const logger = CreateLogCtx(__filename); diff --git a/server/scripts/single-use/create-tables.ts b/server/scripts/single-use/create-tables.ts new file mode 100644 index 000000000..c7e82e131 --- /dev/null +++ b/server/scripts/single-use/create-tables.ts @@ -0,0 +1,44 @@ +import db from "external/mongo/db"; + +(async () => { + const folders = await db.folders.find({}, { sort: { tableIndex: 1 } }); + + for (const folder of folders) { + const { game, playtype } = folder; + + const tableID = ((folder as any).table as string) + .toLowerCase() + .replace(/ /gu, "_") + .replace(/\(/gu, "") + .replace(/\)/gu, ""); + + console.log(game, playtype, tableID, folder.title); + + const existingTable = await db.tables.findOne({ game, playtype, tableID }); + + if (!existingTable) { + await db.tables.insert({ + tableID, + game, + playtype, + description: "todo", + folders: [folder.folderID], + // @ts-expect-error yea + title: folder.table, + }); + } else { + await db.tables.update( + { + game, + playtype, + tableID, + }, + { + $push: { + folders: folder.folderID, + }, + } + ); + } + } +})(); diff --git a/server/src/external/mongo/indexes.ts b/server/src/external/mongo/indexes.ts index 132fbf251..8c7b1d606 100644 --- a/server/src/external/mongo/indexes.ts +++ b/server/src/external/mongo/indexes.ts @@ -79,7 +79,7 @@ const staticIndexes: Partial> = { ], "bms-course-lookup": [index({ md5sums: 1 }, UNIQUE)], "api-tokens": [index({ token: 1 }, UNIQUE), index({ userID: 1 })], - tables: [index({ tableID: 1 }, UNIQUE), index({ game: 1, playtype: 1 })], + tables: [index({ tableID: 1, game: 1, playtype: 1 }, UNIQUE)], "game-stats-snapshots": [index({ timestamp: 1, userID: 1, game: 1, playtype: 1 }, UNIQUE)], "session-view-cache": [ index({ sessionID: 1, ip: 1 }, UNIQUE), diff --git a/server/src/utils/folder.ts b/server/src/utils/folder.ts index 76a0a7762..8e129c9bd 100644 --- a/server/src/utils/folder.ts +++ b/server/src/utils/folder.ts @@ -171,6 +171,9 @@ export async function GetFoldersFromTable(table: TableDocument) { ); } + // we also need to sort folders in their indexed order. + folders.sort((a, b) => table.folders.indexOf(a.folderID) - table.folders.indexOf(b.folderID)); + return folders; } From a12b5104488d4d4376702d67f36b4a995879da92 Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 10 Aug 2021 22:28:24 +0100 Subject: [PATCH 9/9] fix broken tests depending on old config code --- server/src/lib/cdn/cdn.test.ts | 4 +++- server/src/lib/logger/logger.test.ts | 4 +++- server/src/server/router/api/v1/admin/router.test.ts | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/lib/cdn/cdn.test.ts b/server/src/lib/cdn/cdn.test.ts index 413373ae1..eac1b1f83 100644 --- a/server/src/lib/cdn/cdn.test.ts +++ b/server/src/lib/cdn/cdn.test.ts @@ -1,11 +1,13 @@ import t from "tap"; import { CDNDelete, CDNRedirect, CDNRetrieve, CDNStore, CDNStoreOrOverwrite } from "./cdn"; import fs from "fs"; -import { CDN_FILE_ROOT } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; import path from "path"; import { CloseAllConnections } from "test-utils/close-connections"; import expressRequestMock from "express-request-mock"; +const CDN_FILE_ROOT = ServerConfig.CDN_FILE_ROOT; + function getTestTxt() { return fs.readFileSync(path.join(CDN_FILE_ROOT, "test.txt"), "utf-8"); } diff --git a/server/src/lib/logger/logger.test.ts b/server/src/lib/logger/logger.test.ts index e4bfcf51f..878f1a73c 100644 --- a/server/src/lib/logger/logger.test.ts +++ b/server/src/lib/logger/logger.test.ts @@ -1,8 +1,10 @@ -import { LOG_LEVEL } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; import t from "tap"; import { CloseAllConnections } from "test-utils/close-connections"; import CreateLogCtx, { ChangeRootLogLevel, GetLogLevel, rootLogger, Transports } from "./logger"; +const LOG_LEVEL = ServerConfig.LOG_LEVEL; + t.test("Logger Tests", (t) => { const logger = CreateLogCtx(__filename); diff --git a/server/src/server/router/api/v1/admin/router.test.ts b/server/src/server/router/api/v1/admin/router.test.ts index ad57ceefc..f4a1abf8f 100644 --- a/server/src/server/router/api/v1/admin/router.test.ts +++ b/server/src/server/router/api/v1/admin/router.test.ts @@ -1,11 +1,13 @@ import { ONE_MINUTE } from "lib/constants/time"; import { ChangeRootLogLevel, GetLogLevel } from "lib/logger/logger"; -import { LOG_LEVEL } from "lib/setup/config"; +import { ServerConfig } from "lib/setup/config"; import t from "tap"; import { CloseAllConnections } from "test-utils/close-connections"; import { CreateFakeAuthCookie } from "test-utils/fake-auth"; import mockApi from "test-utils/mock-api"; +const LOG_LEVEL = ServerConfig.LOG_LEVEL; + t.test("POST /api/v1/admin/change-log-level", async (t) => { t.beforeEach(() => { ChangeRootLogLevel(LOG_LEVEL);