diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 0344374cd..254af2e57 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -62,7 +62,7 @@ dependencies: express-async-errors: 3.1.1_express@4.17.1 express-session: 1.17.1 fast-json-stable-hash: 1.0.1 - kamaitachi-common: github.com/zkldi/Kamaitachi-common/58204fe537f33cf56c8031aef501e3cc6893902c + kamaitachi-common: github.com/zkldi/Kamaitachi-common/2809c38df01607f5178e438b2cffefe21849dee9 mongodb: 3.6.3 monk: 7.3.4 multer: 1.4.2 @@ -4607,8 +4607,8 @@ packages: '@types/yoga-layout': 1.9.2 dev: true - github.com/zkldi/Kamaitachi-common/58204fe537f33cf56c8031aef501e3cc6893902c: - resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/58204fe537f33cf56c8031aef501e3cc6893902c} + github.com/zkldi/Kamaitachi-common/2809c38df01607f5178e438b2cffefe21849dee9: + resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/2809c38df01607f5178e438b2cffefe21849dee9} name: kamaitachi-common version: 0.1.0 dependencies: diff --git a/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts index 5f39e67fa..b1da8640b 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts @@ -1,9 +1,21 @@ import { ChartDocument } from "kamaitachi-common"; import db from "../../src/db/db"; -import { rootLogger } from "../../src/logger"; +import CreateLogCtx from "../../src/logger"; import MigrateRecords from "./migrate"; +import { gameOrders } from "kamaitachi-common/js/config"; + +const logger = CreateLogCtx("charts-iidx.ts"); + +async function ConvertFn(c: any): Promise> { + let song = await db.songs.iidx.findOne({ + id: c.id, + }); + + if (!song) { + logger.severe(`Cannot find song with ID ${c.id}?`); + throw new Error(`Cannot find song with ID ${c.id}?`); + } -function ConvertFn(c: any): ChartDocument<"iidx:SP" | "iidx:DP"> { const newChartDoc: ChartDocument<"iidx:SP" | "iidx:DP"> = { rgcID: null, chartID: c.chartID, @@ -11,7 +23,6 @@ function ConvertFn(c: any): ChartDocument<"iidx:SP" | "iidx:DP"> { songID: c.id, playtype: c.playtype, levelNum: c.levelNum, - isRemoved: false, level: c.level.toString(), length: c.length, bpmString: "", // sentinel @@ -24,15 +35,26 @@ function ConvertFn(c: any): ChartDocument<"iidx:SP" | "iidx:DP"> { inGameID: c.internals.inGameINTID, notecount: c.notedata.notecount, }, + isPrimary: true, + versions: [], // sentinel }; if (typeof c.length === "number") { let m = Math.floor(c.length / 60_000); - let s = c.length - m * 60_000; + let s = ((c.length - m * 60_000) / 1_000).toFixed(2); newChartDoc.length = `${m}:${s}`; } + let idx = gameOrders.iidx.indexOf(song.firstVersion!); + + if (idx === -1) { + logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`); + newChartDoc.versions = [song.firstVersion!]; + } else { + newChartDoc.versions = gameOrders.iidx.slice(idx); + } + if (c.monoBPM) { newChartDoc.bpmString = c.bpmMin.toString(); } else { @@ -40,7 +62,7 @@ function ConvertFn(c: any): ChartDocument<"iidx:SP" | "iidx:DP"> { } if (newChartDoc.length === null) { - rootLogger.warn(`Length of chart ${newChartDoc.chartID} is null, continuing anyway?`); + logger.warn(`Length of chart ${newChartDoc.chartID} is null, continuing anyway?`); } return newChartDoc; diff --git a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts index 53dad1555..a19820fa2 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/migrate.ts @@ -25,17 +25,20 @@ export default async function MigrateRecords( .get(collectionName) .find({}) // @ts-expect-error it exists. - .each((c) => { + .each(async (c, { pause, resume }) => { i++; + pause(); if (i % 10000 === 0) { logger.info(`Processed ${i} documents.`); } - const newDoc = HandlerFN(c); + const newDoc = await HandlerFN(c); if (newDoc !== null) { newDocuments.push(newDoc); } + + resume(); }); logger.info(`Inserting ${newDocuments.length} documents.`); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores.ts index fa9feb815..9ad92db14 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/scores.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/scores.ts @@ -1,9 +1,11 @@ import { PrivateUserDocument, ScoreDocument } from "kamaitachi-common"; import { grades, lamps, supportedGames, validPlaytypes } from "kamaitachi-common/js/config"; -import db from "../../db/db"; -import { rootLogger } from "../../logger"; -import { CreateScoreID } from "../../score-import/framework/core/score-id"; +import db from "../../src/db/db"; import MigrateRecords from "./migrate"; +import CreateLogCtx from "../../src/logger"; +import { CreateScoreID } from "../../src/score-import/framework/core/score-id"; + +const logger = CreateLogCtx("scores.ts"); // HERE. WE. GO! @@ -15,7 +17,7 @@ function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: s function ConvertFn(c: any): ScoreDocument | null { if (!supportedGames.includes(c.game)) { - rootLogger.warn(`Ignored game ${c.game}`); + logger.warn(`Ignored game ${c.game}`); return null; } @@ -23,9 +25,8 @@ function ConvertFn(c: any): ScoreDocument | null { return null; } - // @ts-expect-error yea if (validPlaytypes[c.game].includes(c.playtype)) { - rootLogger.warn(`Ignored game pt ${c.game}, ${c.playtype}`); + logger.warn(`Ignored game pt ${c.game}, ${c.playtype}`); } let base: Omit = { @@ -46,10 +47,8 @@ function ConvertFn(c: any): ScoreDocument | null { scoreData: { esd: c.scoreData.esd ?? null, grade: c.scoreData.grade, - // @ts-expect-error shut gradeIndex: grades[c.game].indexOf(c.scoreData.grade), lamp: c.scoreData.lamp, - // @ts-expect-error shut lampIndex: lamps[c.game].indexOf(c.scoreData.lamp), percent: c.scoreData.percent, score: c.scoreData.score, diff --git a/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts index d7a310574..b39c06011 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts @@ -1,7 +1,6 @@ import { SongDocument } from "kamaitachi-common"; import { gameOrders } from "kamaitachi-common/js/config"; -import db from "../../db/db"; -import { rootLogger } from "../../logger"; +import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): SongDocument<"iidx"> { @@ -9,27 +8,16 @@ function ConvertFn(c: any): SongDocument<"iidx"> { title: c.title, artist: c.artist, id: c.id, - isAvailable: true, // NOT NECESSARILY firstVersion: c.firstAppearance, "alt-titles": c["alt-titles"].filter((e: string) => e !== c.title), "search-titles": c["search-titles"] .map((e: string) => e.toString()) .filter((e: string) => e !== c.title), - versions: [], // sentinel data: { genre: c.genre, }, }; - let idx = gameOrders.iidx.indexOf(c.firstAppearance); - - if (idx === -1) { - rootLogger.warn(`Invalid firstAppearance of ${c.firstAppearance}, running anyway.`); - newSongDoc.versions = [c.firstAppearance]; - } else { - newSongDoc.versions = gameOrders.iidx.slice(idx); - } - return newSongDoc; } diff --git a/server/scripts/__KT_DATABASE_MIGRATION/users.ts b/server/scripts/__KT_DATABASE_MIGRATION/users.ts index f5b65ea6e..c573c6e73 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/users.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/users.ts @@ -1,5 +1,5 @@ import { PrivateUserDocument } from "kamaitachi-common"; -import db from "../../db/db"; +import db from "../../src/db/db"; import MigrateRecords from "./migrate"; function ConvertFn(c: any): PrivateUserDocument { diff --git a/server/src/core/user-core.test.ts b/server/src/core/user-core.test.ts index e541e6c02..eb7a53eb3 100644 --- a/server/src/core/user-core.test.ts +++ b/server/src/core/user-core.test.ts @@ -2,13 +2,7 @@ import t from "tap"; import { CloseMongoConnection } from "../db/db"; import { PRUDENCE_PRIVATE_USER, PRUDENCE_PUBLIC_USER } from "../db/schemas"; import prAssert from "../test-utils/prassert"; -import { - GetUserCaseInsensitive, - GetUserWithID, - PRIVATEINFO_GetUserCaseInsensitive, - PRIVATEINFO_GetUserWithID, - ResolveUser, -} from "./user-core"; +import { GetUserCaseInsensitive, PRIVATEINFO_GetUserCaseInsensitive } from "./user-core"; t.test("#GetUserCaseInsensitive", (t) => { t.test("Should return the user for an exact username", async (t) => { diff --git a/server/src/data/words.ts b/server/src/data/words.ts index bde7ee3ab..f90619f9a 100644 --- a/server/src/data/words.ts +++ b/server/src/data/words.ts @@ -1,6 +1,3 @@ -import fs from "fs"; -import path from "path"; - // https://raw.githubusercontent.com/hugsy/stuff/master/random-word/english-nouns.txt // and https://gist.github.com/hugsy/8910dc78d208e40de42deb29e62df913 diff --git a/server/src/logger.ts b/server/src/logger.ts index 3ad55f25c..0cb89d42c 100644 --- a/server/src/logger.ts +++ b/server/src/logger.ts @@ -125,7 +125,7 @@ if (IN_TESTING) { ]; } -export const rootLogger = winston.createLogger({ +export const logger = winston.createLogger({ levels: { crit: 0, // entire process termination is necessary severe: 1, // something is wrong, and more than one function is affected (such as a failed assertion that is definitely expected to be true). @@ -140,7 +140,7 @@ export const rootLogger = winston.createLogger({ transports: tports, }); -function CreateLogCtx(context: string, lg = rootLogger): KtLogger { +function CreateLogCtx(context: string, lg = logger): KtLogger { return lg.child({ context }) as KtLogger; } @@ -161,7 +161,7 @@ export function CreateScoreLogger( }; // used so appendLogCtx works - const childLogger = rootLogger.child(meta); + const childLogger = logger.child(meta); childLogger.defaultMeta = meta; diff --git a/server/src/score-import/database-lookup/song-title.ts b/server/src/score-import/database-lookup/song-title.ts index 782570e61..7c3fe3a31 100644 --- a/server/src/score-import/database-lookup/song-title.ts +++ b/server/src/score-import/database-lookup/song-title.ts @@ -2,34 +2,6 @@ import { Game, AnySongDocument } from "kamaitachi-common"; import { FindOneResult } from "monk"; import db from "../../db/db"; -/** - * Finds a song document for the given game with the given title (or alt-title) - * The third parameter - version - is used to distinguish between older versions of charts - * depending on what version of the .csv is currently being imported. - * @param game - The game to search upon. - * @param title - The song title to match. - * @param version - The version a song should be in to be counted. - * @returns AnySongDocument - */ -export function FindSongOnTitleVersion( - game: G, - title: string, - version: string -): Promise> { - // @PERF: Performance should be tested here by having a utility field for all-titles. - return db.songs[game].findOne({ - versions: version, - $or: [ - { - title: title, - }, - { - "alt-titles": title, - }, - ], - }); -} - /** * Finds a song document for the given game with the given title (or alt-title). * This is NOT the preferred way to find a song, as encodings, and typos, make this @@ -42,7 +14,7 @@ export function FindSongOnTitle( game: Game, title: string ): Promise> { - db.songs[game]; + // @PERF: Performance should be tested here by having a utility field for all-titles. return db.songs[game].findOne({ $or: [ { diff --git a/server/src/score-import/import-types/file/csv-eamusement-iidx/converter.ts b/server/src/score-import/import-types/file/csv-eamusement-iidx/converter.ts index a35cf375a..b56f49a70 100644 --- a/server/src/score-import/import-types/file/csv-eamusement-iidx/converter.ts +++ b/server/src/score-import/import-types/file/csv-eamusement-iidx/converter.ts @@ -1,16 +1,7 @@ -import { - config, - ESDCore, - Lamps, - AnySongDocument, - ChartDocument, - SongDocument, -} from "kamaitachi-common"; -import { FindOneResult } from "monk"; +import { config, ESDCore, Lamps, AnySongDocument, ChartDocument } from "kamaitachi-common"; import { Logger } from "winston"; import { DryScore, ConverterFunction, ConverterFnReturn } from "../../../../types"; import { FindChartWithPTDF } from "../../../database-lookup/chart-ptdf"; -import { FindSongOnTitleVersion } from "../../../database-lookup/song-title"; import { KTDataNotFoundFailure, InternalFailure, @@ -233,6 +224,8 @@ const ConverterFn: ConverterFunction => { let isLegacyLeggendaria = false; + + // if pre-HV, leggendarias were stored in a wacky form. if (!context.hasBeginnerAndLegg) { // hack fix for legacy LEGGENDARIA titles if (data.title.match(/(†|†LEGGENDARIA)$/)) { diff --git a/server/src/test-utils/api-common.ts b/server/src/test-utils/api-common.ts index 0793ec3cc..403067317 100644 --- a/server/src/test-utils/api-common.ts +++ b/server/src/test-utils/api-common.ts @@ -1,12 +1,6 @@ import t from "tap"; -import { rootLogger } from "../logger"; -import { CloseAllConnections } from "./close-connections"; import mockApi from "./mock-api"; -type TestCB = Exclude[1], undefined>; - -type Test = Parameters[0]; - export function RequireNeutralAuthentication(url: string, method: "GET" | "POST" = "GET") { t.test(`Testing authentication for ${method} ${url}.`, async (t) => { let res; diff --git a/server/src/test-utils/mock-api.ts b/server/src/test-utils/mock-api.ts index 4d79c9396..0bb929db9 100644 --- a/server/src/test-utils/mock-api.ts +++ b/server/src/test-utils/mock-api.ts @@ -1,12 +1,14 @@ import supertest from "supertest"; -import CreateLogCtx, { rootLogger } from "../logger"; +import CreateLogCtx from "../logger"; + +const logger = CreateLogCtx("mock-api.ts"); import server from "../server"; -rootLogger.info("Creating Mock Server Connection..."); +logger.info("Creating Mock Server Connection..."); const connection = server.listen(8079); -rootLogger.info("Connecting to Supertest..."); +logger.info("Connecting to Supertest..."); const mockApi = supertest(connection); export function CloseServerConnection() { diff --git a/server/src/test-utils/reset-db-state.ts b/server/src/test-utils/reset-db-state.ts index 149d3d722..7582b4c68 100644 --- a/server/src/test-utils/reset-db-state.ts +++ b/server/src/test-utils/reset-db-state.ts @@ -1,7 +1,7 @@ /** * Resets the state of the database. */ -import db, { CloseMongoConnection, ReopenMongoConnection } from "../db/db"; +import db from "../db/db"; import fs from "fs"; import path from "path"; import CreateLogCtx from "../logger";