diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 1abe4e276..d86b0622a 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -15,7 +15,27 @@ not generally be shown to end users. # Changelog -# CURRENT +# 2.1.16 + +- [BUG] EXCEED GEAR scores can now be imported for real for real. + +# 2.1.15 + +- [BUG] EXCEED GEAR scores can now be imported for real. + +# 2.1.14 + +- [BUG] Fixed bug where folders/tables wouldn't sync automatically from the seeds, again. + +# 2.1.13 + +- [BUG] Fixed bug where folders/tables wouldn't sync automatically from the seeds. + +# 2.1.12 + +- [ENH] EXCEED GEAR scores can now be imported. + +# 2.1.11 - [ENH] The LR2Hook now saves courses. - [ENH] The LR2Hook now saves gauge information. diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index d279f9574..86288c8a7 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -95,7 +95,7 @@ dependencies: fast-json-stable-hash: 1.0.3 fast-xml-parser: 4.0.2 helmet: 5.0.2 - html-entities: 2.3.2 + html-entities: 2.3.3 json5: 2.2.0 mkdirp: 1.0.4 mongodb: 3.6.3 @@ -2412,7 +2412,7 @@ packages: resolution: {integrity: sha512-/aUslKhzkTNCQUB2qTX84lVmfia9NyjP3WpDGtj/WxhwBzWBYUV3DgUpurHTme8UTPcPlAD1DJ+b0nN/t50zDQ==} dependencies: boolbase: 1.0.0 - css-what: 5.1.0 + css-what: 6.1.0 domhandler: 4.3.1 domutils: 2.8.0 nth-check: 2.0.1 @@ -4354,7 +4354,7 @@ packages: /node-html-parser/5.3.3: resolution: {integrity: sha512-ncg1033CaX9UexbyA7e1N0aAoAYRDiV8jkTvzEnfd1GDvzFdrsXLzR4p4ik8mwLgnaKP/jyUFWDy9q3jvRT2Jw==} dependencies: - css-select: 4.2.1 + css-select: 4.3.0 he: 1.2.0 dev: false diff --git a/server/src/external/mongo/indexes.ts b/server/src/external/mongo/indexes.ts index cee11b818..b570a3b86 100644 --- a/server/src/external/mongo/indexes.ts +++ b/server/src/external/mongo/indexes.ts @@ -115,6 +115,9 @@ const staticIndexes: Partial> = { index({ "data.hashSHA256": 1, playtype: 1 }, UNIQUE), index({ "data.hashMD5": 1, playtype: 1 }, UNIQUE), ], + "import-locks": [index({ userID: 1 }, UNIQUE)], + "arc-saved-profiles": [index({ userID: 1, forImportType: 1 }, UNIQUE)], + "score-blacklist": [index({ scoreID: 1 }, UNIQUE)], }; const indexes: Partial> = staticIndexes; diff --git a/server/src/lib/score-import/framework/score-importing/score-import-main.ts b/server/src/lib/score-import/framework/score-importing/score-import-main.ts index b4edaa287..7513438c4 100644 --- a/server/src/lib/score-import/framework/score-importing/score-import-main.ts +++ b/server/src/lib/score-import/framework/score-importing/score-import-main.ts @@ -64,20 +64,20 @@ export default async function ScoreImportMain( logger = providedLogger; } + const hasNoOngoingImport = await CheckAndSetOngoingImportLock(user.id); + + if (hasNoOngoingImport) { + logger.info(`User ${userID} made an import while they had one ongoing.`); + // @danger + // Throwing away an import if the user already has one outgoing is *bad*, as in the case + // of degraded performance we might just start throwing scores away. + // Under normal circumstances, there is no scenario where a user would have two ongoing + // imports at the same time - even if they were using single-score imports on a 5 second + // chart, as each score import takes only around ~10-15milliseconds. + throw new ScoreImportFatalError(409, "This user already has an ongoing import."); + } + try { - const hasNoOngoingImport = await CheckAndSetOngoingImportLock(user.id); - - if (hasNoOngoingImport) { - logger.info(`User ${userID} made an import while they had one ongoing.`); - // @danger - // Throwing away an import if the user already has one outgoing is *bad*, as in the case - // of degraded performance we might just start throwing scores away. - // Under normal circumstances, there is no scenario where a user would have two ongoing - // imports at the same time - even if they were using single-score imports on a 5 second - // chart, as each score import takes only around ~10-15milliseconds. - throw new ScoreImportFatalError(409, "This user already has an ongoing import."); - } - const timeStarted = Date.now(); SetJobProgress(job, "Parsing score data."); diff --git a/server/src/lib/score-import/import-types/common/api-kai/sdvx/class-handler.ts b/server/src/lib/score-import/import-types/common/api-kai/sdvx/class-handler.ts index 7549f0dfe..f3c1982ac 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/sdvx/class-handler.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/sdvx/class-handler.ts @@ -35,6 +35,7 @@ export async function CreateKaiSDVXClassHandler( } if (json.skill_level === null || json.skill_level === undefined) { + logger.info(`User has no skill_level. Not updating anything.`); return {}; } diff --git a/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts b/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts index 0ada41108..fe0c329bc 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts @@ -3,12 +3,29 @@ import db from "external/mongo/db"; import CreateLogCtx from "lib/logger/logger"; import t from "tap"; import ResetDBState from "test-utils/resets"; -import { GetKTDataJSON, TestingAlbidaADV, TestingSDVXAlbidaSong } from "test-utils/test-data"; +import { TestingAlbidaADV, TestingSDVXAlbidaSong } from "test-utils/test-data"; import { ConvertAPIKaiSDVX, ConvertDifficulty, ConvertVersion, ResolveKaiLamp } from "./converter"; const logger = CreateLogCtx(__filename); -const sdvxScore = GetKTDataJSON("./api-kai/sdvx-score.json"); +const sdvxScore = { + sdvx_id: 32157055, + music_id: 1, + music_difficulty: 1, + played_version: 6, // exg + clear_type: 2, + max_chain: 179, + score: 9310699, + critical: 1754, + near: 112, + error: 78, + early: 70, + late: 42, + gauge_type: 0, + gauge_rate: 90.01, + timestamp: "2020-08-30T13:08:11Z", + _id: 127108, +}; t.test("#ConvertAPIKaiSDVX", (t) => { t.beforeEach(ResetDBState); @@ -38,7 +55,7 @@ t.test("#ConvertAPIKaiSDVX", (t) => { hitMeta: { fast: 70, slow: 42, - gauge: 90, + gauge: 90.01, maxCombo: 179, }, }, @@ -59,7 +76,7 @@ t.test("#ConvertAPIKaiSDVX", (t) => { logger ), { - message: /Could not find chart with songID 0 \(ADV - Version heaven\)/u, + message: /Could not find chart with songID 0 \(ADV - Version exceed\)/u, } ); @@ -113,7 +130,8 @@ t.test("#ConvertVersion", (t) => { t.equal(ConvertVersion(3), "gw"); t.equal(ConvertVersion(4), "heaven"); t.equal(ConvertVersion(5), "vivid"); - t.throws(() => ConvertVersion(6)); + t.equal(ConvertVersion(6), "exceed"); + t.throws(() => ConvertVersion(7)); t.throws(() => ConvertVersion(0)); t.end(); diff --git a/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.ts b/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.ts index 82eae8615..84805e5ff 100644 --- a/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.ts +++ b/server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.ts @@ -19,7 +19,7 @@ import { KaiContext, KaiSDVXScore } from "../types"; const PR_KaiSDVXScore = { music_id: p.isPositiveInteger, music_difficulty: p.isBoundedInteger(0, 4), - played_version: p.isBoundedInteger(1, 5), + played_version: p.isBoundedInteger(1, 6), clear_type: p.isBoundedInteger(1, 5), max_chain: p.isPositiveInteger, score: p.isBoundedInteger(0, 10_000_000), @@ -28,7 +28,7 @@ const PR_KaiSDVXScore = { error: p.nullable(p.isPositiveInteger), early: p.nullable(p.isPositiveInteger), late: p.nullable(p.isPositiveInteger), - gauge_rate: p.isBoundedInteger(0, 100), + gauge_rate: p.isBetween(0, 100), timestamp: "string", }; @@ -132,6 +132,8 @@ export function ConvertVersion(ver: number) { return "heaven"; case 5: return "vivid"; + case 6: + return "exceed"; } throw new InvalidScoreFailure(`Unknown Game Version ${ver}.`); diff --git a/server/src/scripts/sync-database.ts b/server/src/scripts/sync-database.ts index 219e0318b..1e4dad979 100644 --- a/server/src/scripts/sync-database.ts +++ b/server/src/scripts/sync-database.ts @@ -174,6 +174,18 @@ const syncInstructions: SyncInstructions[] = [ } }, }, + { + pattern: /^songs-(b|p)ms/u, + handler: async (songs: SongDocument[], collection: ICollection, logger) => { + const r = await GenericUpsert(songs, collection, "id", logger, false); + + if (r.thingsChanged) { + await RecalcAllScores({ + songID: { $in: r.changedFields }, + }); + } + }, + }, { pattern: /^songs-/u, handler: async (songs: SongDocument[], collection: ICollection, logger) => { @@ -187,13 +199,19 @@ const syncInstructions: SyncInstructions[] = [ }, }, { - pattern: /^folders$/u, + pattern: /^folders/u, handler: async ( folders: FolderDocument[], collection: ICollection, logger ) => { - const r = await GenericUpsert(folders, collection, "folderID", logger, true); + const r = await GenericUpsert( + folders.filter((e) => TachiConfig.GAMES.includes(e.game)), + collection, + "folderID", + logger, + true + ); if (r) { await InitaliseFolderChartLookup(); @@ -201,17 +219,29 @@ const syncInstructions: SyncInstructions[] = [ }, }, { - pattern: /^tables$/u, + pattern: /^tables/u, handler: (tables: TableDocument[], collection: ICollection, logger) => - GenericUpsert(tables, collection, "tableID", logger, true), + GenericUpsert( + tables.filter((e) => TachiConfig.GAMES.includes(e.game)), + collection, + "tableID", + logger, + true + ), }, { - pattern: /^bms-course-lookup$/u, - handler: ( + pattern: /^bms-course-lookup/u, + handler: async ( bmsCourseDocuments: BMSCourseDocument[], collection: ICollection, logger - ) => GenericUpsert(bmsCourseDocuments, collection, "md5sums", logger), + ) => { + if (TachiConfig.TYPE === "ktchi") { + return; + } + + await GenericUpsert(bmsCourseDocuments, collection, "md5sums", logger); + }, }, ]; diff --git a/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.test.ts b/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.test.ts index facf69ca8..8f116d5f9 100644 --- a/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.test.ts +++ b/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.test.ts @@ -51,6 +51,15 @@ t.test("GET /ir/beatoraja/charts/:chartSHA256/scores", (t) => { t.equal(res.status, 200); + t.hasStrict(res.body.body, [ + { + player: "", + clear: 5, + epg: 617, + lpg: 0, + }, + ]); + t.end(); }); diff --git a/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.ts b/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.ts index 391138cf5..95a449384 100644 --- a/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.ts +++ b/server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.ts @@ -1,6 +1,6 @@ import { RequestHandler, Router } from "express"; import db from "external/mongo/db"; -import { SYMBOL_TachiData } from "lib/constants/tachi"; +import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi"; import { ChartDocument, PBScoreDocument } from "tachi-common"; import { AssignToReqTachiData } from "utils/req-tachi-data"; import { TachiScoreDataToBeatorajaFormat } from "./convert-scores"; @@ -43,6 +43,7 @@ router.use(GetChartDocument); */ router.get("/scores", async (req, res) => { const chart = req[SYMBOL_TachiData]!.beatorajaChartDoc!; + const requestingUserID = req[SYMBOL_TachiAPIAuth].userID; const scores = (await db["personal-bests"].find({ chartID: chart.chartID, @@ -59,7 +60,6 @@ router.get("/scores", async (req, res) => { }, } ); - const userMap = new Map(); for (const user of userDocs) { userMap.set(user.id, user); @@ -72,7 +72,7 @@ router.get("/scores", async (req, res) => { TachiScoreDataToBeatorajaFormat( score, chart.data.hashSHA256, - userMap.get(score.userID).username, + score.userID === requestingUserID ? "" : userMap.get(score.userID).username, chart.data.notecount, 0 // Playcount is always 0 at the moment due to performance concerns. ) diff --git a/server/src/test-utils/mock-db/charts-sdvx.json b/server/src/test-utils/mock-db/charts-sdvx.json index 9195fd563..ce70ac9a2 100644 --- a/server/src/test-utils/mock-db/charts-sdvx.json +++ b/server/src/test-utils/mock-db/charts-sdvx.json @@ -18,6 +18,7 @@ "gw", "heaven", "vivid", + "exceed", "konaste" ], "tierlistInfo": {} diff --git a/server/src/test-utils/test-data.ts b/server/src/test-utils/test-data.ts index 2d09d4762..84d55f76d 100644 --- a/server/src/test-utils/test-data.ts +++ b/server/src/test-utils/test-data.ts @@ -274,7 +274,7 @@ export const TestingAlbidaADV: ChartDocument<"sdvx:Single"> = { arcChartID: "EOZ7FixCDpv", }, isPrimary: true, - versions: ["booth", "inf", "gw", "heaven", "vivid", "konaste"], + versions: ["booth", "inf", "gw", "heaven", "vivid", "exceed", "konaste"], tierlistInfo: {}, }; diff --git a/server/src/test-utils/test-data/api-kai/iidx-score.json b/server/src/test-utils/test-data/api-kai/iidx-score.json deleted file mode 100644 index 3eecb80ea..000000000 --- a/server/src/test-utils/test-data/api-kai/iidx-score.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chart_id": 3848, - "music_id": 1000, - "music_difficulty": 2, - "play_style": "SINGLE", - "difficulty": "ANOTHER", - "iidx_id": 35247879, - "version_played": 26, - "lamp": 5, - "ex_score": 1570, - "grade": "AA", - "miss_count": 24, - "fast_count": null, - "slow_count": null, - "timestamp": "2020-10-31T19:10:50Z", - "_id": 189232 -} \ No newline at end of file diff --git a/server/src/test-utils/test-data/api-kai/sdvx-score.json b/server/src/test-utils/test-data/api-kai/sdvx-score.json deleted file mode 100644 index 9f9d7547f..000000000 --- a/server/src/test-utils/test-data/api-kai/sdvx-score.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "sdvx_id": 32157055, - "music_id": 1, - "music_difficulty": 1, - "played_version": 4, - "clear_type": 2, - "max_chain": 179, - "score": 9310699, - "critical": 1754, - "near": 112, - "error": 78, - "early": 70, - "late": 42, - "gauge_type": 0, - "gauge_rate": 90, - "timestamp": "2020-08-30T13:08:11Z", - "_id": 127108 -} \ No newline at end of file