diff --git a/server/package.json b/server/package.json index f5ab4fe3d..10c7ddc7c 100644 --- a/server/package.json +++ b/server/package.json @@ -82,7 +82,7 @@ "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", "seq-logging": "^1.1.1", - "tachi-common": "0.4.7", + "tachi-common": "0.4.11", "ts-node": "10.0.0", "tsconfig-paths": "3.10.1", "typescript": "4.3.4", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index bbba342d8..22c15f580 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -63,7 +63,7 @@ specifiers: safe-json-stringify: 1.2.0 seq-logging: ^1.1.1 supertest: 6.1.3 - tachi-common: 0.4.7 + tachi-common: 0.4.11 tap: 15.0.9 ts-node: 10.0.0 tsconfig-paths: 3.10.1 @@ -107,7 +107,7 @@ dependencies: rimraf: 3.0.2 safe-json-stringify: 1.2.0 seq-logging: 1.1.1 - tachi-common: 0.4.7 + tachi-common: 0.4.11 ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b tsconfig-paths: 3.10.1 typescript: 4.3.4 @@ -5386,8 +5386,8 @@ packages: strip-ansi: 6.0.0 dev: true - /tachi-common/0.4.7: - resolution: {integrity: sha512-tlntA6tOJSG7l2GNx7byktIma/2upSr+s1tTS1c/t+6IUCED9scsCTwDcqaRSkjzz+fvx3SMg79jyyKIu0Wt9Q==} + /tachi-common/0.4.11: + resolution: {integrity: sha512-jbo432IgMJUXTza7xrDNxBF2dqjGjZuNRRdkfWJI051QbusP2+JZ4+lilKxwpErZnIiZlB07wu/G5iUscFHPVA==} dependencies: monk: 7.3.4 transitivePeerDependencies: diff --git a/server/src/external/mongo/indexes.ts b/server/src/external/mongo/indexes.ts index 61eff41ea..0b7c347c5 100644 --- a/server/src/external/mongo/indexes.ts +++ b/server/src/external/mongo/indexes.ts @@ -85,6 +85,7 @@ const staticIndexes: Partial> = { counters: [index({ counterName: 1 }, UNIQUE)], "class-achievements": [index({ game: 1, playtype: 1, timeAchieved: 1 })], "api-clients": [index({ clientID: 1 }, UNIQUE)], + "charts-popn": [index({ "data.hashSHA256": 1 }, UNIQUE)], }; const indexes: Partial> = staticIndexes; diff --git a/server/src/lib/jobs/update-bpi-poyashi.ts b/server/src/lib/jobs/update-bpi-poyashi.ts deleted file mode 100644 index fbdc37151..000000000 --- a/server/src/lib/jobs/update-bpi-poyashi.ts +++ /dev/null @@ -1,139 +0,0 @@ -/* eslint-disable no-await-in-loop */ -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; -import fetch from "node-fetch"; -import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common"; -import { RecalcGameProfiles } from "utils/calculations/recalc-game-profiles"; -import { RecalcAllScores } from "utils/calculations/recalc-scores"; -import { RecalcSessions } from "utils/calculations/recalc-sessions"; -import { FindChartWithPTDFVersion } from "utils/queries/charts"; -import { FindSongOnTitle } from "utils/queries/songs"; - -const logger = CreateLogCtx(__filename); - -const difficultyResolve: Record = { - 3: ["SP", "HYPER"], - 4: ["SP", "ANOTHER"], - 8: ["DP", "HYPER"], - 9: ["DP", "ANOTHER"], - 10: ["SP", "LEGGENDARIA"], - 11: ["DP", "LEGGENDARIA"], -}; - -async function UpdatePoyashiData() { - logger.info("Fetching BPI Poyashi data from proxy..."); - const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json()); - - logger.info("Fetched data."); - - const data = rj; - - const realData: IIDXBPIData[] = []; - for (const d of data.body) { - const res = difficultyResolve[d.difficulty]; - - if (!res) { - throw new Error(`Unknown difficulty ${d.difficulty}`); - } - - const [playtype, diff] = res; - - const tachiSong = await FindSongOnTitle("iidx", d.title); - - if (!tachiSong) { - logger.warn(`Cannot find song ${d.title}?`); - continue; - } - - const tachiChart = (await FindChartWithPTDFVersion( - "iidx", - tachiSong.id, - playtype as "SP" | "DP", - diff as Difficulties["iidx:DP" | "iidx:SP"], - "29" - )) as ChartDocument<"iidx:SP" | "iidx:DP">; - - if (!tachiChart) { - logger.warn( - `Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?` - ); - continue; - } - - const kavg = Number(d.avg); - - if (kavg < 0) { - logger.warn( - `${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.` - ); - continue; - } - - if (d.removed) { - logger.info(`Skipping removed chart ${tachiSong.title}.`); - continue; - } - - realData.push({ - coef: d.coef === -1 ? null : d.coef, - kavg: Number(d.avg), - wr: Number(d.wr), - chartID: tachiChart.chartID, - }); - } - - const updatedCharts = []; - for (const newData of realData) { - const oldData = await db["iidx-bpi-data"].findOne({ - chartID: newData.chartID, - }); - - if (!oldData) { - updatedCharts.push(newData.chartID); - - await db["iidx-bpi-data"].insert({ - chartID: newData.chartID, - coef: newData.coef, - kavg: newData.kavg, - wr: newData.wr, - }); - } else if ( - oldData.wr !== newData.wr || - oldData.kavg !== newData.kavg || - oldData.coef !== newData.coef - ) { - updatedCharts.push(newData.chartID); - await db["iidx-bpi-data"].update( - { - chartID: newData.chartID, - }, - { - $set: { - wr: newData.wr, - kavg: newData.kavg, - coef: newData.coef, - }, - } - ); - } - } - - if (updatedCharts.length === 0) { - logger.info(`Nothing has changed. Not updating any BPI data.`); - return; - } - - logger.info(`Triggering IIDX Recalc for ${updatedCharts.length} charts.`); - - await RecalcAllScores({ game: "iidx", chartID: { $in: updatedCharts } }); - // @optimisable -- can be filtered to just scores that were recalced. - await RecalcSessions({ game: "iidx" }); - - await RecalcGameProfiles({ game: "iidx" }); - - logger.info(`Done.`); -} - -if (require.main === module) { - UpdatePoyashiData().then(() => process.exit(0)); -} diff --git a/server/src/lib/score-import/import-types/common/batch-manual/converter.test.ts b/server/src/lib/score-import/import-types/common/batch-manual/converter.test.ts index 3ac204b91..fda4b8179 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/converter.test.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/converter.test.ts @@ -158,6 +158,58 @@ t.test("#ResolveMatchTypeToKTData", (t) => { t.end(); }); + t.test("Should resolve for the popn chartHash if the matchType is popnChartHash", async (t) => { + const chartHash = "2c26d666fa7c907e85115dbb279c267c14a263d47b2d46a93f99eae49d779119"; + + const popnContext: BatchManualContext = deepmerge(context, { + game: "popn", + playtype: "9B", + }); + + const res = await ResolveMatchTypeToKTData( + deepmerge(baseBatchManualScore, { + matchType: "popnChartHash", + identifier: chartHash, + }), + popnContext, + importType, + logger + ); + + t.hasStrict( + res, + { + song: { id: 1 }, + chart: { + songID: 1, + data: { hashSHA256: chartHash }, + playtype: "9B", + }, + }, + "Should return the right song and chart." + ); + + t.end(); + }); + + t.test("Should reject if popnChartHash is used while game is not popn", async (t) => { + const chartHash = "2c26d666fa7c907e85115dbb279c267c14a263d47b2d46a93f99eae49d779119"; + + t.rejects(() => + ResolveMatchTypeToKTData( + deepmerge(baseBatchManualScore, { + matchType: "popnChartHash", + identifier: chartHash, + }), + context, + importType, + logger + ) + ); + + t.end(); + }); + t.test("Should resolve for the usc chartHash if the matchType is uscChartHash", async (t) => { const chartHash = "USC_CHART_HASH"; diff --git a/server/src/lib/score-import/import-types/common/batch-manual/converter.ts b/server/src/lib/score-import/import-types/common/batch-manual/converter.ts index 8ef73e380..7fc5e9563 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/converter.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/converter.ts @@ -141,6 +141,35 @@ export async function ResolveMatchTypeToKTData( throw new InternalFailure(`DDR songID ${chart.songID} has charts but no parent song.`); } + return { song, chart }; + } else if (data.matchType === "popnChartHash") { + if (game !== "popn") { + throw new InvalidScoreFailure(`Cannot use popnChartHash lookup on ${game}.`); + } + + const chart = await db.charts.popn.findOne({ + playtype: context.playtype, + "data.hashSHA256": data.identifier, + }); + + if (!chart) { + throw new KTDataNotFoundFailure( + `Cannot find chart for popnChartHash ${data.identifier} (${context.playtype}).`, + importType, + data, + context + ); + } + + const song = await FindSongOnID(game, chart.songID); + + if (!song) { + logger.severe(`Pop'n songID ${chart.songID} has charts but no parent song.`); + throw new InternalFailure( + `Pop'n songID ${chart.songID} has charts but no parent song.` + ); + } + return { song, chart }; } else if (data.matchType === "tachiSongID") { const songID = AssertStrAsPositiveInt( diff --git a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts index 9be2d4a36..441ac8e06 100644 --- a/server/src/lib/score-import/import-types/common/batch-manual/parser.ts +++ b/server/src/lib/score-import/import-types/common/batch-manual/parser.ts @@ -71,6 +71,16 @@ const PR_ScoreMeta = (game: Game, playtype: Playtypes[Game]): PrudenceSchema => return { inSkillAnalyser: "*?boolean", }; + } else if (game === "wacca") { + return { mirror: "*?boolean" }; + } else if (game === "popn") { + return { + hiSpeed: optNull(p.isPositive), + hidden: optNull(p.isInteger), + sudden: optNull(p.isInteger), + random: optNull(p.isIn("NONRAN", "MIRROR", "RANDOM", "S-RANDOM")), + gauge: optNull(p.isIn("NORMAL", "EASY", "HARD", "DANGER")), + }; } return {}; @@ -91,7 +101,7 @@ const PR_HitMeta = (game: Game): PrudenceSchema => { EX_HARD: [p.nullable(p.isBoundedInteger(0, 100))], }), }; - } else if (/* game === "popn" || */ game === "sdvx") { + } else if (game === "sdvx") { return { gauge: optNull(p.isBoundedInteger(0, 100)), }; @@ -112,6 +122,26 @@ const PR_HitMeta = (game: Game): PrudenceSchema => { lpg: optNull(p.isPositiveInteger), epg: optNull(p.isPositiveInteger), }; + } else if (game === "popn") { + return { + gauge: optNull(p.isBoundedInteger(0, 100)), + specificClearType: optNull( + p.isIn( + "failedUnknown", + "failedCircle", + "failedDiamond", + "failedStar", + "easyClear", + "clearCircle", + "clearDiamond", + "clearStar", + "fullComboCircle", + "fullComboDiamond", + "fullComboStar", + "perfect" + ) + ), + }; } return {}; @@ -165,8 +195,6 @@ const PR_BatchManualScore = (game: Game, playtype: Playtypes[Game]): PrudenceSch deepmerge(BaseValidHitMeta, PR_HitMeta(game)) as unknown as ValidSchemaValue ), scoreMeta: optNull(PR_ScoreMeta(game, playtype)), - // scoreMeta: @todo #74 - // more game specific props, maybe? }; }; diff --git a/server/src/test-utils/mock-db/charts-iidx.json b/server/src/test-utils/mock-db/charts-iidx.json index d1f748cec..813e8e803 100644 --- a/server/src/test-utils/mock-db/charts-iidx.json +++ b/server/src/test-utils/mock-db/charts-iidx.json @@ -13,7 +13,10 @@ "notecount": 786, "arcChartID": "CYjwAuz7Yq9", "2dxtraSet": null, - "hashSHA256": null + "hashSHA256": null, + "bpiCoefficient": null, + "kaidenAverage": null, + "worldRecord": null }, "isPrimary": true, "versions": [ diff --git a/server/src/test-utils/mock-db/charts-popn.json b/server/src/test-utils/mock-db/charts-popn.json new file mode 100644 index 000000000..30b8d3307 --- /dev/null +++ b/server/src/test-utils/mock-db/charts-popn.json @@ -0,0 +1,19 @@ +[ + { + "songID": 1, + "chartID": "de5344935963bf24a38a5d5693dd4dd3ddacf8d6", + "difficulty": "Easy", + "playtype": "9B", + "level": "1", + "levelNum": 1, + "rgcID": null, + "isPrimary": true, + "data": { + "hashSHA256": "2c26d666fa7c907e85115dbb279c267c14a263d47b2d46a93f99eae49d779119" + }, + "tierlistInfo": {}, + "versions": [ + "peace" + ] + } +] \ No newline at end of file diff --git a/server/src/test-utils/mock-db/iidx-bpi-data.json b/server/src/test-utils/mock-db/iidx-bpi-data.json deleted file mode 100644 index 2f8ef15b4..000000000 --- a/server/src/test-utils/mock-db/iidx-bpi-data.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "coef": null, - "kavg": 1300, - "wr": 1400, - "chartID": "c2311194e3897ddb5745b1760d2c0141f933e683", - "kesd": 13 - } -] \ No newline at end of file diff --git a/server/src/test-utils/mock-db/songs-popn.json b/server/src/test-utils/mock-db/songs-popn.json new file mode 100644 index 000000000..a3d07ae0b --- /dev/null +++ b/server/src/test-utils/mock-db/songs-popn.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "title": "I REALLY WANT TO HURT YOU", + "artist": "SUGI & REO", + "searchTerms": [], + "altTitles": [], + "data": { + "displayVersion": null, + "genre": "ポップス" + } + } +] \ No newline at end of file diff --git a/server/src/test-utils/resets.ts b/server/src/test-utils/resets.ts index 0362e6923..3fb6c6af3 100644 --- a/server/src/test-utils/resets.ts +++ b/server/src/test-utils/resets.ts @@ -43,6 +43,12 @@ function GetAndCache(filename: string, fileLoc: string) { collection = db[filename]; } + if (!collection) { + throw new Error( + `Panicked when trying to get collection for ${filename}. Does this collection exist?` + ); + } + if (CACHE[filename]) { return { data: CACHE[filename], collection }; } diff --git a/server/src/test-utils/test-data.ts b/server/src/test-utils/test-data.ts index 493368284..d817c03a3 100644 --- a/server/src/test-utils/test-data.ts +++ b/server/src/test-utils/test-data.ts @@ -186,6 +186,9 @@ export const Testing511SPA: ChartDocument<"iidx:SP"> = { arcChartID: "CYjwAuz7Yq9", "2dxtraSet": null, hashSHA256: null, + bpiCoefficient: null, + kaidenAverage: null, + worldRecord: null, }, tierlistInfo: {}, isPrimary: true,