From 90189deea9729ac8a00c4ce0b2f87566c876bafd Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 6 Sep 2021 15:09:29 +0100 Subject: [PATCH 1/4] Change constants array to an enum --- server/src/server/router/ir/usc/router.ts | 56 +++++++++++++++-------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/server/src/server/router/ir/usc/router.ts b/server/src/server/router/ir/usc/router.ts index 7b2771c06..633bc7cc5 100644 --- a/server/src/server/router/ir/usc/router.ts +++ b/server/src/server/router/ir/usc/router.ts @@ -1,4 +1,5 @@ import { Router, RequestHandler } from "express"; +import p from "prudence"; import { FindChartOnSHA256 } from "utils/queries/charts"; import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi"; import db from "external/mongo/db"; @@ -21,20 +22,23 @@ import { CDNStore } from "lib/cdn/cdn"; import { ONE_MEGABYTE } from "lib/constants/filesize"; import { RequirePermissions } from "server/middleware/auth"; import { GetUSCIRReplayURL } from "lib/cdn/url-format"; +import { FormatPrError } from "utils/prudence"; +import { USCClientChart } from "./types"; const logger = CreateLogCtx(__filename); const router: Router = Router({ mergeParams: true }); -const STATUS_CODES = { - UNAUTH: 41, - CHART_REFUSE: 42, - FORBIDDEN: 43, - NOT_FOUND: 44, - SERVER_ERROR: 50, - SUCCESS: 20, - BAD_REQ: 40, -}; +enum STATUS_CODES { + UNAUTH = 41, + CHART_REFUSE = 42, + FORBIDDEN = 43, + NOT_FOUND = 44, + SERVER_ERROR = 50, + SUCCESS = 20, + ACCEPTED = 22, + BAD_REQ = 40, +} const ValidateUSCRequest: RequestHandler = async (req, res, next) => { const token = req.header("Authorization"); @@ -230,29 +234,45 @@ router.get("/charts/:chartHash/leaderboard", RetrieveChart, async (req, res) => }); }); +const PR_USCIRChartDoc = { + chartHash: "string", + artist: "string", + title: "string", + level: p.isBoundedInteger(1, 20), + difficulty: p.isBoundedInteger(0, 3), + effector: "string", + illustrator: "string", + bpm: "string", +}; + /** * Sends a score to the server. * https://uscir.readthedocs.io/en/latest/endpoints/score-submit.html * @name POST /ir/usc/scores */ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => { - if (typeof req.body.chart !== "object" || req.body.chart === null) { + const chartErr = p( + req.body.chart, + PR_USCIRChartDoc, + {}, + { + throwOnNonObject: false, + allowExcessKeys: true, + } + ); + + if (chartErr) { return res.status(200).json({ statusCode: STATUS_CODES.BAD_REQ, - description: "Invalid chart provided.", + description: FormatPrError(chartErr, "Invalid chart."), }); } - if (typeof req.body.chart.chartHash !== "string") { - return res.status(200).json({ - statusCode: STATUS_CODES.BAD_REQ, - description: "Invalid chart provided.", - }); - } + const uscChart = req.body.chart as USCClientChart; const chartDoc = (await FindChartOnSHA256( "usc", - req.body.chart.chartHash + uscChart.chartHash )) as ChartDocument<"usc:Single"> | null; if (!chartDoc) { From 742ce585e9c4cc376e28d32d0b706ef4320037a7 Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 6 Sep 2021 16:40:19 +0100 Subject: [PATCH 2/4] Add USC Chart Orphaning support (and tests!) --- server/package.json | 2 +- server/pnpm-lock.yaml | 8 +- server/scripts/initialise-counters.ts | 22 -- server/scripts/set-counters.ts | 21 ++ server/scripts/single-use/set-counters.ts | 13 - server/src/external/mongo/db.ts | 3 + server/src/lib/logger/logger.ts | 2 +- .../import-types/ir/usc/converter.ts | 27 ++- .../import-types/ir/usc/parser.ts | 4 +- .../score-import/import-types/ir/usc/types.ts | 4 +- server/src/lib/setup/config.ts | 3 + .../src/server/router/ir/usc/orphan-queue.ts | 118 +++++++++ .../src/server/router/ir/usc/router.test.ts | 226 ++++++++++++++++++ server/src/server/router/ir/usc/router.ts | 31 ++- server/src/test-utils/mock-db/charts-usc.json | 2 +- server/src/test-utils/mock-db/counters.json | 4 + server/src/test-utils/mock-db/songs-usc.json | 4 +- 17 files changed, 427 insertions(+), 67 deletions(-) delete mode 100644 server/scripts/initialise-counters.ts create mode 100644 server/scripts/set-counters.ts delete mode 100644 server/scripts/single-use/set-counters.ts create mode 100644 server/src/server/router/ir/usc/orphan-queue.ts diff --git a/server/package.json b/server/package.json index 7973f1e64..aee1e9ce6 100644 --- a/server/package.json +++ b/server/package.json @@ -73,7 +73,7 @@ "redis": "3.1.2", "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", - "tachi-common": "0.2.1", + "tachi-common": "0.2.3", "typescript": "4.3.4", "winston": "3.3.3" }, diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 050d47915..cdbcc1794 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -54,7 +54,7 @@ specifiers: rimraf: 3.0.2 safe-json-stringify: 1.2.0 supertest: 6.1.3 - tachi-common: 0.2.1 + tachi-common: 0.2.3 tap: 15.0.9 ts-node: 10.0.0 tsconfig-paths: 3.10.1 @@ -88,7 +88,7 @@ dependencies: redis: 3.1.2 rimraf: 3.0.2 safe-json-stringify: 1.2.0 - tachi-common: 0.2.1_ts-node@10.0.0+typescript@4.3.4 + tachi-common: 0.2.3_ts-node@10.0.0+typescript@4.3.4 typescript: 4.3.4 winston: 3.3.3 @@ -3681,8 +3681,8 @@ packages: strip-ansi: 6.0.0 dev: true - /tachi-common/0.2.1_ts-node@10.0.0+typescript@4.3.4: - resolution: {integrity: sha512-4Dc9K2o9s9ZDKiETLoegKM+rNC/D7BCTxdKeZnHJX8hcWo4lcTcKZbH51NHLb+hfDTGFYTgfHvm3nXSEcrQm9A==} + /tachi-common/0.2.3_ts-node@10.0.0+typescript@4.3.4: + resolution: {integrity: sha512-l1w5IQfaZejgl/+opB2K5mRkeQstreOq4+6+9A1kkGNVFu0lxvLYZdqSlJwbqYVAPOO7ssYZsi+FSUf81zowpg==} dependencies: monk: 7.3.4 tap: 15.0.9_ts-node@10.0.0+typescript@4.3.4 diff --git a/server/scripts/initialise-counters.ts b/server/scripts/initialise-counters.ts deleted file mode 100644 index 54c0775d1..000000000 --- a/server/scripts/initialise-counters.ts +++ /dev/null @@ -1,22 +0,0 @@ -import db from "../db/db"; -import CreateLogCtx from "../common/logger"; - -const logger = CreateLogCtx(__filename); - -(async () => { - const users = await db.counters.findOne({ - counterName: "users", - }); - - if (users) { - throw new Error(`"users" document already exists, exiting.`); - } - - await db.counters.insert({ - counterName: "users", - value: 1, - }); - - logger.info("Successfully initialised counter documents. Exiting."); - process.exit(0); -})(); diff --git a/server/scripts/set-counters.ts b/server/scripts/set-counters.ts new file mode 100644 index 000000000..294e9f8e8 --- /dev/null +++ b/server/scripts/set-counters.ts @@ -0,0 +1,21 @@ +import { CounterDocument } from "tachi-common"; +import db from "external/mongo/db"; + +(async () => { + const uscSongCount = await db.songs.usc.find({}); + + const Counters: CounterDocument[] = [ + { + counterName: "users", + value: 2, + }, + { + counterName: "usc-song-id", + value: uscSongCount.length + 1, + }, + ]; + + db.counters.insert(Counters).then(() => { + process.exit(0); + }); +})(); diff --git a/server/scripts/single-use/set-counters.ts b/server/scripts/single-use/set-counters.ts deleted file mode 100644 index cefb7b67e..000000000 --- a/server/scripts/single-use/set-counters.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { CounterDocument } from "tachi-common"; -import db from "../../src/external/mongo/db"; - -const Counters: CounterDocument[] = [ - { - counterName: "users", - value: 1, - }, -]; - -db.counters.insert(Counters).then(() => { - process.exit(0); -}); diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index 47a3eb42c..249cc111f 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -34,6 +34,7 @@ import { OAuth2ApplicationDocument, integer, FervidexSettingsDocument, + USCOrphanChart, } from "tachi-common"; import monk, { TMiddleware } from "monk"; import CreateLogCtx from "lib/logger/logger"; @@ -177,6 +178,7 @@ const db = { // i've inlined this one because i don't see it appearing anywhere else. monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"), "fer-settings": monkDB.get("fer-settings"), + "usc-orphan-chart-queue": monkDB.get("usc-orphan-chart-queue"), }; export type StaticDatabases = @@ -208,6 +210,7 @@ export type StaticDatabases = | "oauth2-clients" | "oauth2-auth-codes" | "fer-settings" + | "usc-orphan-chart-queue" | "user-settings"; export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`; diff --git a/server/src/lib/logger/logger.ts b/server/src/lib/logger/logger.ts index e48f63ac5..bb89fbf12 100644 --- a/server/src/lib/logger/logger.ts +++ b/server/src/lib/logger/logger.ts @@ -5,7 +5,7 @@ import { ServerConfig } from "lib/setup/config"; export type KtLogger = Logger & { severe: LeveledLogMethod }; -const level = ServerConfig.LOG_LEVEL; +const level = process.env.LOG_LEVEL ?? ServerConfig.LOG_LEVEL; const IN_TESTING = process.env.NODE_ENV === "test"; diff --git a/server/src/lib/score-import/import-types/ir/usc/converter.ts b/server/src/lib/score-import/import-types/ir/usc/converter.ts index 910d60a21..3a484e748 100644 --- a/server/src/lib/score-import/import-types/ir/usc/converter.ts +++ b/server/src/lib/score-import/import-types/ir/usc/converter.ts @@ -1,7 +1,11 @@ import { USCClientScore } from "server/router/ir/usc/types"; import { FindSongOnID } from "utils/queries/songs"; import { KtLogger } from "lib/logger/logger"; -import { InternalFailure, InvalidScoreFailure } from "../../../framework/common/converter-failures"; +import { + InternalFailure, + InvalidScoreFailure, + KTDataNotFoundFailure, +} from "../../../framework/common/converter-failures"; import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils"; import { IRUSCContext } from "./types"; import { Lamps } from "tachi-common"; @@ -14,6 +18,7 @@ import { USC_DEFAULT_PERFECT, USC_DEFAULT_SLAM, } from "lib/constants/usc-ir"; +import db from "external/mongo/db"; /** * Interprets the "note mod" used based on the USC score. @@ -71,14 +76,22 @@ export const ConverterIRUSC: ConverterFunction = a throw new InvalidScoreFailure(`Autoplay was enabled - Score is invalid.`); } - const song = await FindSongOnID("usc", context.chart.songID); + const chartDoc = await db.charts.usc.findOne({ + "data.hashSHA1": context.chartHash, + }); - if (!song) { - logger.severe(`Song-Chart desync on USCIR ${context.chart.songID}.`); - throw new InternalFailure(`Song-Chart desync on USCIR ${context.chart.songID}.`); + if (!chartDoc) { + throw new KTDataNotFoundFailure("Chart is orphaned.", "ir/usc", data, context); } - const { grade, percent } = GenericGetGradeAndPercent("usc", data.score, context.chart); + const song = await FindSongOnID("usc", chartDoc.songID); + + if (!song) { + logger.severe(`Song-Chart desync on USCIR ${chartDoc.songID}.`); + throw new InternalFailure(`Song-Chart desync on USCIR ${chartDoc.songID}.`); + } + + const { grade, percent } = GenericGetGradeAndPercent("usc", data.score, chartDoc); const dryScore: DryScore<"usc:Single"> = { comment: null, @@ -106,5 +119,5 @@ export const ConverterIRUSC: ConverterFunction = a }, }; - return { chart: context.chart, song, dryScore }; + return { chart: chartDoc, song, dryScore }; }; diff --git a/server/src/lib/score-import/import-types/ir/usc/parser.ts b/server/src/lib/score-import/import-types/ir/usc/parser.ts index 7d53f739c..7535fdb1d 100644 --- a/server/src/lib/score-import/import-types/ir/usc/parser.ts +++ b/server/src/lib/score-import/import-types/ir/usc/parser.ts @@ -24,7 +24,7 @@ const PR_USCIRScore: PrudenceSchema = { export function ParseIRUSC( body: Record, - chart: ChartDocument<"usc:Single">, + chartHash: string, logger: KtLogger ): ParserFunctionReturns { const err = p( @@ -40,7 +40,7 @@ export function ParseIRUSC( return { context: { - chart, + chartHash, }, game: "usc", iterable: [body.score] as USCClientScore[], diff --git a/server/src/lib/score-import/import-types/ir/usc/types.ts b/server/src/lib/score-import/import-types/ir/usc/types.ts index 3d8f51e2e..16e168d0c 100644 --- a/server/src/lib/score-import/import-types/ir/usc/types.ts +++ b/server/src/lib/score-import/import-types/ir/usc/types.ts @@ -1,5 +1,3 @@ -import { ChartDocument } from "tachi-common"; - export interface IRUSCContext { - chart: ChartDocument<"usc:Single">; + chartHash: string; } diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index ec1fc9bc8..2e1a01599 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -72,6 +72,7 @@ export interface TachiConfig { FROM: string; SENDMAIL_BIN?: string; }; + USC_QUEUE_SIZE: integer; } const isValidOauth2 = p.optional({ @@ -107,6 +108,7 @@ const err = p(config, { FROM: "string", SENDMAIL_BIN: "*string", }), + USC_QUEUE_SIZE: p.optional(p.gteInt(2)), }); if (err) { @@ -126,6 +128,7 @@ const tachiConfig = config as TachiConfig; // default rate limit 500 tachiConfig.RATE_LIMIT ??= 500; tachiConfig.OAUTH_CLIENT_CAP ??= 15; +tachiConfig.USC_QUEUE_SIZE ??= 3; if (tachiConfig.EMAIL_CONFIG) { tachiConfig.EMAIL_CONFIG.SENDMAIL_BIN ??= "/usr/bin/sendmail"; diff --git a/server/src/server/router/ir/usc/orphan-queue.ts b/server/src/server/router/ir/usc/orphan-queue.ts new file mode 100644 index 000000000..9ff2e0e9d --- /dev/null +++ b/server/src/server/router/ir/usc/orphan-queue.ts @@ -0,0 +1,118 @@ +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import { ServerConfig } from "lib/setup/config"; +import { ChartDocument, integer, SongDocument } from "tachi-common"; +import { GetNextCounterValue } from "utils/db"; +import { Random20Hex } from "utils/misc"; +import { USCClientChart } from "./types"; + +const logger = CreateLogCtx(__filename); + +/** + * Handles the orphan queue for USC charts. If the provided chart was just + * unorphaned, returns the chart. Else, returns null. + */ +export async function HandleOrphanQueue(uscChartDoc: USCClientChart, userID: integer) { + const chartName = `${uscChartDoc.artist} - ${uscChartDoc.title} (${IndexToDiff( + uscChartDoc.difficulty + )})`; + + logger.debug(`Recieved orphanqueue request for ${chartName}.`); + + const orphanChart = await db["usc-orphan-chart-queue"].findOne({ + "chartDoc.data.hashSHA1": uscChartDoc.chartHash, + }); + + if (!orphanChart) { + logger.verbose(`Recieved unknown chart ${chartName}, orphaning.`); + + const { song, chart } = ConvertUSCChart(uscChartDoc); + + await db["usc-orphan-chart-queue"].insert({ + chartDoc: chart, + songDoc: song, + userIDs: [userID], + }); + + return null; + } + + orphanChart.userIDs.push(userID); + + const uniqueUsersArr = [...new Set(orphanChart.userIDs)]; + + const playcount = uniqueUsersArr.length; + // If N or more people have played this chart while orphaned, unorphan + // it. + if (playcount >= ServerConfig.USC_QUEUE_SIZE) { + logger.info( + `Song ${chartName} was unorphaned by ${uniqueUsersArr.join(", ")} and ${userID}.` + ); + const songID = await GetNextCounterValue("usc-song-id"); + + logger.verbose(`${chartName} has been assigned songID ${songID}.`); + + const { songDoc, chartDoc } = orphanChart; + + songDoc.id = songID; + chartDoc.songID = songID; + + await db.songs.usc.insert(songDoc); + await db.charts.usc.insert(chartDoc); + await db["usc-orphan-chart-queue"].remove({ + "chartDoc.data.hashSHA1": chartDoc.data.hashSHA1, + }); + + return chartDoc; + } + + // otherwise, update the state of this orphan. + + logger.verbose(`UserID ${userID} played ${chartName}, which is now at ${playcount} plays.`); + await db["usc-orphan-chart-queue"].update( + { + "chartDoc.data.hashSHA1": uscChartDoc.chartHash, + }, + { + $set: { + userIDs: uniqueUsersArr, + }, + } + ); + + return null; +} + +function ConvertUSCChart(uscChartDoc: USCClientChart) { + const chart: ChartDocument<"usc:Single"> = { + chartID: Random20Hex(), + difficulty: IndexToDiff(uscChartDoc.difficulty), + isPrimary: true, + level: "?", + levelNum: 0, + playtype: "Single", + rgcID: null, + songID: 0, + versions: [], + data: { + hashSHA1: uscChartDoc.chartHash, + isOfficial: false, + }, + }; + + const song: SongDocument<"usc"> = { + title: uscChartDoc.title, + artist: uscChartDoc.artist, + firstVersion: null, + id: 0, + data: {}, + "alt-titles": [], + "search-titles": [], + }; + + return { chart, song }; +} + +function IndexToDiff(index: 0 | 1 | 2 | 3): ChartDocument<"usc:Single">["difficulty"] { + return (["NOV", "ADV", "EXH", "INF"] as const)[index]; +} diff --git a/server/src/server/router/ir/usc/router.test.ts b/server/src/server/router/ir/usc/router.test.ts index 439187fc4..b0c2a62da 100644 --- a/server/src/server/router/ir/usc/router.test.ts +++ b/server/src/server/router/ir/usc/router.test.ts @@ -457,3 +457,229 @@ t.test("POST /replays", (t) => { t.end(); }); + +t.test("POST /scores", async (t) => { + t.beforeEach(ResetDBState); + t.beforeEach(async () => { + await db["api-tokens"].insert({ + userID: 1, + identifier: "token", + permissions: { submit_score: true }, + token: "token", + }); + }); + + const validRequest = { + chart: { + chartHash: "USC_CHART_HASH", + artist: "test_artist", + title: "test", + level: 5, + difficulty: 0, + effector: "test", + illustrator: "test", + bpm: "test", + }, + score: { + score: 9_000_000, + gauge: 50, + timestamp: 1_000_000, + crit: 5, + near: 4, + error: 3, + options: { + gaugeType: 0, + gaugeOpt: 0, + mirror: true, + random: false, + autoFlags: 0, + }, + windows: { + perfect: 46, + good: 150, + hold: 150, + miss: 300, + slam: 84, + }, + }, + }; + + t.test("Should submit a score from a valid request.", async (t) => { + const res = await mockApi + .post("/ir/usc/scores") + .set("Authorization", "Bearer token") + .send(validRequest); + + t.equal(res.body.statusCode, 20); + + const dbScore = await db.scores.findOne({ + game: "usc", + }); + + t.hasStrict(dbScore, { + scoreData: { + score: 9_000_000, + percent: 90, + lamp: "FAILED", + }, + scoreMeta: { + noteMod: "MIRROR", + gaugeMod: "NORMAL", + }, + }); + + t.end(); + }); + + t.test("Should orphan a score and return 22 if chart has never been seen.", async (t) => { + const res = await mockApi + .post("/ir/usc/scores") + .set("Authorization", "Bearer token") + .send( + deepmerge(validRequest, { + chart: { + chartHash: "NEW_CHART", + }, + }) + ); + + t.equal(res.body.statusCode, 22); + + const dbScore = await db.scores.findOne({ + game: "usc", + }); + + t.equal(dbScore, null, "Should not exist in db."); + + const orphanData = await db["usc-orphan-chart-queue"].findOne({ + "chartDoc.data.hashSHA1": "NEW_CHART", + }); + + t.strictSame(orphanData?.userIDs, [1]); + + t.strictSame(orphanData?.songDoc, { + title: "test", + artist: "test_artist", + firstVersion: null, + id: 0, + "alt-titles": [], + "search-titles": [], + data: {}, + }); + + t.hasStrict(orphanData?.chartDoc, { + songID: 0, + difficulty: "NOV", + data: { + hashSHA1: "NEW_CHART", + }, + isPrimary: true, + level: "?", + levelNum: 0, + }); + + t.end(); + }); + + t.test("Should unorphan charts on their Nth unique user.", async (t) => { + await db["api-tokens"].insert([ + { + userID: 2, + identifier: "token2", + permissions: { submit_score: true }, + token: "token2", + }, + { + userID: 3, + identifier: "token3", + permissions: { submit_score: true }, + token: "token3", + }, + ]); + + await db.users.insert([ + { + id: 2, + username: "foo", + usernameLowercase: "foo", + }, + { + id: 3, + username: "bar", + usernameLowercase: "bar", + }, + ] as PublicUserDocument[]); + + const res = await mockApi + .post("/ir/usc/scores") + .set("Authorization", "Bearer token") + .send( + deepmerge(validRequest, { + chart: { + chartHash: "NEW_CHART", + }, + }) + ); + + t.equal(res.body.statusCode, 22); + + const res2 = await mockApi + .post("/ir/usc/scores") + .set("Authorization", "Bearer token2") + .send( + deepmerge(validRequest, { + chart: { + chartHash: "NEW_CHART", + }, + }) + ); + + t.equal(res2.body.statusCode, 22); + + const orphanData = await db["usc-orphan-chart-queue"].findOne({ + "chartDoc.data.hashSHA1": "NEW_CHART", + }); + + t.strictSame(orphanData?.userIDs, [1, 2]); + + const res3 = await mockApi + .post("/ir/usc/scores") + .set("Authorization", "Bearer token3") + .send( + deepmerge(validRequest, { + chart: { + chartHash: "NEW_CHART", + }, + }) + ); + + t.equal(res3.body.statusCode, 20); + + const orphanData2 = await db["usc-orphan-chart-queue"].findOne({ + "chartDoc.data.hashSHA1": "NEW_CHART", + }); + + t.equal(orphanData2, null, "Orphan data should be removed from the database."); + + const score = await db.scores.findOne({ + game: "usc", + userID: 3, + }); + + t.hasStrict(score, { + scoreData: { + score: 9_000_000, + percent: 90, + lamp: "FAILED", + }, + scoreMeta: { + noteMod: "MIRROR", + gaugeMod: "NORMAL", + }, + }); + + t.end(); + }); + + t.end(); +}); diff --git a/server/src/server/router/ir/usc/router.ts b/server/src/server/router/ir/usc/router.ts index 633bc7cc5..b2ec687b1 100644 --- a/server/src/server/router/ir/usc/router.ts +++ b/server/src/server/router/ir/usc/router.ts @@ -24,6 +24,7 @@ import { RequirePermissions } from "server/middleware/auth"; import { GetUSCIRReplayURL } from "lib/cdn/url-format"; import { FormatPrError } from "utils/prudence"; import { USCClientChart } from "./types"; +import { HandleOrphanQueue } from "./orphan-queue"; const logger = CreateLogCtx(__filename); @@ -99,7 +100,7 @@ router.get("/", (req, res) => ); const RetrieveChart: RequestHandler = async (req, res, next) => { - const chart = await FindChartOnSHA256("usc", req.params.chartHash); + const chart = await db.charts.usc.findOne({ "data.hashSHA1": req.params.chartHash }); AssignToReqTachiData(req, { uscChartDoc: (chart ?? undefined) as ChartDocument<"usc:Single"> | undefined, @@ -270,16 +271,16 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => { const uscChart = req.body.chart as USCClientChart; - const chartDoc = (await FindChartOnSHA256( - "usc", - uscChart.chartHash - )) as ChartDocument<"usc:Single"> | null; + let chartDoc = (await db.charts.usc.findOne({ + "data.hashSHA1": uscChart.chartHash, + })) as ChartDocument<"usc:Single"> | null; + // If the chart doesn't exist, call HandleOrphanQueue. + // If this chart has never been seen before, orphan it. + // If this chart is already orphaned, increase its unique player + // playcount. if (!chartDoc) { - return res.status(200).json({ - statusCode: STATUS_CODES.CHART_REFUSE, - description: "This chart is not supported.", - }); + chartDoc = await HandleOrphanQueue(uscChart, req[SYMBOL_TachiAPIAuth].userID!); } const userDoc = await GetUserWithID(req[SYMBOL_TachiAPIAuth]!.userID!); @@ -292,10 +293,20 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => { }); } - const importParser = (logger: KtLogger) => ParseIRUSC(req.body, chartDoc, logger); + const importParser = (logger: KtLogger) => ParseIRUSC(req.body, uscChart.chartHash, logger); const importRes = await ExpressWrappedScoreImportMain(userDoc, false, "ir/usc", importParser); + // If this was an orphan chart request, return ACCEPTED, + // since it may be unorphaned in the future + if (!chartDoc) { + return res.status(200).json({ + statusCode: STATUS_CODES.ACCEPTED, + description: + "This score has been accepted, but is waiting for more players before its parent chart is accepted.", + }); + } + if (importRes.statusCode === 500) { return res.status(200).json({ statusCode: STATUS_CODES.SERVER_ERROR, diff --git a/server/src/test-utils/mock-db/charts-usc.json b/server/src/test-utils/mock-db/charts-usc.json index 0b7626018..115af8eab 100644 --- a/server/src/test-utils/mock-db/charts-usc.json +++ b/server/src/test-utils/mock-db/charts-usc.json @@ -11,7 +11,7 @@ "CUSTOM": true }, "data": { - "hashSHA256": "USC_CHART_HASH" + "hashSHA1": "USC_CHART_HASH" }, "isPrimary": true, "versions": [] diff --git a/server/src/test-utils/mock-db/counters.json b/server/src/test-utils/mock-db/counters.json index d557444a1..5c28b33ec 100644 --- a/server/src/test-utils/mock-db/counters.json +++ b/server/src/test-utils/mock-db/counters.json @@ -6,5 +6,9 @@ { "counterName": "users", "value": 2 + }, + { + "counterName": "usc-song-id", + "value": 2 } ] \ No newline at end of file diff --git a/server/src/test-utils/mock-db/songs-usc.json b/server/src/test-utils/mock-db/songs-usc.json index c92a9d07d..8ac5ece9e 100644 --- a/server/src/test-utils/mock-db/songs-usc.json +++ b/server/src/test-utils/mock-db/songs-usc.json @@ -6,8 +6,6 @@ "firstVersion": null, "alt-titles": [], "search-titles": [], - "data": { - "sdvxEquiv": null - } + "data": {} } ] \ No newline at end of file From 81e460a46a47bb5a256569be004cc7d0b295c244 Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 6 Sep 2021 16:47:33 +0100 Subject: [PATCH 3/4] fix broken test --- .../lib/score-import/import-types/ir/usc/converter.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/lib/score-import/import-types/ir/usc/converter.test.ts b/server/src/lib/score-import/import-types/ir/usc/converter.test.ts index a02d6c24d..773ed3937 100644 --- a/server/src/lib/score-import/import-types/ir/usc/converter.test.ts +++ b/server/src/lib/score-import/import-types/ir/usc/converter.test.ts @@ -76,7 +76,12 @@ t.test("#DeriveNoteMod", (t) => { }); const dm = (p: Partial) => - ConverterIRUSC(d(uscScore, p), { chart: uscChart }, "ir/usc", logger); + ConverterIRUSC( + d(uscScore, p), + { chartHash: uscChart.data.hashSHA1 as string }, + "ir/usc", + logger + ); t.test("#ConverterIRUSC", (t) => { t.beforeEach(ResetDBState); From df624e13d9f379f28c7c5e8e0aa6aea983e5079a Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 6 Sep 2021 16:58:51 +0100 Subject: [PATCH 4/4] Fix broken parser test --- .../lib/score-import/import-types/ir/usc/parser.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/lib/score-import/import-types/ir/usc/parser.test.ts b/server/src/lib/score-import/import-types/ir/usc/parser.test.ts index fcef0281a..4ca67475e 100644 --- a/server/src/lib/score-import/import-types/ir/usc/parser.test.ts +++ b/server/src/lib/score-import/import-types/ir/usc/parser.test.ts @@ -1,5 +1,4 @@ import t from "tap"; - import { uscChart, uscScore } from "test-utils/test-data"; import CreateLogCtx from "lib/logger/logger"; import { ParseIRUSC } from "./parser"; @@ -10,14 +9,14 @@ t.test("#ParseIRUSC", (t) => { t.test("Should validate and convert a score into an iterable", (t) => { const res = ParseIRUSC( { score: uscScore } as unknown as Record, - uscChart, + uscChart.data.hashSHA1 as string, logger ); t.hasStrict(res, { game: "usc", context: { - chart: uscChart, + chartHash: uscChart.data.hashSHA1, }, iterable: [uscScore], }); @@ -26,7 +25,7 @@ t.test("#ParseIRUSC", (t) => { }); t.test("Should reject empty bodies", (t) => { - t.throws(() => ParseIRUSC({}, uscChart, logger), { + t.throws(() => ParseIRUSC({}, uscChart.data.hashSHA1 as string, logger), { statusCode: 400, message: /invalid usc score/iu, });