From dc15acb18984c09bb2e1aef878514b404bf5ee6b Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 7 Sep 2021 05:59:30 +0100 Subject: [PATCH] Generify Orphan Queue --- server/package.json | 2 +- server/pnpm-lock.yaml | 8 +- server/src/external/mongo/db.ts | 6 +- server/src/lib/orphan-queue/orphan-queue.ts | 97 ++++++++++++++ .../src/server/router/ir/usc/orphan-queue.ts | 118 ------------------ server/src/server/router/ir/usc/router.ts | 30 ++++- server/src/server/router/ir/usc/usc.ts | 42 ++++++- 7 files changed, 170 insertions(+), 133 deletions(-) create mode 100644 server/src/lib/orphan-queue/orphan-queue.ts delete mode 100644 server/src/server/router/ir/usc/orphan-queue.ts diff --git a/server/package.json b/server/package.json index aee1e9ce6..822634212 100644 --- a/server/package.json +++ b/server/package.json @@ -73,7 +73,7 @@ "redis": "3.1.2", "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", - "tachi-common": "0.2.3", + "tachi-common": "0.2.4", "typescript": "4.3.4", "winston": "3.3.3" }, diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index cdbcc1794..e1e3873e7 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -54,7 +54,7 @@ specifiers: rimraf: 3.0.2 safe-json-stringify: 1.2.0 supertest: 6.1.3 - tachi-common: 0.2.3 + tachi-common: 0.2.4 tap: 15.0.9 ts-node: 10.0.0 tsconfig-paths: 3.10.1 @@ -88,7 +88,7 @@ dependencies: redis: 3.1.2 rimraf: 3.0.2 safe-json-stringify: 1.2.0 - tachi-common: 0.2.3_ts-node@10.0.0+typescript@4.3.4 + tachi-common: 0.2.4_ts-node@10.0.0+typescript@4.3.4 typescript: 4.3.4 winston: 3.3.3 @@ -3681,8 +3681,8 @@ packages: strip-ansi: 6.0.0 dev: true - /tachi-common/0.2.3_ts-node@10.0.0+typescript@4.3.4: - resolution: {integrity: sha512-l1w5IQfaZejgl/+opB2K5mRkeQstreOq4+6+9A1kkGNVFu0lxvLYZdqSlJwbqYVAPOO7ssYZsi+FSUf81zowpg==} + /tachi-common/0.2.4_ts-node@10.0.0+typescript@4.3.4: + resolution: {integrity: sha512-5Q6c0qGhQyPKKiCg+BvuYbt9k5q/B1c3aBNR3KaX+aEKSQ8ZbuYTcKPKnQlDccjDfcGNj2OAmELx3oHz4UJdGA==} dependencies: monk: 7.3.4 tap: 15.0.9_ts-node@10.0.0+typescript@4.3.4 diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index 249cc111f..6988848e9 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -34,7 +34,7 @@ import { OAuth2ApplicationDocument, integer, FervidexSettingsDocument, - USCOrphanChart, + OrphanChart, } from "tachi-common"; import monk, { TMiddleware } from "monk"; import CreateLogCtx from "lib/logger/logger"; @@ -178,7 +178,7 @@ const db = { // i've inlined this one because i don't see it appearing anywhere else. monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"), "fer-settings": monkDB.get("fer-settings"), - "usc-orphan-chart-queue": monkDB.get("usc-orphan-chart-queue"), + "orphan-chart-queue": monkDB.get("orphan-chart-queue"), }; export type StaticDatabases = @@ -210,7 +210,7 @@ export type StaticDatabases = | "oauth2-clients" | "oauth2-auth-codes" | "fer-settings" - | "usc-orphan-chart-queue" + | "orphan-chart-queue" | "user-settings"; export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`; diff --git a/server/src/lib/orphan-queue/orphan-queue.ts b/server/src/lib/orphan-queue/orphan-queue.ts new file mode 100644 index 000000000..10f123930 --- /dev/null +++ b/server/src/lib/orphan-queue/orphan-queue.ts @@ -0,0 +1,97 @@ +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import { ChartDocument, IDStrings, IDStringToGame, integer, SongDocument } from "tachi-common"; +import { GetNextCounterValue } from "utils/db"; +import { FilterQuery } from "mongodb"; + +const logger = CreateLogCtx(__filename); + +/** + * Handles an orphan queue request. + * + * If the chart has never been seen before, add it to the orphan queue + * and return null. + * + * If the chart has been seen before, and has less than N unique players + * who have played it, return null. + * + * If the chart has been seen before, and has >= N unique players who have + * played it, unorphan the chart, and return it. + */ +export async function HandleOrphanQueue( + idString: I, + game: IDStringToGame[I], + chartDoc: ChartDocument, + songDoc: SongDocument, + orphanMatchCriteria: FilterQuery>, + queueSize: integer, + userID: integer, + chartName: string +) { + logger.debug(`Recieved orphanqueue request for ${chartName}.`); + + const orphanChart = await db["orphan-chart-queue"].findOne( + Object.assign({ idString }, orphanMatchCriteria), + { + projectID: true, + } + ); + + if (!orphanChart) { + logger.verbose(`Recieved unknown chart ${chartName}, orphaning.`); + + await db["orphan-chart-queue"].insert({ + idString, + chartDoc, + songDoc, + userIDs: [userID], + }); + + return null; + } + + orphanChart.userIDs.push(userID); + + const uniqueUsersArr = [...new Set(orphanChart.userIDs)]; + + const playcount = uniqueUsersArr.length; + // If N or more people have played this chart while orphaned, unorphan + // it. + if (playcount >= queueSize) { + logger.info( + `Song ${chartName} was unorphaned by ${uniqueUsersArr.join(", ")} and ${userID}.` + ); + const songID = await GetNextCounterValue(`${game}-song-id`); + + logger.verbose(`${chartName} has been assigned songID ${songID}.`); + + const { songDoc, chartDoc } = orphanChart; + + songDoc.id = songID; + chartDoc.songID = songID; + + await db.songs[game].insert(songDoc); + await db.charts[game].insert(chartDoc); + await db["orphan-chart-queue"].remove({ + _id: orphanChart._id, + }); + + return chartDoc as ChartDocument; + } + + // otherwise, update the state of this orphan. + + logger.verbose(`UserID ${userID} played ${chartName}, which is now at ${playcount} plays.`); + await db["orphan-chart-queue"].update( + { + _id: orphanChart._id, + }, + { + $set: { + userIDs: uniqueUsersArr, + }, + } + ); + + return null; +} diff --git a/server/src/server/router/ir/usc/orphan-queue.ts b/server/src/server/router/ir/usc/orphan-queue.ts deleted file mode 100644 index 9ff2e0e9d..000000000 --- a/server/src/server/router/ir/usc/orphan-queue.ts +++ /dev/null @@ -1,118 +0,0 @@ -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; -import { ServerConfig } from "lib/setup/config"; -import { ChartDocument, integer, SongDocument } from "tachi-common"; -import { GetNextCounterValue } from "utils/db"; -import { Random20Hex } from "utils/misc"; -import { USCClientChart } from "./types"; - -const logger = CreateLogCtx(__filename); - -/** - * Handles the orphan queue for USC charts. If the provided chart was just - * unorphaned, returns the chart. Else, returns null. - */ -export async function HandleOrphanQueue(uscChartDoc: USCClientChart, userID: integer) { - const chartName = `${uscChartDoc.artist} - ${uscChartDoc.title} (${IndexToDiff( - uscChartDoc.difficulty - )})`; - - logger.debug(`Recieved orphanqueue request for ${chartName}.`); - - const orphanChart = await db["usc-orphan-chart-queue"].findOne({ - "chartDoc.data.hashSHA1": uscChartDoc.chartHash, - }); - - if (!orphanChart) { - logger.verbose(`Recieved unknown chart ${chartName}, orphaning.`); - - const { song, chart } = ConvertUSCChart(uscChartDoc); - - await db["usc-orphan-chart-queue"].insert({ - chartDoc: chart, - songDoc: song, - userIDs: [userID], - }); - - return null; - } - - orphanChart.userIDs.push(userID); - - const uniqueUsersArr = [...new Set(orphanChart.userIDs)]; - - const playcount = uniqueUsersArr.length; - // If N or more people have played this chart while orphaned, unorphan - // it. - if (playcount >= ServerConfig.USC_QUEUE_SIZE) { - logger.info( - `Song ${chartName} was unorphaned by ${uniqueUsersArr.join(", ")} and ${userID}.` - ); - const songID = await GetNextCounterValue("usc-song-id"); - - logger.verbose(`${chartName} has been assigned songID ${songID}.`); - - const { songDoc, chartDoc } = orphanChart; - - songDoc.id = songID; - chartDoc.songID = songID; - - await db.songs.usc.insert(songDoc); - await db.charts.usc.insert(chartDoc); - await db["usc-orphan-chart-queue"].remove({ - "chartDoc.data.hashSHA1": chartDoc.data.hashSHA1, - }); - - return chartDoc; - } - - // otherwise, update the state of this orphan. - - logger.verbose(`UserID ${userID} played ${chartName}, which is now at ${playcount} plays.`); - await db["usc-orphan-chart-queue"].update( - { - "chartDoc.data.hashSHA1": uscChartDoc.chartHash, - }, - { - $set: { - userIDs: uniqueUsersArr, - }, - } - ); - - return null; -} - -function ConvertUSCChart(uscChartDoc: USCClientChart) { - const chart: ChartDocument<"usc:Single"> = { - chartID: Random20Hex(), - difficulty: IndexToDiff(uscChartDoc.difficulty), - isPrimary: true, - level: "?", - levelNum: 0, - playtype: "Single", - rgcID: null, - songID: 0, - versions: [], - data: { - hashSHA1: uscChartDoc.chartHash, - isOfficial: false, - }, - }; - - const song: SongDocument<"usc"> = { - title: uscChartDoc.title, - artist: uscChartDoc.artist, - firstVersion: null, - id: 0, - data: {}, - "alt-titles": [], - "search-titles": [], - }; - - return { chart, song }; -} - -function IndexToDiff(index: 0 | 1 | 2 | 3): ChartDocument<"usc:Single">["difficulty"] { - return (["NOV", "ADV", "EXH", "INF"] as const)[index]; -} diff --git a/server/src/server/router/ir/usc/router.ts b/server/src/server/router/ir/usc/router.ts index b2ec687b1..09ff6ac4c 100644 --- a/server/src/server/router/ir/usc/router.ts +++ b/server/src/server/router/ir/usc/router.ts @@ -1,6 +1,5 @@ import { Router, RequestHandler } from "express"; import p from "prudence"; -import { FindChartOnSHA256 } from "utils/queries/charts"; import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi"; import db from "external/mongo/db"; import { @@ -11,7 +10,12 @@ import { } from "tachi-common"; import { AssertStrAsPositiveNonZeroInt } from "lib/score-import/framework/common/string-asserts"; import CreateLogCtx, { KtLogger } from "lib/logger/logger"; -import { CreatePOSTScoresResponseBody, TachiScoreToServerScore } from "./usc"; +import { + ConvertUSCChart, + CreatePOSTScoresResponseBody, + TachiScoreToServerScore, + USCChartIndexToDiff, +} from "./usc"; import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; import { GetUserWithID } from "utils/user"; import { ParseIRUSC } from "lib/score-import/import-types/ir/usc/parser"; @@ -24,7 +28,8 @@ import { RequirePermissions } from "server/middleware/auth"; import { GetUSCIRReplayURL } from "lib/cdn/url-format"; import { FormatPrError } from "utils/prudence"; import { USCClientChart } from "./types"; -import { HandleOrphanQueue } from "./orphan-queue"; +import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue"; +import { ServerConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); @@ -280,7 +285,24 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => { // If this chart is already orphaned, increase its unique player // playcount. if (!chartDoc) { - chartDoc = await HandleOrphanQueue(uscChart, req[SYMBOL_TachiAPIAuth].userID!); + const { song, chart } = ConvertUSCChart(uscChart); + + const uscChartName = `${uscChart.artist} - ${uscChart.title} (${USCChartIndexToDiff( + uscChart.difficulty + )})`; + + chartDoc = await HandleOrphanQueue( + "usc:Single", + "usc", + chart, + song, + { + "data.hashSHA1": uscChart.chartHash, + }, + ServerConfig.USC_QUEUE_SIZE, + req[SYMBOL_TachiAPIAuth].userID!, + uscChartName + ); } const userDoc = await GetUserWithID(req[SYMBOL_TachiAPIAuth]!.userID!); diff --git a/server/src/server/router/ir/usc/usc.ts b/server/src/server/router/ir/usc/usc.ts index a46f49a55..e1d149098 100644 --- a/server/src/server/router/ir/usc/usc.ts +++ b/server/src/server/router/ir/usc/usc.ts @@ -1,10 +1,10 @@ -import { integer, PBScoreDocument, ScoreDocument, ChartDocument } from "tachi-common"; +import { integer, PBScoreDocument, ScoreDocument, ChartDocument, SongDocument } from "tachi-common"; import CreateLogCtx from "lib/logger/logger"; import { GetPBOnChart, GetServerRecordOnChart } from "utils/scores"; -import { MStoS } from "utils/misc"; +import { MStoS, Random20Hex } from "utils/misc"; import { USCIR_ADJACENT_SCORE_N } from "lib/constants/usc-ir"; import db from "external/mongo/db"; -import { USCServerScore } from "./types"; +import { USCClientChart, USCServerScore } from "./types"; const logger = CreateLogCtx(__filename); @@ -201,3 +201,39 @@ export interface POSTScoresResponseBody { isServerRecord: boolean; sendReplay: string; } + +export function ConvertUSCChart(uscChartDoc: USCClientChart) { + const chart: ChartDocument<"usc:Single"> = { + chartID: Random20Hex(), + difficulty: USCChartIndexToDiff(uscChartDoc.difficulty), + isPrimary: true, + level: "?", + levelNum: 0, + playtype: "Single", + rgcID: null, + songID: 0, + versions: [], + data: { + hashSHA1: uscChartDoc.chartHash, + isOfficial: false, + }, + }; + + const song: SongDocument<"usc"> = { + title: uscChartDoc.title, + artist: uscChartDoc.artist, + firstVersion: null, + id: 0, + data: {}, + "alt-titles": [], + "search-titles": [], + }; + + return { chart, song }; +} + +export function USCChartIndexToDiff( + index: 0 | 1 | 2 | 3 +): ChartDocument<"usc:Single">["difficulty"] { + return (["NOV", "ADV", "EXH", "INF"] as const)[index]; +}