From e89ee76dc67d7c67be77c3cdbf78c6e8d316301f Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 7 Sep 2021 07:16:25 +0100 Subject: [PATCH] Add bms chart orphaning support --- server/scripts/set-counters.ts | 5 + server/src/lib/orphan-queue/orphan-queue.ts | 11 +- .../import-types/ir/beatoraja/converter.ts | 90 +++++++++++-- .../import-types/ir/beatoraja/parser.ts | 3 + .../import-types/ir/beatoraja/types.ts | 2 + server/src/lib/setup/config.ts | 3 + .../router/ir/beatoraja/charts/router.ts | 1 + .../server/router/ir/beatoraja/router.test.ts | 127 ++++++++++++++++++ .../src/server/router/ir/beatoraja/router.ts | 17 ++- server/src/test-utils/mock-db/counters.json | 4 + 10 files changed, 247 insertions(+), 16 deletions(-) diff --git a/server/scripts/set-counters.ts b/server/scripts/set-counters.ts index 294e9f8e8..024f14a91 100644 --- a/server/scripts/set-counters.ts +++ b/server/scripts/set-counters.ts @@ -3,6 +3,7 @@ import db from "external/mongo/db"; (async () => { const uscSongCount = await db.songs.usc.find({}); + const bmsSongCount = await db.songs.bms.find({}); const Counters: CounterDocument[] = [ { @@ -13,6 +14,10 @@ import db from "external/mongo/db"; counterName: "usc-song-id", value: uscSongCount.length + 1, }, + { + counterName: "bms-song-id", + value: bmsSongCount.length + 1, + }, ]; db.counters.insert(Counters).then(() => { diff --git a/server/src/lib/orphan-queue/orphan-queue.ts b/server/src/lib/orphan-queue/orphan-queue.ts index 10f123930..ddc66a559 100644 --- a/server/src/lib/orphan-queue/orphan-queue.ts +++ b/server/src/lib/orphan-queue/orphan-queue.ts @@ -1,6 +1,13 @@ import db from "external/mongo/db"; import CreateLogCtx from "lib/logger/logger"; -import { ChartDocument, IDStrings, IDStringToGame, integer, SongDocument } from "tachi-common"; +import { + ChartDocument, + IDStrings, + IDStringToGame, + integer, + OrphanChart, + SongDocument, +} from "tachi-common"; import { GetNextCounterValue } from "utils/db"; import { FilterQuery } from "mongodb"; @@ -23,7 +30,7 @@ export async function HandleOrphanQueue( game: IDStringToGame[I], chartDoc: ChartDocument, songDoc: SongDocument, - orphanMatchCriteria: FilterQuery>, + orphanMatchCriteria: FilterQuery>, queueSize: integer, userID: integer, chartName: string diff --git a/server/src/lib/score-import/import-types/ir/beatoraja/converter.ts b/server/src/lib/score-import/import-types/ir/beatoraja/converter.ts index 9e7f64054..734176059 100644 --- a/server/src/lib/score-import/import-types/ir/beatoraja/converter.ts +++ b/server/src/lib/score-import/import-types/ir/beatoraja/converter.ts @@ -8,8 +8,11 @@ import { import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils"; import { DryScore } from "../../../framework/common/types"; import { ConverterFunction } from "../../common/types"; -import { BeatorajaContext, BeatorajaScore } from "./types"; -import { ChartDocument } from "tachi-common"; +import { BeatorajaContext, BeatorajaScore, BeatorajaChart } from "./types"; +import { ChartDocument, SongDocument } from "tachi-common"; +import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue"; +import { Random20Hex } from "utils/misc"; +import { ServerConfig, ServerTypeInfo } from "lib/setup/config"; const LAMP_LOOKUP = { NoPlay: "NO PLAY", @@ -41,19 +44,50 @@ export const ConverterIRBeatoraja: ConverterFunction | null; if (!chart) { - // @todo #141 Import charts into the Kamaitachi Database if the chart - // doesn't exist. - throw new KTDataNotFoundFailure( - `Could not find chart ${context.chart.title}.`, - importType, - data, - context + const chartName = `${context.chart.artist} (${context.chart.subartist})- ${context.chart.title} (${context.chart.subtitle})`; + + if (context.chart.hasRandom) { + // If you're someone forking tachi looking to remove this + // check, remember to change the entire score import + // framework and database to be able to handle variable notecounts. + logger.verbose(`Declined to orphan chart ${chartName} as it has #RANDOM declarations.`); + throw new InvalidScoreFailure( + `${ServerTypeInfo.name} will not support #RANDOM charts.` + ); + } + + const idString = context.chart.mode === "BEAT_7K" ? "bms:7K" : "bms:14K"; + + const { chartDoc, songDoc } = ConvertBeatorajaChartToTachi(context.chart); + + chart = await HandleOrphanQueue( + idString, + "bms", + chartDoc, + songDoc, + { + "chartDoc.data.hashSHA256": context.chart.sha256, + }, + ServerConfig.BEATORAJA_QUEUE_SIZE, + context.userID, + chartName ); + + // If chart wasn't unorphaned as a result of this request + // orphan this score and return ktdnf + if (!chart) { + throw new KTDataNotFoundFailure( + `This chart is orphaned.`, + "ir/beatoraja", + data, + context + ); + } } const song = await FindSongOnID("bms", chart.songID); @@ -127,3 +161,39 @@ export const ConverterIRBeatoraja: ConverterFunction = { + chartID: Random20Hex(), + difficulty: "CHART", + isPrimary: true, + level: "?", + levelNum: 0, + playtype: chart.mode === "BEAT_7K" ? "7K" : "14K", + rgcID: null, + songID: 0, + versions: [], + data: { + hashMD5: chart.md5, + hashSHA256: chart.sha256, + notecount: chart.notes, + tableFolders: [], + }, + }; + + const songDoc: SongDocument<"bms"> = { + artist: chart.artist, + title: chart.title, + firstVersion: "", + id: 0, + "alt-titles": [], + "search-titles": [], + data: { + genre: chart.genre, + subartist: chart.subartist, + subtitle: chart.subtitle, + }, + }; + + return { songDoc, chartDoc }; +} diff --git a/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts b/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts index af8e1e82b..ab85ab0e6 100644 --- a/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts +++ b/server/src/lib/score-import/import-types/ir/beatoraja/parser.ts @@ -4,6 +4,7 @@ import ScoreImportFatalError from "../../../framework/score-importing/score-impo import { FormatPrError } from "utils/prudence"; import { BeatorajaChart, BeatorajaContext, BeatorajaScore } from "./types"; import { ParserFunctionReturns } from "../../common/types"; +import { integer } from "tachi-common"; const PR_BeatorajaScore = { sha256: "string", @@ -63,6 +64,7 @@ const PR_BeatorajaChart = { export function ParseBeatorajaSingle( body: Record, + userID: integer, logger: KtLogger ): ParserFunctionReturns { const err = p( @@ -104,6 +106,7 @@ export function ParseBeatorajaSingle( context: { client, chart: body.chart as BeatorajaChart, + userID, }, game: "bms", iterable: [body.score] as unknown as BeatorajaScore[], diff --git a/server/src/lib/score-import/import-types/ir/beatoraja/types.ts b/server/src/lib/score-import/import-types/ir/beatoraja/types.ts index 175d5d1a5..5380fc747 100644 --- a/server/src/lib/score-import/import-types/ir/beatoraja/types.ts +++ b/server/src/lib/score-import/import-types/ir/beatoraja/types.ts @@ -3,6 +3,7 @@ import { integer } from "tachi-common"; export interface BeatorajaContext { client: "lr2oraja"; chart: BeatorajaChart; + userID: integer; // unexpectedly necessary for orphan code! } export interface BeatorajaChart { @@ -18,6 +19,7 @@ export interface BeatorajaChart { mode: "BEAT_7K" | "BEAT_14K"; judge: number; notes: integer; + hasRandom: boolean; } export interface BeatorajaScore { diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index 2e1a01599..ca8139074 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -73,6 +73,7 @@ export interface TachiConfig { SENDMAIL_BIN?: string; }; USC_QUEUE_SIZE: integer; + BEATORAJA_QUEUE_SIZE: integer; } const isValidOauth2 = p.optional({ @@ -109,6 +110,7 @@ const err = p(config, { SENDMAIL_BIN: "*string", }), USC_QUEUE_SIZE: p.optional(p.gteInt(2)), + BEATORAJA_QUEUE_SIZE: p.optional(p.gteInt(2)), }); if (err) { @@ -129,6 +131,7 @@ const tachiConfig = config as TachiConfig; tachiConfig.RATE_LIMIT ??= 500; tachiConfig.OAUTH_CLIENT_CAP ??= 15; tachiConfig.USC_QUEUE_SIZE ??= 3; +tachiConfig.BEATORAJA_QUEUE_SIZE ??= 3; if (tachiConfig.EMAIL_CONFIG) { tachiConfig.EMAIL_CONFIG.SENDMAIL_BIN ??= "/usr/bin/sendmail"; diff --git a/server/src/server/router/ir/beatoraja/charts/router.ts b/server/src/server/router/ir/beatoraja/charts/router.ts index 78edacc63..5d73c8a05 100644 --- a/server/src/server/router/ir/beatoraja/charts/router.ts +++ b/server/src/server/router/ir/beatoraja/charts/router.ts @@ -28,6 +28,7 @@ router.use(GetChartDocument); /** * Retrieves scores for the given chart. + * * @name GET /ir/beatoraja/chart/:chartSHA256/scores */ router.get("/scores", async (req, res) => { diff --git a/server/src/server/router/ir/beatoraja/router.test.ts b/server/src/server/router/ir/beatoraja/router.test.ts index 36f11346e..5edc86e5e 100644 --- a/server/src/server/router/ir/beatoraja/router.test.ts +++ b/server/src/server/router/ir/beatoraja/router.test.ts @@ -5,6 +5,7 @@ import mockApi from "test-utils/mock-api"; import ResetDBState from "test-utils/resets"; import { GetKTDataJSON } from "test-utils/test-data"; import deepmerge from "deepmerge"; +import { PublicUserDocument } from "tachi-common"; t.test("POST /ir/beatoraja/submit-score", (t) => { t.beforeEach(ResetDBState); @@ -103,6 +104,132 @@ t.test("POST /ir/beatoraja/submit-score", (t) => { t.end(); }); + t.test("Should defer a chart to the orphan queue if not found.", async (t) => { + const res = await mockApi + .post("/ir/beatoraja/submit-score") + .set("X-TachiIR-Version", "2.0.0") + .set("Authorization", "Bearer mock_token") + .send( + deepmerge(scoreReq, { + chart: { sha256: "new_chart", md5: "new_md5" }, + score: { sha256: "new_chart", md5: "new_md5" }, + }) + ); + + t.equal(res.status, 202); + + t.equal(res.body.success, true); + t.match(res.body.description, /Chart and score have been orphaned/u); + + const orphanChart = await db["orphan-chart-queue"].findOne({ + "chartDoc.data.hashSHA256": "new_chart", + }); + + t.hasStrict(orphanChart?.chartDoc, { + data: { + hashSHA256: "new_chart", + hashMD5: "new_md5", + }, + }); + + t.end(); + }); + + t.test("Should eventually unorphan a chart.", async (t) => { + await db["api-tokens"].insert([ + { + userID: 2, + identifier: "token2", + permissions: { submit_score: true }, + token: "token2", + }, + { + userID: 3, + identifier: "token3", + permissions: { submit_score: true }, + token: "token3", + }, + ]); + + await db.users.insert([ + { + id: 2, + username: "foo", + usernameLowercase: "foo", + }, + { + id: 3, + username: "bar", + usernameLowercase: "bar", + }, + ] as PublicUserDocument[]); + + const res = await mockApi + .post("/ir/beatoraja/submit-score") + .set("X-TachiIR-Version", "2.0.0") + .set("Authorization", "Bearer mock_token") + .send( + deepmerge(scoreReq, { + chart: { sha256: "new_chart", md5: "new_md5" }, + score: { sha256: "new_chart", md5: "new_md5" }, + }) + ); + + t.equal(res.statusCode, 202); + + const res2 = await mockApi + .post("/ir/beatoraja/submit-score") + .set("X-TachiIR-Version", "2.0.0") + .set("Authorization", "Bearer token2") + .send( + deepmerge(scoreReq, { + chart: { sha256: "new_chart", md5: "new_md5" }, + score: { sha256: "new_chart", md5: "new_md5" }, + }) + ); + + t.equal(res2.statusCode, 202); + + const orphanData = await db["orphan-chart-queue"].findOne({ + "chartDoc.data.hashSHA256": "new_chart", + }); + + t.strictSame(orphanData?.userIDs, [1, 2]); + + const res3 = await mockApi + .post("/ir/beatoraja/submit-score") + .set("X-TachiIR-Version", "2.0.0") + .set("Authorization", "Bearer token3") + .send( + deepmerge(scoreReq, { + chart: { sha256: "new_chart", md5: "new_md5" }, + score: { sha256: "new_chart", md5: "new_md5" }, + }) + ); + + t.equal(res3.statusCode, 200); + + const orphanData2 = await db["orphan-chart-queue"].findOne({ + "chartDoc.data.hashSHA256": "new_chart", + }); + + t.equal(orphanData2, null, "Orphan data should be removed from the database."); + + const score = await db.scores.findOne({ + game: "bms", + userID: 3, + }); + + t.hasStrict(score, { + scoreData: { + score: 1004, + }, + importType: "ir/beatoraja", + }); + + t.end(); + }); + t.end(); }); diff --git a/server/src/server/router/ir/beatoraja/router.ts b/server/src/server/router/ir/beatoraja/router.ts index 8cba74500..93b2d83dc 100644 --- a/server/src/server/router/ir/beatoraja/router.ts +++ b/server/src/server/router/ir/beatoraja/router.ts @@ -4,6 +4,7 @@ import { SYMBOL_TachiAPIAuth } from "lib/constants/tachi"; import CreateLogCtx, { KtLogger } from "lib/logger/logger"; import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper"; import { ParseBeatorajaSingle } from "lib/score-import/import-types/ir/beatoraja/parser"; +import { ServerConfig } from "lib/setup/config"; import { UpdateClassIfGreater } from "utils/class"; import { GetUserWithIDGuaranteed } from "utils/user"; import { ValidateIRClientVersion } from "./auth"; @@ -16,14 +17,14 @@ const router: Router = Router({ mergeParams: true }); router.use(ValidateIRClientVersion); /** - * Submits a beatoraja score to Kamaitachi. If the chart is unavailable, - * store it as a new chart alongside the new score. + * Submits a beatoraja score to Tachi. + * * @name POST /ir/beatoraja/submit-score */ router.post("/submit-score", async (req, res) => { const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TachiAPIAuth]!.userID!); - const ParserFunction = (logger: KtLogger) => ParseBeatorajaSingle(req.body, logger); + const ParserFunction = (logger: KtLogger) => ParseBeatorajaSingle(req.body, userDoc.id, logger); const importRes = await ExpressWrappedScoreImportMain( userDoc, @@ -35,6 +36,13 @@ router.post("/submit-score", async (req, res) => { if (!importRes.body.success) { return res.status(400).json(importRes.body); } else if (importRes.body.body.errors.length !== 0) { + if (importRes.body.body.errors[0].type === "KTDataNotFound") { + return res.status(202).json({ + success: true, + description: `Chart and score have been orphaned. This score will reify when atleast ${ServerConfig.BEATORAJA_QUEUE_SIZE} players have played the chart.`, + }); + } + // since we're only ever importing one score, we can guarantee // that this means the score we tried to import was skipped. return res.status(400).json({ @@ -78,8 +86,9 @@ router.post("/submit-score", async (req, res) => { }); /** - * Submits a course result to Kamaitachi. This only accepts a limited set of + * Submits a course result to Tachi. This only accepts a limited set of * courses - all of which are dans. + * * @name POST /ir/beatoraja/submit-course */ router.post("/submit-course", async (req, res) => { diff --git a/server/src/test-utils/mock-db/counters.json b/server/src/test-utils/mock-db/counters.json index 5c28b33ec..b7587808e 100644 --- a/server/src/test-utils/mock-db/counters.json +++ b/server/src/test-utils/mock-db/counters.json @@ -10,5 +10,9 @@ { "counterName": "usc-song-id", "value": 2 + }, + { + "counterName": "bms-song-id", + "value": 2 } ] \ No newline at end of file