Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-02 03:48:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

Generify Orphan Queue

Browse Source
This commit is contained in:
zkldi
2021-09-07 05:59:30 +01:00
parent c374ee0473
commit dc15acb189
7 changed files with 170 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/package.json
+1 -1
View File
@@ -73,7 +73,7 @@
"redis": "3.1.2",
"rimraf": "3.0.2",
"safe-json-stringify": "1.2.0",
"tachi-common": "0.2.3",
"tachi-common": "0.2.4",
"typescript": "4.3.4",
"winston": "3.3.3"
},
server/pnpm-lock.yaml
Generated
+4 -4
View File
@@ -54,7 +54,7 @@ specifiers:
rimraf: 3.0.2
safe-json-stringify: 1.2.0
supertest: 6.1.3
tachi-common: 0.2.3
tachi-common: 0.2.4
tap: 15.0.9
ts-node: 10.0.0
tsconfig-paths: 3.10.1
@@ -88,7 +88,7 @@ dependencies:
redis: 3.1.2
rimraf: 3.0.2
safe-json-stringify: 1.2.0
tachi-common: 0.2.3_ts-node@10.0.0+typescript@4.3.4
tachi-common: 0.2.4_ts-node@10.0.0+typescript@4.3.4
typescript: 4.3.4
winston: 3.3.3
@@ -3681,8 +3681,8 @@ packages:
strip-ansi: 6.0.0
dev: true
/tachi-common/0.2.3_ts-node@10.0.0+typescript@4.3.4:
resolution: {integrity: sha512-l1w5IQfaZejgl/+opB2K5mRkeQstreOq4+6+9A1kkGNVFu0lxvLYZdqSlJwbqYVAPOO7ssYZsi+FSUf81zowpg==}
/tachi-common/0.2.4_ts-node@10.0.0+typescript@4.3.4:
resolution: {integrity: sha512-5Q6c0qGhQyPKKiCg+BvuYbt9k5q/B1c3aBNR3KaX+aEKSQ8ZbuYTcKPKnQlDccjDfcGNj2OAmELx3oHz4UJdGA==}
dependencies:
monk: 7.3.4
tap: 15.0.9_ts-node@10.0.0+typescript@4.3.4
server/src/external/mongo/db.ts
Vendored
+3 -3
View File
@@ -34,7 +34,7 @@ import {
OAuth2ApplicationDocument,
integer,
FervidexSettingsDocument,
USCOrphanChart,
OrphanChart,
} from "tachi-common";
import monk, { TMiddleware } from "monk";
import CreateLogCtx from "lib/logger/logger";
@@ -178,7 +178,7 @@ const db = {
// i've inlined this one because i don't see it appearing anywhere else.
monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"),
"fer-settings": monkDB.get<FervidexSettingsDocument>("fer-settings"),
"usc-orphan-chart-queue": monkDB.get<USCOrphanChart>("usc-orphan-chart-queue"),
"orphan-chart-queue": monkDB.get<OrphanChart>("orphan-chart-queue"),
};
export type StaticDatabases =
@@ -210,7 +210,7 @@ export type StaticDatabases =
| "oauth2-clients"
| "oauth2-auth-codes"
| "fer-settings"
| "usc-orphan-chart-queue"
| "orphan-chart-queue"
| "user-settings";
export type Databases = StaticDatabases | `songs-${Game}` | `charts-${Game}`;
server/src/lib/orphan-queue/orphan-queue.ts
+97
View File
@@ -0,0 +1,97 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { ChartDocument, IDStrings, IDStringToGame, integer, SongDocument } from "tachi-common";
import { GetNextCounterValue } from "utils/db";
import { FilterQuery } from "mongodb";
const logger = CreateLogCtx(__filename);
/**
* Handles an orphan queue request.
*
* If the chart has never been seen before, add it to the orphan queue
* and return null.
*
* If the chart has been seen before, and has less than N unique players
* who have played it, return null.
*
* If the chart has been seen before, and has >= N unique players who have
* played it, unorphan the chart, and return it.
*/
export async function HandleOrphanQueue<I extends IDStrings>(
idString: I,
game: IDStringToGame[I],
chartDoc: ChartDocument<I>,
songDoc: SongDocument<IDStringToGame[I]>,
orphanMatchCriteria: FilterQuery<ChartDocument<I>>,
queueSize: integer,
userID: integer,
chartName: string
) {
logger.debug(`Recieved orphanqueue request for ${chartName}.`);
const orphanChart = await db["orphan-chart-queue"].findOne(
Object.assign({ idString }, orphanMatchCriteria),
{
projectID: true,
}
);
if (!orphanChart) {
logger.verbose(`Recieved unknown chart ${chartName}, orphaning.`);
await db["orphan-chart-queue"].insert({
idString,
chartDoc,
songDoc,
userIDs: [userID],
});
return null;
}
orphanChart.userIDs.push(userID);
const uniqueUsersArr = [...new Set(orphanChart.userIDs)];
const playcount = uniqueUsersArr.length;
// If N or more people have played this chart while orphaned, unorphan
// it.
if (playcount >= queueSize) {
logger.info(
`Song ${chartName} was unorphaned by ${uniqueUsersArr.join(", ")} and ${userID}.`
);
const songID = await GetNextCounterValue(`${game}-song-id`);
logger.verbose(`${chartName} has been assigned songID ${songID}.`);
const { songDoc, chartDoc } = orphanChart;
songDoc.id = songID;
chartDoc.songID = songID;
await db.songs[game].insert(songDoc);
await db.charts[game].insert(chartDoc);
await db["orphan-chart-queue"].remove({
_id: orphanChart._id,
});
return chartDoc as ChartDocument<I>;
}
// otherwise, update the state of this orphan.
logger.verbose(`UserID ${userID} played ${chartName}, which is now at ${playcount} plays.`);
await db["orphan-chart-queue"].update(
{
_id: orphanChart._id,
},
{
$set: {
userIDs: uniqueUsersArr,
},
}
);
return null;
}
server/src/server/router/ir/usc/orphan-queue.ts
-118
View File
@@ -1,118 +0,0 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import { ChartDocument, integer, SongDocument } from "tachi-common";
import { GetNextCounterValue } from "utils/db";
import { Random20Hex } from "utils/misc";
import { USCClientChart } from "./types";
const logger = CreateLogCtx(__filename);
/**
* Handles the orphan queue for USC charts. If the provided chart was just
* unorphaned, returns the chart. Else, returns null.
*/
export async function HandleOrphanQueue(uscChartDoc: USCClientChart, userID: integer) {
const chartName = `${uscChartDoc.artist} - ${uscChartDoc.title} (${IndexToDiff(
uscChartDoc.difficulty
)})`;
logger.debug(`Recieved orphanqueue request for ${chartName}.`);
const orphanChart = await db["usc-orphan-chart-queue"].findOne({
"chartDoc.data.hashSHA1": uscChartDoc.chartHash,
});
if (!orphanChart) {
logger.verbose(`Recieved unknown chart ${chartName}, orphaning.`);
const { song, chart } = ConvertUSCChart(uscChartDoc);
await db["usc-orphan-chart-queue"].insert({
chartDoc: chart,
songDoc: song,
userIDs: [userID],
});
return null;
}
orphanChart.userIDs.push(userID);
const uniqueUsersArr = [...new Set(orphanChart.userIDs)];
const playcount = uniqueUsersArr.length;
// If N or more people have played this chart while orphaned, unorphan
// it.
if (playcount >= ServerConfig.USC_QUEUE_SIZE) {
logger.info(
`Song ${chartName} was unorphaned by ${uniqueUsersArr.join(", ")} and ${userID}.`
);
const songID = await GetNextCounterValue("usc-song-id");
logger.verbose(`${chartName} has been assigned songID ${songID}.`);
const { songDoc, chartDoc } = orphanChart;
songDoc.id = songID;
chartDoc.songID = songID;
await db.songs.usc.insert(songDoc);
await db.charts.usc.insert(chartDoc);
await db["usc-orphan-chart-queue"].remove({
"chartDoc.data.hashSHA1": chartDoc.data.hashSHA1,
});
return chartDoc;
}
// otherwise, update the state of this orphan.
logger.verbose(`UserID ${userID} played ${chartName}, which is now at ${playcount} plays.`);
await db["usc-orphan-chart-queue"].update(
{
"chartDoc.data.hashSHA1": uscChartDoc.chartHash,
},
{
$set: {
userIDs: uniqueUsersArr,
},
}
);
return null;
}
function ConvertUSCChart(uscChartDoc: USCClientChart) {
const chart: ChartDocument<"usc:Single"> = {
chartID: Random20Hex(),
difficulty: IndexToDiff(uscChartDoc.difficulty),
isPrimary: true,
level: "?",
levelNum: 0,
playtype: "Single",
rgcID: null,
songID: 0,
versions: [],
data: {
hashSHA1: uscChartDoc.chartHash,
isOfficial: false,
},
};
const song: SongDocument<"usc"> = {
title: uscChartDoc.title,
artist: uscChartDoc.artist,
firstVersion: null,
id: 0,
data: {},
"alt-titles": [],
"search-titles": [],
};
return { chart, song };
}
function IndexToDiff(index: 0 | 1 | 2 | 3): ChartDocument<"usc:Single">["difficulty"] {
return (["NOV", "ADV", "EXH", "INF"] as const)[index];
}
server/src/server/router/ir/usc/router.ts
+26 -4
View File
@@ -1,6 +1,5 @@
import { Router, RequestHandler } from "express";
import p from "prudence";
import { FindChartOnSHA256 } from "utils/queries/charts";
import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi";
import db from "external/mongo/db";
import {
@@ -11,7 +10,12 @@ import {
} from "tachi-common";
import { AssertStrAsPositiveNonZeroInt } from "lib/score-import/framework/common/string-asserts";
import CreateLogCtx, { KtLogger } from "lib/logger/logger";
import { CreatePOSTScoresResponseBody, TachiScoreToServerScore } from "./usc";
import {
ConvertUSCChart,
CreatePOSTScoresResponseBody,
TachiScoreToServerScore,
USCChartIndexToDiff,
} from "./usc";
import { ExpressWrappedScoreImportMain } from "lib/score-import/framework/express-wrapper";
import { GetUserWithID } from "utils/user";
import { ParseIRUSC } from "lib/score-import/import-types/ir/usc/parser";
@@ -24,7 +28,8 @@ import { RequirePermissions } from "server/middleware/auth";
import { GetUSCIRReplayURL } from "lib/cdn/url-format";
import { FormatPrError } from "utils/prudence";
import { USCClientChart } from "./types";
import { HandleOrphanQueue } from "./orphan-queue";
import { HandleOrphanQueue } from "lib/orphan-queue/orphan-queue";
import { ServerConfig } from "lib/setup/config";
const logger = CreateLogCtx(__filename);
@@ -280,7 +285,24 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => {
// If this chart is already orphaned, increase its unique player
// playcount.
if (!chartDoc) {
chartDoc = await HandleOrphanQueue(uscChart, req[SYMBOL_TachiAPIAuth].userID!);
const { song, chart } = ConvertUSCChart(uscChart);
const uscChartName = `${uscChart.artist} - ${uscChart.title} (${USCChartIndexToDiff(
uscChart.difficulty
)})`;
chartDoc = await HandleOrphanQueue(
"usc:Single",
"usc",
chart,
song,
{
"data.hashSHA1": uscChart.chartHash,
},
ServerConfig.USC_QUEUE_SIZE,
req[SYMBOL_TachiAPIAuth].userID!,
uscChartName
);
}
const userDoc = await GetUserWithID(req[SYMBOL_TachiAPIAuth]!.userID!);
server/src/server/router/ir/usc/usc.ts
+39 -3
View File
@@ -1,10 +1,10 @@
import { integer, PBScoreDocument, ScoreDocument, ChartDocument } from "tachi-common";
import { integer, PBScoreDocument, ScoreDocument, ChartDocument, SongDocument } from "tachi-common";
import CreateLogCtx from "lib/logger/logger";
import { GetPBOnChart, GetServerRecordOnChart } from "utils/scores";
import { MStoS } from "utils/misc";
import { MStoS, Random20Hex } from "utils/misc";
import { USCIR_ADJACENT_SCORE_N } from "lib/constants/usc-ir";
import db from "external/mongo/db";
import { USCServerScore } from "./types";
import { USCClientChart, USCServerScore } from "./types";
const logger = CreateLogCtx(__filename);
@@ -201,3 +201,39 @@ export interface POSTScoresResponseBody {
isServerRecord: boolean;
sendReplay: string;
}
export function ConvertUSCChart(uscChartDoc: USCClientChart) {
const chart: ChartDocument<"usc:Single"> = {
chartID: Random20Hex(),
difficulty: USCChartIndexToDiff(uscChartDoc.difficulty),
isPrimary: true,
level: "?",
levelNum: 0,
playtype: "Single",
rgcID: null,
songID: 0,
versions: [],
data: {
hashSHA1: uscChartDoc.chartHash,
isOfficial: false,
},
};
const song: SongDocument<"usc"> = {
title: uscChartDoc.title,
artist: uscChartDoc.artist,
firstVersion: null,
id: 0,
data: {},
"alt-titles": [],
"search-titles": [],
};
return { chart, song };
}
export function USCChartIndexToDiff(
index: 0 | 1 | 2 | 3
): ChartDocument<"usc:Single">["difficulty"] {
return (["NOV", "ADV", "EXH", "INF"] as const)[index];
}