add some new endpoints for the user API

This commit is contained in:
zkldi
2021-06-17 20:51:53 +01:00
parent b82638e96b
commit af19cc05af
8 changed files with 414 additions and 196 deletions
+5 -2
View File
@@ -18,6 +18,9 @@ scripts/validate-database-errs/*
scripts/__JSON-TO-KTDB/**/*.json
/src/lib/env/config.json
src/lib/env/config.json
src/datasets/text-files/splash-text.txt
src/datasets/text-files/pseudonyms.txt
local-cdn
/local-cdn
+4 -3
View File
@@ -60,7 +60,6 @@ const staticIndexes: Partial<Record<ValidDatabases, Index[]>> = {
index({ timestamp: 1 }),
index({ total: 1 }),
],
// @todo #96 Add more indexes to the users collection.
users: [index({ id: 1 }, UNIQUE)],
tierlists: [
index({ tierlistID: 1 }, UNIQUE),
@@ -105,12 +104,13 @@ for (const game of CONF_INFO.SUPPORTED_GAMES) {
if (indexes[`songs-${game}` as ValidDatabases]) {
indexes[`songs-${game}` as ValidDatabases]!.push(
index({ id: 1 }, UNIQUE),
index({ title: 1 })
index({ title: "text", artist: "text", "alt-titles": "text", "search-titles": "text" })
);
} else {
indexes[`songs-${game}` as ValidDatabases] = [
index({ id: 1 }, UNIQUE),
index({ title: 1 }),
index({ title: "text", artist: "text", "alt-titles": "text", "search-titles": "text" }),
];
}
}
@@ -124,6 +124,7 @@ export async function SetIndexes(dbst: string) {
if (options.reset) {
// eslint-disable-next-line no-await-in-loop
await db.get(collection).dropIndexes();
logger.info(`Reset ${collection}.`);
}
// @ts-expect-error dru(n)kts
@@ -142,5 +143,5 @@ export async function SetIndexes(dbst: string) {
// if calling this as a script -- similar to pythons if __name__ == "__main__"
if (require.main === module) {
SetIndexes(options.db ?? "ktblackdb").then(process.exit(0));
SetIndexes(options.db ?? "ktblackdb").then(() => process.exit(0));
}
@@ -35,193 +35,219 @@ export default async function ScoreImportMain<D, C>(
InputParser: ImportInputParser<D, C>,
providedImportObjects?: { logger: KtLogger; importID: string }
) {
const lock = await GetOrSetUserLock(user.id);
// in the event of any error, we remove the user lock.
try {
const lock = await GetOrSetUserLock(user.id);
if (lock) {
// @danger
// Throwing away an import if the user already has one outgoing is *bad*, as in the case
// of degraded performance we might just start throwing scores away. This is obviously
// not great, but any other solution involves making a queue, which can't be done because
// InputParser is a very dynamic function that cannot be stored in redis or something.
//
// Under normal circumstances, there is no scenario where a user would have two ongoing
// imports at the same time - even if they were using single-score imports on a 5 second
// chart, as each score import takes only around ~10-15miliseconds.
throw new ScoreImportFatalError(409, "This user already has an ongoing import.");
if (lock) {
// @danger
// Throwing away an import if the user already has one outgoing is *bad*, as in the case
// of degraded performance we might just start throwing scores away. This is obviously
// not great, but any other solution involves making a queue, which can't be done because
// InputParser is a very dynamic function that cannot be stored in redis or something.
//
// Under normal circumstances, there is no scenario where a user would have two ongoing
// imports at the same time - even if they were using single-score imports on a 5 second
// chart, as each score import takes only around ~10-15miliseconds.
throw new ScoreImportFatalError(409, "This user already has an ongoing import.");
}
const timeStarted = Date.now();
let importID;
let logger;
if (!providedImportObjects) {
// If they weren't given to us -
// we create an "import logger".
// this holds a reference to the user's name, ID, and type
// of score import for any future debugging.
({ importID, logger } = CreateImportLoggerAndID(user, importType));
logger.debug("Received import request.");
} else {
({ importID, logger } = providedImportObjects);
}
// --- 1. Parsing ---
// We get an iterable from the provided parser function, alongside some context and a converter function.
// This iterable does not have to be an array - it's anything that's iterable, like a generator.
const parseTimeStart = process.hrtime.bigint();
const { iterable, context, game, classHandler } = await InputParser(logger);
const parseTime = GetMilisecondsSince(parseTimeStart);
logger.debug(`Parsing took ${parseTime} miliseconds.`);
// We have to cast here due to typescript generic confusions. This is guaranteed` to be correct.
const ConverterFunction = Converters[importType] as unknown as ConverterFunction<D, C>;
// --- 2. Importing ---
// ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data.
const importTimeStart = process.hrtime.bigint();
const importInfo = await ImportAllIterableData(
user.id,
importType,
iterable,
ConverterFunction,
context,
logger
);
const importTime = GetMilisecondsSince(importTimeStart);
const importTimeRel = importTime / importInfo.length;
logger.debug(`Importing took ${importTime} miliseconds. (${importTimeRel}ms/doc)`);
// --- 3. ParseImportInfo ---
// ImportInfo is a relatively complex structure. We need some information from it for subsequent steps
// such as the list of chartIDs involved in this import.
const importParseTimeStart = process.hrtime.bigint();
const { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo);
const importParseTime = GetMilisecondsSince(importParseTimeStart);
const importParseTimeRel = importParseTime / importInfo.length;
logger.debug(
`Import Parsing took ${importParseTime} miliseconds. (${importParseTimeRel}ms/doc)`
);
// --- 4. Sessions ---
// We create (or update existing) sessions here. This uses the aforementioned parsed import info
// to determine what goes where.
const sessionTimeStart = process.hrtime.bigint();
const sessionInfo = await CreateSessions(
user.id,
importType,
game,
scorePlaytypeMap,
logger
);
const sessionTime = GetMilisecondsSince(sessionTimeStart);
const sessionTimeRel = sessionTime / sessionInfo.length;
logger.debug(
`Session Processing took ${sessionTime} miliseconds (${sessionTimeRel}ms/doc).`
);
// --- 5. PersonalBests ---
// We want to keep an updated reference of a users best score on a given chart.
// This function also handles conjoining different scores together (such as unioning best lamp and
// best score).
const pbTimeStart = process.hrtime.bigint();
await ProcessPBs(user.id, chartIDs, logger);
const pbTime = GetMilisecondsSince(pbTimeStart);
const pbTimeRel = pbTime / chartIDs.size;
logger.debug(`PB Processing took ${pbTime} miliseconds (${pbTimeRel}ms/doc)`);
const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][];
// --- 6. Game Stats ---
// This function updates the users "stats" for this game - such as their profile rating or their classes.
const ugsTimeStart = process.hrtime.bigint();
const classDeltas = await UpdateUsersGameStats(
game,
playtypes,
user.id,
classHandler,
logger
);
const ugsTime = GetMilisecondsSince(ugsTimeStart);
logger.debug(`UGS Processing took ${ugsTime} miliseconds.`);
// --- 7. Goals ---
// Evaluate and update the users goals. This returns information about goals that have changed.
const goalTimeStart = process.hrtime.bigint();
const goalInfo = await GetAndUpdateUsersGoals(game, user.id, chartIDs, logger);
const goalTime = GetMilisecondsSince(goalTimeStart);
logger.debug(`Goal Processing took ${goalTime} miliseconds.`);
// --- 8. Milestones ---
// Evaluate and update the users milestones. This returns...
const milestoneTimeStart = process.hrtime.bigint();
const milestoneInfo = await UpdateUsersMilestones(
goalInfo,
game,
playtypes,
user.id,
logger
);
const milestoneTime = GetMilisecondsSince(milestoneTimeStart);
logger.debug(`Milestone Processing took ${milestoneTime} miliseconds.`);
// --- 9. Finalise Import Document ---
// Create and Save an import document to the database, and finish everything up!
const ImportDocument: ImportDocument = {
importType,
idStrings: playtypes.map((e) => `${game}:${e}`) as IDStrings[],
scoreIDs,
errors,
importID,
timeFinished: Date.now(),
timeStarted,
createdSessions: sessionInfo,
userID: user.id,
classDeltas,
goalInfo,
milestoneInfo,
userIntent,
};
const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${
importInfo.length
} documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${
sessionInfo.length
}). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`;
// I only really want to log "big" imports. The others are here for debugging purposes.
if (scoreIDs.length > 500) {
logger.info(logMessage);
} else if (scoreIDs.length > 1) {
logger.verbose(logMessage);
} else {
logger.debug(logMessage);
}
await db.imports.insert(ImportDocument);
// we don't await this because we don't
// particularly care about waiting for it.
db["import-timings"].insert({
importID,
timestamp: Date.now(),
total: ImportDocument.timeFinished - timeStarted,
rel: {
import: importTimeRel,
importParse: importParseTimeRel,
pb: pbTimeRel,
session: sessionTimeRel,
},
abs: {
parse: parseTime,
import: importTime,
importParse: importParseTime,
session: sessionTime,
pb: pbTime,
ugs: ugsTime,
goal: goalTime,
milestone: milestoneTime,
},
});
await RemoveUserLock(user.id);
return ImportDocument;
} catch (err) {
await RemoveUserLock(user.id);
throw err;
}
const timeStarted = Date.now();
let importID;
let logger;
if (!providedImportObjects) {
// If they weren't given to us -
// we create an "import logger".
// this holds a reference to the user's name, ID, and type
// of score import for any future debugging.
({ importID, logger } = CreateImportLoggerAndID(user, importType));
logger.debug("Received import request.");
} else {
({ importID, logger } = providedImportObjects);
}
// --- 1. Parsing ---
// We get an iterable from the provided parser function, alongside some context and a converter function.
// This iterable does not have to be an array - it's anything that's iterable, like a generator.
const parseTimeStart = process.hrtime.bigint();
const { iterable, context, game, classHandler } = await InputParser(logger);
const parseTime = GetMilisecondsSince(parseTimeStart);
logger.debug(`Parsing took ${parseTime} miliseconds.`);
// We have to cast here due to typescript generic confusions. This is guaranteed` to be correct.
const ConverterFunction = Converters[importType] as unknown as ConverterFunction<D, C>;
// --- 2. Importing ---
// ImportAllIterableData iterates over the iterable, applying the converter function to each bit of data.
const importTimeStart = process.hrtime.bigint();
const importInfo = await ImportAllIterableData(
user.id,
importType,
iterable,
ConverterFunction,
context,
logger
);
const importTime = GetMilisecondsSince(importTimeStart);
const importTimeRel = importTime / importInfo.length;
logger.debug(`Importing took ${importTime} miliseconds. (${importTimeRel}ms/doc)`);
// --- 3. ParseImportInfo ---
// ImportInfo is a relatively complex structure. We need some information from it for subsequent steps
// such as the list of chartIDs involved in this import.
const importParseTimeStart = process.hrtime.bigint();
const { scorePlaytypeMap, errors, scoreIDs, chartIDs } = ParseImportInfo(importInfo);
const importParseTime = GetMilisecondsSince(importParseTimeStart);
const importParseTimeRel = importParseTime / importInfo.length;
logger.debug(
`Import Parsing took ${importParseTime} miliseconds. (${importParseTimeRel}ms/doc)`
);
// --- 4. Sessions ---
// We create (or update existing) sessions here. This uses the aforementioned parsed import info
// to determine what goes where.
const sessionTimeStart = process.hrtime.bigint();
const sessionInfo = await CreateSessions(user.id, importType, game, scorePlaytypeMap, logger);
const sessionTime = GetMilisecondsSince(sessionTimeStart);
const sessionTimeRel = sessionTime / sessionInfo.length;
logger.debug(`Session Processing took ${sessionTime} miliseconds (${sessionTimeRel}ms/doc).`);
// --- 5. PersonalBests ---
// We want to keep an updated reference of a users best score on a given chart.
// This function also handles conjoining different scores together (such as unioning best lamp and
// best score).
const pbTimeStart = process.hrtime.bigint();
await ProcessPBs(user.id, chartIDs, logger);
const pbTime = GetMilisecondsSince(pbTimeStart);
const pbTimeRel = pbTime / chartIDs.size;
logger.debug(`PB Processing took ${pbTime} miliseconds (${pbTimeRel}ms/doc)`);
const playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][];
// --- 6. Game Stats ---
// This function updates the users "stats" for this game - such as their profile rating or their classes.
const ugsTimeStart = process.hrtime.bigint();
const classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, classHandler, logger);
const ugsTime = GetMilisecondsSince(ugsTimeStart);
logger.debug(`UGS Processing took ${ugsTime} miliseconds.`);
// --- 7. Goals ---
// Evaluate and update the users goals. This returns information about goals that have changed.
const goalTimeStart = process.hrtime.bigint();
const goalInfo = await GetAndUpdateUsersGoals(game, user.id, chartIDs, logger);
const goalTime = GetMilisecondsSince(goalTimeStart);
logger.debug(`Goal Processing took ${goalTime} miliseconds.`);
// --- 8. Milestones ---
// Evaluate and update the users milestones. This returns...
const milestoneTimeStart = process.hrtime.bigint();
const milestoneInfo = await UpdateUsersMilestones(goalInfo, game, playtypes, user.id, logger);
const milestoneTime = GetMilisecondsSince(milestoneTimeStart);
logger.debug(`Milestone Processing took ${milestoneTime} miliseconds.`);
// --- 9. Finalise Import Document ---
// Create and Save an import document to the database, and finish everything up!
const ImportDocument: ImportDocument = {
importType,
idStrings: playtypes.map((e) => `${game}:${e}`) as IDStrings[],
scoreIDs,
errors,
importID,
timeFinished: Date.now(),
timeStarted,
createdSessions: sessionInfo,
userID: user.id,
classDeltas,
goalInfo,
milestoneInfo,
userIntent,
};
const logMessage = `Import took: ${ImportDocument.timeFinished - timeStarted}ms, with ${
importInfo.length
} documents (Fails: ${errors.length}, Successes: ${scoreIDs.length}, Sessions: ${
sessionInfo.length
}). Aprx ${(ImportDocument.timeFinished - timeStarted) / importInfo.length}ms/doc`;
// I only really want to log "big" imports. The others are here for debugging purposes.
if (scoreIDs.length > 500) {
logger.info(logMessage);
} else if (scoreIDs.length > 1) {
logger.verbose(logMessage);
} else {
logger.debug(logMessage);
}
await db.imports.insert(ImportDocument);
// we don't await this because we don't
// particularly care about waiting for it.
db["import-timings"].insert({
importID,
timestamp: Date.now(),
total: ImportDocument.timeFinished - timeStarted,
rel: {
import: importTimeRel,
importParse: importParseTimeRel,
pb: pbTimeRel,
session: sessionTimeRel,
},
abs: {
parse: parseTime,
import: importTime,
importParse: importParseTime,
session: sessionTime,
pb: pbTime,
ugs: ugsTime,
goal: goalTime,
milestone: milestoneTime,
},
});
await RemoveUserLock(user.id);
return ImportDocument;
}
/**
@@ -39,6 +39,8 @@ t.test("#UpdateUsersGamePlaytypeStats", (t) => {
});
t.test("Should update UserGameStats if the user has one", async (t) => {
await db["game-stats"].remove({});
await db["game-stats"].insert({
game: "iidx",
playtype: "SP",
@@ -84,6 +86,8 @@ t.test("#UpdateUsersGamePlaytypeStats", (t) => {
});
t.test("Should return class deltas", async (t) => {
await db["game-stats"].remove({});
await db["game-stats"].insert({
game: "iidx",
playtype: "SP",
@@ -134,6 +138,8 @@ t.test("#UpdateUsersGamePlaytypeStats", (t) => {
});
t.test("Should return updated class deltas", async (t) => {
await db["game-stats"].remove({});
await db["game-stats"].insert({
game: "iidx",
playtype: "SP",
@@ -0,0 +1,40 @@
import t from "tap";
import db from "../../external/mongo/db";
import { CloseAllConnections } from "../../test-utils/close-connections";
import ResetDBState from "../../test-utils/resets";
import { LoadKTBlackIIDXData } from "../../test-utils/test-data";
import { SearchGameSongs } from "./songs-charts";
t.test("#SearchGameSongs", (t) => {
t.beforeEach(ResetDBState);
t.beforeEach(LoadKTBlackIIDXData);
t.beforeEach(async () => {
await db.songs.iidx.dropIndexes();
await db.songs.iidx.createIndex(
{
title: "text",
artist: "text",
"alt-titles": "text",
"search-titles": "text",
} as any /* known bug with monk */
);
});
t.test("Should return songs like the query.", async (t) => {
const res = await SearchGameSongs("iidx", "amuro");
t.strictSame(
// for simplicity of testing (and because the
// return order is ambiguous) we sort on
// songID here and expect this.
res.sort((a, b) => a.id - b.id).map((e) => e.title),
["A", "AA", "冥", "F", "HAERETICUS", "ZZ", "X", "AA -rebuild-", "∀"]
);
t.end();
});
t.end();
});
t.teardown(CloseAllConnections);
+55
View File
@@ -0,0 +1,55 @@
import { Game, Playtypes, SongDocument, AnyChartDocument } from "tachi-common";
import db from "../../external/mongo/db";
import { FilterQuery } from "mongodb";
export type SongSearchReturn = {
__textScore: number;
} & SongDocument<Game>;
export async function SearchGameSongs(
game: Game,
search: string,
limit = 100
): Promise<SongSearchReturn[]> {
const res = await db.songs[game].aggregate([
{ $match: { $text: { $search: search } } },
// This is a weird optimisation, but generally
// the less data we return the better
// we're projecting __textScore here, and we
// use that opportunity to limit our returns
// generously.
{
$addFields: {
__textScore: { $meta: "textScore" },
},
},
// sort by quality of match
{ $sort: { __textScore: -1 } },
// hide nonsense
{ $match: { __textScore: { $gt: 0.25 } } },
{ $limit: limit },
]);
return res;
}
export async function SearchGameSongsAndCharts(
game: Game,
search: string,
playtype?: Playtypes[Game],
limit = 100
) {
const songs = await SearchGameSongs(game, search, limit);
const chartQuery: FilterQuery<AnyChartDocument> = {
songID: { $in: songs.map((e) => e.id) },
};
if (playtype) {
chartQuery.playtype = playtype;
}
const charts = await db.charts[game].find(chartQuery);
return { songs, charts };
}
@@ -246,7 +246,7 @@ t.test("GET /api/v1/users/:userID/games/:game/:playtype/milestones", (t) => {
t.end();
});
t.test("GET /api/v1/users/:userID/games/:game/:playtype/recent-scores", (t) => {
t.test("GET /api/v1/users/:userID/games/:game/:playtype/scores/recent", (t) => {
t.beforeEach(ResetDBState);
t.test("Should return a users 100 most recent scores.", async (t) => {
@@ -270,7 +270,7 @@ t.test("GET /api/v1/users/:userID/games/:game/:playtype/recent-scores", (t) => {
delete sc._id; // lol
}
const res = await mockApi.get("/api/v1/users/test_zkldi/games/iidx/SP/recent-scores");
const res = await mockApi.get("/api/v1/users/test_zkldi/games/iidx/SP/scores/recent");
t.hasStrict(res.body, {
success: true,
@@ -6,6 +6,7 @@ import { GetDefaultScoreRatingAlg, GetUsersRanking } from "../../../../../../../
import { CheckUserPlayedGamePlaytype } from "./middleware";
import { FilterQuery } from "mongodb";
import { UserGoalDocument, UserMilestoneDocument } from "tachi-common";
import { SearchGameSongsAndCharts } from "../../../../../../../../../lib/search/songs-charts";
const router: Router = Router({ mergeParams: true });
@@ -146,9 +147,9 @@ router.get("/milestones", async (req, res) => {
/**
* Returns a users recent 100 scores for this game.
*
* @name GET /api/v1/users/:userID/games/:game/:playtype/recent-scores
* @name GET /api/v1/users/:userID/games/:game/:playtype/scores/recent
*/
router.get("/recent-scores", async (req, res) => {
router.get("/scores/recent", async (req, res) => {
const user = req[SYMBOL_TachiData]!.requestedUser!;
const game = req[SYMBOL_TachiData]!.game!;
const playtype = req[SYMBOL_TachiData]!.playtype!;
@@ -180,12 +181,98 @@ router.get("/recent-scores", async (req, res) => {
});
});
/**
* Searches a user's individual scores.
*
* @name GET /api/v1/users/:userID/games/:game/:playtype/scores
*/
router.get("/scores", async (req, res) => {
const user = req[SYMBOL_TachiData]!.requestedUser!;
const game = req[SYMBOL_TachiData]!.game!;
const playtype = req[SYMBOL_TachiData]!.playtype!;
if (typeof req.query.search !== "string") {
return res.status(400).json({
success: false,
description: `Invalid value of ${req.query.search} for search parameter.`,
});
}
const { songs, charts } = await SearchGameSongsAndCharts(game, req.query.search, playtype);
const scores = await db.scores.find(
{
chartID: { $in: charts.map((e) => e.chartID) },
userID: user.id,
},
{
sort: {
timeAchieved: -1,
},
limit: 30,
}
);
return res.status(200).json({
success: true,
description: `Retrieved ${scores.length} scores.`,
body: {
scores,
songs,
charts,
},
});
});
/**
* Searches a user's personal bests.
*
* @name GET /api/v1/users/:userID/games/:game/:playtype/scores
*/
router.get("/pbs", async (req, res) => {
const user = req[SYMBOL_TachiData]!.requestedUser!;
const game = req[SYMBOL_TachiData]!.game!;
const playtype = req[SYMBOL_TachiData]!.playtype!;
if (typeof req.query.search !== "string") {
return res.status(400).json({
success: false,
description: `Invalid value of ${req.query.search} for search parameter.`,
});
}
const { songs, charts } = await SearchGameSongsAndCharts(game, req.query.search, playtype);
const pbs = await db["personal-bests"].find(
{
chartID: { $in: charts.map((e) => e.chartID) },
userID: user.id,
},
{
sort: {
timeAchieved: -1,
},
limit: 30,
}
);
return res.status(200).json({
success: true,
description: `Retrieved ${pbs.length} personal bests.`,
body: {
pbs,
songs,
charts,
},
});
});
/**
* Returns a users best 100 personal-bests for this game.
*
* @name GET /api/v1/users/:userID/games/:game/:playtype/best
* @name GET /api/v1/users/:userID/games/:game/:playtype/pbs/best
*/
router.get("/best", async (req, res) => {
router.get("/pbs/best", async (req, res) => {
const user = req[SYMBOL_TachiData]!.requestedUser!;
const game = req[SYMBOL_TachiData]!.game!;
const playtype = req[SYMBOL_TachiData]!.playtype!;