Add some more tests and fix existing broken ones

This commit is contained in:
zkldi
2021-06-18 16:37:47 +01:00
parent ba085c18fa
commit f2946fee96
8 changed files with 455 additions and 150 deletions
+3 -135
View File
@@ -1,11 +1,6 @@
import monk from "monk";
/* eslint-disable no-await-in-loop */
import { Command } from "commander";
import CreateLogCtx from "../src/lib/logger/logger";
import { IndexOptions } from "mongodb";
import { ValidDatabases } from "tachi-common";
import { CONF_INFO } from "../src/lib/setup/config";
const logger = CreateLogCtx(__filename);
import { SetIndexes } from "../src/external/mongo/indexes";
const program = new Command();
@@ -18,131 +13,4 @@ program.option(
program.parse(process.argv);
const options = program.opts();
interface Index {
fields: Record<string, unknown>;
options?: IndexOptions;
}
function index(fields: Record<string, unknown>, options?: IndexOptions) {
return { fields, options };
}
const UNIQUE = { unique: true };
const staticIndexes: Partial<Record<ValidDatabases, Index[]>> = {
scores: [index({ scoreID: 1 }, UNIQUE)],
"personal-bests": [
index({ chartID: 1, userID: 1 }, UNIQUE),
index({ chartID: 1, "scoreData.percent": 1 }),
],
sessions: [
// lol
index({ timeStarted: 1, timeEnded: 1, userID: 1, game: 1, playtype: 1 }),
index({ name: "text" }),
],
"game-stats": [index({ userID: 1, game: 1, playtype: 1 }, UNIQUE)],
"folder-chart-lookup": [index({ chartID: 1, folderID: 1 }, UNIQUE)],
"tierlist-data": [
index({ tierlistDataID: 1 }, UNIQUE),
index({ chartID: 1, tierlistID: 1 }),
index({ chartID: 1, tierlistID: 1, type: 1 }),
index({ chartID: 1, tierlistID: 1, type: 1, key: 1 }, UNIQUE),
],
goals: [index({ goalID: 1 }, UNIQUE)],
"user-goals": [index({ goalID: 1, userID: 1 }, UNIQUE), index({ goalID: 1 })],
milestones: [index({ milestoneID: 1 }, UNIQUE), index({ group: 1, game: 1, playtype: 1 })],
"user-milestones": [
index({ milestoneID: 1, userID: 1 }, UNIQUE),
index({ userID: 1, game: 1, playtype: 1 }),
],
imports: [index({ importID: 1 }, UNIQUE)],
"import-timings": [
index({ importID: 1 }, UNIQUE),
index({ timestamp: 1 }),
index({ total: 1 }),
],
users: [index({ id: 1 }, UNIQUE)],
tierlists: [
index({ tierlistID: 1 }, UNIQUE),
index({ game: 1, playtype: 1, isDefault: 1 }, UNIQUE),
],
folders: [
index({ folderID: 1 }, UNIQUE),
index({ game: 1, playtype: 1 }),
index({ game: 1, playtype: 1, table: 1 }),
index({ game: 1, playtype: 1, table: 1, tableIndex: 1 }),
],
"kai-auth-tokens": [index({ userID: 1, service: 1 }, UNIQUE)],
"charts-iidx": [
index(
{ "data.arcChartID": 1 },
{ unique: true, partialFilterExpression: { "data.arcChartID": { $type: "string" } } }
),
index({ "data.hashSHA256": 1 }),
],
"bms-course-lookup": [index({ md5sums: 1 }, UNIQUE)],
"api-tokens": [index({ token: 1 }, UNIQUE)],
};
const indexes: Partial<Record<ValidDatabases, Index[]>> = staticIndexes;
for (const game of CONF_INFO.SUPPORTED_GAMES) {
if (indexes[`charts-${game}` as ValidDatabases]) {
indexes[`charts-${game}` as ValidDatabases]!.push(
index({ chartID: 1 }, UNIQUE),
index(
{ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 },
{ unique: true, partialFilterExpression: { isPrimary: { $eq: true } } }
)
);
} else {
indexes[`charts-${game}` as ValidDatabases] = [
index({ chartID: 1 }, UNIQUE),
index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE),
];
}
if (indexes[`songs-${game}` as ValidDatabases]) {
indexes[`songs-${game}` as ValidDatabases]!.push(
index({ id: 1 }, UNIQUE),
index({ title: "text", artist: "text", "alt-titles": "text", "search-titles": "text" })
);
} else {
indexes[`songs-${game}` as ValidDatabases] = [
index({ id: 1 }, UNIQUE),
index({ title: 1 }),
index({ title: "text", artist: "text", "alt-titles": "text", "search-titles": "text" }),
];
}
}
export async function SetIndexes(dbst: string) {
const db = monk(`${process.env.MONGO_BASE_URL ?? "127.0.0.1"}/${dbst}`);
logger.info(`Starting indexing for ${options.db ?? "ktblackdb"}...`);
for (const collection in indexes) {
if (options.reset) {
// eslint-disable-next-line no-await-in-loop
await db.get(collection).dropIndexes();
logger.info(`Reset ${collection}.`);
}
// @ts-expect-error dru(n)kts
for (const index of indexes[collection]) {
// eslint-disable-next-line no-await-in-loop
const r = await db.get(collection).createIndex(index.fields, index.options);
logger.info(r);
}
}
db.close();
logger.info("Done.");
}
// if calling this as a script -- similar to pythons if __name__ == "__main__"
if (require.main === module) {
SetIndexes(options.db ?? "ktblackdb").then(() => process.exit(0));
}
SetIndexes(options.db, options.reset).then(() => process.exit(0));
+137
View File
@@ -0,0 +1,137 @@
/* eslint-disable no-await-in-loop */
import monk from "monk";
import { IndexOptions } from "mongodb";
import { ValidDatabases } from "tachi-common";
import CreateLogCtx from "../../lib/logger/logger";
import { CONF_INFO } from "../../lib/setup/config";
const logger = CreateLogCtx(__filename);
interface Index {
fields: Record<string, unknown>;
options?: IndexOptions;
}
function index(fields: Record<string, unknown>, options?: IndexOptions) {
return { fields, options };
}
const UNIQUE = { unique: true };
const staticIndexes: Partial<Record<ValidDatabases, Index[]>> = {
scores: [index({ scoreID: 1 }, UNIQUE)],
"personal-bests": [
index({ chartID: 1, userID: 1 }, UNIQUE),
index({ chartID: 1, "scoreData.percent": 1 }),
],
sessions: [
// lol
index({ timeStarted: 1, timeEnded: 1, userID: 1, game: 1, playtype: 1 }),
index({ name: "text" }),
],
"game-stats": [index({ userID: 1, game: 1, playtype: 1 }, UNIQUE)],
"folder-chart-lookup": [index({ chartID: 1, folderID: 1 }, UNIQUE)],
"tierlist-data": [
index({ tierlistDataID: 1 }, UNIQUE),
index({ chartID: 1, tierlistID: 1 }),
index({ chartID: 1, tierlistID: 1, type: 1 }),
index({ chartID: 1, tierlistID: 1, type: 1, key: 1 }, UNIQUE),
],
goals: [index({ goalID: 1 }, UNIQUE)],
"user-goals": [index({ goalID: 1, userID: 1 }, UNIQUE), index({ goalID: 1 })],
milestones: [index({ milestoneID: 1 }, UNIQUE), index({ group: 1, game: 1, playtype: 1 })],
"user-milestones": [
index({ milestoneID: 1, userID: 1 }, UNIQUE),
index({ userID: 1, game: 1, playtype: 1 }),
],
imports: [index({ importID: 1 }, UNIQUE)],
"import-timings": [
index({ importID: 1 }, UNIQUE),
index({ timestamp: 1 }),
index({ total: 1 }),
],
users: [index({ id: 1 }, UNIQUE)],
tierlists: [
index({ tierlistID: 1 }, UNIQUE),
index({ game: 1, playtype: 1, isDefault: 1 }, UNIQUE),
],
folders: [
index({ folderID: 1 }, UNIQUE),
index({ game: 1, playtype: 1 }),
index({ game: 1, playtype: 1, table: 1 }),
index({ game: 1, playtype: 1, table: 1, tableIndex: 1 }),
],
"kai-auth-tokens": [index({ userID: 1, service: 1 }, UNIQUE)],
"charts-iidx": [
index(
{ "data.arcChartID": 1 },
{ unique: true, partialFilterExpression: { "data.arcChartID": { $type: "string" } } }
),
index({ "data.hashSHA256": 1 }),
],
"bms-course-lookup": [index({ md5sums: 1 }, UNIQUE)],
"api-tokens": [index({ token: 1 }, UNIQUE)],
};
const indexes: Partial<Record<ValidDatabases, Index[]>> = staticIndexes;
for (const game of CONF_INFO.SUPPORTED_GAMES) {
if (indexes[`charts-${game}` as ValidDatabases]) {
indexes[`charts-${game}` as ValidDatabases]!.push(
index({ chartID: 1 }, UNIQUE),
index(
{ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 },
{ unique: true, partialFilterExpression: { isPrimary: { $eq: true } } }
)
);
} else {
indexes[`charts-${game}` as ValidDatabases] = [
index({ chartID: 1 }, UNIQUE),
index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE),
];
}
if (indexes[`songs-${game}` as ValidDatabases]) {
indexes[`songs-${game}` as ValidDatabases]!.push(
index({ id: 1 }, UNIQUE),
index({ title: "text", artist: "text", "alt-titles": "text", "search-titles": "text" })
);
} else {
indexes[`songs-${game}` as ValidDatabases] = [
index({ id: 1 }, UNIQUE),
index({ title: 1 }),
index({ title: "text", artist: "text", "alt-titles": "text", "search-titles": "text" }),
];
}
}
export async function SetIndexes(mongoURL: string, reset: boolean) {
const db = monk(mongoURL);
logger.debug(`Starting indexing for ${mongoURL}...`);
const collections = (await db.listCollections()).map((e) => e.name);
for (const collection in indexes) {
if (!collections.includes(collection)) {
// this creates a collection, i cant find the createCollection
// call.
await db.get(collection).insert({});
await db.get(collection).remove({});
}
if (reset) {
await db.get(collection).dropIndexes();
logger.debug(`Reset ${collection}.`);
}
// @ts-expect-error dru(n)kts
for (const index of indexes[collection]) {
const r = await db.get(collection).createIndex(index.fields, index.options);
logger.debug(r);
}
}
logger.debug("Done.");
}
+1 -1
View File
@@ -88,7 +88,7 @@ export async function SearchGameSongsAndCharts(
chartQuery.playtype = playtype;
}
const charts = await db.charts[game].find(chartQuery);
const charts = (await db.charts[game].find(chartQuery)) as AnyChartDocument[];
return { songs, charts };
}
@@ -9,17 +9,13 @@ import {
import { CloseAllConnections } from "../../../../../test-utils/close-connections";
import { RequireNeutralAuthentication } from "../../../../../test-utils/api-common";
import { CreateFakeAuthCookie } from "../../../../../test-utils/fake-auth";
import ResetDBState from "../../../../../test-utils/resets";
import ResetDBState, { SetIndexesForDB } from "../../../../../test-utils/resets";
import db from "../../../../../external/mongo/db";
import { SetIndexes } from "../../../../../../scripts/set-indexes";
t.test("POST /api/v1/import/file", async (t) => {
const cookie = await CreateFakeAuthCookie(mockApi);
await SetIndexes(
process.env.TACHI_PARALLEL_TESTS ? `test-ephemeral-${process.pid.toString()}` : "testingdb"
);
t.before(SetIndexesForDB);
t.beforeEach(ResetDBState);
RequireNeutralAuthentication("/api/v1/import/file", "POST");
@@ -2,8 +2,9 @@ import t from "tap";
import db from "../../../../../../../../../external/mongo/db";
import { CloseAllConnections } from "../../../../../../../../../test-utils/close-connections";
import mockApi from "../../../../../../../../../test-utils/mock-api";
import ResetDBState from "../../../../../../../../../test-utils/resets";
import ResetDBState, { SetIndexesForDB } from "../../../../../../../../../test-utils/resets";
import {
SessionDocument,
ScoreDocument,
UserGoalDocument,
GoalDocument,
@@ -11,7 +12,14 @@ import {
UserMilestoneDocument,
PBScoreDocument,
} from "tachi-common";
import { Testing511Song, Testing511SPA } from "../../../../../../../../../test-utils/test-data";
import {
GetKTDataJSON,
LoadKTBlackIIDXData,
Testing511Song,
Testing511SPA,
} from "../../../../../../../../../test-utils/test-data";
t.before(SetIndexesForDB);
t.test("GET /api/v1/users/:userID/games/:game/:playtype/", (t) => {
t.beforeEach(ResetDBState);
@@ -332,4 +340,253 @@ t.test("GET /api/v1/users/:userID/games/:game/:playtype/pbs/best", (t) => {
t.end();
});
t.test("GET /api/v1/users/:userID/games/:game/:playtype/pbs", (t) => {
t.beforeEach(ResetDBState);
t.beforeEach(LoadKTBlackIIDXData);
t.test("Should return 400 if no search param is given", async (t) => {
const res = await mockApi.get("/api/v1/users/test_zkldi/games/iidx/SP/pbs");
t.equal(res.statusCode, 400);
t.equal(res.body.success, false);
t.end();
});
t.test("Should return 400 if invalid search param is given", async (t) => {
const res = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/pbs?search=foo&search=bar"
);
t.equal(res.statusCode, 400);
t.equal(res.body.success, false);
const res2 = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/pbs?search[$where]=process.exit(1)"
);
t.equal(res2.statusCode, 400);
t.equal(res2.body.success, false);
t.end();
});
t.test("Should search a user's personal bests.", async (t) => {
const mockPBs: PBScoreDocument[] = [];
const charts = GetKTDataJSON("./tachi/ktblack-charts-iidx.json");
for (let i = 0; i < 200; i++) {
mockPBs.push({
userID: 1,
game: "iidx",
playtype: "SP",
isPrimary: true,
chartID: charts[i].chartID,
songID: charts[i].songID,
calculatedData: {
ktRating: i,
},
} as PBScoreDocument);
}
await db["personal-bests"].insert(mockPBs);
const res = await mockApi.get("/api/v1/users/test_zkldi/games/iidx/SP/pbs?search=5.1.1.");
t.hasStrict(res.body, {
success: true,
description: "Retrieved 2 personal bests.",
body: {
pbs: [
{
chartID: "c2311194e3897ddb5745b1760d2c0141f933e683",
},
{
chartID: "c641238220d73faf82659513ba03bde71b0b45f0",
},
],
songs: [
{
title: "5.1.1.",
},
],
charts: [
{
chartID: "c2311194e3897ddb5745b1760d2c0141f933e683",
},
{
chartID: "c641238220d73faf82659513ba03bde71b0b45f0",
},
],
},
});
t.end();
});
t.end();
});
t.test("GET /api/v1/users/:userID/games/:game/:playtype/scores", (t) => {
t.beforeEach(ResetDBState);
t.beforeEach(LoadKTBlackIIDXData);
t.test("Should return 400 if no search param is given", async (t) => {
const res = await mockApi.get("/api/v1/users/test_zkldi/games/iidx/SP/scores");
t.equal(res.statusCode, 400);
t.equal(res.body.success, false);
t.end();
});
t.test("Should return 400 if invalid search param is given", async (t) => {
const res = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/scores?search=foo&search=bar"
);
t.equal(res.statusCode, 400);
t.equal(res.body.success, false);
// evil eval attempts
const res2 = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/scores?search[$where]=process.exit(1)"
);
t.equal(res2.statusCode, 400);
t.equal(res2.body.success, false);
t.end();
});
t.test("Should search a user's scores.", async (t) => {
const mockScores: ScoreDocument[] = [];
const charts = GetKTDataJSON("./tachi/ktblack-charts-iidx.json");
for (let i = 0; i < 200; i++) {
mockScores.push({
scoreID: i.toString(),
userID: 1,
game: "iidx",
playtype: "SP",
isPrimary: true,
chartID: charts[i].chartID,
songID: charts[i].songID,
calculatedData: {
ktRating: i,
},
} as ScoreDocument);
}
await db.scores.insert(mockScores);
const res = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/scores?search=5.1.1."
);
t.equal(res.body.body.scores.length, 3);
t.equal(res.body.body.songs.length, 1);
t.equal(res.body.body.charts.length, 2);
t.hasStrict(res.body, {
success: true,
description: "Retrieved 3 scores.",
body: {
scores: [
{
songID: 1,
},
{
songID: 1,
},
{
songID: 1,
},
],
songs: [
{
title: "5.1.1.",
},
],
charts: [
{
songID: 1,
},
{
songID: 1,
},
],
},
});
t.end();
});
t.end();
});
t.test("GET /api/v1/users/:userID/games/:game/:playtype/sessions", (t) => {
t.beforeEach(ResetDBState);
t.beforeEach(LoadKTBlackIIDXData);
t.test("Should return 400 if no search param is given", async (t) => {
const res = await mockApi.get("/api/v1/users/test_zkldi/games/iidx/SP/sessions");
t.equal(res.statusCode, 400);
t.equal(res.body.success, false);
t.end();
});
t.test("Should return 400 if invalid search param is given", async (t) => {
const res = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/sessions?search=foo&search=bar"
);
t.equal(res.statusCode, 400);
t.equal(res.body.success, false);
// evil eval attempts
const res2 = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/sessions?search[$where]=process.exit(1)"
);
t.equal(res2.statusCode, 400);
t.equal(res2.body.success, false);
t.end();
});
t.test("Should search a user's sessions.", async (t) => {
await db.sessions.insert(
["Epic Session", "Session Of Epic", "Epic Gaming", "something else", "bad session"].map(
(e) => ({
userID: 1,
game: "iidx",
playtype: "SP",
name: e,
desc: "something",
sessionID: e, // hack to avoid db nonsense
})
) as SessionDocument[]
);
const res = await mockApi.get(
"/api/v1/users/test_zkldi/games/iidx/SP/sessions?search=Epic"
);
t.hasStrict(res.body, {
success: true,
description: "Retrieved 3 sessions.",
body: [],
});
t.end();
});
t.end();
});
t.teardown(CloseAllConnections);
@@ -14,6 +14,7 @@ import {
SearchGameSongsAndCharts,
SearchSessions,
} from "../../../../../../../../../lib/search/search";
import { FilterChartsAndSongs } from "../../../../../../../../../utils/scores";
const router: Router = Router({ mergeParams: true });
@@ -168,11 +169,15 @@ router.get("/scores", async (req, res) => {
});
}
const { songs, charts } = await SearchGameSongsAndCharts(game, req.query.search, playtype);
const { songs: allSongs, charts: allCharts } = await SearchGameSongsAndCharts(
game,
req.query.search,
playtype
);
const scores = await db.scores.find(
{
chartID: { $in: charts.map((e) => e.chartID) },
chartID: { $in: allCharts.map((e) => e.chartID) },
userID: user.id,
},
{
@@ -183,6 +188,8 @@ router.get("/scores", async (req, res) => {
}
);
const { songs, charts } = FilterChartsAndSongs(scores, allCharts, allSongs);
return res.status(200).json({
success: true,
description: `Retrieved ${scores.length} scores.`,
@@ -248,11 +255,15 @@ router.get("/pbs", async (req, res) => {
});
}
const { songs, charts } = await SearchGameSongsAndCharts(game, req.query.search, playtype);
const { songs: allSongs, charts: allCharts } = await SearchGameSongsAndCharts(
game,
req.query.search,
playtype
);
const pbs = await db["personal-bests"].find(
{
chartID: { $in: charts.map((e) => e.chartID) },
chartID: { $in: allCharts.map((e) => e.chartID) },
userID: user.id,
},
{
@@ -263,6 +274,8 @@ router.get("/pbs", async (req, res) => {
}
);
const { songs, charts } = FilterChartsAndSongs(pbs, allCharts, allSongs);
return res.status(200).json({
success: true,
description: `Retrieved ${pbs.length} personal bests.`,
@@ -335,7 +348,7 @@ router.get("/sessions", async (req, res) => {
return res.status(200).json({
success: true,
description: `Successfully retrieved ${sessions.length} sessions.`,
description: `Retrieved ${sessions.length} sessions.`,
body: sessions,
});
});
+8
View File
@@ -8,6 +8,7 @@ import CreateLogCtx from "../lib/logger/logger";
// im installing an entire library for rm rf...
import rimraf from "rimraf";
import { KTCDN_ROOT } from "../lib/setup/config";
import { SetIndexes } from "../external/mongo/indexes";
const logger = CreateLogCtx(__filename);
@@ -84,3 +85,10 @@ export function ResetCDN() {
})
);
}
export function SetIndexesForDB() {
return SetIndexes(
process.env.TACHI_PARALLEL_TESTS ? `test-ephemeral-${process.pid.toString()}` : "testingdb",
true
);
}
+27 -1
View File
@@ -1,5 +1,11 @@
import db from "../external/mongo/db";
import { integer } from "tachi-common";
import {
integer,
PBScoreDocument,
ScoreDocument,
AnyChartDocument,
AnySongDocument,
} from "tachi-common";
export function GetPBOnChart(userID: integer, chartID: string) {
return db["personal-bests"].findOne({
@@ -14,3 +20,23 @@ export function GetServerRecordOnChart(chartID: string) {
"rankingData.rank": 1,
});
}
export function FilterChartsAndSongs(
scores: (PBScoreDocument | ScoreDocument)[],
charts: AnyChartDocument[],
songs: AnySongDocument[]
) {
const chartIDs = new Set();
const songIDs = new Set();
for (const score of scores) {
chartIDs.add(score.chartID);
songIDs.add(score.songID);
}
// filter out irrelevant songs and charts
return {
songs: songs.filter((e) => songIDs.has(e.id)),
charts: charts.filter((e) => chartIDs.has(e.chartID)),
};
}