Merge pull request #522 from TNG-dev:release-2.0.1

Release 2.0.1

server/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "tachi-server",
-	"version": "2.0.0-prerelease",
+	"version": "2.0.1",
 	"description": "A score tracking server.",
 	"main": "js/index.js",
 	"private": true,

server/scripts/sync-database/sync-database.ts

@@ -11,7 +11,13 @@ import { BulkWriteOperation } from "mongodb";
 import { ICollection } from "monk";
 import os from "os";
 import path from "path";
-import { ChartDocument, FolderDocument, SongDocument, TableDocument } from "tachi-common";
+import {
+	ChartDocument,
+	FolderDocument,
+	SongDocument,
+	TableDocument,
+	BMSCourseDocument,
+} from "tachi-common";
 import { InitaliseFolderChartLookup } from "utils/folder";
 
 interface SyncInstructions {
@@ -112,14 +118,16 @@ const syncInstructions: SyncInstructions[] = [
 			// Since the USC and BMS databases are managed Bokutachi-side, we
 			// shouldn't be honoring any sort of updates from tachi-database-seeds
 			// aside from an initial one.
-			const isInitial = (await collection.findOne()) === null;
+			//
+			// However, in practice, these syncs are the best way to actually update
+			// issues in the database.
+			// We're going to disable this anyway.
+			// const isInitial = (await collection.findOne()) === null;
 
-			if (isInitial) {
-				const r = await GenericUpsert(charts, collection, "chartID", logger, true);
+			const r = await GenericUpsert(charts, collection, "chartID", logger, false);
 
-				if (r) {
-					await InitaliseFolderChartLookup();
-				}
+			if (r) {
+				await InitaliseFolderChartLookup();
 			}
 		},
 	},
@@ -227,6 +235,14 @@ const syncInstructions: SyncInstructions[] = [
 			}
 		},
 	},
+	{
+		pattern: /^bms-course-lookup$/u,
+		handler: (
+			bmsCourseDocuments: BMSCourseDocument[],
+			collection: ICollection<BMSCourseDocument>,
+			logger
+		) => GenericUpsert(bmsCourseDocuments, collection, "md5sums", logger),
+	},
 ];
 
 const logger = CreateLogCtx("Database Sync");

server/src/lib/constants/bms-tables.ts

@@ -1,6 +1,6 @@
 export const BMS_TABLES = {
-	insane: "🟊",
-	overjoy: "🟊🟊",
+	insane: "★",
+	overjoy: "★★",
 	normal: "☆",
 	normal2: "▽",
 	insane2: "▼",

server/src/lib/constants/version.ts

@@ -4,7 +4,7 @@
 
 const MAJOR = 2;
 const MINOR = 0;
-const PATCH = 0;
+const PATCH = 1;
 
 // As is with all front-facing zkldi projects, the version names for tachi-server
 // are from an album I like. In this case, the album is Portishead - Dummy.

server/src/lib/delete-scores/delete-scores.ts

@@ -110,7 +110,7 @@ export async function DeleteScore(score: ScoreDocument, blacklist = false) {
 }
 
 export async function DeleteMultipleScores(scores: ScoreDocument[], blacklist = false) {
-	logger.info(`Recieved request to delete ${scores.length} (Blacklist: ${blacklist}).`);
+	logger.info(`received request to delete ${scores.length} (Blacklist: ${blacklist}).`);
 
 	const scoreIDs = scores.map((e) => e.scoreID);
 

server/src/lib/email/formats.ts

@@ -6,9 +6,9 @@ import { ServerConfig, TachiConfig } from "lib/setup/config";
 export function EmailFormatResetPassword(username: string, resetCode: string, ipAddr: string) {
 	return {
 		html: MainHTMLWrapper(
-			`Hey ${username}, you've recieved a password reset request.<br/><a href="${ServerConfig.OUR_URL}/reset-password?code=${resetCode}">Click here</a> to perform the reset.<br/>If you did not request this reset, report this! This reset request was made by ${ipAddr}.`
+			`Hey ${username}, you've received a password reset request.<br/><a href="${ServerConfig.OUR_URL}/reset-password?code=${resetCode}">Click here</a> to perform the reset.<br/>If you did not request this reset, report this! This reset request was made by ${ipAddr}.`
 		),
-		text: `Hey ${username}, you've recieved a password reset request. This link
+		text: `Hey ${username}, you've received a password reset request. This link
 ${ServerConfig.OUR_URL}/reset-password?code=${resetCode} will perform the reset.
 If you did not request this reset, report this! This reset request was made by ${ipAddr}.`,
 	};

server/src/lib/jobs/deorphan-scores.ts

@@ -18,19 +18,23 @@ export async function DeoprhanScores() {
 	let success = 0;
 	let removed = 0;
 
-	await Promise.all(
-		orphans.map((or) =>
-			ReprocessOrphan(or, blacklist, logger).then((r) => {
-				if (r === null) {
-					removed++;
-				} else if (r === false) {
-					failed++;
-				} else {
-					success++;
-				}
-			})
-		)
-	);
+	for (const or of orphans) {
+		// We have to await like this to avoid mid-air race conditions,
+		// where two orphans attempt to deorphan to the same scoreID
+		// at the same time.
+		// See #511.
+
+		// eslint-disable-next-line no-await-in-loop
+		const r = await ReprocessOrphan(or, blacklist, logger);
+
+		if (r === null) {
+			removed++;
+		} else if (r === false) {
+			failed++;
+		} else {
+			success++;
+		}
+	}
 
 	logger.info(`Finished attempting deorphaning.`);
 

server/src/lib/orphan-queue/orphan-queue.ts

@@ -35,7 +35,7 @@ export async function HandleOrphanQueue<I extends IDStrings>(
 	userID: integer,
 	chartName: string
 ) {
-	logger.debug(`Recieved orphanqueue request for ${chartName}.`);
+	logger.debug(`received orphanqueue request for ${chartName}.`);
 
 	const orphanChart = await db["orphan-chart-queue"].findOne(
 		Object.assign({ idString }, orphanMatchCriteria),
@@ -45,7 +45,7 @@ export async function HandleOrphanQueue<I extends IDStrings>(
 	);
 
 	if (!orphanChart) {
-		logger.verbose(`Recieved unknown chart ${chartName}, orphaning.`);
+		logger.verbose(`received unknown chart ${chartName}, orphaning.`);
 
 		await db["orphan-chart-queue"].insert({
 			idString,

server/src/lib/score-import/framework/orphans/orphans.ts

@@ -112,7 +112,7 @@ export async function ReprocessOrphan(
 		return false;
 	} else if (res instanceof ConverterFailure) {
 		logger.warn(
-			`Recieved ConverterFailure ${res.message} on orphan ${orphan.orphanID}. Removing orphan.`
+			`received ConverterFailure ${res.message} on orphan ${orphan.orphanID}. Removing orphan.`
 		);
 
 		// @danger - This could go terribly, if there's a mistake in the converterFN we might accidentally

server/src/lib/score-import/framework/score-importing/score-importing.ts

@@ -98,7 +98,7 @@ export async function ImportAllIterableData<D, C>(
 
 	logger.debug(`Removed null from results.`);
 
-	logger.verbose(`Recieved ${datapoints.length} returns, from ${processedResults.length} data.`);
+	logger.verbose(`received ${datapoints.length} returns, from ${processedResults.length} data.`);
 
 	// Flush the score queue out after finishing most of the import. This ensures no scores get left in the
 	// queue.

server/src/lib/score-import/framework/user-game-stats/update-ugs.ts

@@ -62,6 +62,21 @@ export async function UpdateUsersGamePlaytypeStats(
 			}
 		);
 	} else {
+		const hasAnyScores = await db.scores.findOne({
+			game,
+			playtype,
+			userID,
+		});
+
+		if (!hasAnyScores) {
+			logger.debug("Not creating new game stats for user with no scores.", {
+				userID,
+				game,
+				playtype,
+			});
+			return deltas;
+		}
+
 		const newStats: UserGameStats = {
 			game,
 			playtype,

server/src/lib/score-import/import-types/common/api-kai/sdvx/class-handler.ts

@@ -36,7 +36,7 @@ export async function CreateKaiSDVXClassHandler(
 
 		const sdvxDan: number | null = json.skill_level - 1;
 
-		if (sdvxDan === null) {
+		if (sdvxDan === null || sdvxDan === undefined) {
 			return {};
 		}
 
@@ -52,6 +52,12 @@ export async function CreateKaiSDVXClassHandler(
 			return {};
 		}
 
+		// Kai APIs return -1 to indicate no dan. They also sometimes return undefined.
+		// I'm not too sure why.
+		if (sdvxDan === -1) {
+			return {};
+		}
+
 		if (sdvxDan < SDVXDans.DAN_1) {
 			logger.warn(
 				`${baseUrl} returned a dan of ${sdvxDan}, which was less than DAN_1 (${SDVXDans.DAN_1}.)`

server/src/lib/score-import/import-types/common/api-kai/traverse-api.ts

@@ -75,10 +75,10 @@ export async function* TraverseKaiAPI(
 				},
 			});
 		} catch (err) {
-			logger.error(`Recieved invalid response from ${url}.`, { err });
+			logger.error(`received invalid response from ${url}.`, { err });
 			throw new ScoreImportFatalError(
 				500,
-				`Recieved invalid response from their API. Are they down?`
+				`received invalid response from their API. Are they down?`
 			);
 		}
 
@@ -113,21 +113,21 @@ export async function* TraverseKaiAPI(
 			json = await res.json();
 		} catch (err) {
 			logger.error(
-				`Recieved invalid (non-json) response from ${url}. Status code was ${res.status}.`,
+				`received invalid (non-json) response from ${url}. Status code was ${res.status}.`,
 				{ err }
 			);
 			throw new ScoreImportFatalError(
 				500,
-				`Recieved invalid response from their API. Are they down?`
+				`received invalid response from their API. Are they down?`
 			);
 		}
 
 		if (json._links === null || typeof json._links !== "object") {
-			logger.error(`Recieved invalid JSON from ${url}. Invalid _links.`, { body: json });
+			logger.error(`received invalid JSON from ${url}. Invalid _links.`, { body: json });
 
 			throw new ScoreImportFatalError(
 				500,
-				`Recieved no _links prop from their API. This is not an error with ${TachiConfig.NAME}.`
+				`received no _links prop from their API. This is not an error with ${TachiConfig.NAME}.`
 			);
 		}
 
@@ -149,22 +149,22 @@ export async function* TraverseKaiAPI(
 			// exit the loop after this, we're on the last page.
 			fetchMoreData = false;
 		} else {
-			logger.error(`Recieved invalid response from ${url}. Invalid _links._next.`, {
+			logger.error(`received invalid response from ${url}. Invalid _links._next.`, {
 				body: json,
 			});
 
 			throw new ScoreImportFatalError(
 				500,
-				`Recieved invalid _links._next prop from their API.`
+				`received invalid _links._next prop from their API.`
 			);
 		}
 
 		if (!Array.isArray(json._items)) {
-			logger.error(`Recieved invalid response from ${url}. Invalid _items.`, {
+			logger.error(`received invalid response from ${url}. Invalid _items.`, {
 				body: json,
 			});
 
-			throw new ScoreImportFatalError(500, `Recieved invalid _items from their API.`);
+			throw new ScoreImportFatalError(500, `received invalid _items from their API.`);
 		}
 
 		// yield everything out of the score array

server/src/lib/score-import/import-types/common/batch-manual/converter.test.ts

@@ -358,6 +358,38 @@ t.test("#ConverterFn", (t) => {
 		t.end();
 	});
 
+	t.test("Should produce a with timeAchieved null if timeAchieved is 0", async (t) => {
+		const res = await ConverterBatchManual(
+			deepmerge(baseBatchManualScore, { timeAchieved: 0 }),
+			{ game: "iidx", service: "foo", playtype: "SP", version: null },
+			importType,
+			logger
+		);
+
+		t.hasStrict(res, {
+			chart: Testing511SPA,
+			song: { id: 1 },
+			dryScore: {
+				game: "iidx",
+				service: "foo (BATCH-MANUAL)",
+				comment: null,
+				importType: "file/batch-manual",
+				timeAchieved: null,
+				scoreData: {
+					lamp: "HARD CLEAR",
+					score: 500,
+					grade: "E",
+					// percent: 31.5, -- ish, FPA is hard.
+					judgements: {},
+					hitMeta: {},
+				},
+				scoreMeta: {},
+			},
+		});
+
+		t.end();
+	});
+
 	t.test("Should reject a score with > 100%", (t) => {
 		t.rejects(
 			() =>

server/src/lib/score-import/import-types/common/batch-manual/converter.ts

@@ -53,7 +53,7 @@ export const ConverterBatchManual: ConverterFunction<BatchManualScore, BatchManu
 		service,
 		comment: data.comment ?? null,
 		importType,
-		timeAchieved: data.timeAchieved ?? null,
+		timeAchieved: data.timeAchieved || null,
 		scoreData: {
 			lamp: data.lamp,
 			score: data.score,

server/src/lib/score-import/import-types/common/batch-manual/parser.test.ts

@@ -43,7 +43,7 @@ t.test("#ParserFn", (t) => {
 			() => ParserFn(false, "file/batch-manual", logger),
 			new ScoreImportFatalError(
 				400,
-				"Invalid BATCH-MANUAL (Not an object, recieved boolean.)"
+				"Invalid BATCH-MANUAL (Not an object, received boolean.)"
 			),
 			"Should throw an error."
 		);
@@ -402,6 +402,32 @@ t.test("#ParserFn", (t) => {
 			t.end();
 		});
 
+		t.test("TimeAchieved of 0 should be legal.", (t) => {
+			const res = ParserFn(dm({ timeAchieved: 0 }), "file/batch-manual", logger);
+
+			t.hasStrict(res, {
+				game: "iidx",
+				context: {
+					service: "foo",
+					game: "iidx",
+					playtype: "SP",
+					version: null,
+				},
+				iterable: [
+					{
+						score: 1000,
+						lamp: "HARD CLEAR",
+						matchType: "tachiSongID",
+						identifier: "123",
+						difficulty: "ANOTHER",
+						timeAchieved: 0,
+					},
+				],
+			});
+
+			t.end();
+		});
+
 		t.test("Invalid Identifier", (t) => {
 			// this is not a valid playtype for IIDX
 			const fn = () => ParserFn(dm({ identifier: null }), "file/batch-manual", logger);

server/src/lib/score-import/import-types/common/batch-manual/parser.ts

@@ -138,6 +138,7 @@ const PR_BatchManualScore = (game: Game, playtype: Playtypes[Game]): PrudenceSch
 		timeAchieved: optNull(
 			(self) =>
 				(typeof self === "number" && self > 1_000_000_000_000) ||
+				self === 0 ||
 				"Expected a number greater than 1 Trillion - did you pass unix seconds instead of milliseconds?"
 		),
 		judgements: optNull((self) => {
@@ -196,7 +197,7 @@ export function ParseBatchManualFromObject(
 	if (typeof object !== "object" || object === null) {
 		throw new ScoreImportFatalError(
 			400,
-			`Invalid BATCH-MANUAL (Not an object, recieved ${
+			`Invalid BATCH-MANUAL (Not an object, received ${
 				object === null ? "null" : typeof object
 			}.)`
 		);

server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.test.ts

@@ -82,7 +82,7 @@ t.test("#ParseEamusementCSV", (t) => {
 		t.equal(
 			version,
 			"27",
-			"Should pick largest version regardless of order scores are recieved in."
+			"Should pick largest version regardless of order scores are received in."
 		);
 
 		t.end();

server/src/lib/score-import/import-types/common/eamusement-iidx-csv/parser.ts

@@ -74,12 +74,12 @@ const HV_HEADER_COUNT = 41;
 
 export function ResolveHeaders(headers: string[], logger: KtLogger) {
 	if (headers.length === PRE_HV_HEADER_COUNT) {
-		logger.verbose("PRE_HV csv recieved.");
+		logger.verbose("PRE_HV csv received.");
 		return {
 			hasBeginnerAndLegg: false,
 		};
 	} else if (headers.length === HV_HEADER_COUNT) {
-		logger.verbose("HV+ csv recieved.");
+		logger.verbose("HV+ csv received.");
 		return {
 			hasBeginnerAndLegg: true,
 		};

server/src/lib/score-import/import-types/ir/fervidex-static/class-handler.ts

@@ -21,7 +21,7 @@ export function CreateFerStaticClassHandler(body: Record<string, unknown>): Clas
 		}
 
 		if (!Number.isInteger(index)) {
-			logger.info(`Recieved invalid fer-static class of ${index} (${playtype}).`, { body });
+			logger.info(`received invalid fer-static class of ${index} (${playtype}).`, { body });
 			return;
 		}
 

server/src/lib/score-import/worker/worker.ts

@@ -77,7 +77,7 @@ export const worker = new Worker(
 
 		job.data.parserArguments = processedArgs;
 
-		logger.debug(`Recieved score import job ${job.id}`, { job });
+		logger.debug(`received score import job ${job.id}`, { job });
 
 		const InputParser = GetInputParser(job.data);
 

server/src/server/middleware/prudence-validate.test.ts

@@ -36,7 +36,7 @@ t.test("#PrudenceMiddleware", (t) => {
 		t.equal(
 			json.description,
 			"[foo] example error message (Received nothing)",
-			"Should return error message with recieved nothing"
+			"Should return error message with received nothing"
 		);
 
 		t.end();

server/src/server/router/api/v1/auth/auth.test.ts

@@ -58,13 +58,31 @@ t.test("#AddNewInvite", (t) => {
 
 t.test("#ValidateCaptcha", async (t) => {
 	t.equal(
-		await ValidateCaptcha("200", "bar", MockJSONFetch({ success: true })),
+		await ValidateCaptcha(
+			"200",
+			"bar",
+			MockJSONFetch({
+				"https://www.google.com/recaptcha/api/siteverify?secret=unused&response=200&remoteip=bar":
+					{
+						success: true,
+					},
+			})
+		),
 		true,
 		"Validates captcha when sucess return is true"
 	);
 
 	t.equal(
-		await ValidateCaptcha("400", "bar", MockJSONFetch({ success: false })),
+		await ValidateCaptcha(
+			"400",
+			"bar",
+			MockJSONFetch({
+				"https://www.google.com/recaptcha/api/siteverify?secret=unused&response=400&remoteip=bar":
+					{
+						success: false,
+					},
+			})
+		),
 		false,
 		"Invalidates captcha when success return is not true"
 	);

server/src/server/router/api/v1/auth/router.ts

@@ -68,7 +68,7 @@ router.post(
 			});
 		}
 
-		logger.verbose(`Recieved login request with username ${req.body.username} (${req.ip})`);
+		logger.verbose(`received login request with username ${req.body.username} (${req.ip})`);
 
 		/* istanbul ignore next */
 		if (Environment.nodeEnv === "production" || Environment.nodeEnv === "staging") {
@@ -181,7 +181,7 @@ router.post(
 		"verbose"
 	),
 	async (req, res) => {
-		logger.verbose(`Recieved register request with username ${req.body.username} (${req.ip})`);
+		logger.verbose(`received register request with username ${req.body.username} (${req.ip})`);
 
 		/* istanbul ignore next */
 		if (Environment.nodeEnv === "production" || Environment.nodeEnv === "staging") {
@@ -433,7 +433,7 @@ router.post(
 			});
 		}
 
-		logger.debug(`Recieved password reset request for ${req.body.email}.`);
+		logger.debug(`received password reset request for ${req.body.email}.`);
 		// For timing attack and infosec reasons, we can't do anything but **immediately** return here.
 		res.status(202).json({
 			success: true,

server/src/server/router/api/v1/clients/router.ts

@@ -275,7 +275,7 @@ router.post(
 		const client = req[SYMBOL_TachiData]!.apiClientDoc!;
 		const clientName = `${client.name} (${client.clientID})`;
 
-		logger.info(`Recieved request to reset client secret for ${clientName}`);
+		logger.info(`received request to reset client secret for ${clientName}`);
 
 		const newSecret = Random20Hex();
 
@@ -308,7 +308,7 @@ router.delete("/:clientID", GetClientFromID, RequireOwnershipOfClient, async (re
 
 	const clientName = `${client.name} (${client.clientID})`;
 
-	logger.info(`Recieved request to destroy API Client ${client.name} (${client.clientID})`);
+	logger.info(`received request to destroy API Client ${client.name} (${client.clientID})`);
 
 	logger.verbose(`Removing API Client ${clientName}.`);
 	await db["api-clients"].remove({

server/src/server/router/api/v1/imports/router.ts

@@ -135,7 +135,7 @@ router.get("/:importID/poll-status", async (req, res) => {
 		} else {
 			return res.status(content.statusCode).json({
 				success: false,
-				description: content.message,
+				description: content.description,
 			});
 		}
 	}

server/src/server/router/api/v1/users/_userID/api-tokens/router.ts

@@ -152,7 +152,7 @@ router.delete("/:token", async (req, res) => {
 	const user = req[SYMBOL_TachiData]!.requestedUser!;
 
 	logger.info(
-		`Recieved request from ${FormatUserDoc(user)} to delete token ${req.params.token}.`
+		`received request from ${FormatUserDoc(user)} to delete token ${req.params.token}.`
 	);
 
 	const token = await db["api-tokens"].findOne({

server/src/server/router/api/v1/users/_userID/banner/router.test.ts

@@ -29,7 +29,7 @@ t.test("GET /api/v1/users/:userID/banner", (t) => {
 		t.equal(res.statusCode, 200, "Should return 200.");
 
 		if (res.statusCode !== 200) {
-			rootLogger.info("Unexpected non-200 in CDN tests, Recieved this as a body.", {
+			rootLogger.info("Unexpected non-200 in CDN tests, received this as a body.", {
 				body: res.body,
 			});
 		}

server/src/server/router/ir/kshook/sv3c/router.ts

@@ -40,7 +40,7 @@ const ValidateHeaders: RequestHandler = (req, res, next) => {
 
 	if (!softID) {
 		logger.debug(
-			`Recieved request without X-Software-Model from ${req[SYMBOL_TachiAPIAuth].userID!}.`
+			`received request without X-Software-Model from ${req[SYMBOL_TachiAPIAuth].userID!}.`
 		);
 		return res.status(400).json({
 			success: false,
@@ -53,7 +53,7 @@ const ValidateHeaders: RequestHandler = (req, res, next) => {
 
 		if (modelInfo.model !== MODEL_SDVX3_KONASTE) {
 			logger.info(
-				`Recieved unexpected softID ${softID}. Expected ${MODEL_SDVX3_KONASTE} as model.`
+				`received unexpected softID ${softID}. Expected ${MODEL_SDVX3_KONASTE} as model.`
 			);
 			return res.status(400).json({
 				success: false,