Merge pull request #552 from TNG-dev/release-v2.0.2

server/.gitignore

@@ -30,7 +30,7 @@ local-cdn
 test-cdn
 
 # Weird Redis Dumps
-dump.rdb
+*.rdb
 
 scripts/quick-insert/insert.ts
 scripts/bpi-poyashi-data/output.json

server/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "tachi-server",
-	"version": "2.0.1",
+	"version": "2.0.2",
 	"description": "A score tracking server.",
 	"main": "js/index.js",
 	"private": true,
@@ -51,6 +51,7 @@
 	},
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.41.0",
+		"@aws-sdk/signature-v4-crt": "^3.41.0",
 		"@types/mocha": "^9.0.0",
 		"@valuabletouch/winston-seq": "^1.2.0",
 		"bcryptjs": "^2.4.3",
@@ -82,7 +83,7 @@
 		"rimraf": "3.0.2",
 		"safe-json-stringify": "1.2.0",
 		"seq-logging": "^1.1.1",
-		"tachi-common": "0.3.10",
+		"tachi-common": "0.3.13",
 		"ts-node": "10.0.0",
 		"typescript": "4.3.4",
 		"winston": "3.3.3",

server/src/external/mongo/schemas.ts

@@ -209,7 +209,7 @@ const GetScoreMeta = (game: Game, playtype: Playtypes[Game]): PrudenceSchema =>
 	} else if (game === "usc") {
 		return {
 			noteMod: optNull(p.isIn("NORMAL", "MIRROR", "RANDOM", "MIR-RAN")),
-			gaugeMod: optNull(p.isIn("NORMAL", "HARD")),
+			gaugeMod: optNull(p.isIn("NORMAL", "HARD", "PERMISSIVE")),
 		};
 	} else if (game === "bms" && playtype === "7K") {
 		return {
@@ -771,7 +771,7 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
 		return prSchemaify(
 			PR_ChartDocument("iidx", playtype, {
 				notecount: p.isPositiveNonZeroInteger,
-				inGameID: p.isPositiveNonZeroInteger,
+				inGameID: p.or(p.isPositiveNonZeroInteger, [p.isPositiveNonZeroInteger]),
 				arcChartID: "?string",
 				hashSHA256: "?string",
 				"2dxtraSet": "?string",

server/src/lib/constants/ea3id.ts

@@ -6,6 +6,7 @@ export const MODEL_IIDX_LIGHTNING = "TDJ";
 export const REV_2DXTRA = "E";
 export const REV_2DXBMS = "Z";
 export const REV_OMNIMIX = "X";
+export const REV_NORMAL = "A";
 
 export const EXT_HEROIC_VERSE = "2020092900";
 export const EXT_BISTROVER = "2021091500";

server/src/lib/constants/version.ts

@@ -4,7 +4,7 @@
 
 const MAJOR = 2;
 const MINOR = 0;
-const PATCH = 1;
+const PATCH = 2;
 
 // As is with all front-facing zkldi projects, the version names for tachi-server
 // are from an album I like. In this case, the album is Portishead - Dummy.

server/src/lib/orphan-queue/orphan-queue.ts

@@ -10,6 +10,7 @@ import {
 	SongDocument,
 } from "tachi-common";
 import { GetNextCounterValue } from "utils/db";
+import { DedupeArr } from "utils/misc";
 
 const logger = CreateLogCtx(__filename);
 
@@ -35,7 +36,7 @@ export async function HandleOrphanQueue<I extends IDStrings>(
 	userID: integer,
 	chartName: string
 ) {
-	logger.debug(`received orphanqueue request for ${chartName}.`);
+	logger.debug(`Received orphanqueue request for ${chartName}.`);
 
 	const orphanChart = await db["orphan-chart-queue"].findOne(
 		Object.assign({ idString }, orphanMatchCriteria),
@@ -45,7 +46,7 @@ export async function HandleOrphanQueue<I extends IDStrings>(
 	);
 
 	if (!orphanChart) {
-		logger.verbose(`received unknown chart ${chartName}, orphaning.`);
+		logger.verbose(`Received unknown chart ${chartName}, orphaning.`);
 
 		await db["orphan-chart-queue"].insert({
 			idString,
@@ -59,7 +60,7 @@ export async function HandleOrphanQueue<I extends IDStrings>(
 
 	orphanChart.userIDs.push(userID);
 
-	const uniqueUsersArr = [...new Set(orphanChart.userIDs)];
+	const uniqueUsersArr = DedupeArr(orphanChart.userIDs);
 
 	const playcount = uniqueUsersArr.length;
 	// If N or more people have played this chart while orphaned, unorphan

server/src/lib/score-import/import-types/common/api-kai/iidx/class-handler.ts

@@ -45,7 +45,7 @@ export async function CreateKaiIIDXClassHandler(
 			return {};
 		}
 
-		if (iidxDan === null) {
+		if (iidxDan === null || iidxDan === undefined) {
 			return {};
 		}
 

server/src/lib/score-import/import-types/common/api-kai/sdvx/class-handler.ts

@@ -36,7 +36,7 @@ export async function CreateKaiSDVXClassHandler(
 
 		const sdvxDan: number | null = json.skill_level - 1;
 
-		if (sdvxDan === null || sdvxDan === undefined) {
+		if (json.skill_level === null || json.skill_level === undefined) {
 			return {};
 		}
 

server/src/lib/score-import/import-types/common/batch-manual/parser.ts

@@ -65,7 +65,7 @@ const PR_ScoreMeta = (game: Game, playtype: Playtypes[Game]): PrudenceSchema =>
 	} else if (game === "usc") {
 		return {
 			noteMod: optNull(p.isIn("NORMAL", "MIRROR", "RANDOM", "MIR-RAN")),
-			gaugeMod: optNull(p.isIn("NORMAL", "HARD")),
+			gaugeMod: optNull(p.isIn("NORMAL", "HARD", "PERMISSIVE")),
 		};
 	} else if (game === "sdvx") {
 		return {

server/src/lib/score-import/import-types/common/types.ts

@@ -16,11 +16,11 @@ import { ClassHandler } from "../../framework/user-game-stats/types";
 import { SDVXEamusementCSVData } from "../file/eamusement-sdvx-csv/types";
 import { MerScore } from "../file/mer-iidx/types";
 import { S3Score } from "../file/solid-state-squad/types";
-import { BarbatosScore } from "../ir/barbatos/types";
+import { BarbatosContext, BarbatosScore } from "../ir/barbatos/types";
 import { BeatorajaContext, BeatorajaScore } from "../ir/beatoraja/types";
 import { FervidexStaticContext, FervidexStaticScore } from "../ir/fervidex-static/types";
 import { FervidexContext, FervidexScore } from "../ir/fervidex/types";
-import { KsHookSV3CScore } from "../ir/kshook-sv3c/types";
+import { KsHookSV3CContext, KsHookSV3CScore } from "../ir/kshook-sv3c/types";
 import { IRUSCContext } from "../ir/usc/types";
 import { KaiContext } from "./api-kai/types";
 import { BatchManualContext } from "./batch-manual/types";
@@ -62,12 +62,12 @@ export interface ImportTypeContextMap {
 	"file/pli-iidx-csv": IIDXEamusementCSVContext;
 
 	"ir/direct-manual": BatchManualContext;
-	"ir/barbatos": EmptyObject;
+	"ir/barbatos": BarbatosContext;
 	"ir/fervidex": FervidexContext;
 	"ir/fervidex-static": FervidexStaticContext;
 	"ir/beatoraja": BeatorajaContext;
 	"ir/usc": IRUSCContext;
-	"ir/kshook-sv3c": EmptyObject;
+	"ir/kshook-sv3c": KsHookSV3CContext;
 
 	"api/arc-iidx": EmptyObject;
 	"api/arc-sdvx": EmptyObject;

server/src/lib/score-import/import-types/file/solid-state-squad/parser.ts

@@ -66,7 +66,6 @@ export function ParseSolidStateXML(
 	}
 
 	if (!parsedXML?.s3data?.scoredata?.song) {
-		logger.crit("foo", { data: fileData.buffer });
 		throw new ScoreImportFatalError(400, `Invalid S3 XML, no s3data -> scoredata -> song?`);
 	}
 

server/src/lib/score-import/import-types/ir/barbatos/converter.test.ts

@@ -40,7 +40,12 @@ t.test("#ConverterIRBarbatos", (t) => {
 	};
 
 	t.test("Should convert a BarbatosScore into a Dry Score", async (t) => {
-		const res = await ConverterIRBarbatos(barbScore, {}, "ir/barbatos", logger);
+		const res = await ConverterIRBarbatos(
+			barbScore,
+			{ timeReceived: 10 },
+			"ir/barbatos",
+			logger
+		);
 
 		t.hasStrict(res, {
 			song: albidaSong,
@@ -82,7 +87,7 @@ t.test("#ConverterIRBarbatos", (t) => {
 			() =>
 				ConverterIRBarbatos(
 					deepmerge(barbScore, { song_id: 1000 }) as BarbatosScore,
-					{},
+					{ timeReceived: 10 },
 					"ir/barbatos",
 					logger
 				),
@@ -97,9 +102,12 @@ t.test("#ConverterIRBarbatos", (t) => {
 	t.test("Should throw InternalFailure if song-chart desync.", async (t) => {
 		await db.songs.sdvx.remove({ id: 1 }); // force a song-chart desync
 
-		t.rejects(() => ConverterIRBarbatos(barbScore, {}, "ir/barbatos", logger), {
-			message: /Song 1 \(sdvx\) has no parent song/u,
-		});
+		t.rejects(
+			() => ConverterIRBarbatos(barbScore, { timeReceived: 10 }, "ir/barbatos", logger),
+			{
+				message: /Song 1 \(sdvx\) has no parent song/u,
+			}
+		);
 
 		t.end();
 	});

server/src/lib/score-import/import-types/ir/barbatos/converter.ts

@@ -1,7 +1,6 @@
 import { Lamps } from "tachi-common";
 import { FindSDVXChartOnInGameID } from "utils/queries/charts";
 import { FindSongOnID } from "utils/queries/songs";
-import { EmptyObject } from "utils/types";
 import {
 	InternalFailure,
 	KTDataNotFoundFailure,
@@ -9,7 +8,7 @@ import {
 import { GenericGetGradeAndPercent } from "../../../framework/common/score-utils";
 import { DryScore } from "../../../framework/common/types";
 import { ConverterFunction } from "../../common/types";
-import { BarbatosScore } from "./types";
+import { BarbatosContext, BarbatosScore } from "./types";
 
 const LAMP_LOOKUP = {
 	1: "FAILED",
@@ -27,7 +26,7 @@ const DIFFICULTY_LOOKUP = {
 	4: "MXM",
 };
 
-export const ConverterIRBarbatos: ConverterFunction<BarbatosScore, EmptyObject> = async (
+export const ConverterIRBarbatos: ConverterFunction<BarbatosScore, BarbatosContext> = async (
 	data,
 	context,
 	importType,
@@ -65,7 +64,7 @@ export const ConverterIRBarbatos: ConverterFunction<BarbatosScore, EmptyObject>
 		service: "Barbatos",
 		comment: null,
 		importType: "ir/barbatos",
-		timeAchieved: Date.now(),
+		timeAchieved: context.timeReceived,
 		scoreData: {
 			score: data.score,
 			percent,
@@ -84,7 +83,7 @@ export const ConverterIRBarbatos: ConverterFunction<BarbatosScore, EmptyObject>
 			},
 		},
 		scoreMeta: {
-			inSkillAnalyser: data.is_skill_analyser,
+			inSkillAnalyser: data.is_skill_analyzer,
 		},
 	};
 

server/src/lib/score-import/import-types/ir/barbatos/parser.ts

@@ -21,7 +21,7 @@ const PR_Barbatos = {
 	did_fail: "boolean",
 	clear_type: p.isIn(1, 2, 3, 4, 5),
 	gauge_type: p.isIn(0, 1, 2, 3),
-	is_skill_analyser: "boolean",
+	is_skill_analyzer: "boolean",
 };
 
 export function ParseBarbatosSingle(

server/src/lib/score-import/import-types/ir/barbatos/types.ts

@@ -15,5 +15,9 @@ export interface BarbatosScore {
 	did_fail: boolean;
 	clear_type: 1 | 2 | 3 | 4 | 5;
 	gauge_type: 1 | 2;
-	is_skill_analyser: boolean;
+	is_skill_analyzer: boolean;
+}
+
+export interface BarbatosContext {
+	timeReceived: number;
 }

server/src/lib/score-import/import-types/ir/beatoraja/converter.ts

@@ -182,7 +182,7 @@ export const ConverterIRBeatoraja: ConverterFunction<BeatorajaScore, BeatorajaCo
 			inputDevice: data.deviceType,
 			random,
 		},
-		timeAchieved: Date.now(),
+		timeAchieved: context.timeReceived,
 		service: "Beatoraja IR",
 	};
 

server/src/lib/score-import/import-types/ir/beatoraja/parser.ts

@@ -110,12 +110,13 @@ export function ParseBeatorajaSingle(
 		throw new ScoreImportFatalError(400, `Unsupported client ${client}`);
 	}
 
-	// asserted using prudence.
 	return {
 		context: {
 			client: "lr2oraja",
+			// asserted using prudence.
 			chart: body.chart as BeatorajaChart,
 			userID,
+			timeReceived: Date.now(),
 		},
 		game: "bms",
 		iterable: [body.score] as unknown as BeatorajaScore[],

server/src/lib/score-import/import-types/ir/beatoraja/types.ts

@@ -4,6 +4,7 @@ export interface BeatorajaContext {
 	client: "lr2oraja";
 	chart: BeatorajaChart;
 	userID: integer; // unexpectedly necessary for orphan code!
+	timeReceived: number;
 }
 
 export interface BeatorajaChart {

server/src/lib/score-import/import-types/ir/fervidex/converter.test.ts

@@ -125,7 +125,7 @@ t.test("#ConverterIRFervidex", (t) => {
 	t.test("Should convert a valid fervidex score into a dry score.", async (t) => {
 		const res = await ConverterIRFervidex(
 			baseFervidexScore,
-			{ version: "27" },
+			{ version: "27", timeReceived: 10 },
 			"ir/fervidex",
 			logger
 		);
@@ -159,7 +159,7 @@ t.test("#ConverterIRFervidex", (t) => {
 
 		const res = await ConverterIRFervidex(
 			deepmerge(baseFervidexScore, { option: { style_2p: "R_RANDOM" }, chart: "dpa" }),
-			{ version: "27" },
+			{ version: "27", timeReceived: 10 },
 			"ir/fervidex",
 			logger
 		);
@@ -182,7 +182,7 @@ t.test("#ConverterIRFervidex", (t) => {
 	t.test("Should null BP if the user died pre-emptively.", async (t) => {
 		const res = await ConverterIRFervidex(
 			deepmerge(baseFervidexScore, { dead: { measure: 1, note: 10 } }),
-			{ version: "27" },
+			{ version: "27", timeReceived: 10 },
 			"ir/fervidex",
 			logger
 		);
@@ -204,7 +204,7 @@ t.test("#ConverterIRFervidex", (t) => {
 		t.rejects(
 			ConverterIRFervidex(
 				deepmerge(baseFervidexScore, { chart: "spl" }),
-				{ version: "27" },
+				{ version: "27", timeReceived: 10 },
 				"ir/fervidex",
 				logger
 			),
@@ -218,7 +218,12 @@ t.test("#ConverterIRFervidex", (t) => {
 		await db.songs.iidx.remove({}); // this forces desync
 
 		t.rejects(
-			ConverterIRFervidex(baseFervidexScore, { version: "27" }, "ir/fervidex", logger),
+			ConverterIRFervidex(
+				baseFervidexScore,
+				{ version: "27", timeReceived: 10 },
+				"ir/fervidex",
+				logger
+			),
 			{ message: /Song 1 \(iidx\) has no parent song/giu }
 		);
 
@@ -258,7 +263,7 @@ t.test("#ConverterIRFervidex", (t) => {
 	t.test("Should convert undeflow gauge to null.", async (t) => {
 		const res = await ConverterIRFervidex(
 			deepmerge(baseFervidexScore, { gauge: [10, 5, 249, 248] }),
-			{ version: "27" },
+			{ version: "27", timeReceived: 10 },
 			"ir/fervidex",
 			logger
 		);

server/src/lib/score-import/import-types/ir/fervidex/converter.ts

@@ -23,7 +23,7 @@ export const FERVIDEX_LAMP_LOOKUP = {
 };
 
 export function TachifyAssist(
-	assist: FervidexScore["option"]["assist"]
+	assist: Required<FervidexScore>["option"]["assist"]
 ): DryScore<"iidx:SP" | "iidx:DP">["scoreMeta"]["assist"] {
 	switch (assist) {
 		case "FULL_ASSIST":
@@ -40,7 +40,7 @@ export function TachifyAssist(
 }
 
 export function TachifyGauge(
-	gauge: FervidexScore["option"]["gauge"]
+	gauge: Required<FervidexScore>["option"]["gauge"]
 ): DryScore<"iidx:SP" | "iidx:DP">["scoreMeta"]["gauge"] {
 	switch (gauge) {
 		case "ASSISTED_EASY":
@@ -58,7 +58,7 @@ export function TachifyGauge(
 }
 
 export function TachifyRange(
-	gauge: FervidexScore["option"]["range"]
+	gauge: Required<FervidexScore>["option"]["range"]
 ): DryScore<"iidx:SP" | "iidx:DP">["scoreMeta"]["range"] {
 	switch (gauge) {
 		case "HIDDEN_PLUS":
@@ -77,7 +77,7 @@ export function TachifyRange(
 	}
 }
 
-export function TachifyRandom(gauge: FervidexScore["option"]["style"]) {
+export function TachifyRandom(gauge: Required<FervidexScore>["option"]["style"]) {
 	switch (gauge) {
 		case "RANDOM":
 			return "RANDOM";
@@ -186,7 +186,7 @@ export const ConverterIRFervidex: ConverterFunction<FervidexScore, FervidexConte
 		service: "Fervidex",
 		comment: null,
 		importType: "ir/fervidex",
-		timeAchieved: Date.now(),
+		timeAchieved: context.timeReceived,
 		scoreData: {
 			score: data.ex_score,
 			percent,
@@ -212,15 +212,15 @@ export const ConverterIRFervidex: ConverterFunction<FervidexScore, FervidexConte
 			},
 		},
 		scoreMeta: {
-			assist: TachifyAssist(data.option.assist),
-			gauge: TachifyGauge(data.option.gauge),
+			assist: TachifyAssist(data.option?.assist),
+			gauge: TachifyGauge(data.option?.gauge),
 			// @ts-expect-error Awkward expansion of iidx:SP|DP strings here causes this
 			// to complain that [x,x] is not assignable to SP randoms. This is a lazy ignore!
 			random:
 				chart.playtype === "SP"
-					? TachifyRandom(data.option.style)
-					: [TachifyRandom(data.option.style), TachifyRandom(data.option.style_2p)],
-			range: TachifyRange(data.option.range),
+					? TachifyRandom(data.option?.style)
+					: [TachifyRandom(data.option?.style), TachifyRandom(data.option?.style_2p)],
+			range: TachifyRange(data.option?.range),
 		},
 	};
 

server/src/lib/score-import/import-types/ir/fervidex/parser.test.ts

@@ -0,0 +1,33 @@
+import CreateLogCtx from "lib/logger/logger";
+import t from "tap";
+import { SoftwareIDToVersion } from "./parser";
+
+const logger = CreateLogCtx(__filename);
+
+t.test("#SoftwareIDToVersion", (t) => {
+	const f = (sid: string) => SoftwareIDToVersion(sid, logger);
+
+	t.throws(() => f("a"), {}, "Should throw on invalid string.");
+	t.throws(() => f("LDJ:J:B:Q:2020092900"), {}, "Should throw on unknown HV revision.");
+	t.throws(() => f("LDJ:J:B:Q:2021091500"), {}, "Should throw on unknown BST revision.");
+	t.throws(() => f("XDJ:J:B:A:2020092900"), {}, "Should throw on unknown model.");
+	t.throws(() => f("LDJ:J:B:A:2099092900"), {}, "Should throw on unknown ext.");
+
+	t.equal(f("LDJ:J:B:A:2020092900"), "27");
+	t.equal(f("LDJ:J:B:X:2020092900"), "27-omni");
+	t.equal(f("LDJ:J:B:E:2020092900"), "27-2dxtra");
+	t.equal(f("LDJ:J:B:A:2021091500"), "28");
+	t.equal(f("LDJ:J:B:X:2021091500"), "28-omni");
+	t.equal(f("LDJ:J:B:E:2021091500"), "28-2dxtra");
+
+	t.equal(f("TDJ:J:B:A:2020092900"), "27");
+	t.equal(f("TDJ:J:B:X:2020092900"), "27-omni");
+	t.equal(f("TDJ:J:B:E:2020092900"), "27-2dxtra");
+	t.equal(f("TDJ:J:B:A:2021091500"), "28");
+	t.equal(f("TDJ:J:B:X:2021091500"), "28-omni");
+	t.equal(f("TDJ:J:B:E:2021091500"), "28-2dxtra");
+
+	t.equal(f("P2D:J:B:A:2020092900"), "inf");
+
+	t.end();
+});

server/src/lib/score-import/import-types/ir/fervidex/parser.ts

@@ -2,8 +2,10 @@ import {
 	EXT_BISTROVER,
 	EXT_HEROIC_VERSE,
 	MODEL_IIDX,
+	MODEL_IIDX_LIGHTNING,
 	MODEL_INFINITAS_2,
 	REV_2DXTRA,
+	REV_NORMAL,
 	REV_OMNIMIX,
 } from "lib/constants/ea3id";
 import { KtLogger } from "lib/logger/logger";
@@ -42,7 +44,7 @@ const PR_Fervidex: PrudenceSchema = {
 		note: optNull(p.isPositiveInteger),
 	} as unknown as ValidSchemaValue),
 
-	option: {
+	option: p.optional({
 		gauge: optNull(p.isIn("ASSISTED_EASY", "EASY", "HARD", "EX_HARD")),
 		range: optNull(
 			p.isIn("SUDDEN_PLUS", "HIDDEN_PLUS", "SUD_PLUS_HID_PLUS", "LIFT", "LIFT_SUD_PLUS")
@@ -50,7 +52,7 @@ const PR_Fervidex: PrudenceSchema = {
 		style: optNull(p.isIn("RANDOM", "R_RANDOM", "S_RANDOM", "MIRROR")),
 		style_2p: optNull(p.isIn("RANDOM", "R_RANDOM", "S_RANDOM", "MIRROR")),
 		assist: optNull(p.isIn("AUTO_SCRATCH", "LEGACY_NOTE", "ASCR_LEGACY", "FULL_ASSIST")),
-	},
+	}),
 
 	// we dont use it and we dont care.
 	pacemaker: p.any,
@@ -73,24 +75,25 @@ export function SoftwareIDToVersion(model: string, logger: KtLogger) {
 
 		if (data.model === MODEL_INFINITAS_2) {
 			return "inf";
-		} else if (data.model === MODEL_IIDX) {
-			// only HV. everything else is disabled deliberately.
+		} else if (data.model === MODEL_IIDX || data.model === MODEL_IIDX_LIGHTNING) {
+			// pretty icky yandere-dev tier if statements, but hey
+			// that's just how it works...
 			if (data.ext === EXT_HEROIC_VERSE) {
 				if (data.rev === REV_OMNIMIX) {
 					return "27-omni";
-				}
-
-				if (data.rev === REV_2DXTRA) {
+				} else if (data.rev === REV_2DXTRA) {
 					return "27-2dxtra";
+				} else if (data.rev === REV_NORMAL) {
+					return "27";
 				}
-
-				return "27";
 			} else if (data.ext === EXT_BISTROVER) {
 				if (data.rev === REV_OMNIMIX) {
 					return "28-omni";
+				} else if (data.rev === REV_2DXTRA) {
+					return "28-2dxtra";
+				} else if (data.rev === REV_NORMAL) {
+					return "28";
 				}
-
-				return "28";
 			}
 		}
 
@@ -122,7 +125,7 @@ export function ParseFervidexSingle(
 
 	// asserted using prudence.
 	return {
-		context: { version },
+		context: { version, timeReceived: Date.now() },
 		game: "iidx",
 		iterable: [body] as unknown as FervidexScore[],
 		classHandler: null,

server/src/lib/score-import/import-types/ir/fervidex/types.ts

@@ -30,7 +30,7 @@ export interface FervidexScore {
 		note?: integer | null;
 	};
 
-	option: {
+	option?: {
 		gauge?: "ASSISTED_EASY" | "EASY" | "HARD" | "EX_HARD" | null;
 		range?:
 			| "SUDDEN_PLUS"
@@ -60,4 +60,5 @@ export interface FervidexScore {
 
 export interface FervidexContext {
 	version: GPTSupportedVersions["iidx:SP" | "iidx:DP"];
+	timeReceived: number;
 }

server/src/lib/score-import/import-types/ir/kshook-sv3c/converter.test.ts

@@ -12,7 +12,7 @@ t.test("#ConverterIRKsHookSV3C", (t) => {
 	t.test("Should match a score with its song and chart.", async (t) => {
 		const res = await ConverterIRKsHookSV3C(
 			TestingKsHookSV3CScore,
-			{},
+			{ timeReceived: 10 },
 			"ir/kshook-sv3c",
 			logger
 		);
@@ -57,7 +57,7 @@ t.test("#ConverterIRKsHookSV3C", (t) => {
 			() =>
 				ConverterIRKsHookSV3C(
 					Object.assign({}, TestingKsHookSV3CScore, { music_id: 10000 }),
-					{},
+					{ timeReceived: 10 },
 					"ir/kshook-sv3c",
 					logger
 				),

server/src/lib/score-import/import-types/ir/kshook-sv3c/converter.ts

@@ -7,11 +7,10 @@ import { DryScore } from "lib/score-import/framework/common/types";
 import { Lamps } from "tachi-common";
 import { FindSDVXChartOnInGameIDVersion } from "utils/queries/charts";
 import { FindSongOnID } from "utils/queries/songs";
-import { EmptyObject } from "utils/types";
 import { ConverterFunction } from "../../common/types";
-import { KsHookSV3CScore } from "./types";
+import { KsHookSV3CContext, KsHookSV3CScore } from "./types";
 
-export const ConverterIRKsHookSV3C: ConverterFunction<KsHookSV3CScore, EmptyObject> = async (
+export const ConverterIRKsHookSV3C: ConverterFunction<KsHookSV3CScore, KsHookSV3CContext> = async (
 	data,
 	context,
 	importType,
@@ -44,7 +43,7 @@ export const ConverterIRKsHookSV3C: ConverterFunction<KsHookSV3CScore, EmptyObje
 		service: "kshook SV3C",
 		comment: null,
 		importType: "ir/kshook-sv3c",
-		timeAchieved: Date.now(),
+		timeAchieved: context.timeReceived,
 		scoreData: {
 			score: data.score,
 			percent,

server/src/lib/score-import/import-types/ir/kshook-sv3c/parser.test.ts

@@ -38,7 +38,7 @@ t.test("#ParseKsHookSV3C", (t) => {
 			const res = ParseKsHookSV3C(data, logger);
 
 			t.equal(res.game, "sdvx");
-			t.strictSame(res.context, {});
+			t.type(res.context.timeReceived, "number");
 			t.ok(Array.isArray(res.iterable));
 			t.type(res.classHandler, "function");
 		} catch (err) {

server/src/lib/score-import/import-types/ir/kshook-sv3c/parser.ts

@@ -3,9 +3,8 @@ import { KtLogger } from "lib/logger/logger";
 import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error";
 import p, { PrudenceSchema } from "prudence";
 import { FormatPrError } from "utils/prudence";
-import { EmptyObject } from "utils/types";
 import { ParserFunctionReturns } from "../../common/types";
-import { KsHookSV3CScore } from "./types";
+import { KsHookSV3CContext, KsHookSV3CScore } from "./types";
 
 const PR_KsHookSV3C: PrudenceSchema = {
 	appeal_id: p.isPositiveInteger,
@@ -75,7 +74,7 @@ const PR_KsHookSV3C: PrudenceSchema = {
 export function ParseKsHookSV3C(
 	body: Record<string, unknown>,
 	logger: KtLogger
-): ParserFunctionReturns<KsHookSV3CScore, EmptyObject> {
+): ParserFunctionReturns<KsHookSV3CScore, KsHookSV3CContext> {
 	// Ignore excess keys, as SV3C might add more features in the future.
 	const err = p(body, PR_KsHookSV3C, undefined, { allowExcessKeys: true });
 
@@ -88,7 +87,9 @@ export function ParseKsHookSV3C(
 	return {
 		game: "sdvx",
 		iterable: [score],
-		context: {},
+		context: {
+			timeReceived: Date.now(),
+		},
 		classHandler: () => {
 			const skillLevel = ConvertSkillLevel(score.skill_level);
 

server/src/lib/score-import/import-types/ir/kshook-sv3c/types.ts

@@ -56,3 +56,7 @@ export interface KsHookSV3CScore {
 	skill_name: integer;
 	track_no: integer;
 }
+
+export interface KsHookSV3CContext {
+	timeReceived: number;
+}

server/src/lib/score-import/import-types/ir/usc/converter.test.ts

@@ -16,6 +16,17 @@ t.test("#DeriveLamp", (t) => {
 
 	t.equal(DeriveLamp(d(uscScore, { options: { gaugeType: 0 }, gauge: 0.7 }), logger), "CLEAR");
 
+	t.equal(DeriveLamp(d(uscScore, { options: { gaugeType: 2 }, gauge: 1 }), logger), "FAILED");
+	t.equal(DeriveLamp(d(uscScore, { options: { gaugeType: 2 }, gauge: 0 }), logger), "FAILED");
+	t.equal(
+		DeriveLamp(d(uscScore, { options: { gaugeType: 2 }, gauge: 1, score: 10_000_000 }), logger),
+		"PERFECT ULTIMATE CHAIN"
+	);
+	t.equal(
+		DeriveLamp(d(uscScore, { options: { gaugeType: 2 }, gauge: 1, error: 0 }), logger),
+		"ULTIMATE CHAIN"
+	);
+
 	t.equal(
 		DeriveLamp(d(uscScore, { options: { gaugeType: 1 }, gauge: 0.1 }), logger),
 		"EXCESSIVE CLEAR"
@@ -58,7 +69,7 @@ t.test("#DeriveLamp", (t) => {
 		"ULTIMATE CHAIN"
 	);
 
-	t.throws(() => DeriveLamp(d(uscScore, { options: { gaugeType: 2 } }), logger));
+	t.throws(() => DeriveLamp(d(uscScore, { options: { gaugeType: 3 } }), logger));
 
 	t.end();
 });
@@ -77,7 +88,7 @@ t.test("#DeriveNoteMod", (t) => {
 const dm = (p: Partial<USCClientScore>) =>
 	ConverterIRUSC(
 		d(uscScore, p),
-		{ chartHash: uscChart.data.hashSHA1 as string, playtype: "Controller" },
+		{ chartHash: uscChart.data.hashSHA1 as string, playtype: "Controller", timeReceived: 10 },
 		"ir/usc",
 		logger
 	);

server/src/lib/score-import/import-types/ir/usc/converter.ts

@@ -50,6 +50,8 @@ export function DeriveLamp(
 		return scoreDoc.gauge >= 0.7 ? "CLEAR" : "FAILED";
 	} else if (scoreDoc.options.gaugeType === 1) {
 		return scoreDoc.gauge > 0 ? "EXCESSIVE CLEAR" : "FAILED";
+	} else if (scoreDoc.options.gaugeType === 2) {
+		return "FAILED";
 	}
 
 	logger.error(`Could not derive Lamp from Score Document`, { scoreDoc });
@@ -106,7 +108,7 @@ export const ConverterIRUSC: ConverterFunction<USCClientScore, IRUSCContext> = a
 		comment: null,
 		game: "usc",
 		importType,
-		timeAchieved: Date.now(),
+		timeAchieved: context.timeReceived,
 		service: "USC-IR",
 		scoreData: {
 			grade,
@@ -126,7 +128,12 @@ export const ConverterIRUSC: ConverterFunction<USCClientScore, IRUSCContext> = a
 			},
 		},
 		scoreMeta: {
-			gaugeMod: data.options.gaugeType === 0 ? "NORMAL" : "HARD",
+			gaugeMod:
+				data.options.gaugeType === 0
+					? "NORMAL"
+					: data.options.gaugeType === 1
+					? "HARD"
+					: "PERMISSIVE",
 			noteMod: DeriveNoteMod(data),
 		},
 	};

server/src/lib/score-import/import-types/ir/usc/parser.ts

@@ -18,7 +18,7 @@ const PR_USCIRScore: PrudenceSchema = {
 	late: p.optional(p.isPositiveInteger),
 	combo: p.optional(p.isPositiveInteger),
 	options: {
-		gaugeType: p.isIn(0, 1),
+		gaugeType: p.isIn(0, 1, 2),
 		mirror: "boolean",
 		random: "boolean",
 		autoFlags: p.isInteger,
@@ -62,6 +62,7 @@ export function ParseIRUSC(
 		context: {
 			chartHash,
 			playtype,
+			timeReceived: Date.now(),
 		},
 		game: "usc",
 		iterable: [score] as USCClientScore[],

server/src/lib/score-import/import-types/ir/usc/types.ts

@@ -3,4 +3,5 @@ import { Playtypes } from "tachi-common";
 export interface IRUSCContext {
 	chartHash: string;
 	playtype: Playtypes["usc"];
+	timeReceived: number;
 }

server/src/server/router/api/v1/admin/router.ts

@@ -12,6 +12,7 @@ import { Game, UserAuthLevels } from "tachi-common";
 import db from "external/mongo/db";
 import { DeleteScore } from "lib/delete-scores/delete-scores";
 import { UpdateAllPBs } from "utils/calculations/recalc-scores";
+import DestroyUserGamePlaytypeData from "utils/reset-state/destroy-ugpt";
 
 const logger = CreateLogCtx(__filename);
 
@@ -260,6 +261,44 @@ router.post("/delete-score", prValidate({ scoreID: "string" }), async (req, res)
 	});
 });
 
+/**
+ * Destroys a users UGPT profile and forces a site recalc.
+ *
+ * @POST /api/v1/admin/destroy-ugpt
+ */
+router.post(
+	"/destroy-ugpt",
+	prValidate({
+		userID: p.isInteger,
+		game: p.isIn(TachiConfig.GAMES),
+		playtype: "string", // lazy
+	}),
+	async (req, res) => {
+		const { userID, game, playtype } = req.body;
+
+		const ugpt = await db["game-stats"].findOne({
+			userID,
+			game,
+			playtype,
+		});
+
+		if (!ugpt) {
+			return res.status(404).json({
+				success: false,
+				description: `No stats for ${userID} (${game} ${playtype}) exist.`,
+			});
+		}
+
+		await DestroyUserGamePlaytypeData(userID, game, playtype);
+
+		return res.status(200).json({
+			success: true,
+			description: `Completely destroyed UGPT for ${userID} (${game} ${playtype}).`,
+			body: {},
+		});
+	}
+);
+
 /**
  * Destroy a chart and all of its scores (and sessions).
  *

server/src/server/router/ir/fervidex/router.test.ts

@@ -5,6 +5,7 @@ import mockApi from "test-utils/mock-api";
 import ResetDBState from "test-utils/resets";
 import { GetKTDataJSON } from "test-utils/test-data";
 import { Sleep } from "utils/misc";
+import deepmerge from "deepmerge";
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function TestHeaders(url: string, data: any) {
@@ -301,6 +302,27 @@ t.test("POST /ir/fervidex/score/submit", (t) => {
 		t.end();
 	});
 
+	t.test("Should import a valid score with undefined options", async (t) => {
+		const res = await mockApi
+			.post("/ir/fervidex/score/submit")
+			.set("Authorization", "Bearer mock_token")
+			.set("User-Agent", "fervidex/1.3.0")
+			.set("X-Software-Model", "LDJ:J:B:A:2020092900")
+			.send(deepmerge(GetKTDataJSON("./fervidex/base.json"), { option: undefined }));
+
+		t.equal(res.body.success, true, "Should be successful");
+
+		t.equal(res.body.body.errors.length, 0, "Should have 0 failed scores.");
+
+		const scores = await db.scores.count({
+			service: "Fervidex",
+		});
+
+		t.equal(scores, 1, "Should import 1 score.");
+
+		t.end();
+	});
+
 	t.test("Should import a valid score with 2dx-gsm", async (t) => {
 		const res = await mockApi
 			.post("/ir/fervidex/score/submit")

server/src/server/router/ir/fervidex/router.ts

@@ -60,7 +60,7 @@ const ValidateFervidexHeader: RequestHandler = (req, res, next) => {
 	// version.minor
 	if (versions[1] < 3) {
 		logger.debug(
-			`Rejected outdated fervidex client from user ${req[SYMBOL_TachiAPIAuth].userID!}.`
+			`Rejected outdated fervidex client from user ${req[SYMBOL_TachiAPIAuth].userID}.`
 		);
 		return res.status(400).json({
 			success: false,

server/src/server/router/ir/usc/_playtype/router.test.ts

@@ -582,6 +582,12 @@ t.test("POST /scores", (t) => {
 
 		t.equal(res.body.statusCode, 22);
 
+		t.match(
+			res.body.description,
+			"(1/3)",
+			"Should contain (required/outOf) in the response description."
+		);
+
 		const dbScore = await db.scores.findOne({
 			game: "usc",
 		});

server/src/server/router/ir/usc/_playtype/router.ts

@@ -21,6 +21,7 @@ import {
 	Playtypes,
 	SuccessfulAPIResponse,
 } from "tachi-common";
+import { DedupeArr } from "utils/misc";
 import { FormatPrError } from "utils/prudence";
 import { GetBlacklist } from "utils/queries/blacklist";
 import { AssignToReqTachiData } from "utils/req-tachi-data";
@@ -319,6 +320,7 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => {
 			song,
 			{
 				"chartDoc.data.hashSHA1": uscChart.chartHash,
+				"chartDoc.playtype": playtype,
 			},
 			ServerConfig.USC_QUEUE_SIZE,
 			req[SYMBOL_TachiAPIAuth].userID!,
@@ -332,9 +334,12 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => {
 				"context.playtype": playtype,
 			});
 
-			await Promise.all(
-				scoresToDeorphan.map((score) => ReprocessOrphan(score, blacklist, logger))
-			);
+			for (const score of scoresToDeorphan) {
+				// Has to be like this to avoid race conditions where two
+				// orphans resolve to the same scoreID and collide in mid-air!
+				// eslint-disable-next-line no-await-in-loop
+				await ReprocessOrphan(score, blacklist, logger);
+			}
 		}
 	}
 
@@ -346,16 +351,6 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => {
 		playtype,
 	]);
 
-	// If this was an orphan chart request, return ACCEPTED,
-	// since it may be unorphaned in the future
-	if (!chartDoc) {
-		return res.status(200).json({
-			statusCode: STATUS_CODES.ACCEPTED,
-			description:
-				"This score has been accepted, but is waiting for more players before its parent chart is accepted.",
-		});
-	}
-
 	if (importRes.statusCode === 500) {
 		return res.status(200).json({
 			statusCode: STATUS_CODES.SERVER_ERROR,
@@ -370,6 +365,71 @@ router.post("/scores", RequirePermissions("submit_score"), async (req, res) => {
 
 	const importDoc = (importRes.body as SuccessfulAPIResponse).body as ImportDocument;
 
+	// If the import failed, AND the import failure WAS NOT that the chart didnt exist
+	// report that error instead.
+	if (importDoc.errors[0]?.type && importDoc.errors[0].type !== "KTDataNotFound") {
+		logger.info(`USC Import Failed ${importDoc.errors[0].message}`, {
+			importDoc,
+			userID,
+		});
+
+		return res.status(200).json({
+			statusCode: STATUS_CODES.BAD_REQ,
+			description: `${importDoc.errors[0].type} ${importDoc.errors[0].message}`,
+		});
+	}
+
+	// If this was an orphan chart request, return ACCEPTED,
+	// since it may be unorphaned in the future
+	if (!chartDoc) {
+		const orphanChart = await db["orphan-chart-queue"].findOne({
+			"chartDoc.data.hashSHA1": uscChart.chartHash,
+			"chartDoc.playtype": playtype,
+		});
+
+		if (!orphanChart) {
+			logger.severe(
+				`No orphan chart for USC ${uscChart.chartHash} (${playtype})? One was expected.`,
+				{ uscChart, playtype }
+			);
+			return res.status(200).json({
+				statusCode: STATUS_CODES.SERVER_ERROR,
+				description: `An internal server error has occured.`,
+			});
+		}
+
+		const players = DedupeArr([...orphanChart.userIDs, userID]);
+
+		return res.status(200).json({
+			statusCode: STATUS_CODES.ACCEPTED,
+			description: `This score has been accepted, but is waiting for more players before its parent chart is accepted. (${players.length}/${ServerConfig.USC_QUEUE_SIZE})`,
+		});
+	}
+
+	// If the chartDoc exists, any error is a failure here.
+	if (importDoc.errors[0]) {
+		logger.info(`USC Import Failed ${importDoc.errors[0].message}`, {
+			importDoc,
+			userID,
+		});
+
+		return res.status(200).json({
+			statusCode: STATUS_CODES.BAD_REQ,
+			description: `${importDoc.errors[0].type} ${importDoc.errors[0].message}`,
+		});
+	}
+
+	if (!importDoc.scoreIDs[0]) {
+		logger.warn(`No scoreID, but import was successful?`, {
+			importDoc,
+			userID,
+		});
+		return res.status(200).json({
+			statusCode: STATUS_CODES.BAD_REQ,
+			description: "No score was imported.",
+		});
+	}
+
 	if (importDoc.errors[0]) {
 		logger.info(`USC Import Failed ${importDoc.errors[0].message}`, {
 			importDoc,

server/src/server/router/ir/usc/_playtype/types.ts

@@ -8,7 +8,7 @@ export interface USCServerScore {
 	near: integer;
 	error: integer;
 	ranking: integer;
-	gaugeMod: "NORMAL" | "HARD";
+	gaugeMod: "NORMAL" | "HARD" | "PERMISSIVE";
 	noteMod: "NORMAL" | "MIRROR" | "RANDOM" | "MIR-RAN";
 	username: string;
 }
@@ -24,7 +24,7 @@ export interface USCClientScore {
 	late: integer | null;
 	combo: integer | null;
 	options: {
-		gaugeType: 0 | 1;
+		gaugeType: 0 | 1 | 2;
 		gaugeOpt: integer;
 		mirror: boolean;
 		random: boolean;

server/src/test-utils/test-data.ts

@@ -195,7 +195,7 @@ export const barbScore: BarbatosScore = {
 	near_fast: 40,
 	near_slow: 10,
 	gauge_type: 1,
-	is_skill_analyser: false,
+	is_skill_analyzer: false,
 	level: 10,
 	max_chain: 100,
 	percent: 90,

server/src/test-utils/test-data/barbatos/base.json

@@ -13,5 +13,5 @@
 	"did_fail": false,
 	"clear_type": 2,
 	"gauge_type": 0,
-	"is_skill_analyser": false
+	"is_skill_analyzer": false
 }

server/src/utils/folder.ts

@@ -59,7 +59,7 @@ export async function ResolveFolderToCharts(
 		const folderDataTransposed: Record<string, unknown> = {};
 
 		for (const key in folder.data) {
-			folderDataTransposed[key.replace(/¬/gu, ".").replace("~", "$")] = folder.data[key];
+			folderDataTransposed[key.replace(/¬/gu, ".").replace(/~/gu, "$")] = folder.data[key];
 		}
 
 		const fx = deepmerge.all([filter, { playtype: folder.playtype }, folderDataTransposed]);

server/src/utils/reset-state/destroy-ugpt.ts

@@ -0,0 +1,94 @@
+import db from "external/mongo/db";
+import { integer, Game, Playtypes } from "tachi-common";
+import { UpdateAllPBs } from "utils/calculations/recalc-scores";
+
+/**
+ * Completely resets a UGPT profile.
+ *
+ * This function is dangerous! Should only be ran by admins.
+ */
+export default async function DestroyUserGamePlaytypeData(
+	userID: integer,
+	game: Game,
+	playtype: Playtypes[Game]
+) {
+	await db["game-stats-snapshots"].remove({
+		userID,
+		game,
+		playtype,
+	});
+
+	await db.scores.remove({
+		userID,
+		game,
+		playtype,
+	});
+
+	const chartIDs = (
+		await db["personal-bests"].find(
+			{
+				userID,
+				game,
+				playtype,
+			},
+			{
+				projection: {
+					chartID: 1,
+				},
+			}
+		)
+	).map((e) => e.chartID);
+
+	await db["personal-bests"].remove({
+		userID,
+		game,
+		playtype,
+	});
+
+	await UpdateAllPBs(undefined, {
+		chartID: { $in: chartIDs },
+	});
+
+	const sessionIDs = (
+		await db.sessions.find(
+			{
+				userID,
+				game,
+				playtype,
+			},
+			{
+				projection: {
+					sessionID: 1,
+				},
+			}
+		)
+	).map((e) => e.sessionID);
+
+	await db["session-view-cache"].remove({
+		sessionID: { $in: sessionIDs },
+	});
+
+	await db.sessions.remove({
+		userID,
+		game,
+		playtype,
+	});
+
+	await db.imports.remove({
+		userID,
+		game,
+		playtype,
+	});
+
+	await db["game-settings"].remove({
+		userID,
+		game,
+		playtype,
+	});
+
+	await db["game-stats"].remove({
+		userID,
+		game,
+		playtype,
+	});
+}