Merge branch 'develop' into zkldi/issue-673

server/CHANGELOG.md

@@ -15,7 +15,27 @@ not generally be shown to end users.
 
 # Changelog
 
-# CURRENT
+# 2.1.16
+
+- [BUG] EXCEED GEAR scores can now be imported for real for real.
+
+# 2.1.15
+
+- [BUG] EXCEED GEAR scores can now be imported for real.
+
+# 2.1.14
+
+- [BUG] Fixed bug where folders/tables wouldn't sync automatically from the seeds, again.
+
+# 2.1.13
+
+- [BUG] Fixed bug where folders/tables wouldn't sync automatically from the seeds.
+
+# 2.1.12
+
+- [ENH] EXCEED GEAR scores can now be imported.
+
+# 2.1.11
 
 - [ENH] The LR2Hook now saves courses.
 - [ENH] The LR2Hook now saves gauge information.

server/pnpm-lock.yaml

@@ -95,7 +95,7 @@ dependencies:
   fast-json-stable-hash: 1.0.3
   fast-xml-parser: 4.0.2
   helmet: 5.0.2
-  html-entities: 2.3.2
+  html-entities: 2.3.3
   json5: 2.2.0
   mkdirp: 1.0.4
   mongodb: 3.6.3
@@ -2412,7 +2412,7 @@ packages:
     resolution: {integrity: sha512-/aUslKhzkTNCQUB2qTX84lVmfia9NyjP3WpDGtj/WxhwBzWBYUV3DgUpurHTme8UTPcPlAD1DJ+b0nN/t50zDQ==}
     dependencies:
       boolbase: 1.0.0
-      css-what: 5.1.0
+      css-what: 6.1.0
       domhandler: 4.3.1
       domutils: 2.8.0
       nth-check: 2.0.1
@@ -4354,7 +4354,7 @@ packages:
   /node-html-parser/5.3.3:
     resolution: {integrity: sha512-ncg1033CaX9UexbyA7e1N0aAoAYRDiV8jkTvzEnfd1GDvzFdrsXLzR4p4ik8mwLgnaKP/jyUFWDy9q3jvRT2Jw==}
     dependencies:
-      css-select: 4.2.1
+      css-select: 4.3.0
       he: 1.2.0
     dev: false
 

server/src/external/mongo/indexes.ts

@@ -115,6 +115,9 @@ const staticIndexes: Partial<Record<Databases, Index[]>> = {
 		index({ "data.hashSHA256": 1, playtype: 1 }, UNIQUE),
 		index({ "data.hashMD5": 1, playtype: 1 }, UNIQUE),
 	],
+	"import-locks": [index({ userID: 1 }, UNIQUE)],
+	"arc-saved-profiles": [index({ userID: 1, forImportType: 1 }, UNIQUE)],
+	"score-blacklist": [index({ scoreID: 1 }, UNIQUE)],
 };
 
 const indexes: Partial<Record<Databases, Index[]>> = staticIndexes;

server/src/lib/score-import/framework/score-importing/score-import-main.ts

@@ -64,20 +64,20 @@ export default async function ScoreImportMain<D, C>(
 		logger = providedLogger;
 	}
 
+	const hasNoOngoingImport = await CheckAndSetOngoingImportLock(user.id);
+
+	if (hasNoOngoingImport) {
+		logger.info(`User ${userID} made an import while they had one ongoing.`);
+		// @danger
+		// Throwing away an import if the user already has one outgoing is *bad*, as in the case
+		// of degraded performance we might just start throwing scores away.
+		// Under normal circumstances, there is no scenario where a user would have two ongoing
+		// imports at the same time - even if they were using single-score imports on a 5 second
+		// chart, as each score import takes only around ~10-15milliseconds.
+		throw new ScoreImportFatalError(409, "This user already has an ongoing import.");
+	}
+
 	try {
-		const hasNoOngoingImport = await CheckAndSetOngoingImportLock(user.id);
-
-		if (hasNoOngoingImport) {
-			logger.info(`User ${userID} made an import while they had one ongoing.`);
-			// @danger
-			// Throwing away an import if the user already has one outgoing is *bad*, as in the case
-			// of degraded performance we might just start throwing scores away.
-			// Under normal circumstances, there is no scenario where a user would have two ongoing
-			// imports at the same time - even if they were using single-score imports on a 5 second
-			// chart, as each score import takes only around ~10-15milliseconds.
-			throw new ScoreImportFatalError(409, "This user already has an ongoing import.");
-		}
-
 		const timeStarted = Date.now();
 
 		SetJobProgress(job, "Parsing score data.");

server/src/lib/score-import/import-types/common/api-kai/sdvx/class-handler.ts

@@ -35,6 +35,7 @@ export async function CreateKaiSDVXClassHandler(
 		}
 
 		if (json.skill_level === null || json.skill_level === undefined) {
+			logger.info(`User has no skill_level. Not updating anything.`);
 			return {};
 		}
 

server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.test.ts

@@ -3,12 +3,29 @@ import db from "external/mongo/db";
 import CreateLogCtx from "lib/logger/logger";
 import t from "tap";
 import ResetDBState from "test-utils/resets";
-import { GetKTDataJSON, TestingAlbidaADV, TestingSDVXAlbidaSong } from "test-utils/test-data";
+import { TestingAlbidaADV, TestingSDVXAlbidaSong } from "test-utils/test-data";
 import { ConvertAPIKaiSDVX, ConvertDifficulty, ConvertVersion, ResolveKaiLamp } from "./converter";
 
 const logger = CreateLogCtx(__filename);
 
-const sdvxScore = GetKTDataJSON("./api-kai/sdvx-score.json");
+const sdvxScore = {
+	sdvx_id: 32157055,
+	music_id: 1,
+	music_difficulty: 1,
+	played_version: 6, // exg
+	clear_type: 2,
+	max_chain: 179,
+	score: 9310699,
+	critical: 1754,
+	near: 112,
+	error: 78,
+	early: 70,
+	late: 42,
+	gauge_type: 0,
+	gauge_rate: 90.01,
+	timestamp: "2020-08-30T13:08:11Z",
+	_id: 127108,
+};
 
 t.test("#ConvertAPIKaiSDVX", (t) => {
 	t.beforeEach(ResetDBState);
@@ -38,7 +55,7 @@ t.test("#ConvertAPIKaiSDVX", (t) => {
 					hitMeta: {
 						fast: 70,
 						slow: 42,
-						gauge: 90,
+						gauge: 90.01,
 						maxCombo: 179,
 					},
 				},
@@ -59,7 +76,7 @@ t.test("#ConvertAPIKaiSDVX", (t) => {
 					logger
 				),
 			{
-				message: /Could not find chart with songID 0 \(ADV - Version heaven\)/u,
+				message: /Could not find chart with songID 0 \(ADV - Version exceed\)/u,
 			}
 		);
 
@@ -113,7 +130,8 @@ t.test("#ConvertVersion", (t) => {
 	t.equal(ConvertVersion(3), "gw");
 	t.equal(ConvertVersion(4), "heaven");
 	t.equal(ConvertVersion(5), "vivid");
-	t.throws(() => ConvertVersion(6));
+	t.equal(ConvertVersion(6), "exceed");
+	t.throws(() => ConvertVersion(7));
 	t.throws(() => ConvertVersion(0));
 
 	t.end();

server/src/lib/score-import/import-types/common/api-kai/sdvx/converter.ts

@@ -19,7 +19,7 @@ import { KaiContext, KaiSDVXScore } from "../types";
 const PR_KaiSDVXScore = {
 	music_id: p.isPositiveInteger,
 	music_difficulty: p.isBoundedInteger(0, 4),
-	played_version: p.isBoundedInteger(1, 5),
+	played_version: p.isBoundedInteger(1, 6),
 	clear_type: p.isBoundedInteger(1, 5),
 	max_chain: p.isPositiveInteger,
 	score: p.isBoundedInteger(0, 10_000_000),
@@ -28,7 +28,7 @@ const PR_KaiSDVXScore = {
 	error: p.nullable(p.isPositiveInteger),
 	early: p.nullable(p.isPositiveInteger),
 	late: p.nullable(p.isPositiveInteger),
-	gauge_rate: p.isBoundedInteger(0, 100),
+	gauge_rate: p.isBetween(0, 100),
 	timestamp: "string",
 };
 
@@ -132,6 +132,8 @@ export function ConvertVersion(ver: number) {
 			return "heaven";
 		case 5:
 			return "vivid";
+		case 6:
+			return "exceed";
 	}
 
 	throw new InvalidScoreFailure(`Unknown Game Version ${ver}.`);

server/src/scripts/sync-database.ts

@@ -174,6 +174,18 @@ const syncInstructions: SyncInstructions[] = [
 			}
 		},
 	},
+	{
+		pattern: /^songs-(b|p)ms/u,
+		handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
+			const r = await GenericUpsert(songs, collection, "id", logger, false);
+
+			if (r.thingsChanged) {
+				await RecalcAllScores({
+					songID: { $in: r.changedFields },
+				});
+			}
+		},
+	},
 	{
 		pattern: /^songs-/u,
 		handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
@@ -187,13 +199,19 @@ const syncInstructions: SyncInstructions[] = [
 		},
 	},
 	{
-		pattern: /^folders$/u,
+		pattern: /^folders/u,
 		handler: async (
 			folders: FolderDocument[],
 			collection: ICollection<FolderDocument>,
 			logger
 		) => {
-			const r = await GenericUpsert(folders, collection, "folderID", logger, true);
+			const r = await GenericUpsert(
+				folders.filter((e) => TachiConfig.GAMES.includes(e.game)),
+				collection,
+				"folderID",
+				logger,
+				true
+			);
 
 			if (r) {
 				await InitaliseFolderChartLookup();
@@ -201,17 +219,29 @@ const syncInstructions: SyncInstructions[] = [
 		},
 	},
 	{
-		pattern: /^tables$/u,
+		pattern: /^tables/u,
 		handler: (tables: TableDocument[], collection: ICollection<TableDocument>, logger) =>
-			GenericUpsert(tables, collection, "tableID", logger, true),
+			GenericUpsert(
+				tables.filter((e) => TachiConfig.GAMES.includes(e.game)),
+				collection,
+				"tableID",
+				logger,
+				true
+			),
 	},
 	{
-		pattern: /^bms-course-lookup$/u,
-		handler: (
+		pattern: /^bms-course-lookup/u,
+		handler: async (
 			bmsCourseDocuments: BMSCourseDocument[],
 			collection: ICollection<BMSCourseDocument>,
 			logger
-		) => GenericUpsert(bmsCourseDocuments, collection, "md5sums", logger),
+		) => {
+			if (TachiConfig.TYPE === "ktchi") {
+				return;
+			}
+
+			await GenericUpsert(bmsCourseDocuments, collection, "md5sums", logger);
+		},
 	},
 ];
 

server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.test.ts

@@ -51,6 +51,15 @@ t.test("GET /ir/beatoraja/charts/:chartSHA256/scores", (t) => {
 
 		t.equal(res.status, 200);
 
+		t.hasStrict(res.body.body, [
+			{
+				player: "",
+				clear: 5,
+				epg: 617,
+				lpg: 0,
+			},
+		]);
+
 		t.end();
 	});
 

server/src/server/router/ir/beatoraja/charts/_chartSHA256/router.ts

@@ -1,6 +1,6 @@
 import { RequestHandler, Router } from "express";
 import db from "external/mongo/db";
-import { SYMBOL_TachiData } from "lib/constants/tachi";
+import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi";
 import { ChartDocument, PBScoreDocument } from "tachi-common";
 import { AssignToReqTachiData } from "utils/req-tachi-data";
 import { TachiScoreDataToBeatorajaFormat } from "./convert-scores";
@@ -43,6 +43,7 @@ router.use(GetChartDocument);
  */
 router.get("/scores", async (req, res) => {
 	const chart = req[SYMBOL_TachiData]!.beatorajaChartDoc!;
+	const requestingUserID = req[SYMBOL_TachiAPIAuth].userID;
 
 	const scores = (await db["personal-bests"].find({
 		chartID: chart.chartID,
@@ -59,7 +60,6 @@ router.get("/scores", async (req, res) => {
 			},
 		}
 	);
-
 	const userMap = new Map();
 	for (const user of userDocs) {
 		userMap.set(user.id, user);
@@ -72,7 +72,7 @@ router.get("/scores", async (req, res) => {
 			TachiScoreDataToBeatorajaFormat(
 				score,
 				chart.data.hashSHA256,
-				userMap.get(score.userID).username,
+				score.userID === requestingUserID ? "" : userMap.get(score.userID).username,
 				chart.data.notecount,
 				0 // Playcount is always 0 at the moment due to performance concerns.
 			)

server/src/test-utils/mock-db/charts-sdvx.json

@@ -18,6 +18,7 @@
 			"gw",
 			"heaven",
 			"vivid",
+			"exceed",
 			"konaste"
 		],
 		"tierlistInfo": {}

server/src/test-utils/test-data.ts

@@ -274,7 +274,7 @@ export const TestingAlbidaADV: ChartDocument<"sdvx:Single"> = {
 		arcChartID: "EOZ7FixCDpv",
 	},
 	isPrimary: true,
-	versions: ["booth", "inf", "gw", "heaven", "vivid", "konaste"],
+	versions: ["booth", "inf", "gw", "heaven", "vivid", "exceed", "konaste"],
 	tierlistInfo: {},
 };
 

server/src/test-utils/test-data/api-kai/iidx-score.json

@@ -1,17 +0,0 @@
-{
-	"chart_id": 3848,
-	"music_id": 1000,
-	"music_difficulty": 2,
-	"play_style": "SINGLE",
-	"difficulty": "ANOTHER",
-	"iidx_id": 35247879,
-	"version_played": 26,
-	"lamp": 5,
-	"ex_score": 1570,
-	"grade": "AA",
-	"miss_count": 24,
-	"fast_count": null,
-	"slow_count": null,
-	"timestamp": "2020-10-31T19:10:50Z",
-	"_id": 189232
-}

server/src/test-utils/test-data/api-kai/sdvx-score.json

@@ -1,18 +0,0 @@
-{
-	"sdvx_id": 32157055,
-	"music_id": 1,
-	"music_difficulty": 1,
-	"played_version": 4,
-	"clear_type": 2,
-	"max_chain": 179,
-	"score": 9310699,
-	"critical": 1754,
-	"near": 112,
-	"error": 78,
-	"early": 70,
-	"late": 42,
-	"gauge_type": 0,
-	"gauge_rate": 90,
-	"timestamp": "2020-08-30T13:08:11Z",
-	"_id": 127108
-}