Add DP Tier support.

server/package.json

@@ -67,6 +67,7 @@
 		"fast-json-stable-hash": "1.0.3",
 		"fast-xml-parser": "4.0.2",
 		"helmet": "5.0.2",
+		"html-entities": "^2.3.2",
 		"json5": "2.2.0",
 		"mkdirp": "1.0.4",
 		"mongodb": "3.6.3",
@@ -74,6 +75,7 @@
 		"multer": "1.4.4",
 		"node-cache": "5.1.2",
 		"node-fetch": "2.6.7",
+		"node-html-parser": "^5.3.3",
 		"nodemailer": "6.7.2",
 		"prudence": "0.9.7",
 		"rate-limit-redis": "2.1.0",
@@ -82,7 +84,7 @@
 		"rimraf": "3.0.2",
 		"safe-json-stringify": "1.2.0",
 		"seq-logging": "1.1.1",
-		"tachi-common": "0.7.1",
+		"tachi-common": "0.7.3",
 		"ts-node": "10.4.0",
 		"tsconfig-paths": "3.12.0",
 		"typescript": "4.5.5",

server/src/lib/jobs/update-dp-tiers.ts

@@ -0,0 +1,121 @@
+/* eslint-disable no-await-in-loop */
+import fetch from "utils/fetch";
+import { parse } from "node-html-parser";
+import CreateLogCtx from "lib/logger/logger";
+import { FindSongOnTitle } from "utils/queries/songs";
+import db from "external/mongo/db";
+import { RecalcAllScores } from "utils/calculations/recalc-scores";
+import { decode } from "html-entities";
+
+const logger = CreateLogCtx(__filename);
+
+async function UpdateDPTiers() {
+	const rawHTML = await fetch("https://zasa.sakura.ne.jp/dp/run.php").then((r) => r.text());
+
+	// time for some fun html parsing.
+
+	const data = parse(rawHTML);
+
+	const rows = data.querySelectorAll("tr");
+
+	const parsedData = [];
+
+	for (const row of rows) {
+		// We only care about rows with 4 children.
+		// that's because the data is formatted like
+		// [HYPER, ANOTHER, LEGGENDARIA, SONG_TITLE]
+		// but has interspersed headers for game versions.
+		if (row.childNodes.length !== 4) {
+			continue;
+		}
+
+		parsedData.push({
+			HYPER: ParseTierStr(row.childNodes[0].innerText),
+			ANOTHER: ParseTierStr(row.childNodes[1].innerText),
+			LEGGENDARIA: ParseTierStr(row.childNodes[2].innerText),
+			songTitle: decode(row.childNodes[3].innerText),
+		});
+	}
+
+	logger.info(`Got DP tier data. Applying it.`);
+
+	const updatedSongIDs = new Set();
+
+	await Promise.all(
+		parsedData.map(async (d) => {
+			const song = await FindSongOnTitle("iidx", d.songTitle);
+
+			if (!song) {
+				logger.warn(`Couldn't find song with title ${d.songTitle}.`);
+				return;
+			}
+
+			for (const difficulty of ["HYPER", "ANOTHER", "LEGGENDARIA"] as const) {
+				if (d[difficulty] === null) {
+					continue;
+				}
+
+				const result = await db.charts.iidx.update(
+					{
+						versions: "29",
+						playtype: "DP",
+						songID: song.id,
+						difficulty,
+					},
+					{
+						$set: {
+							"tierlistInfo.dp-tier": {
+								text: d[difficulty]!.toString(),
+								value: d[difficulty]!,
+							},
+						},
+					}
+				);
+
+				if (result.nModified) {
+					updatedSongIDs.add(song.id);
+				}
+			}
+		})
+	);
+
+	if (updatedSongIDs.size !== 0) {
+		logger.info(
+			`${updatedSongIDs.size} songs were changed. Recalcing the relevant scores now.`
+		);
+
+		await RecalcAllScores({
+			game: "iidx",
+			playtype: "DP",
+			songID: { $in: [...updatedSongIDs.values()] },
+		});
+
+		logger.info(`Recalced those scores.`);
+	}
+
+	logger.info("Done.");
+}
+
+/**
+ * Parses a formatted string on the zasa website into a number. Returns null
+ * if the string is "-", which indicates nothing is present.
+ */
+function ParseTierStr(tierStr: string) {
+	if (tierStr === "-") {
+		return null;
+	}
+
+	const result = tierStr.match(/\((.*)\)$/u);
+
+	if (result && result[1]) {
+		return Number(result[1]);
+	}
+
+	throw new Error(`Can't parse tierStr ${tierStr}.`);
+}
+
+if (require.main === module) {
+	UpdateDPTiers().then(() => {
+		process.exit(0);
+	});
+}

server/src/lib/score-import/framework/calculated-data/calculated-data.ts

@@ -19,7 +19,8 @@ import {
 } from "tachi-common";
 import { DryScore } from "../common/types";
 import {
-	CalculateKTLampRatingIIDX,
+	CalculateKTLampRatingIIDXDP,
+	CalculateKTLampRatingIIDXSP,
 	CalculateKTRating,
 	CalculateMFCP,
 	CalculateSieglinde,
@@ -115,12 +116,14 @@ function CalculateDataIIDX(
 		bpi = null;
 	}
 
+	const ktLampRating =
+		chart.playtype === "SP"
+			? CalculateKTLampRatingIIDXSP(dryScore, chart as ChartDocument<"iidx:SP">)
+			: CalculateKTLampRatingIIDXDP(dryScore, chart as ChartDocument<"iidx:DP">);
+
 	return {
 		BPI: bpi,
-		ktLampRating: CalculateKTLampRatingIIDX(
-			dryScore,
-			chart as ChartDocument<"iidx:SP" | "iidx:DP">
-		),
+		ktLampRating,
 	};
 }
 

server/src/lib/score-import/framework/calculated-data/stats.test.ts

@@ -3,9 +3,9 @@ import { ChartDocument, Difficulties, Lamps, ScoreDocument } from "tachi-common"
 import t from "tap";
 import { Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data";
 import { DryScore } from "../common/types";
-import { CalculateKTLampRatingIIDX, CalculateMFCP } from "./stats";
+import { CalculateKTLampRatingIIDXDP, CalculateKTLampRatingIIDXSP, CalculateMFCP } from "./stats";
 
-t.test("#CalculateKTLampRatingIIDX", (t) => {
+t.test("#CalculateKTLampRatingIIDXSP", (t) => {
 	function c(
 		nc: number | null,
 		hc: number | null,
@@ -35,61 +35,61 @@ t.test("#CalculateKTLampRatingIIDX", (t) => {
 	}
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("CLEAR"), c(10.5, 10.6, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("CLEAR"), c(10.5, 10.6, 10.7)),
 		10.5,
 		"Should return NC if the score was NC."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("HARD CLEAR"), c(10.5, 10.6, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(10.5, 10.6, 10.7)),
 		10.6,
 		"Should return HC if the score was HC."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("EX HARD CLEAR"), c(10.5, 10.6, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("EX HARD CLEAR"), c(10.5, 10.6, 10.7)),
 		10.7,
 		"Should return EXHC if the score was EXHC."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("CLEAR"), c(null, 10.6, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("CLEAR"), c(null, 10.6, 10.7)),
 		0,
 		"Should return 0 if the score was NC but no NC was available."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("HARD CLEAR"), c(null, null, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(null, null, 10.7)),
 		0,
 		"Should return 0 if the score was HC but no HC or NC was available."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("HARD CLEAR"), c(10.5, null, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(10.5, null, 10.7)),
 		10.5,
 		"Should return NC if the score was HC but no HC was available."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("HARD CLEAR"), c(10.9, 10.5, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(10.9, 10.5, 10.7)),
 		10.9,
 		"Should return NC if the score was HC but NC was worth more."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("EX HARD CLEAR"), c(10.9, 10.5, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("EX HARD CLEAR"), c(10.9, 10.5, 10.7)),
 		10.9,
 		"Should return NC if the score was EXHC but NC was worth more."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("EX HARD CLEAR"), c(10.4, 10.9, 10.7)),
+		CalculateKTLampRatingIIDXSP(s("EX HARD CLEAR"), c(10.4, 10.9, 10.7)),
 		10.9,
 		"Should return HC if the score was EXHC but HC was worth more."
 	);
 
 	t.equal(
-		CalculateKTLampRatingIIDX(s("CLEAR"), c(null, null, null)),
+		CalculateKTLampRatingIIDXSP(s("CLEAR"), c(null, null, null)),
 		10,
 		"Should return chart level if the chart has no tierlist data."
 	);
@@ -97,6 +97,55 @@ t.test("#CalculateKTLampRatingIIDX", (t) => {
 	t.end();
 });
 
+t.test("#CalculateKTLampRatingIIDXDP", (t) => {
+	function c(dpTier: number | null): ChartDocument<"iidx:DP"> {
+		return Object.assign({}, Testing511SPA, {
+			playtype: "DP",
+			tierlistInfo: {
+				"dp-tier": {
+					value: dpTier,
+				},
+			},
+			// hack over-assert this type, since we're merging the SPA into being
+			// a DPA.
+		}) as unknown as ChartDocument<"iidx:DP">;
+	}
+
+	function s(lamp: Lamps["iidx:SP" | "iidx:DP"]): DryScore<"iidx:SP" | "iidx:DP"> {
+		return Object.assign({}, TestingIIDXSPDryScore, {
+			scoreData: {
+				lamp,
+			},
+		});
+	}
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("CLEAR"), c(10.4)),
+		10.4,
+		"Should return 10.4 if chart has tierlist value of 10.4 and was clear."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("FAILED"), c(10.4)),
+		0,
+		"Should return 0 if chart has tierlist value of 10.4 and was not cleared."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("CLEAR"), c(null)),
+		10,
+		"Should return chart level if the chart has no tierlist data."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("FAILED"), c(null)),
+		0,
+		"Should return 0 if the chart has no tierlist data and was failed."
+	);
+
+	t.end();
+});
+
 const logger = CreateLogCtx(__filename);
 
 t.test("#CalculateMFCP", (t) => {

server/src/lib/score-import/framework/calculated-data/stats.ts

@@ -144,10 +144,7 @@ export function CalculateKTRating(
 	return KTRatingCalcV1(dryScore.scoreData.percent, levelNum, parameters, logger);
 }
 
-export function CalculateKTLampRatingIIDX(
-	dryScore: DryScore,
-	chart: ChartDocument<"iidx:SP" | "iidx:DP">
-) {
+export function CalculateKTLampRatingIIDXSP(dryScore: DryScore, chart: ChartDocument<"iidx:SP">) {
 	const ncValue = chart.tierlistInfo["kt-NC"]?.value ?? 0;
 	const hcValue = Math.max(chart.tierlistInfo["kt-HC"]?.value ?? 0, ncValue);
 	const exhcValue = Math.max(chart.tierlistInfo["kt-EXHC"]?.value ?? 0, hcValue);
@@ -173,6 +170,26 @@ export function CalculateKTLampRatingIIDX(
 	return 0;
 }
 
+export function CalculateKTLampRatingIIDXDP(dryScore: DryScore, chart: ChartDocument<"iidx:DP">) {
+	const tierlistValue = chart.tierlistInfo["dp-tier"]?.value ?? 0;
+
+	if (!tierlistValue) {
+		return LampRatingNoTierlistInfo(dryScore, "iidx", chart.playtype, chart);
+	}
+
+	const lamp = dryScore.scoreData.lamp;
+
+	const gptConfig = GetGamePTConfig("iidx", chart.playtype);
+
+	const lampIndex = gptConfig.lamps.indexOf(lamp);
+
+	if (lampIndex >= IIDX_LAMPS.CLEAR) {
+		return tierlistValue;
+	}
+
+	return 0;
+}
+
 function LampRatingNoTierlistInfo(
 	dryScore: DryScore,
 	game: Game,