add converter tests

common/src/types.ts

@@ -1212,7 +1212,9 @@ export type FileUploadImportTypes =
 	| "file/eamusement-sdvx-csv"
 	| "file/mer-iidx"
 	| "file/pli-iidx-csv"
-	| "file/solid-state-squad";
+	| "file/solid-state-squad"
+	| "file/mypagescraper-records-csv"
+	| "file/mypagescraper-player-csv" ;
 
 export type APIImportTypes =
 	| "api/eag-iidx"

server/src/lib/score-import/import-types/common/types.ts

@@ -22,6 +22,7 @@ import type {
 import type { KaiContext } from "./api-kai/types";
 import type { BatchManualContext } from "./batch-manual/types";
 import type { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./eamusement-iidx-csv/types";
+import type { MyPageRecordsParsedPB } from "./types";
 import type { KtLogger } from "lib/logger/logger";
 import type { USCClientScore } from "server/router/ir/usc/_playtype/types";
 import type {
@@ -42,6 +43,8 @@ export interface ImportTypeDataMap {
 	"file/solid-state-squad": S3Score;
 	"file/mer-iidx": MerScore;
 	"file/pli-iidx-csv": IIDXEamusementCSVData;
+	"file/mypagescraper-records-csv": MyPageRecordsParsedPB;
+	"file/mypagescraper-player-csv": never;
 
 	"ir/direct-manual": BatchManualScore;
 	"ir/barbatos": BarbatosScore | BarbatosSDVX6Score;
@@ -81,6 +84,8 @@ export interface ImportTypeContextMap {
 	"file/solid-state-squad": EmptyObject;
 	"file/mer-iidx": EmptyObject;
 	"file/pli-iidx-csv": IIDXEamusementCSVContext;
+	"file/mypagescraper-records-csv": EmptyObject;
+	"file/mypagescraper-player-csv": EmptyObject;
 
 	"ir/direct-manual": BatchManualContext;
 	"ir/barbatos": BarbatosContext;

server/src/lib/score-import/import-types/converters.ts

@@ -8,6 +8,7 @@ import ConvertEamIIDXCSV from "./common/eamusement-iidx-csv/converter";
 import ConvertEamSDVXCSV from "./file/eamusement-sdvx-csv/converter";
 import { ConvertFileMerIIDX } from "./file/mer-iidx/converter";
 import { ConvertFileS3 } from "./file/solid-state-squad/converter";
+import ConvertMyPageScraperRecordsCSV from "./file/wacca-mypage-scraper/converter";
 import { ConverterIRBarbatos } from "./ir/barbatos/converter";
 import { ConverterIRBeatoraja } from "./ir/beatoraja/converter";
 import { ConverterIRFervidexStatic } from "./ir/fervidex-static/converter";
@@ -16,6 +17,7 @@ import { ConverterKsHookSV6CStatic } from "./ir/kshook-sv6c-static/converter";
 import { ConverterIRKsHookSV6C } from "./ir/kshook-sv6c/converter";
 import { ConverterLR2Hook } from "./ir/lr2hook/converter";
 import { ConverterIRUSC } from "./ir/usc/converter";
+import { Never } from "utils/misc";
 import type { ConverterFunction, ImportTypeContextMap, ImportTypeDataMap } from "./common/types";
 import type { ImportTypes } from "tachi-common";
 
@@ -30,6 +32,8 @@ export const Converters: ConverterMap = {
 	"file/pli-iidx-csv": ConvertEamIIDXCSV,
 	"file/eamusement-iidx-csv": ConvertEamIIDXCSV,
 	"file/eamusement-sdvx-csv": ConvertEamSDVXCSV,
+	"file/mypagescraper-records-csv": ConvertMyPageScraperRecordsCSV,
+	"file/mypagescraper-player-csv": Never,
 
 	"api/eag-iidx": ConvertAPIKaiIIDX,
 	"api/eag-sdvx": ConvertAPIKaiSDVX,

server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.test.ts

@@ -1 +1,84 @@
 import ConvertMyPageScraperRecordsCSV from "./converter";
+import deepmerge from "deepmerge";
+import CreateLogCtx from "lib/logger/logger";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+import { TestingWaccaPupaExp, TestingWaccaPupaSong } from "test-utils/test-data";
+import type { MyPageRecordsParsedPB } from "./types";
+
+const logger = CreateLogCtx(__filename);
+
+const parsedScore: MyPageRecordsParsedPB = {
+	// 2085,PUPA,モリモリあつし,4,"[4,8,13+,0]","[0,0,12]","[0,0,996827]","[0,0,2]"
+	songId: 2085,
+	songTitle: "PUPA",
+	diffIndex: 2,
+	level: "13+",
+	score: 996827,
+	lamp: 2,
+};
+
+t.test("#ConvertMyPageScraperRecordsCSV", (t) => {
+	t.before(ResetDBState);
+
+	function conv(g: Partial<MyPageRecordsParsedPB> = {}) {
+		return ConvertMyPageScraperRecordsCSV(
+			deepmerge(parsedScore, g),
+			{},
+			"file/mypagescraper-records-csv",
+			logger
+		);
+	}
+
+	t.test("Should return a dryScore on valid input.", async (t) => {
+		const res = await conv();
+
+		t.strictSame(res, {
+			song: TestingWaccaPupaSong,
+			chart: TestingWaccaPupaExp,
+			dryScore: {
+				service: "mypage-scraper",
+				game: "wacca",
+				scoreMeta: {},
+				timeAchieved: null,
+				comment: null,
+				importType: "file/mypagescraper-records-csv",
+				scoreData: {
+					score: 996827,
+					lamp: "MISSLESS",
+
+					percent: 99.6827, // floating point
+					grade: "SSS+",
+					judgements: {},
+					hitMeta: {},
+				},
+			},
+		});
+
+		t.end();
+	});
+
+	t.test("Should reject out of bounds diffIndex", (t) => {
+		t.rejects(() => conv({ diffIndex: 4 }), {
+			message: /We somehow got an invalid difficulty index 4\./u,
+		});
+
+		t.end();
+	});
+
+	t.test("Should throw KTDataNotFound on missing song", (t) => {
+		t.rejects(() => conv({ songTitle: "INVALID SONG" }), {
+			message: /Could not find song for INVALID SONG\./u,
+		});
+
+		t.end();
+	});
+
+	t.test("Should reject incorrect level", (t) => {
+		t.rejects(() => conv({ level: "12" }), {
+			message: /PUPA \[EXPERT\] - Should be level 13\+, but found level 12\./u,
+		});
+
+		t.end();
+	});
+});

server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.ts

@@ -26,7 +26,7 @@ const ConvertMyPageScraperRecordsCSV: ConverterFunction<
 	MyPageRecordsParsedPB,
 	EmptyObject
 > = async (data, context, importType, logger) => {
-	// If we can verify that songId matches in-game (data) id, we should use that instead.
+	// TODO: If we can verify that songId matches in-game (data) id, we should use that instead.
 	const song = await FindSongOnTitle("wacca", data.songTitle);
 
 	if (song === null) {

server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.test.ts

@@ -1,7 +1,7 @@
 import { ParseMyPageScraperRecordsCsv, ParseMyPageScraperPlayerCsv } from "./parser";
+import { WACCA_STAGEUPS } from "lib/constants/classes";
 import CreateLogCtx from "lib/logger/logger";
 import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error";
-import { WACCA_STAGEUPS } from "lib/constants/classes";
 import t from "tap";
 import { MockMulterFile } from "test-utils/mock-multer";
 import { TestingWaccaMyPageScraperRecordsCSV } from "test-utils/test-data";
@@ -29,7 +29,9 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => {
 	});
 
 	t.test("Correctly parse CSV", (t) => {
-		const buffer = Buffer.from("music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,\"[3,7+,12+,0]\",\"[0,0,12]\",\"[0,0,996952]\",\"[0,0,3]\"");
+		const buffer = Buffer.from(
+			'music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,"[3,7+,12+,0]","[0,0,12]","[0,0,996952]","[0,0,3]"'
+		);
 
 		const file = MockMulterFile(buffer, "records.csv");
 
@@ -37,27 +39,34 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => {
 
 		t.equal(game, "wacca");
 
-		t.strictSame(iterable, [{
-			songId: 3080,
-			songTitle: "Avenue",
-			diffIndex: 2,
-			level: "12+",
-			score: 996952,
-			lamp: 3,
-		}]);
+		t.strictSame(iterable, [
+			{
+				songId: 3080,
+				songTitle: "Avenue",
+				diffIndex: 2,
+				level: "12+",
+				score: 996952,
+				lamp: 3,
+			},
+		]);
 
 		t.end();
 	});
 
 	t.test("Malformed CSV", (t) => {
 		// Missing music_id in the record
-		const buffer = Buffer.from("music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\nAvenue,aran,6,\"[3,7+,12+,0]\",\"[0,0,12]\",\"[0,0,996952]\",\"[0,0,3]\"");
+		const buffer = Buffer.from(
+			'music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\nAvenue,aran,6,"[3,7+,12+,0]","[0,0,12]","[0,0,996952]","[0,0,3]"'
+		);
 
 		const file = MockMulterFile(buffer, "records.csv");
 
 		t.throws(
 			() => ParseMyPageScraperRecordsCsv(file, {}, logger),
-			new ScoreImportFatalError(400, "Failed to parse CSV: Invalid Record Length: columns length is 8, got 7 on line 2")
+			new ScoreImportFatalError(
+				400,
+				"Failed to parse CSV: Invalid Record Length: columns length is 8, got 7 on line 2"
+			)
 		);
 
 		t.end();
@@ -65,13 +74,18 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => {
 
 	t.test("CSV with wrong headers", (t) => {
 		// not_music_title instead of music_title
-		const buffer = Buffer.from("music_id,not_music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,\"[3,7+,12+,0]\",\"[0,0,12]\",\"[0,0,996952]\",\"[0,0,3]\"");
+		const buffer = Buffer.from(
+			'music_id,not_music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,"[3,7+,12+,0]","[0,0,12]","[0,0,996952]","[0,0,3]"'
+		);
 
 		const file = MockMulterFile(buffer, "records.csv");
 
 		t.throws(
 			() => ParseMyPageScraperRecordsCsv(file, {}, logger),
-			new ScoreImportFatalError(400, "Malformed CSV, invalid column(s) (music_title: undefined): Expected string.")
+			new ScoreImportFatalError(
+				400,
+				"Malformed CSV, invalid column(s) (music_title: undefined): Expected string."
+			)
 		);
 
 		t.end();
@@ -79,13 +93,18 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => {
 
 	t.test("CSV with missing headers", (t) => {
 		// missing music_achieves
-		const buffer = Buffer.from("music_id,music_title,music_levels,music_scores\n3080,Avenue,\"[3,7+,12+,0]\",\"[0,0,996952]\"");
+		const buffer = Buffer.from(
+			'music_id,music_title,music_levels,music_scores\n3080,Avenue,"[3,7+,12+,0]","[0,0,996952]"'
+		);
 
 		const file = MockMulterFile(buffer, "records.csv");
 
 		t.throws(
 			() => ParseMyPageScraperRecordsCsv(file, {}, logger),
-			new ScoreImportFatalError(400, "Malformed CSV, invalid column(s) (music_achieves: undefined): Expected string.")
+			new ScoreImportFatalError(
+				400,
+				"Malformed CSV, invalid column(s) (music_achieves: undefined): Expected string."
+			)
 		);
 
 		t.end();
@@ -97,8 +116,9 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => {
 t.test("#ParseMyPageScraperPlayerCSV", (t) => {
 	t.test("Valid CSV", (t) => {
 		// This file is small so we just inline it.
-		const buffer = Buffer.from("player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,\"[12,ステージXII,2]\",1274,57,0,2088515,531325");
-
+		const buffer = Buffer.from(
+			'player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,"[12,ステージXII,2]",1274,57,0,2088515,531325'
+		);
 
 		const file = MockMulterFile(buffer, "player.csv");
 
@@ -112,20 +132,27 @@ t.test("#ParseMyPageScraperPlayerCSV", (t) => {
 
 		// There's no good way to test that the classHandler got a valid
 		// MyPagePlayerStage, so we just call it to see.
-		t.strictSame(classHandler!("wacca", "Single", 0, {}, logger), {stageUp: WACCA_STAGEUPS.XII});
+		t.strictSame(classHandler!("wacca", "Single", 0, {}, logger), {
+			stageUp: WACCA_STAGEUPS.XII,
+		});
 
 		t.end();
 	});
 
 	t.test("Malformed CSV", (t) => {
 		// Missing player_level in record
-		const buffer = Buffer.from("player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,2704,\"[12,ステージXII,2]\",1274,57,0,2088515,531325");
+		const buffer = Buffer.from(
+			'player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,2704,"[12,ステージXII,2]",1274,57,0,2088515,531325'
+		);
 
 		const file = MockMulterFile(buffer, "player.csv");
 
 		t.throws(
 			() => ParseMyPageScraperPlayerCsv(file, {}, logger),
-			new ScoreImportFatalError(400, "Failed to parse CSV: Invalid Record Length: columns length is 9, got 8 on line 2")
+			new ScoreImportFatalError(
+				400,
+				"Failed to parse CSV: Invalid Record Length: columns length is 9, got 8 on line 2"
+			)
 		);
 
 		t.end();
@@ -133,7 +160,9 @@ t.test("#ParseMyPageScraperPlayerCSV", (t) => {
 
 	t.test("CSV missing player_stage", (t) => {
 		// This is the only value we actually care about.
-		const buffer = Buffer.from("player_name,player_level,player_rate,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,1274,57,0,2088515,531325");
+		const buffer = Buffer.from(
+			"player_name,player_level,player_rate,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,1274,57,0,2088515,531325"
+		);
 
 		const file = MockMulterFile(buffer, "player.csv");
 
@@ -147,7 +176,7 @@ t.test("#ParseMyPageScraperPlayerCSV", (t) => {
 
 	t.test("malformed player_stage array", (t) => {
 		// Missing the stage grade (third element)
-		const buffer = Buffer.from("player_stage\n\"[12,ステージXII]\"");
+		const buffer = Buffer.from('player_stage\n"[12,ステージXII]"');
 
 		const file = MockMulterFile(buffer, "player.csv");
 

server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.ts

@@ -66,7 +66,10 @@ export function ParseMyPageScraperRecordsCsv(
 
 		if (err) {
 			// TODO: add some context so we don't get things like just "Expected string."
-			throw new ScoreImportFatalError(400, `Malformed CSV, invalid column(s) (${err.keychain}: ${err.userVal}): ${err.message}`);
+			throw new ScoreImportFatalError(
+				400,
+				`Malformed CSV, invalid column(s) (${err.keychain}: ${err.userVal}): ${err.message}`
+			);
 		}
 	}
 

server/src/lib/score-import/import-types/parsers.ts

@@ -19,6 +19,10 @@ import ParseEamusementSDVXCSV from "./file/eamusement-sdvx-csv/parser";
 import { ParseMerIIDX } from "./file/mer-iidx/parser";
 import ParsePLIIIDXCSV from "./file/pli-iidx-csv/parser";
 import { ParseSolidStateXML } from "./file/solid-state-squad/parser";
+import {
+	ParseMyPageScraperRecordsCsv,
+	ParseMyPageScraperPlayerCsv,
+} from "./file/wacca-mypage-scraper/parser";
 import { ParseBarbatosSingle } from "./ir/barbatos/parser";
 import { ParseBeatorajaSingle } from "./ir/beatoraja/parser";
 import ParseDirectManual from "./ir/direct-manual/parser";
@@ -36,6 +40,8 @@ export const Parsers = {
 	"file/pli-iidx-csv": ParsePLIIIDXCSV,
 	"file/eamusement-iidx-csv": ParseEamusementIIDXCSV,
 	"file/eamusement-sdvx-csv": ParseEamusementSDVXCSV,
+	"file/mypagescraper-records-csv": ParseMyPageScraperRecordsCsv,
+	"file/mypagescraper-player-csv": ParseMyPageScraperPlayerCsv,
 
 	"api/eag-iidx": ParseEagIIDX,
 	"api/eag-sdvx": ParseEagSDVX,

server/src/test-utils/mock-db/charts-wacca.json

@@ -1 +1,19 @@
-[]
+[
+	{
+		"chartID": "2abcf2da86f3d968dbe07695ab7ebf6b79cb0841",
+		"data": {
+			"isHot": false
+		},
+		"difficulty": "EXPERT",
+		"isPrimary": true,
+		"level": "13+",
+		"levelNum": 13.7,
+		"playtype": "Single",
+		"rgcID": null,
+		"songID": 77,
+		"tierlistInfo": {},
+		"versions": [
+			"reverse"
+		]
+	}
+]

server/src/test-utils/mock-db/songs-wacca.json

@@ -1 +1,15 @@
-[]
+[
+	{
+		"altTitles": [],
+		"artist": "モリモリあつし",
+		"data": {
+			"artistJP": "モリモリアツシ",
+			"displayVersion": "lilyr",
+			"genre": "バラエティ",
+			"titleJP": "ピューパ"
+		},
+		"id": 77,
+		"searchTerms": [],
+		"title": "PUPA"
+	}
+]

server/src/test-utils/test-data.ts

@@ -415,6 +415,17 @@ export const TestingAlbidaADV: ChartDocument<"sdvx:Single"> = {
 	tierlistInfo: {},
 };
 
+export const TestingSDVXAlbidaSong: SongDocument<"sdvx"> = {
+	title: "ALBIDA Powerless Mix",
+	artist: "無力P",
+	id: 1,
+	altTitles: [],
+	searchTerms: ["albida_muryoku", "ｱﾙﾋﾞﾀﾞﾊﾟﾜｰﾚｽﾐｯｸｽ"],
+	data: {
+		displayVersion: "booth",
+	},
+};
+
 export const BMSGazerChart: ChartDocument<"bms:7K"> = {
 	songID: 27339,
 	chartID: "88eb6cc5683e2740cbd07f588a5f3db1db8d467b",
@@ -470,17 +481,6 @@ export const CHUNITHMBBKKChart: ChartDocument<"chunithm:Single"> = {
 	versions: ["paradiselost"],
 };
 
-export const TestingSDVXAlbidaSong: SongDocument<"sdvx"> = {
-	title: "ALBIDA Powerless Mix",
-	artist: "無力P",
-	id: 1,
-	altTitles: [],
-	searchTerms: ["albida_muryoku", "ｱﾙﾋﾞﾀﾞﾊﾟﾜｰﾚｽﾐｯｸｽ"],
-	data: {
-		displayVersion: "booth",
-	},
-};
-
 export const TestingDoraChart: ChartDocument<"gitadora:Dora"> = {
 	songID: 0,
 	chartID: "29f0bfab357ba54e3fd0176fb3cbc578c9ec8df5",
@@ -497,6 +497,36 @@ export const TestingDoraChart: ChartDocument<"gitadora:Dora"> = {
 	versions: ["konaste"],
 };
 
+export const TestingWaccaPupaExp: ChartDocument<"wacca:Single"> = {
+	songID: 77,
+	chartID: "2abcf2da86f3d968dbe07695ab7ebf6b79cb0841",
+	difficulty: "EXPERT",
+	playtype: "Single",
+	levelNum: 13.7,
+	level: "13+",
+	data: {
+		isHot: false,
+	},
+	isPrimary: true,
+	rgcID: null,
+	tierlistInfo: {},
+	versions: ["reverse"],
+};
+
+export const TestingWaccaPupaSong: SongDocument<"wacca"> = {
+	id: 77,
+	title: "PUPA",
+	artist: "モリモリあつし",
+	data: {
+		artistJP: "モリモリアツシ",
+		displayVersion: "lilyr",
+		genre: "バラエティ",
+		titleJP: "ピューパ",
+	},
+	searchTerms: [],
+	altTitles: [],
+};
+
 export const TestingSDVXSingleDryScore: DryScore<"sdvx:Single"> = {
 	service: "e-amusement",
 	game: "sdvx",

server/src/utils/misc.ts

@@ -352,3 +352,11 @@ export function AddToSetInRecord<T extends string, V>(
 		obj[key] = new Set([toAdd]);
 	}
 }
+
+// This function can be used in places where we need a function that takes never
+// as a parameter (for instance, an iterator over Array<never>). Such a function
+// cannot ever be called. See lib/score-import/import-types/converters.ts for an
+// example.
+export function Never(_: never): never {
+	throw new Error("Unreachable");
+}