diff --git a/common/src/types.ts b/common/src/types.ts index dbdd32253..36f23c4b7 100644 --- a/common/src/types.ts +++ b/common/src/types.ts @@ -1212,7 +1212,9 @@ export type FileUploadImportTypes = | "file/eamusement-sdvx-csv" | "file/mer-iidx" | "file/pli-iidx-csv" - | "file/solid-state-squad"; + | "file/solid-state-squad" + | "file/mypagescraper-records-csv" + | "file/mypagescraper-player-csv" ; export type APIImportTypes = | "api/eag-iidx" diff --git a/server/src/lib/score-import/import-types/common/types.ts b/server/src/lib/score-import/import-types/common/types.ts index 2d9e04b49..72000e758 100644 --- a/server/src/lib/score-import/import-types/common/types.ts +++ b/server/src/lib/score-import/import-types/common/types.ts @@ -22,6 +22,7 @@ import type { import type { KaiContext } from "./api-kai/types"; import type { BatchManualContext } from "./batch-manual/types"; import type { IIDXEamusementCSVContext, IIDXEamusementCSVData } from "./eamusement-iidx-csv/types"; +import type { MyPageRecordsParsedPB } from "./types"; import type { KtLogger } from "lib/logger/logger"; import type { USCClientScore } from "server/router/ir/usc/_playtype/types"; import type { @@ -42,6 +43,8 @@ export interface ImportTypeDataMap { "file/solid-state-squad": S3Score; "file/mer-iidx": MerScore; "file/pli-iidx-csv": IIDXEamusementCSVData; + "file/mypagescraper-records-csv": MyPageRecordsParsedPB; + "file/mypagescraper-player-csv": never; "ir/direct-manual": BatchManualScore; "ir/barbatos": BarbatosScore | BarbatosSDVX6Score; @@ -81,6 +84,8 @@ export interface ImportTypeContextMap { "file/solid-state-squad": EmptyObject; "file/mer-iidx": EmptyObject; "file/pli-iidx-csv": IIDXEamusementCSVContext; + "file/mypagescraper-records-csv": EmptyObject; + "file/mypagescraper-player-csv": EmptyObject; "ir/direct-manual": BatchManualContext; "ir/barbatos": BarbatosContext; diff --git a/server/src/lib/score-import/import-types/converters.ts b/server/src/lib/score-import/import-types/converters.ts index 8d100954a..96513630c 100644 --- a/server/src/lib/score-import/import-types/converters.ts +++ b/server/src/lib/score-import/import-types/converters.ts @@ -8,6 +8,7 @@ import ConvertEamIIDXCSV from "./common/eamusement-iidx-csv/converter"; import ConvertEamSDVXCSV from "./file/eamusement-sdvx-csv/converter"; import { ConvertFileMerIIDX } from "./file/mer-iidx/converter"; import { ConvertFileS3 } from "./file/solid-state-squad/converter"; +import ConvertMyPageScraperRecordsCSV from "./file/wacca-mypage-scraper/converter"; import { ConverterIRBarbatos } from "./ir/barbatos/converter"; import { ConverterIRBeatoraja } from "./ir/beatoraja/converter"; import { ConverterIRFervidexStatic } from "./ir/fervidex-static/converter"; @@ -16,6 +17,7 @@ import { ConverterKsHookSV6CStatic } from "./ir/kshook-sv6c-static/converter"; import { ConverterIRKsHookSV6C } from "./ir/kshook-sv6c/converter"; import { ConverterLR2Hook } from "./ir/lr2hook/converter"; import { ConverterIRUSC } from "./ir/usc/converter"; +import { Never } from "utils/misc"; import type { ConverterFunction, ImportTypeContextMap, ImportTypeDataMap } from "./common/types"; import type { ImportTypes } from "tachi-common"; @@ -30,6 +32,8 @@ export const Converters: ConverterMap = { "file/pli-iidx-csv": ConvertEamIIDXCSV, "file/eamusement-iidx-csv": ConvertEamIIDXCSV, "file/eamusement-sdvx-csv": ConvertEamSDVXCSV, + "file/mypagescraper-records-csv": ConvertMyPageScraperRecordsCSV, + "file/mypagescraper-player-csv": Never, "api/eag-iidx": ConvertAPIKaiIIDX, "api/eag-sdvx": ConvertAPIKaiSDVX, diff --git a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.test.ts b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.test.ts index c91b62f6f..9acb8372d 100644 --- a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.test.ts +++ b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.test.ts @@ -1 +1,84 @@ import ConvertMyPageScraperRecordsCSV from "./converter"; +import deepmerge from "deepmerge"; +import CreateLogCtx from "lib/logger/logger"; +import t from "tap"; +import ResetDBState from "test-utils/resets"; +import { TestingWaccaPupaExp, TestingWaccaPupaSong } from "test-utils/test-data"; +import type { MyPageRecordsParsedPB } from "./types"; + +const logger = CreateLogCtx(__filename); + +const parsedScore: MyPageRecordsParsedPB = { + // 2085,PUPA,モリモリあつし,4,"[4,8,13+,0]","[0,0,12]","[0,0,996827]","[0,0,2]" + songId: 2085, + songTitle: "PUPA", + diffIndex: 2, + level: "13+", + score: 996827, + lamp: 2, +}; + +t.test("#ConvertMyPageScraperRecordsCSV", (t) => { + t.before(ResetDBState); + + function conv(g: Partial = {}) { + return ConvertMyPageScraperRecordsCSV( + deepmerge(parsedScore, g), + {}, + "file/mypagescraper-records-csv", + logger + ); + } + + t.test("Should return a dryScore on valid input.", async (t) => { + const res = await conv(); + + t.strictSame(res, { + song: TestingWaccaPupaSong, + chart: TestingWaccaPupaExp, + dryScore: { + service: "mypage-scraper", + game: "wacca", + scoreMeta: {}, + timeAchieved: null, + comment: null, + importType: "file/mypagescraper-records-csv", + scoreData: { + score: 996827, + lamp: "MISSLESS", + + percent: 99.6827, // floating point + grade: "SSS+", + judgements: {}, + hitMeta: {}, + }, + }, + }); + + t.end(); + }); + + t.test("Should reject out of bounds diffIndex", (t) => { + t.rejects(() => conv({ diffIndex: 4 }), { + message: /We somehow got an invalid difficulty index 4\./u, + }); + + t.end(); + }); + + t.test("Should throw KTDataNotFound on missing song", (t) => { + t.rejects(() => conv({ songTitle: "INVALID SONG" }), { + message: /Could not find song for INVALID SONG\./u, + }); + + t.end(); + }); + + t.test("Should reject incorrect level", (t) => { + t.rejects(() => conv({ level: "12" }), { + message: /PUPA \[EXPERT\] - Should be level 13\+, but found level 12\./u, + }); + + t.end(); + }); +}); diff --git a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.ts b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.ts index 68547569f..05292a622 100644 --- a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.ts +++ b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/converter.ts @@ -26,7 +26,7 @@ const ConvertMyPageScraperRecordsCSV: ConverterFunction< MyPageRecordsParsedPB, EmptyObject > = async (data, context, importType, logger) => { - // If we can verify that songId matches in-game (data) id, we should use that instead. + // TODO: If we can verify that songId matches in-game (data) id, we should use that instead. const song = await FindSongOnTitle("wacca", data.songTitle); if (song === null) { diff --git a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.test.ts b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.test.ts index 92dd75c1f..942f945b1 100644 --- a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.test.ts +++ b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.test.ts @@ -1,7 +1,7 @@ import { ParseMyPageScraperRecordsCsv, ParseMyPageScraperPlayerCsv } from "./parser"; +import { WACCA_STAGEUPS } from "lib/constants/classes"; import CreateLogCtx from "lib/logger/logger"; import ScoreImportFatalError from "lib/score-import/framework/score-importing/score-import-error"; -import { WACCA_STAGEUPS } from "lib/constants/classes"; import t from "tap"; import { MockMulterFile } from "test-utils/mock-multer"; import { TestingWaccaMyPageScraperRecordsCSV } from "test-utils/test-data"; @@ -29,7 +29,9 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => { }); t.test("Correctly parse CSV", (t) => { - const buffer = Buffer.from("music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,\"[3,7+,12+,0]\",\"[0,0,12]\",\"[0,0,996952]\",\"[0,0,3]\""); + const buffer = Buffer.from( + 'music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,"[3,7+,12+,0]","[0,0,12]","[0,0,996952]","[0,0,3]"' + ); const file = MockMulterFile(buffer, "records.csv"); @@ -37,27 +39,34 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => { t.equal(game, "wacca"); - t.strictSame(iterable, [{ - songId: 3080, - songTitle: "Avenue", - diffIndex: 2, - level: "12+", - score: 996952, - lamp: 3, - }]); + t.strictSame(iterable, [ + { + songId: 3080, + songTitle: "Avenue", + diffIndex: 2, + level: "12+", + score: 996952, + lamp: 3, + }, + ]); t.end(); }); t.test("Malformed CSV", (t) => { // Missing music_id in the record - const buffer = Buffer.from("music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\nAvenue,aran,6,\"[3,7+,12+,0]\",\"[0,0,12]\",\"[0,0,996952]\",\"[0,0,3]\""); + const buffer = Buffer.from( + 'music_id,music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\nAvenue,aran,6,"[3,7+,12+,0]","[0,0,12]","[0,0,996952]","[0,0,3]"' + ); const file = MockMulterFile(buffer, "records.csv"); t.throws( () => ParseMyPageScraperRecordsCsv(file, {}, logger), - new ScoreImportFatalError(400, "Failed to parse CSV: Invalid Record Length: columns length is 8, got 7 on line 2") + new ScoreImportFatalError( + 400, + "Failed to parse CSV: Invalid Record Length: columns length is 8, got 7 on line 2" + ) ); t.end(); @@ -65,13 +74,18 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => { t.test("CSV with wrong headers", (t) => { // not_music_title instead of music_title - const buffer = Buffer.from("music_id,not_music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,\"[3,7+,12+,0]\",\"[0,0,12]\",\"[0,0,996952]\",\"[0,0,3]\""); + const buffer = Buffer.from( + 'music_id,not_music_title,music_artist,music_genre,music_levels,music_play_counts,music_scores,music_achieves\n3080,Avenue,aran,6,"[3,7+,12+,0]","[0,0,12]","[0,0,996952]","[0,0,3]"' + ); const file = MockMulterFile(buffer, "records.csv"); t.throws( () => ParseMyPageScraperRecordsCsv(file, {}, logger), - new ScoreImportFatalError(400, "Malformed CSV, invalid column(s) (music_title: undefined): Expected string.") + new ScoreImportFatalError( + 400, + "Malformed CSV, invalid column(s) (music_title: undefined): Expected string." + ) ); t.end(); @@ -79,13 +93,18 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => { t.test("CSV with missing headers", (t) => { // missing music_achieves - const buffer = Buffer.from("music_id,music_title,music_levels,music_scores\n3080,Avenue,\"[3,7+,12+,0]\",\"[0,0,996952]\""); + const buffer = Buffer.from( + 'music_id,music_title,music_levels,music_scores\n3080,Avenue,"[3,7+,12+,0]","[0,0,996952]"' + ); const file = MockMulterFile(buffer, "records.csv"); t.throws( () => ParseMyPageScraperRecordsCsv(file, {}, logger), - new ScoreImportFatalError(400, "Malformed CSV, invalid column(s) (music_achieves: undefined): Expected string.") + new ScoreImportFatalError( + 400, + "Malformed CSV, invalid column(s) (music_achieves: undefined): Expected string." + ) ); t.end(); @@ -97,8 +116,9 @@ t.test("#ParseMyPageScraperRecordsCSV", (t) => { t.test("#ParseMyPageScraperPlayerCSV", (t) => { t.test("Valid CSV", (t) => { // This file is small so we just inline it. - const buffer = Buffer.from("player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,\"[12,ステージXII,2]\",1274,57,0,2088515,531325"); - + const buffer = Buffer.from( + 'player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,"[12,ステージXII,2]",1274,57,0,2088515,531325' + ); const file = MockMulterFile(buffer, "player.csv"); @@ -112,20 +132,27 @@ t.test("#ParseMyPageScraperPlayerCSV", (t) => { // There's no good way to test that the classHandler got a valid // MyPagePlayerStage, so we just call it to see. - t.strictSame(classHandler!("wacca", "Single", 0, {}, logger), {stageUp: WACCA_STAGEUPS.XII}); + t.strictSame(classHandler!("wacca", "Single", 0, {}, logger), { + stageUp: WACCA_STAGEUPS.XII, + }); t.end(); }); t.test("Malformed CSV", (t) => { // Missing player_level in record - const buffer = Buffer.from("player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,2704,\"[12,ステージXII,2]\",1274,57,0,2088515,531325"); + const buffer = Buffer.from( + 'player_name,player_level,player_rate,player_stage,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,2704,"[12,ステージXII,2]",1274,57,0,2088515,531325' + ); const file = MockMulterFile(buffer, "player.csv"); t.throws( () => ParseMyPageScraperPlayerCsv(file, {}, logger), - new ScoreImportFatalError(400, "Failed to parse CSV: Invalid Record Length: columns length is 9, got 8 on line 2") + new ScoreImportFatalError( + 400, + "Failed to parse CSV: Invalid Record Length: columns length is 9, got 8 on line 2" + ) ); t.end(); @@ -133,7 +160,9 @@ t.test("#ParseMyPageScraperPlayerCSV", (t) => { t.test("CSV missing player_stage", (t) => { // This is the only value we actually care about. - const buffer = Buffer.from("player_name,player_level,player_rate,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,1274,57,0,2088515,531325"); + const buffer = Buffer.from( + "player_name,player_level,player_rate,player_play_count,player_play_count_versus,player_play_count_coop,player_total_rp_earned,player_total_rp_spent\ncg505,120,2704,1274,57,0,2088515,531325" + ); const file = MockMulterFile(buffer, "player.csv"); @@ -147,7 +176,7 @@ t.test("#ParseMyPageScraperPlayerCSV", (t) => { t.test("malformed player_stage array", (t) => { // Missing the stage grade (third element) - const buffer = Buffer.from("player_stage\n\"[12,ステージXII]\""); + const buffer = Buffer.from('player_stage\n"[12,ステージXII]"'); const file = MockMulterFile(buffer, "player.csv"); diff --git a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.ts b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.ts index bdbc34cf7..17d0fe298 100644 --- a/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.ts +++ b/server/src/lib/score-import/import-types/file/wacca-mypage-scraper/parser.ts @@ -66,7 +66,10 @@ export function ParseMyPageScraperRecordsCsv( if (err) { // TODO: add some context so we don't get things like just "Expected string." - throw new ScoreImportFatalError(400, `Malformed CSV, invalid column(s) (${err.keychain}: ${err.userVal}): ${err.message}`); + throw new ScoreImportFatalError( + 400, + `Malformed CSV, invalid column(s) (${err.keychain}: ${err.userVal}): ${err.message}` + ); } } diff --git a/server/src/lib/score-import/import-types/parsers.ts b/server/src/lib/score-import/import-types/parsers.ts index 54fd2fce7..4992ba6fc 100644 --- a/server/src/lib/score-import/import-types/parsers.ts +++ b/server/src/lib/score-import/import-types/parsers.ts @@ -19,6 +19,10 @@ import ParseEamusementSDVXCSV from "./file/eamusement-sdvx-csv/parser"; import { ParseMerIIDX } from "./file/mer-iidx/parser"; import ParsePLIIIDXCSV from "./file/pli-iidx-csv/parser"; import { ParseSolidStateXML } from "./file/solid-state-squad/parser"; +import { + ParseMyPageScraperRecordsCsv, + ParseMyPageScraperPlayerCsv, +} from "./file/wacca-mypage-scraper/parser"; import { ParseBarbatosSingle } from "./ir/barbatos/parser"; import { ParseBeatorajaSingle } from "./ir/beatoraja/parser"; import ParseDirectManual from "./ir/direct-manual/parser"; @@ -36,6 +40,8 @@ export const Parsers = { "file/pli-iidx-csv": ParsePLIIIDXCSV, "file/eamusement-iidx-csv": ParseEamusementIIDXCSV, "file/eamusement-sdvx-csv": ParseEamusementSDVXCSV, + "file/mypagescraper-records-csv": ParseMyPageScraperRecordsCsv, + "file/mypagescraper-player-csv": ParseMyPageScraperPlayerCsv, "api/eag-iidx": ParseEagIIDX, "api/eag-sdvx": ParseEagSDVX, diff --git a/server/src/test-utils/mock-db/charts-wacca.json b/server/src/test-utils/mock-db/charts-wacca.json index 0637a088a..8ca9fb07b 100644 --- a/server/src/test-utils/mock-db/charts-wacca.json +++ b/server/src/test-utils/mock-db/charts-wacca.json @@ -1 +1,19 @@ -[] \ No newline at end of file +[ + { + "chartID": "2abcf2da86f3d968dbe07695ab7ebf6b79cb0841", + "data": { + "isHot": false + }, + "difficulty": "EXPERT", + "isPrimary": true, + "level": "13+", + "levelNum": 13.7, + "playtype": "Single", + "rgcID": null, + "songID": 77, + "tierlistInfo": {}, + "versions": [ + "reverse" + ] + } +] diff --git a/server/src/test-utils/mock-db/songs-wacca.json b/server/src/test-utils/mock-db/songs-wacca.json index 0637a088a..f3807753c 100644 --- a/server/src/test-utils/mock-db/songs-wacca.json +++ b/server/src/test-utils/mock-db/songs-wacca.json @@ -1 +1,15 @@ -[] \ No newline at end of file +[ + { + "altTitles": [], + "artist": "モリモリあつし", + "data": { + "artistJP": "モリモリアツシ", + "displayVersion": "lilyr", + "genre": "バラエティ", + "titleJP": "ピューパ" + }, + "id": 77, + "searchTerms": [], + "title": "PUPA" + } +] diff --git a/server/src/test-utils/test-data.ts b/server/src/test-utils/test-data.ts index 52b3a7958..9473d85c7 100644 --- a/server/src/test-utils/test-data.ts +++ b/server/src/test-utils/test-data.ts @@ -415,6 +415,17 @@ export const TestingAlbidaADV: ChartDocument<"sdvx:Single"> = { tierlistInfo: {}, }; +export const TestingSDVXAlbidaSong: SongDocument<"sdvx"> = { + title: "ALBIDA Powerless Mix", + artist: "無力P", + id: 1, + altTitles: [], + searchTerms: ["albida_muryoku", "アルビダパワーレスミックス"], + data: { + displayVersion: "booth", + }, +}; + export const BMSGazerChart: ChartDocument<"bms:7K"> = { songID: 27339, chartID: "88eb6cc5683e2740cbd07f588a5f3db1db8d467b", @@ -470,17 +481,6 @@ export const CHUNITHMBBKKChart: ChartDocument<"chunithm:Single"> = { versions: ["paradiselost"], }; -export const TestingSDVXAlbidaSong: SongDocument<"sdvx"> = { - title: "ALBIDA Powerless Mix", - artist: "無力P", - id: 1, - altTitles: [], - searchTerms: ["albida_muryoku", "アルビダパワーレスミックス"], - data: { - displayVersion: "booth", - }, -}; - export const TestingDoraChart: ChartDocument<"gitadora:Dora"> = { songID: 0, chartID: "29f0bfab357ba54e3fd0176fb3cbc578c9ec8df5", @@ -497,6 +497,36 @@ export const TestingDoraChart: ChartDocument<"gitadora:Dora"> = { versions: ["konaste"], }; +export const TestingWaccaPupaExp: ChartDocument<"wacca:Single"> = { + songID: 77, + chartID: "2abcf2da86f3d968dbe07695ab7ebf6b79cb0841", + difficulty: "EXPERT", + playtype: "Single", + levelNum: 13.7, + level: "13+", + data: { + isHot: false, + }, + isPrimary: true, + rgcID: null, + tierlistInfo: {}, + versions: ["reverse"], +}; + +export const TestingWaccaPupaSong: SongDocument<"wacca"> = { + id: 77, + title: "PUPA", + artist: "モリモリあつし", + data: { + artistJP: "モリモリアツシ", + displayVersion: "lilyr", + genre: "バラエティ", + titleJP: "ピューパ", + }, + searchTerms: [], + altTitles: [], +}; + export const TestingSDVXSingleDryScore: DryScore<"sdvx:Single"> = { service: "e-amusement", game: "sdvx", diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 77a6d6bb6..d5bb5f194 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -352,3 +352,11 @@ export function AddToSetInRecord( obj[key] = new Set([toAdd]); } } + +// This function can be used in places where we need a function that takes never +// as a parameter (for instance, an iterator over Array). Such a function +// cannot ever be called. See lib/score-import/import-types/converters.ts for an +// example. +export function Never(_: never): never { + throw new Error("Unreachable"); +}