diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 83e0efe65..a69248c12 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -62,7 +62,7 @@ dependencies: express-async-errors: 3.1.1_express@4.17.1 express-session: 1.17.1 fast-json-stable-hash: 1.0.2 - kamaitachi-common: github.com/zkldi/Kamaitachi-common/a3a77f6c5e8c870a01652c92515dfe7f8030863d + kamaitachi-common: github.com/zkldi/Kamaitachi-common/d00963e4f8d40089e98be1b52e03b6f4a42d6a1a mongodb: 3.6.6 monk: 7.3.4 multer: 1.4.2 @@ -4638,8 +4638,8 @@ packages: '@types/yoga-layout': 1.9.2 dev: true - github.com/zkldi/Kamaitachi-common/a3a77f6c5e8c870a01652c92515dfe7f8030863d: - resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/a3a77f6c5e8c870a01652c92515dfe7f8030863d} + github.com/zkldi/Kamaitachi-common/d00963e4f8d40089e98be1b52e03b6f4a42d6a1a: + resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/d00963e4f8d40089e98be1b52e03b6f4a42d6a1a} name: kamaitachi-common version: 0.1.0 dependencies: diff --git a/server/scripts/init-folder-cache/folder-cache.ts b/server/scripts/init-folder-cache/folder-cache.ts index f6f348199..09684ebb9 100644 --- a/server/scripts/init-folder-cache/folder-cache.ts +++ b/server/scripts/init-folder-cache/folder-cache.ts @@ -1,24 +1,3 @@ -import db from "../../src/db/db"; -import CreateLogCtx from "../../src/logger"; -import { CreateFolderChartLookup } from "../../src/core/folder-core"; - -const logger = CreateLogCtx("folder-cache.ts"); - -/** - * Creates the "folder-chart-lookup" cache. This is used to optimise - * common use cases, such as retrieving chartIDs from a folder. - */ -export async function InitaliseFolderChartLookup() { - logger.info(`Started InitialiseFolderChartLookup`); - await db["folder-chart-lookup"].remove({}); - logger.info(`Flushed Cache.`); - - let folders = await db.folders.find({}); - logger.info(`Reloading ${folders.length} folders.`); - - await Promise.all(folders.map(CreateFolderChartLookup)); - - logger.info(`Completed InitialiseFolderChartLookup.`); -} +import { InitaliseFolderChartLookup } from "../../src/core/folder-core"; InitaliseFolderChartLookup(); diff --git a/server/src/core/folder-core.test.ts b/server/src/core/folder-core.test.ts index 704c7a6ec..e8e2a0c5c 100644 --- a/server/src/core/folder-core.test.ts +++ b/server/src/core/folder-core.test.ts @@ -14,6 +14,9 @@ import deepmerge from "deepmerge"; const logger = CreateLogCtx("folder-core.test.ts"); +t.todo("#CreateFolderChartLookup"); +t.todo("#GetFolderCharts"); + t.test("#ResolveFolderToCharts", (t) => { t.beforeEach(ResetDBState); diff --git a/server/src/core/folder-core.ts b/server/src/core/folder-core.ts index e3d2e315d..eec997027 100644 --- a/server/src/core/folder-core.ts +++ b/server/src/core/folder-core.ts @@ -43,7 +43,14 @@ export async function ResolveFolderToCharts( }) ); } else if (folder.type === "charts") { - let fx = deepmerge.all([filter, { playtype: folder.playtype }, folder.data]); + let folderDataTransposed: Record = {}; + + for (const key in folder.data) { + folderDataTransposed[key.replace(/¬/gu, ".")] = folder.data[key]; + } + + let fx = deepmerge.all([filter, { playtype: folder.playtype }, folderDataTransposed]); + charts = await db.charts[folder.game].find(fx); } else { // @ts-expect-error This is already a weird scenario. Shouldn't fail, though. @@ -114,7 +121,6 @@ export async function GetFolderChartIDs(folderID: string) { return chartIDs.map((e) => e.chartID); } - export async function CreateFolderChartLookup(folder: FolderDocument) { let { charts } = await ResolveFolderToCharts(folder, {}, false); @@ -125,3 +131,20 @@ export async function CreateFolderChartLookup(folder: FolderDocument) { })) ); } + +/** + * Creates the "folder-chart-lookup" cache. This is used to optimise + * common use cases, such as retrieving chartIDs from a folder. + */ +export async function InitaliseFolderChartLookup() { + logger.info(`Started InitialiseFolderChartLookup`); + await db["folder-chart-lookup"].remove({}); + logger.info(`Flushed Cache.`); + + let folders = await db.folders.find({}); + logger.info(`Reloading ${folders.length} folders.`); + + await Promise.all(folders.map(CreateFolderChartLookup)); + + logger.info(`Completed InitialiseFolderChartLookup.`); +} diff --git a/server/src/core/goal-core.ts b/server/src/core/goal-core.ts index 9cfc6116c..7abfd3aac 100644 --- a/server/src/core/goal-core.ts +++ b/server/src/core/goal-core.ts @@ -1,5 +1,6 @@ // idk what needs to go here just yet - need to write the goal support in score import - zkldi -import { GoalDocument, integer, PBScoreDocument } from "kamaitachi-common"; +import { GoalDocument, integer, PBScoreDocument, Game } from "kamaitachi-common"; +import { grades, lamps } from "kamaitachi-common/js/config"; import db from "../db/db"; import { KtLogger } from "../types"; import { GetFolderChartIDs } from "./folder-core"; @@ -7,7 +8,7 @@ import { FilterQuery } from "mongodb"; export interface EvaluatedGoalReturn { achieved: boolean; - progress: number; + progress: number | null; outOf: number; progressHuman: string; outOfHuman: string; @@ -38,34 +39,70 @@ export async function EvaluateGoalForUser( }; if (chartIDs) { - scoreQuery.chartIDs = { $in: chartIDs }; + scoreQuery.chartID = { $in: chartIDs }; } // Next, we need to figure out our criteria. if (goal.criteria.mode === "single") { let res = await db["score-pbs"].findOne(scoreQuery); + // hack, but guaranteed to work. + let scoreDataKey = goal.criteria.key.split(".")[1] as + | "lampIndex" + | "gradeIndex" + | "score" + | "percent"; + + const outOfHuman = HumaniseGoalSingle(goal.game, goal.criteria.key, goal.criteria.value); if (res) { - // hack, but guaranteed to work. - let scoreDataKey = goal.criteria.key.split(".")[1] as - | "lampIndex" - | "gradeIndex" - | "score" - | "percent"; - return { achieved: true, outOf: goal.criteria.value, progress: res.scoreData[scoreDataKey], - outOfHuman: "", // @todo - progressHuman: "", + outOfHuman, + progressHuman: HumaniseGoalSingle( + goal.game, + goal.criteria.key, + res.scoreData[scoreDataKey] + ), }; } // if we weren't successful, we have to get the users next best score and put it up here // this is made infinitely easier by the existance of score-pbs. + let nextBestQuery: FilterQuery = { + userID, + }; - // @todo + if (chartIDs) { + nextBestQuery.chartID = { $in: chartIDs }; + } + + let nextBestScore = await db["score-pbs"].findOne(nextBestQuery, { + sort: { [goal.criteria.key]: -1 }, + }); + + if (!nextBestScore) { + return { + achieved: false, + outOf: goal.criteria.value, + progress: null, + outOfHuman, + progressHuman: "NO DATA", + }; + } + + return { + achieved: false, + outOf: goal.criteria.value, + outOfHuman, + progress: nextBestScore.scoreData[scoreDataKey], + progressHuman: HumaniseGoalSingle( + goal.game, + goal.criteria.key, + nextBestScore.scoreData[scoreDataKey] + ), + }; } else if (goal.criteria.mode === "abs" || goal.criteria.mode === "proportion") { let count; @@ -125,3 +162,21 @@ function ResolveGoalCharts(goal: GoalDocument): Promise | string[] | n return null; // special case. } } + +type GoalKeys = GoalDocument["criteria"]["key"]; + +// @todo, improve this (add things like BP for iidx, maybe, percents for scores?) +function HumaniseGoalSingle(game: Game, key: GoalKeys, value: number): string { + switch (key) { + case "scoreData.gradeIndex": + return grades[game][value]; + case "scoreData.lampIndex": + return lamps[game][value]; + case "scoreData.percent": + return `${value.toFixed(2)}%`; + case "scoreData.score": + return value.toString(); + default: + throw new Error(`Broken goal - invalid key ${key}.`); + } +} diff --git a/server/src/score-import/framework/calculated-data/game-specific-stats.test.ts b/server/src/score-import/framework/calculated-data/game-specific-stats.test.ts index 61e6c8eab..dc514694b 100644 --- a/server/src/score-import/framework/calculated-data/game-specific-stats.test.ts +++ b/server/src/score-import/framework/calculated-data/game-specific-stats.test.ts @@ -240,4 +240,9 @@ t.test("#CalculateMFCP", (t) => { t.end(); }); +t.todo("#CalculateCHUNITHMRating"); +t.todo("#CalculateVF5"); +t.todo("#CalculateVF4"); +t.todo("#KaidenPercentile"); + t.teardown(CloseMongoConnection); diff --git a/server/src/score-import/framework/goals/goals.test.ts b/server/src/score-import/framework/goals/goals.test.ts new file mode 100644 index 000000000..863f8dff0 --- /dev/null +++ b/server/src/score-import/framework/goals/goals.test.ts @@ -0,0 +1,291 @@ +import t from "tap"; +import db, { CloseMongoConnection } from "../../../db/db"; +import ResetDBState from "../../../test-utils/reset-db-state"; +import { + GetRelevantFolderGoals, + GetRelevantGoals, + ProcessGoalsForUser, + UpdateUsersGoals, + ProcessGoal, +} from "./goals"; +import { GoalDocument, FolderDocument, UserGoalDocument } from "kamaitachi-common"; +import { CreateFolderChartLookup } from "../../../core/folder-core"; +import { GetKTDataJSON, Testing511SPA, TestingIIDXSPScorePB } from "../../../test-utils/test-data"; +import deepmerge from "deepmerge"; +import CreateLogCtx from "../../../logger"; +import crypto from "crypto"; +import { lamps } from "kamaitachi-common/js/config"; + +const logger = CreateLogCtx("goals.test.ts"); + +const HC511Goal: GoalDocument = { + charts: { + type: "single", + data: Testing511SPA.chartID, + }, + createdBy: 1, + game: "iidx", + goalID: "mock_goalID", + playtype: "SP", + timeAdded: 0, + title: "HC 5.1.1. SPA", + criteria: { + mode: "single", + value: lamps.iidx.indexOf("HARD CLEAR"), + key: "scoreData.lampIndex", + }, +}; + +const HC511UserGoal: UserGoalDocument = { + achieved: false, + timeAchieved: null, + game: "iidx", + playtype: "SP", + goalID: "mock_goalID", + lastInteraction: null, + outOf: 5, + outOfHuman: "HARD CLEAR", + progress: null, + progressHuman: "NO DATA", + timeSet: Date.now(), + userID: 1, +}; + +t.test("#GetRelevantFolderGoals", (t) => { + t.beforeEach(ResetDBState); + + const sp10folder: FolderDocument = { + title: "Level 10", + game: "iidx", + playtype: "SP", + type: "charts", + folderID: "ed9d8c734447ce67d7135c0067441a98cc81aeaf", + table: "Levels", + tableIndex: 10, + data: { + level: "10", + "flags¬IN BASE GAME": true, + }, + }; + + const fakeFolderGoalDocument: GoalDocument = { + charts: { + type: "folder", + data: "ed9d8c734447ce67d7135c0067441a98cc81aeaf", + }, + createdBy: 1, + game: "iidx", + goalID: "fake_goal_id", + playtype: "SP", + timeAdded: 0, + title: "get > 1 ex score on any level 10.", + criteria: { + mode: "single", + value: 1, + key: "scoreData.score", + }, + }; + + const notFolderGoalDocument: GoalDocument = { + charts: { + type: "folder", + data: "some_fake_folder_id", + }, + createdBy: 1, + game: "iidx", + goalID: "fake_bad_goal_id", + playtype: "SP", + timeAdded: 0, + title: "get > 1 ex score on some other folder.", + criteria: { + mode: "single", + value: 1, + key: "scoreData.score", + }, + }; + + t.beforeEach(async () => { + const sp11folder = deepmerge(sp10folder, { data: { level: "11" } }); + await db.folders.insert(sp10folder); + await db.folders.insert(sp11folder); + await db.goals.insert(fakeFolderGoalDocument); + await db.goals.insert(notFolderGoalDocument); + await CreateFolderChartLookup(sp10folder); + await CreateFolderChartLookup(sp11folder); + }); + + t.test("Should correctly find the goals on this folder.", async (t) => { + let res = await GetRelevantFolderGoals( + ["fake_goal_id", "fake_bad_goal_id"], + [Testing511SPA.chartID] + ); + + t.strictSame( + res, + [fakeFolderGoalDocument], + "Should correctly return only the 511 goal document." + ); + + t.end(); + }); + + t.end(); +}); + +t.test("#GetRelevantGoals", (t) => { + t.beforeEach(ResetDBState); + t.beforeEach(async () => { + // use the real data so we have enough charts loaded for this to test properly. + await db.songs.iidx.remove({}); + await db.charts.iidx.remove({}); + await db.charts.iidx.insert(GetKTDataJSON("./kamaitachi/ktblack-charts-iidx.json")); + await db.songs.iidx.insert(GetKTDataJSON("./kamaitachi/ktblack-songs-iidx.json")); + + let lotsOfCharts = await db.charts.iidx.find({}, { limit: 20 }); + let goals: GoalDocument[] = lotsOfCharts.map((e) => ({ + charts: { + type: "single", + data: e.chartID, + }, + createdBy: 1, + game: "iidx", + goalID: crypto.randomBytes(20).toString("hex"), + playtype: "SP", + timeAdded: 0, + title: "get > 1 ex score on some other folder.", + criteria: { + mode: "single", + value: 1, + key: "scoreData.score", + }, + })); + + await db.goals.insert(goals); + + await db["user-goals"].insert( + goals.map((e) => ({ + achieved: false, + timeAchieved: null, + game: "iidx", + playtype: "SP", + goalID: e.goalID, + lastInteraction: null, + outOf: 5, + outOfHuman: "HARD CLEAR", + progress: null, + progressHuman: "NO DATA", + timeSet: Date.now(), + userID: 1, + })) + ); + }); + + t.test( + "Should successfully filter goals to only those that are relevant to the session.", + async (t) => { + // lets pretend our session had scores on charts 0->4 and 20->25. We also only have goals on + // charts 1->20, so only 5 of these should resolve. + let ourCharts = [ + ...(await db.charts.iidx.find({}, { limit: 5 })), + ...(await db.charts.iidx.find({}, { skip: 20, limit: 5 })), + ]; + + let chartIDs = new Set(ourCharts.map((e) => e.chartID)); + + let res = await GetRelevantGoals("iidx", 1, chartIDs, logger); + + t.equal(res.goals.length, 5, "Should correctly resolve to 5 goals."); + t.equal(res.userGoalsMap.size, 5, "Should also return 5 userGoals."); + + t.end(); + } + ); + + t.end(); +}); + +t.todo("#ProcessGoalsForUser", (t) => { + t.beforeEach(ResetDBState); + + t.end(); +}); + +t.todo("#UpdateUsersGoals", (t) => { + t.beforeEach(ResetDBState); + + t.end(); +}); + +t.test("#ProcessGoal", (t) => { + t.beforeEach(ResetDBState); + t.beforeEach(async () => { + await db.goals.insert(HC511Goal); + }); + + t.test("Should process the users goal if a score has changed.", async (t) => { + await db["user-goals"].insert(HC511UserGoal); + await db["score-pbs"].insert(TestingIIDXSPScorePB); // score is EX HARD CLEAR by default. + + let res = await ProcessGoal(HC511Goal, HC511UserGoal, 1, logger); + + t.not(res, undefined, "Should NOT return undefined."); + + t.strictSame( + res!.import, + { + goalID: "mock_goalID", + old: { + progress: null, + progressHuman: "NO DATA", + outOf: 5, + outOfHuman: "HARD CLEAR", + achieved: false, + }, + new: { + progress: 6, + progressHuman: "EX HARD CLEAR", + outOf: 5, + outOfHuman: "HARD CLEAR", + achieved: true, + }, + }, + "Should return a valid import goal fragment." + ); + + t.end(); + }); + + t.test("Should return undefined if there's no score.", async (t) => { + await db["user-goals"].insert(HC511UserGoal); + + let res = await ProcessGoal(HC511Goal, HC511UserGoal, 1, logger); + + t.equal(res, undefined, "Should return undefined."); + + t.end(); + }); + + t.test("Should return undefined if the progress has not changed.", async (t) => { + await db["user-goals"].insert(HC511UserGoal); + await db["score-pbs"].insert(TestingIIDXSPScorePB); + + let firstUpdate = await ProcessGoal(HC511Goal, HC511UserGoal, 1, logger); + + // ignore this one + t.not(firstUpdate, undefined, "Should NOT return undefined."); + + await db["user-goals"].bulkWrite([firstUpdate!.bwrite]); + + let userGoal = await db["user-goals"].findOne({ userID: 1, goalID: HC511Goal.goalID }); + + let secondUpdate = await ProcessGoal(HC511Goal, userGoal!, 1, logger); + + t.equal(secondUpdate, undefined, "Should return undefined."); + + t.end(); + }); + + t.end(); +}); + +t.teardown(CloseMongoConnection); diff --git a/server/src/score-import/framework/goals/goals.ts b/server/src/score-import/framework/goals/goals.ts index f86bfd5c9..45b87d266 100644 --- a/server/src/score-import/framework/goals/goals.ts +++ b/server/src/score-import/framework/goals/goals.ts @@ -21,10 +21,10 @@ export async function UpdateUsersGoals( logger.verbose(`Found ${goals.length} relevant goals.`); - return ProcessGoals(goals, userGoalsMap, userID, logger); + return ProcessGoalsForUser(goals, userGoalsMap, userID, logger); } -export async function ProcessGoals( +export async function ProcessGoalsForUser( goals: GoalDocument[], userGoalsMap: Map, userID: integer, @@ -74,7 +74,7 @@ export async function ProcessGoals( * @returns undefined on error (i.e. EvaluateGoalForUser) OR if there's nothing * to say (i.e. user didnt raise the goal). */ -async function ProcessGoal( +export async function ProcessGoal( goal: GoalDocument, userGoal: UserGoalDocument, userID: integer, @@ -87,9 +87,16 @@ async function ProcessGoal( return; } - // if the user has improved their progress on the goal - if (userGoal.progress < res.progress) { - // @todo emit something + // if the user has changed their progress on the goal + if (userGoal.progress !== res.progress) { + // if the user has improved their progress on the goal + // if userGoal.progress is null, then res.progress must be non-null, and therefore an improvement. + if ( + userGoal.progress === null || + (res.progress !== null && userGoal.progress < res.progress) + ) { + // @todo emit something + } } else if (userGoal.outOf === res.outOf) { // if the users progress hasn't changed AND the outOf hasn't // then nothing has changed. @@ -152,7 +159,7 @@ async function ProcessGoal( * evaluate the ones we need to. * @returns An array of Goals, and an array of userGoals. */ -async function GetRelevantGoals( +export async function GetRelevantGoals( game: Game, userID: integer, chartIDs: Set, @@ -173,33 +180,35 @@ async function GetRelevantGoals( chartIDsArr.push(c); } + const goals = await Promise.all([ + // this gets the relevantGoals for direct and multi + db.goals.find({ + "charts.type": { $in: ["single", "multi"] }, + "charts.data": { $in: chartIDsArr }, + goalID: { $in: goalIDs }, + }), + db.goals.find({ + "charts.type": "any", + goalID: { $in: goalIDs }, + }), + GetRelevantFolderGoals(goalIDs, chartIDsArr), + ]).then((r) => r.flat(1)); + + let goalSet = new Set(goals.map((e) => e.goalID)); + const userGoalsMap: Map = new Map(); for (const userGoal of userGoals) { + if (!goalSet.has(userGoal.goalID)) { + continue; + } // since these are guaranteed to be unique, lets make a hot map of goalID -> userGoalDocument, so we can // pull them in for post-processing and filter out the userGoalDocuments that aren't relevant. userGoalsMap.set(userGoal.goalID, userGoal); } return { - goals: await Promise.all([ - // this gets the relevantGoals for direct and multi - db.goals.find({ - charts: { - // @ts-expect-error monk tries her hardest - type: { $in: ["direct", "multi"] }, - data: { $in: chartIDsArr }, - }, - goalID: { $in: goalIDs }, - }), - db.goals.find({ - charts: { - type: "any", - }, - goalID: { $in: goalIDs }, - }), - GetRelevantFolderGoals(goalIDs, chartIDsArr), - ]).then((r) => r.flat(1)), + goals, userGoalsMap, }; } @@ -208,7 +217,7 @@ async function GetRelevantGoals( * Returns the set of goals where its folder contains any member * of chartIDsArr. */ -function GetRelevantFolderGoals(goalIDs: string[], chartIDsArr: string[]) { +export function GetRelevantFolderGoals(goalIDs: string[], chartIDsArr: string[]) { // Slightly black magic - this is kind of like doing an SQL join. // it's weird to do this in mongodb, but this seems like the right // way to actually handle this. @@ -217,9 +226,7 @@ function GetRelevantFolderGoals(goalIDs: string[], chartIDsArr: string[]) { return db.goals.aggregate([ { $match: { - charts: { - type: "folder", - }, + "charts.type": "folder", goalID: { $in: goalIDs }, }, }, diff --git a/server/src/score-import/framework/pb/create-pb-doc.test.ts b/server/src/score-import/framework/pb/create-pb-doc.test.ts index 61d0b1c1d..f9959850e 100644 --- a/server/src/score-import/framework/pb/create-pb-doc.test.ts +++ b/server/src/score-import/framework/pb/create-pb-doc.test.ts @@ -1,12 +1,7 @@ import t from "tap"; import db, { CloseMongoConnection } from "../../../db/db"; import ResetDBState from "../../../test-utils/reset-db-state"; -import { - GetKTDataJSON, - Testing511SPA, - TestingIIDXSPScore, - TestingIIDXSPScorePB, -} from "../../../test-utils/test-data"; +import { GetKTDataJSON, Testing511SPA, TestingIIDXSPScore } from "../../../test-utils/test-data"; import { CreatePBDoc } from "./create-pb-doc"; import deepmerge from "deepmerge"; import CreateLogCtx from "../../../logger"; diff --git a/server/src/score-import/framework/score-import-main.ts b/server/src/score-import/framework/score-import-main.ts index 01c92e715..3d2f67462 100644 --- a/server/src/score-import/framework/score-import-main.ts +++ b/server/src/score-import/framework/score-import-main.ts @@ -84,11 +84,12 @@ export default async function ScoreImportMain( let playtypes = Object.keys(scorePlaytypeMap) as Playtypes[Game][]; // --- 6. Game Stats --- - // This function updates the users "stats" for this game - such as their profile rating, or their classes. + // This function updates the users "stats" for this game - such as their profile rating or their classes. let classDeltas = await UpdateUsersGameStats(game, playtypes, user.id, logger); // --- 7. Goals --- - let unknown = await UpdateUsersGoals(game, user.id, chartIDs, logger); + // Evaluate and update the users goals. This returns information about goals that have changed. + let goalInfo = await UpdateUsersGoals(game, user.id, chartIDs, logger); // --- 8. Finalise Import Document --- // Create and Save an import document to the database, and finish everything up! @@ -103,6 +104,7 @@ export default async function ScoreImportMain( createdSessions: sessionInfo, userID: user.id, classDeltas, + goalInfo, userIntent, };