From 19fd8132b250bc0f7f65e62f4d7b2b1ccea2af7c Mon Sep 17 00:00:00 2001 From: zkldi <20380519+zkldi@users.noreply.github.com> Date: Thu, 18 Nov 2021 19:48:24 +0000 Subject: [PATCH] Add BPI Recalc Update Job --- .../state-sync/recalc-all-scores.ts | 52 +--- .../state-sync/update-all-pbs.ts | 25 +- .../bpi-poyashi-data/update-bpi-poyashi.ts | 236 +++++++++--------- server/src/lib/jobs/update-bpi-poyashi.ts | 127 ++++++++++ .../src/utils/calculations/recalc-scores.ts | 113 +++++++++ .../utils}/efficient-db-iterate.ts | 0 6 files changed, 360 insertions(+), 193 deletions(-) create mode 100644 server/src/lib/jobs/update-bpi-poyashi.ts create mode 100644 server/src/utils/calculations/recalc-scores.ts rename server/{scripts/util => src/utils}/efficient-db-iterate.ts (100%) diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts index 6ba363c1f..88723303f 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts @@ -1,54 +1,4 @@ -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; -import { CreateCalculatedData } from "lib/score-import/framework/calculated-data/calculated-data"; -import { EfficientDBIterate } from "../../util/efficient-db-iterate"; -import { UpdateAllPBs } from "./update-all-pbs"; - -const logger = CreateLogCtx(__filename); - -export async function RecalcAllScores(filter = {}) { - logger.info(`Recalcing All Scores.`); - - await EfficientDBIterate( - db.scores, - async (c) => { - const chart = await db.charts[c.game].findOne({ chartID: c.chartID }); - - if (!chart) { - logger.error(`Can't find chartID ${c.chartID} ${c.scoreID} (${c.game})`, { - score: c, - }); - - throw new Error(`screwed`); - } - const calculatedData = await CreateCalculatedData(c, chart, c.scoreData.esd, logger); - - return { scoreID: c.scoreID, calculatedData }; - }, - async (updates) => { - await db.scores.bulkWrite( - updates.map((e) => ({ - updateOne: { - filter: { - scoreID: e.scoreID, - }, - update: { - $set: { - calculatedData: e.calculatedData, - }, - }, - }, - })) - ); - }, - filter - ); - - logger.info("Reprocessing PBs..."); - await UpdateAllPBs(filter); - - logger.info(`Done!`); -} +import { RecalcAllScores } from "utils/calculations/recalc-scores"; if (require.main === module) { RecalcAllScores(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts index 3e1428fa8..4e28a917a 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts @@ -1,28 +1,5 @@ /* eslint-disable no-await-in-loop */ -import deepmerge from "deepmerge"; -import db from "external/mongo/db"; -import CreateLogCtx from "lib/logger/logger"; -import { ProcessPBs } from "lib/score-import/framework/pb/process-pbs"; -import { FormatUserDoc } from "utils/user"; - -const logger = CreateLogCtx(__filename); - -export async function UpdateAllPBs(filter = {}) { - const allUsers = await db.users.find({}); - - for (const user of allUsers) { - logger.info(`Finding ${FormatUserDoc(user)}'s scores.`); - - const scores = await db.scores.find(deepmerge({ userID: user.id }, filter), { - projection: { chartID: 1 }, - }); - - logger.info(`PBing ${FormatUserDoc(user)}'s scores.`); - await ProcessPBs(user.id, new Set(scores.map((e) => e.chartID)), logger); - } - - logger.info(`Done!`); -} +import { UpdateAllPBs } from "utils/calculations/recalc-scores"; if (require.main === module) { UpdateAllPBs(); diff --git a/server/scripts/bpi-poyashi-data/update-bpi-poyashi.ts b/server/scripts/bpi-poyashi-data/update-bpi-poyashi.ts index 587d94193..e1e6dfb84 100644 --- a/server/scripts/bpi-poyashi-data/update-bpi-poyashi.ts +++ b/server/scripts/bpi-poyashi-data/update-bpi-poyashi.ts @@ -1,118 +1,118 @@ -/* eslint-disable no-await-in-loop */ -import { Command } from "commander"; -import db from "external/mongo/db"; -import fs from "fs"; -import CreateLogCtx from "lib/logger/logger"; -import fetch from "node-fetch"; -import path from "path"; -import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common"; -import { FindChartWithPTDFVersion } from "utils/queries/charts"; -import { FindSongOnTitle } from "utils/queries/songs"; -import { RecalcAllScores } from "../__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores"; - -const program = new Command(); - -const logger = CreateLogCtx(__filename); - -program.option("-f, --fetch", "Fetch the latest data from the poyashi repo."); - -program.parse(process.argv); -const options = program.opts(); - -const dataLoc = path.join(__dirname, "./output.json"); - -const difficultyResolve: Record = { - 3: ["SP", "HYPER"], - 4: ["SP", "ANOTHER"], - 8: ["DP", "HYPER"], - 9: ["DP", "ANOTHER"], - 10: ["SP", "LEGGENDARIA"], - 11: ["DP", "LEGGENDARIA"], -}; - -async function UpdatePoyashiData() { - let data; - if (options.fetch) { - // lol! - logger.info("Fetching data from github..."); - const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json()); - - logger.info("Fetched data."); - fs.writeFileSync(dataLoc, JSON.stringify(rj)); - logger.info("Saved data to output.json."); - - data = rj; - } else { - logger.info("Reading data from output.json..."); - data = JSON.parse(fs.readFileSync(dataLoc, "utf-8")); - } - - const realData: IIDXBPIData[] = []; - for (const d of data.body) { - const res = difficultyResolve[d.difficulty]; - - if (!res) { - throw new Error(`Unknown difficulty ${d.difficulty}`); - } - - const [playtype, diff] = res; - - const tachiSong = await FindSongOnTitle("iidx", d.title); - - if (!tachiSong) { - logger.warn(`Cannot find song ${d.title}?`); - continue; - } - - const tachiChart = (await FindChartWithPTDFVersion( - "iidx", - tachiSong.id, - playtype as "SP" | "DP", - diff as Difficulties["iidx:DP" | "iidx:SP"], - "27" - )) as ChartDocument<"iidx:SP" | "iidx:DP">; - - if (!tachiChart) { - logger.warn( - `Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?` - ); - continue; - } - - const kavg = Number(d.avg); - - if (kavg < 0) { - logger.warn( - `${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.` - ); - continue; - } - - if (d.removed) { - logger.info(`Skipping removed chart ${tachiSong.title}.`); - continue; - } - - realData.push({ - coef: d.coef === -1 ? null : d.coef, - kavg: Number(d.avg), - wr: Number(d.wr), - chartID: tachiChart.chartID, - }); - } - - await db["iidx-bpi-data"].remove({}); - await db["iidx-bpi-data"].insert(realData); - - logger.info(`Removed all, and inserted ${realData.length} documents.`); - - logger.info(`Triggering IIDX Recalc.`); - - await RecalcAllScores({ game: "iidx" }); - - logger.info(`Done.`); -} - -if (require.main === module) { - UpdatePoyashiData(); -} +/* eslint-disable no-await-in-loop */ +import { Command } from "commander"; +import db from "external/mongo/db"; +import fs from "fs"; +import CreateLogCtx from "lib/logger/logger"; +import fetch from "node-fetch"; +import path from "path"; +import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common"; +import { FindChartWithPTDFVersion } from "utils/queries/charts"; +import { FindSongOnTitle } from "utils/queries/songs"; +import { RecalcAllScores } from "../__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores"; + +const program = new Command(); + +const logger = CreateLogCtx(__filename); + +program.option("-f, --fetch", "Fetch the latest data from the poyashi repo."); + +program.parse(process.argv); +const options = program.opts(); + +const dataLoc = path.join(__dirname, "./output.json"); + +const difficultyResolve: Record = { + 3: ["SP", "HYPER"], + 4: ["SP", "ANOTHER"], + 8: ["DP", "HYPER"], + 9: ["DP", "ANOTHER"], + 10: ["SP", "LEGGENDARIA"], + 11: ["DP", "LEGGENDARIA"], +}; + +async function UpdatePoyashiData() { + let data; + if (options.fetch) { + // lol! + logger.info("Fetching data from github..."); + const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json()); + + logger.info("Fetched data."); + fs.writeFileSync(dataLoc, JSON.stringify(rj)); + logger.info("Saved data to output.json."); + + data = rj; + } else { + logger.info("Reading data from output.json..."); + data = JSON.parse(fs.readFileSync(dataLoc, "utf-8")); + } + + const realData: IIDXBPIData[] = []; + for (const d of data.body) { + const res = difficultyResolve[d.difficulty]; + + if (!res) { + throw new Error(`Unknown difficulty ${d.difficulty}`); + } + + const [playtype, diff] = res; + + const tachiSong = await FindSongOnTitle("iidx", d.title); + + if (!tachiSong) { + logger.warn(`Cannot find song ${d.title}?`); + continue; + } + + const tachiChart = (await FindChartWithPTDFVersion( + "iidx", + tachiSong.id, + playtype as "SP" | "DP", + diff as Difficulties["iidx:DP" | "iidx:SP"], + "27" + )) as ChartDocument<"iidx:SP" | "iidx:DP">; + + if (!tachiChart) { + logger.warn( + `Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?` + ); + continue; + } + + const kavg = Number(d.avg); + + if (kavg < 0) { + logger.warn( + `${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.` + ); + continue; + } + + if (d.removed) { + logger.info(`Skipping removed chart ${tachiSong.title}.`); + continue; + } + + realData.push({ + coef: d.coef === -1 ? null : d.coef, + kavg: Number(d.avg), + wr: Number(d.wr), + chartID: tachiChart.chartID, + }); + } + + await db["iidx-bpi-data"].remove({}); + await db["iidx-bpi-data"].insert(realData); + + logger.info(`Removed all, and inserted ${realData.length} documents.`); + + logger.info(`Triggering IIDX Recalc.`); + + await RecalcAllScores({ game: "iidx" }); + + logger.info(`Done.`); +} + +if (require.main === module) { + UpdatePoyashiData(); +} diff --git a/server/src/lib/jobs/update-bpi-poyashi.ts b/server/src/lib/jobs/update-bpi-poyashi.ts new file mode 100644 index 000000000..671c163cc --- /dev/null +++ b/server/src/lib/jobs/update-bpi-poyashi.ts @@ -0,0 +1,127 @@ +/* eslint-disable no-await-in-loop */ +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import fetch from "node-fetch"; +import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common"; +import { RecalcAllScores } from "utils/calculations/recalc-scores"; +import { FindChartWithPTDFVersion } from "utils/queries/charts"; +import { FindSongOnTitle } from "utils/queries/songs"; + +const logger = CreateLogCtx(__filename); + +const difficultyResolve: Record = { + 3: ["SP", "HYPER"], + 4: ["SP", "ANOTHER"], + 8: ["DP", "HYPER"], + 9: ["DP", "ANOTHER"], + 10: ["SP", "LEGGENDARIA"], + 11: ["DP", "LEGGENDARIA"], +}; + +async function UpdatePoyashiData() { + logger.info("Fetching BPI Poyashi data from proxy..."); + const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json()); + + logger.info("Fetched data."); + logger.info("Saved data to output.json."); + + const data = rj; + + const realData: IIDXBPIData[] = []; + for (const d of data.body) { + const res = difficultyResolve[d.difficulty]; + + if (!res) { + throw new Error(`Unknown difficulty ${d.difficulty}`); + } + + const [playtype, diff] = res; + + const tachiSong = await FindSongOnTitle("iidx", d.title); + + if (!tachiSong) { + logger.warn(`Cannot find song ${d.title}?`); + continue; + } + + const tachiChart = (await FindChartWithPTDFVersion( + "iidx", + tachiSong.id, + playtype as "SP" | "DP", + diff as Difficulties["iidx:DP" | "iidx:SP"], + "29" + )) as ChartDocument<"iidx:SP" | "iidx:DP">; + + if (!tachiChart) { + logger.warn( + `Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?` + ); + continue; + } + + const kavg = Number(d.avg); + + if (kavg < 0) { + logger.warn( + `${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.` + ); + continue; + } + + if (d.removed) { + logger.info(`Skipping removed chart ${tachiSong.title}.`); + continue; + } + + realData.push({ + coef: d.coef === -1 ? null : d.coef, + kavg: Number(d.avg), + wr: Number(d.wr), + chartID: tachiChart.chartID, + }); + } + + const updatedCharts = []; + for (const newData of realData) { + const oldData = await db["iidx-bpi-data"].findOne({ + chartID: newData.chartID, + }); + + if (!oldData) { + updatedCharts.push(newData.chartID); + } else if ( + oldData.wr !== newData.wr || + oldData.kavg !== newData.kavg || + oldData.coef !== newData.coef + ) { + updatedCharts.push(newData.chartID); + await db["iidx-bpi-data"].update( + { + chartID: newData.chartID, + }, + { + $set: { + wr: newData.wr, + kavg: newData.kavg, + coef: newData.coef, + }, + } + ); + } + } + + if (updatedCharts.length === 0) { + logger.info(`Nothing has changed. Not updating any BPI data.`); + return; + } + + logger.info(`Triggering IIDX Recalc for ${updatedCharts.length} charts.`); + + await RecalcAllScores({ game: "iidx", chartID: { $in: updatedCharts } }); + + logger.info(`Done.`); +} + +if (require.main === module) { + UpdatePoyashiData(); +} diff --git a/server/src/utils/calculations/recalc-scores.ts b/server/src/utils/calculations/recalc-scores.ts new file mode 100644 index 000000000..53c407225 --- /dev/null +++ b/server/src/utils/calculations/recalc-scores.ts @@ -0,0 +1,113 @@ +/* eslint-disable no-await-in-loop */ +import db from "external/mongo/db"; +import { Game, integer, Playtypes, PublicUserDocument } from "tachi-common"; +import CreateLogCtx from "lib/logger/logger"; +import { CreateCalculatedData } from "lib/score-import/framework/calculated-data/calculated-data"; +import { UpdateUsersGamePlaytypeStats } from "lib/score-import/framework/user-game-stats/update-ugs"; +import { GetAndUpdateUsersGoals } from "lib/score-import/framework/goals/goals"; +import { UpdateUsersMilestones } from "lib/score-import/framework/milestones/milestones"; +import { EfficientDBIterate } from "utils/efficient-db-iterate"; +import { ProcessPBs } from "lib/score-import/framework/pb/process-pbs"; +import { FormatUserDoc } from "utils/user"; +import deepmerge from "deepmerge"; + +const logger = CreateLogCtx(__filename); + +export async function RecalcAllScores(filter = {}) { + logger.info(`Recalcing All Scores.`); + + const modifiedUsers = new Set(); + const modifiedUserIDs = new Set(); + const chartIDs = new Set(); + + await EfficientDBIterate( + db.scores, + async (c) => { + const chart = await db.charts[c.game].findOne({ chartID: c.chartID }); + + if (!chart) { + logger.error(`Can't find chartID ${c.chartID} ${c.scoreID} (${c.game})`, { + score: c, + }); + + throw new Error(`screwed`); + } + + chartIDs.add(chart.chartID); + + modifiedUsers.add(`${c.game}-${c.playtype}-${c.userID}`); + modifiedUserIDs.add(c.userID); + + const calculatedData = await CreateCalculatedData(c, chart, c.scoreData.esd, logger); + + return { scoreID: c.scoreID, calculatedData }; + }, + async (updates) => { + await db.scores.bulkWrite( + updates.map((e) => ({ + updateOne: { + filter: { + scoreID: e.scoreID, + }, + update: { + $set: { + calculatedData: e.calculatedData, + }, + }, + }, + })) + ); + }, + filter + ); + + logger.info("Reprocessing PBs..."); + await UpdateAllPBs([...modifiedUserIDs.values()], filter); + + logger.info(`Updating Profiles for ${modifiedUsers.size} users...`); + + for (const userInfo of modifiedUsers.values()) { + const [game, playtype, strUserID] = userInfo.split("-") as [Game, Playtypes[Game], string]; + + const userID = Number(strUserID); + + await UpdateUsersGamePlaytypeStats( + game as Game, + playtype as Playtypes[Game], + userID, + null, + logger + ); + + const goalInfo = await GetAndUpdateUsersGoals(game, userID, chartIDs, logger); + + await UpdateUsersMilestones(goalInfo, game, [playtype], userID, logger); + } + + logger.info(`Done!`); +} + +export async function UpdateAllPBs(userIDs?: integer[], filter = {}) { + let allUsers: PublicUserDocument[]; + + if (!userIDs) { + allUsers = await db.users.find({}); + } else { + allUsers = await db.users.find({ + id: { $in: userIDs }, + }); + } + + for (const user of allUsers) { + logger.info(`Finding ${FormatUserDoc(user)}'s scores.`); + + const scores = await db.scores.find(deepmerge({ userID: user.id }, filter), { + projection: { chartID: 1 }, + }); + + logger.info(`PBing ${FormatUserDoc(user)}'s scores.`); + await ProcessPBs(user.id, new Set(scores.map((e) => e.chartID)), logger); + } + + logger.info(`Done!`); +} diff --git a/server/scripts/util/efficient-db-iterate.ts b/server/src/utils/efficient-db-iterate.ts similarity index 100% rename from server/scripts/util/efficient-db-iterate.ts rename to server/src/utils/efficient-db-iterate.ts