mirror of
https://github.com/zkldi/Tachi.git
synced 2026-10-14 01:59:05 +03:00
Add BPI Recalc Update Job
This commit is contained in:
zkldi
@@ -1,54 +1,4 @@
|
||||
import db from "external/mongo/db";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import { CreateCalculatedData } from "lib/score-import/framework/calculated-data/calculated-data";
|
||||
import { EfficientDBIterate } from "../../util/efficient-db-iterate";
|
||||
import { UpdateAllPBs } from "./update-all-pbs";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
export async function RecalcAllScores(filter = {}) {
|
||||
logger.info(`Recalcing All Scores.`);
|
||||
|
||||
await EfficientDBIterate(
|
||||
db.scores,
|
||||
async (c) => {
|
||||
const chart = await db.charts[c.game].findOne({ chartID: c.chartID });
|
||||
|
||||
if (!chart) {
|
||||
logger.error(`Can't find chartID ${c.chartID} ${c.scoreID} (${c.game})`, {
|
||||
score: c,
|
||||
});
|
||||
|
||||
throw new Error(`screwed`);
|
||||
}
|
||||
const calculatedData = await CreateCalculatedData(c, chart, c.scoreData.esd, logger);
|
||||
|
||||
return { scoreID: c.scoreID, calculatedData };
|
||||
},
|
||||
async (updates) => {
|
||||
await db.scores.bulkWrite(
|
||||
updates.map((e) => ({
|
||||
updateOne: {
|
||||
filter: {
|
||||
scoreID: e.scoreID,
|
||||
},
|
||||
update: {
|
||||
$set: {
|
||||
calculatedData: e.calculatedData,
|
||||
},
|
||||
},
|
||||
},
|
||||
}))
|
||||
);
|
||||
},
|
||||
filter
|
||||
);
|
||||
|
||||
logger.info("Reprocessing PBs...");
|
||||
await UpdateAllPBs(filter);
|
||||
|
||||
logger.info(`Done!`);
|
||||
}
|
||||
import { RecalcAllScores } from "utils/calculations/recalc-scores";
|
||||
|
||||
if (require.main === module) {
|
||||
RecalcAllScores();
|
||||
|
||||
@@ -1,28 +1,5 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import deepmerge from "deepmerge";
|
||||
import db from "external/mongo/db";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import { ProcessPBs } from "lib/score-import/framework/pb/process-pbs";
|
||||
import { FormatUserDoc } from "utils/user";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
export async function UpdateAllPBs(filter = {}) {
|
||||
const allUsers = await db.users.find({});
|
||||
|
||||
for (const user of allUsers) {
|
||||
logger.info(`Finding ${FormatUserDoc(user)}'s scores.`);
|
||||
|
||||
const scores = await db.scores.find(deepmerge({ userID: user.id }, filter), {
|
||||
projection: { chartID: 1 },
|
||||
});
|
||||
|
||||
logger.info(`PBing ${FormatUserDoc(user)}'s scores.`);
|
||||
await ProcessPBs(user.id, new Set(scores.map((e) => e.chartID)), logger);
|
||||
}
|
||||
|
||||
logger.info(`Done!`);
|
||||
}
|
||||
import { UpdateAllPBs } from "utils/calculations/recalc-scores";
|
||||
|
||||
if (require.main === module) {
|
||||
UpdateAllPBs();
|
||||
|
||||
@@ -1,118 +1,118 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import { Command } from "commander";
|
||||
import db from "external/mongo/db";
|
||||
import fs from "fs";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import fetch from "node-fetch";
|
||||
import path from "path";
|
||||
import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common";
|
||||
import { FindChartWithPTDFVersion } from "utils/queries/charts";
|
||||
import { FindSongOnTitle } from "utils/queries/songs";
|
||||
import { RecalcAllScores } from "../__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
program.option("-f, --fetch", "Fetch the latest data from the poyashi repo.");
|
||||
|
||||
program.parse(process.argv);
|
||||
const options = program.opts();
|
||||
|
||||
const dataLoc = path.join(__dirname, "./output.json");
|
||||
|
||||
const difficultyResolve: Record<string, [string, string]> = {
|
||||
3: ["SP", "HYPER"],
|
||||
4: ["SP", "ANOTHER"],
|
||||
8: ["DP", "HYPER"],
|
||||
9: ["DP", "ANOTHER"],
|
||||
10: ["SP", "LEGGENDARIA"],
|
||||
11: ["DP", "LEGGENDARIA"],
|
||||
};
|
||||
|
||||
async function UpdatePoyashiData() {
|
||||
let data;
|
||||
if (options.fetch) {
|
||||
// lol!
|
||||
logger.info("Fetching data from github...");
|
||||
const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json());
|
||||
|
||||
logger.info("Fetched data.");
|
||||
fs.writeFileSync(dataLoc, JSON.stringify(rj));
|
||||
logger.info("Saved data to output.json.");
|
||||
|
||||
data = rj;
|
||||
} else {
|
||||
logger.info("Reading data from output.json...");
|
||||
data = JSON.parse(fs.readFileSync(dataLoc, "utf-8"));
|
||||
}
|
||||
|
||||
const realData: IIDXBPIData[] = [];
|
||||
for (const d of data.body) {
|
||||
const res = difficultyResolve[d.difficulty];
|
||||
|
||||
if (!res) {
|
||||
throw new Error(`Unknown difficulty ${d.difficulty}`);
|
||||
}
|
||||
|
||||
const [playtype, diff] = res;
|
||||
|
||||
const tachiSong = await FindSongOnTitle("iidx", d.title);
|
||||
|
||||
if (!tachiSong) {
|
||||
logger.warn(`Cannot find song ${d.title}?`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const tachiChart = (await FindChartWithPTDFVersion(
|
||||
"iidx",
|
||||
tachiSong.id,
|
||||
playtype as "SP" | "DP",
|
||||
diff as Difficulties["iidx:DP" | "iidx:SP"],
|
||||
"27"
|
||||
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
|
||||
|
||||
if (!tachiChart) {
|
||||
logger.warn(
|
||||
`Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const kavg = Number(d.avg);
|
||||
|
||||
if (kavg < 0) {
|
||||
logger.warn(
|
||||
`${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (d.removed) {
|
||||
logger.info(`Skipping removed chart ${tachiSong.title}.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
realData.push({
|
||||
coef: d.coef === -1 ? null : d.coef,
|
||||
kavg: Number(d.avg),
|
||||
wr: Number(d.wr),
|
||||
chartID: tachiChart.chartID,
|
||||
});
|
||||
}
|
||||
|
||||
await db["iidx-bpi-data"].remove({});
|
||||
await db["iidx-bpi-data"].insert(realData);
|
||||
|
||||
logger.info(`Removed all, and inserted ${realData.length} documents.`);
|
||||
|
||||
logger.info(`Triggering IIDX Recalc.`);
|
||||
|
||||
await RecalcAllScores({ game: "iidx" });
|
||||
|
||||
logger.info(`Done.`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
UpdatePoyashiData();
|
||||
}
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import { Command } from "commander";
|
||||
import db from "external/mongo/db";
|
||||
import fs from "fs";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import fetch from "node-fetch";
|
||||
import path from "path";
|
||||
import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common";
|
||||
import { FindChartWithPTDFVersion } from "utils/queries/charts";
|
||||
import { FindSongOnTitle } from "utils/queries/songs";
|
||||
import { RecalcAllScores } from "../__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
program.option("-f, --fetch", "Fetch the latest data from the poyashi repo.");
|
||||
|
||||
program.parse(process.argv);
|
||||
const options = program.opts();
|
||||
|
||||
const dataLoc = path.join(__dirname, "./output.json");
|
||||
|
||||
const difficultyResolve: Record<string, [string, string]> = {
|
||||
3: ["SP", "HYPER"],
|
||||
4: ["SP", "ANOTHER"],
|
||||
8: ["DP", "HYPER"],
|
||||
9: ["DP", "ANOTHER"],
|
||||
10: ["SP", "LEGGENDARIA"],
|
||||
11: ["DP", "LEGGENDARIA"],
|
||||
};
|
||||
|
||||
async function UpdatePoyashiData() {
|
||||
let data;
|
||||
if (options.fetch) {
|
||||
// lol!
|
||||
logger.info("Fetching data from github...");
|
||||
const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json());
|
||||
|
||||
logger.info("Fetched data.");
|
||||
fs.writeFileSync(dataLoc, JSON.stringify(rj));
|
||||
logger.info("Saved data to output.json.");
|
||||
|
||||
data = rj;
|
||||
} else {
|
||||
logger.info("Reading data from output.json...");
|
||||
data = JSON.parse(fs.readFileSync(dataLoc, "utf-8"));
|
||||
}
|
||||
|
||||
const realData: IIDXBPIData[] = [];
|
||||
for (const d of data.body) {
|
||||
const res = difficultyResolve[d.difficulty];
|
||||
|
||||
if (!res) {
|
||||
throw new Error(`Unknown difficulty ${d.difficulty}`);
|
||||
}
|
||||
|
||||
const [playtype, diff] = res;
|
||||
|
||||
const tachiSong = await FindSongOnTitle("iidx", d.title);
|
||||
|
||||
if (!tachiSong) {
|
||||
logger.warn(`Cannot find song ${d.title}?`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const tachiChart = (await FindChartWithPTDFVersion(
|
||||
"iidx",
|
||||
tachiSong.id,
|
||||
playtype as "SP" | "DP",
|
||||
diff as Difficulties["iidx:DP" | "iidx:SP"],
|
||||
"27"
|
||||
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
|
||||
|
||||
if (!tachiChart) {
|
||||
logger.warn(
|
||||
`Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const kavg = Number(d.avg);
|
||||
|
||||
if (kavg < 0) {
|
||||
logger.warn(
|
||||
`${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (d.removed) {
|
||||
logger.info(`Skipping removed chart ${tachiSong.title}.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
realData.push({
|
||||
coef: d.coef === -1 ? null : d.coef,
|
||||
kavg: Number(d.avg),
|
||||
wr: Number(d.wr),
|
||||
chartID: tachiChart.chartID,
|
||||
});
|
||||
}
|
||||
|
||||
await db["iidx-bpi-data"].remove({});
|
||||
await db["iidx-bpi-data"].insert(realData);
|
||||
|
||||
logger.info(`Removed all, and inserted ${realData.length} documents.`);
|
||||
|
||||
logger.info(`Triggering IIDX Recalc.`);
|
||||
|
||||
await RecalcAllScores({ game: "iidx" });
|
||||
|
||||
logger.info(`Done.`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
UpdatePoyashiData();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import db from "external/mongo/db";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import fetch from "node-fetch";
|
||||
import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common";
|
||||
import { RecalcAllScores } from "utils/calculations/recalc-scores";
|
||||
import { FindChartWithPTDFVersion } from "utils/queries/charts";
|
||||
import { FindSongOnTitle } from "utils/queries/songs";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
const difficultyResolve: Record<string, [string, string]> = {
|
||||
3: ["SP", "HYPER"],
|
||||
4: ["SP", "ANOTHER"],
|
||||
8: ["DP", "HYPER"],
|
||||
9: ["DP", "ANOTHER"],
|
||||
10: ["SP", "LEGGENDARIA"],
|
||||
11: ["DP", "LEGGENDARIA"],
|
||||
};
|
||||
|
||||
async function UpdatePoyashiData() {
|
||||
logger.info("Fetching BPI Poyashi data from proxy...");
|
||||
const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json());
|
||||
|
||||
logger.info("Fetched data.");
|
||||
logger.info("Saved data to output.json.");
|
||||
|
||||
const data = rj;
|
||||
|
||||
const realData: IIDXBPIData[] = [];
|
||||
for (const d of data.body) {
|
||||
const res = difficultyResolve[d.difficulty];
|
||||
|
||||
if (!res) {
|
||||
throw new Error(`Unknown difficulty ${d.difficulty}`);
|
||||
}
|
||||
|
||||
const [playtype, diff] = res;
|
||||
|
||||
const tachiSong = await FindSongOnTitle("iidx", d.title);
|
||||
|
||||
if (!tachiSong) {
|
||||
logger.warn(`Cannot find song ${d.title}?`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const tachiChart = (await FindChartWithPTDFVersion(
|
||||
"iidx",
|
||||
tachiSong.id,
|
||||
playtype as "SP" | "DP",
|
||||
diff as Difficulties["iidx:DP" | "iidx:SP"],
|
||||
"29"
|
||||
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
|
||||
|
||||
if (!tachiChart) {
|
||||
logger.warn(
|
||||
`Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const kavg = Number(d.avg);
|
||||
|
||||
if (kavg < 0) {
|
||||
logger.warn(
|
||||
`${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (d.removed) {
|
||||
logger.info(`Skipping removed chart ${tachiSong.title}.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
realData.push({
|
||||
coef: d.coef === -1 ? null : d.coef,
|
||||
kavg: Number(d.avg),
|
||||
wr: Number(d.wr),
|
||||
chartID: tachiChart.chartID,
|
||||
});
|
||||
}
|
||||
|
||||
const updatedCharts = [];
|
||||
for (const newData of realData) {
|
||||
const oldData = await db["iidx-bpi-data"].findOne({
|
||||
chartID: newData.chartID,
|
||||
});
|
||||
|
||||
if (!oldData) {
|
||||
updatedCharts.push(newData.chartID);
|
||||
} else if (
|
||||
oldData.wr !== newData.wr ||
|
||||
oldData.kavg !== newData.kavg ||
|
||||
oldData.coef !== newData.coef
|
||||
) {
|
||||
updatedCharts.push(newData.chartID);
|
||||
await db["iidx-bpi-data"].update(
|
||||
{
|
||||
chartID: newData.chartID,
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
wr: newData.wr,
|
||||
kavg: newData.kavg,
|
||||
coef: newData.coef,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (updatedCharts.length === 0) {
|
||||
logger.info(`Nothing has changed. Not updating any BPI data.`);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Triggering IIDX Recalc for ${updatedCharts.length} charts.`);
|
||||
|
||||
await RecalcAllScores({ game: "iidx", chartID: { $in: updatedCharts } });
|
||||
|
||||
logger.info(`Done.`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
UpdatePoyashiData();
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import db from "external/mongo/db";
|
||||
import { Game, integer, Playtypes, PublicUserDocument } from "tachi-common";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import { CreateCalculatedData } from "lib/score-import/framework/calculated-data/calculated-data";
|
||||
import { UpdateUsersGamePlaytypeStats } from "lib/score-import/framework/user-game-stats/update-ugs";
|
||||
import { GetAndUpdateUsersGoals } from "lib/score-import/framework/goals/goals";
|
||||
import { UpdateUsersMilestones } from "lib/score-import/framework/milestones/milestones";
|
||||
import { EfficientDBIterate } from "utils/efficient-db-iterate";
|
||||
import { ProcessPBs } from "lib/score-import/framework/pb/process-pbs";
|
||||
import { FormatUserDoc } from "utils/user";
|
||||
import deepmerge from "deepmerge";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
export async function RecalcAllScores(filter = {}) {
|
||||
logger.info(`Recalcing All Scores.`);
|
||||
|
||||
const modifiedUsers = new Set<string>();
|
||||
const modifiedUserIDs = new Set<integer>();
|
||||
const chartIDs = new Set<string>();
|
||||
|
||||
await EfficientDBIterate(
|
||||
db.scores,
|
||||
async (c) => {
|
||||
const chart = await db.charts[c.game].findOne({ chartID: c.chartID });
|
||||
|
||||
if (!chart) {
|
||||
logger.error(`Can't find chartID ${c.chartID} ${c.scoreID} (${c.game})`, {
|
||||
score: c,
|
||||
});
|
||||
|
||||
throw new Error(`screwed`);
|
||||
}
|
||||
|
||||
chartIDs.add(chart.chartID);
|
||||
|
||||
modifiedUsers.add(`${c.game}-${c.playtype}-${c.userID}`);
|
||||
modifiedUserIDs.add(c.userID);
|
||||
|
||||
const calculatedData = await CreateCalculatedData(c, chart, c.scoreData.esd, logger);
|
||||
|
||||
return { scoreID: c.scoreID, calculatedData };
|
||||
},
|
||||
async (updates) => {
|
||||
await db.scores.bulkWrite(
|
||||
updates.map((e) => ({
|
||||
updateOne: {
|
||||
filter: {
|
||||
scoreID: e.scoreID,
|
||||
},
|
||||
update: {
|
||||
$set: {
|
||||
calculatedData: e.calculatedData,
|
||||
},
|
||||
},
|
||||
},
|
||||
}))
|
||||
);
|
||||
},
|
||||
filter
|
||||
);
|
||||
|
||||
logger.info("Reprocessing PBs...");
|
||||
await UpdateAllPBs([...modifiedUserIDs.values()], filter);
|
||||
|
||||
logger.info(`Updating Profiles for ${modifiedUsers.size} users...`);
|
||||
|
||||
for (const userInfo of modifiedUsers.values()) {
|
||||
const [game, playtype, strUserID] = userInfo.split("-") as [Game, Playtypes[Game], string];
|
||||
|
||||
const userID = Number(strUserID);
|
||||
|
||||
await UpdateUsersGamePlaytypeStats(
|
||||
game as Game,
|
||||
playtype as Playtypes[Game],
|
||||
userID,
|
||||
null,
|
||||
logger
|
||||
);
|
||||
|
||||
const goalInfo = await GetAndUpdateUsersGoals(game, userID, chartIDs, logger);
|
||||
|
||||
await UpdateUsersMilestones(goalInfo, game, [playtype], userID, logger);
|
||||
}
|
||||
|
||||
logger.info(`Done!`);
|
||||
}
|
||||
|
||||
export async function UpdateAllPBs(userIDs?: integer[], filter = {}) {
|
||||
let allUsers: PublicUserDocument[];
|
||||
|
||||
if (!userIDs) {
|
||||
allUsers = await db.users.find({});
|
||||
} else {
|
||||
allUsers = await db.users.find({
|
||||
id: { $in: userIDs },
|
||||
});
|
||||
}
|
||||
|
||||
for (const user of allUsers) {
|
||||
logger.info(`Finding ${FormatUserDoc(user)}'s scores.`);
|
||||
|
||||
const scores = await db.scores.find(deepmerge({ userID: user.id }, filter), {
|
||||
projection: { chartID: 1 },
|
||||
});
|
||||
|
||||
logger.info(`PBing ${FormatUserDoc(user)}'s scores.`);
|
||||
await ProcessPBs(user.id, new Set(scores.map((e) => e.chartID)), logger);
|
||||
}
|
||||
|
||||
logger.info(`Done!`);
|
||||
}
|
||||
Reference in New Issue
Block a user