mirror of
https://github.com/zkldi/Tachi.git
synced 2026-09-29 02:18:09 +03:00
Move BMS Table updating to jobs.
This commit is contained in:
zkldi
@@ -1,174 +0,0 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import db from "external/mongo/db";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import fetch from "node-fetch";
|
||||
import { ChartDocument } from "tachi-common";
|
||||
import { CreateFolderID } from "utils/folder";
|
||||
|
||||
// this seems to be all we care about
|
||||
interface TableJSONDoc {
|
||||
md5: string;
|
||||
title?: string;
|
||||
level: string;
|
||||
}
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
async function ImportTableLevels(tableJSON: TableJSONDoc[], prefix: string) {
|
||||
let failures = 0;
|
||||
let success = 0;
|
||||
const total = tableJSON.length;
|
||||
|
||||
for (const td of tableJSON) {
|
||||
const chart = (await db.charts.bms.findOne({ "data.hashMD5": td.md5 })) as ChartDocument<
|
||||
"bms:7K" | "bms:14K"
|
||||
>;
|
||||
|
||||
if (!chart) {
|
||||
logger.warn(
|
||||
`No chart exists in table for ${td.md5} Possible title: ${td.title} ${prefix}${td.level}`
|
||||
);
|
||||
failures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const tableFolders = chart.data.tableFolders.filter((e) => e.table !== prefix);
|
||||
|
||||
tableFolders.push({ table: prefix, level: td.level.toString() });
|
||||
|
||||
await db.charts.bms.update(
|
||||
{
|
||||
chartID: chart.chartID,
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
"data.tableFolders": tableFolders,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
success++;
|
||||
}
|
||||
|
||||
logger.info(`Finished updating table ${prefix}.`);
|
||||
logger.info(`${success} Success | ${failures} Failures | ${total} Total.`);
|
||||
}
|
||||
|
||||
export interface BMSTablesDataset {
|
||||
url: string;
|
||||
name: string;
|
||||
description: string;
|
||||
humanisedPrefix: string;
|
||||
prefix: string;
|
||||
playtype: "7K" | "14K";
|
||||
}
|
||||
|
||||
interface BMSTableChart {
|
||||
title: string;
|
||||
artist: string;
|
||||
url: string;
|
||||
url_diff: string;
|
||||
md5: string;
|
||||
sha256: string;
|
||||
level: string;
|
||||
}
|
||||
|
||||
const RETRY_COUNT = 3;
|
||||
|
||||
export async function UpdateTable(table: BMSTablesDataset) {
|
||||
const res = await fetch(table.url);
|
||||
|
||||
const statusCode = res.status;
|
||||
|
||||
let retries = 0;
|
||||
|
||||
// instead of using res.json() we have to use res.text() here
|
||||
// so we have proper logging when some of these bms urls
|
||||
// spontaneously return HTML.
|
||||
let tableJSON: BMSTableChart[] | undefined;
|
||||
|
||||
// We need to have a retry counter because some of the bms tables
|
||||
// randomly return useless html for no reason.
|
||||
|
||||
// i hate bms.
|
||||
while (!tableJSON) {
|
||||
let text;
|
||||
try {
|
||||
text = await res.text();
|
||||
tableJSON = JSON.parse(text);
|
||||
} catch (err) {
|
||||
logger.error(`Failed to fetch ${table.url}`, { err, res, statusCode, text });
|
||||
retries++;
|
||||
|
||||
if (retries > RETRY_COUNT) {
|
||||
logger.error(`SKIPPING TABLE!`);
|
||||
return;
|
||||
}
|
||||
logger.info(`Retrying ${table.url}.`);
|
||||
}
|
||||
}
|
||||
|
||||
const tableID = `bms-${table.playtype}-${table.humanisedPrefix}`;
|
||||
|
||||
const folderIDs = [];
|
||||
const levels = [...new Set(tableJSON.map((e) => e.level))];
|
||||
|
||||
for (const level of levels) {
|
||||
const query = {
|
||||
"data¬tableFolders": {
|
||||
table: table.prefix,
|
||||
// just incase some dude tries numbers
|
||||
level: level.toString(),
|
||||
},
|
||||
};
|
||||
|
||||
const folderID = CreateFolderID(query, "bms", table.playtype);
|
||||
|
||||
folderIDs.push(folderID);
|
||||
|
||||
const exists = await db.folders.findOne({
|
||||
folderID,
|
||||
});
|
||||
|
||||
if (exists) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await db.folders.insert({
|
||||
title: `${table.prefix}${level}`,
|
||||
inactive: false,
|
||||
folderID,
|
||||
game: "bms",
|
||||
playtype: table.playtype,
|
||||
type: "charts",
|
||||
data: query,
|
||||
searchTerms: [`${table.name} ${level}`],
|
||||
});
|
||||
|
||||
logger.info(`Inserted new table ${table.prefix}${level}.`);
|
||||
}
|
||||
|
||||
await db.tables.update(
|
||||
{ tableID },
|
||||
{
|
||||
$set: {
|
||||
tableID,
|
||||
game: "bms",
|
||||
playtype: table.playtype,
|
||||
folders: folderIDs,
|
||||
title: table.name,
|
||||
description: table.description,
|
||||
inactive: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
upsert: true,
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Bumped table ${table.name}.`);
|
||||
|
||||
logger.info(`Bumping levels...`);
|
||||
await ImportTableLevels(tableJSON, table.prefix);
|
||||
logger.info(`Levels bumped.`);
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
import { BMS_TABLES } from "lib/constants/bms-tables";
|
||||
import { BMSTablesDataset, UpdateTable } from "./table-sync";
|
||||
|
||||
// I have no confidence in half of these links surviving.
|
||||
// Eventually, some of these are going to disappear, and we'll have to find
|
||||
// something else. probably.
|
||||
const registeredTables: BMSTablesDataset[] = [
|
||||
{
|
||||
name: "Insane",
|
||||
humanisedPrefix: "insane",
|
||||
playtype: "7K",
|
||||
prefix: BMS_TABLES.insane,
|
||||
description: "The 7K GENOSIDE insane table.",
|
||||
url: "http://www.ribbit.xyz/bms/tables/insane_body.json",
|
||||
},
|
||||
{
|
||||
name: "Normal",
|
||||
humanisedPrefix: "normal",
|
||||
playtype: "7K",
|
||||
prefix: BMS_TABLES.normal,
|
||||
description: "The 7K GENOSIDE normal table.",
|
||||
url: "http://www.ribbit.xyz/bms/tables/normal_body.json",
|
||||
},
|
||||
{
|
||||
name: "Stella",
|
||||
humanisedPrefix: "st",
|
||||
playtype: "7K",
|
||||
prefix: BMS_TABLES.stella,
|
||||
description: "The stella table, from Stellaverse.",
|
||||
url: "https://stellabms.xyz/st/score.json",
|
||||
},
|
||||
{
|
||||
name: "Satellite",
|
||||
humanisedPrefix: "sl",
|
||||
playtype: "7K",
|
||||
prefix: BMS_TABLES.satellite,
|
||||
description: "The satellite table, from Stellaverse.",
|
||||
url: "https://stellabms.xyz/sl/score.json",
|
||||
},
|
||||
{
|
||||
name: "Insane2",
|
||||
humanisedPrefix: "insane2",
|
||||
playtype: "7K",
|
||||
prefix: BMS_TABLES.insane2,
|
||||
description: "A successor to the original insane table, but is generally less consistent.",
|
||||
url: "http://rattoto10.jounin.jp/js/insane_data.json",
|
||||
},
|
||||
{
|
||||
name: "Normal2",
|
||||
humanisedPrefix: "normal2",
|
||||
playtype: "7K",
|
||||
prefix: BMS_TABLES.normal2,
|
||||
description: "A successor to the original normal table.",
|
||||
url: "http://rattoto10.jounin.jp/js/score.json",
|
||||
},
|
||||
{
|
||||
name: "Overjoy",
|
||||
description: "The overjoy table. Level 1 is roughly equivalent to Insane 20.",
|
||||
humanisedPrefix: "overjoy",
|
||||
playtype: "7K",
|
||||
prefix: BMS_TABLES.overjoy,
|
||||
url: "http://lr2.sakura.ne.jp/data/score.json",
|
||||
},
|
||||
{
|
||||
name: "DP Insane",
|
||||
description: "The 14K Insane table.",
|
||||
humanisedPrefix: "dpInsane",
|
||||
prefix: BMS_TABLES.dpInsane,
|
||||
playtype: "14K",
|
||||
url: "http://dpbmsdelta.web.fc2.com/table/data/insane_data.json",
|
||||
},
|
||||
{
|
||||
name: "DP Normal",
|
||||
description: "The 14K Normal table, Sometimes called the delta table.",
|
||||
humanisedPrefix: "dpNormal",
|
||||
prefix: BMS_TABLES.dpNormal,
|
||||
playtype: "14K",
|
||||
url: "http://dpbmsdelta.web.fc2.com/table/data/dpdelta_data.json",
|
||||
},
|
||||
{
|
||||
name: "DP Satellite",
|
||||
description: "The 14K Satellite table.",
|
||||
humanisedPrefix: "dpSatellite",
|
||||
prefix: BMS_TABLES.satellite,
|
||||
playtype: "14K",
|
||||
url: "https://stellabms.xyz/dp/score.json",
|
||||
},
|
||||
];
|
||||
|
||||
(async () => {
|
||||
for (const table of registeredTables) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await UpdateTable(table);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
})();
|
||||
Reference in New Issue
Block a user