mirror of
https://github.com/zkldi/Tachi.git
synced 2026-09-27 09:28:12 +03:00
feat: update how bms-table-sync works
This commit is contained in:
zkldi
Generated
+4
-4
@@ -336,7 +336,7 @@ importers:
|
||||
'@types/tap': 15.0.5
|
||||
'@valuabletouch/winston-seq': 1.3.0
|
||||
bcryptjs: 2.4.3
|
||||
bms-table-loader: ^1.0.0
|
||||
bms-table-loader: 1.0.1
|
||||
bullmq: 1.68.4
|
||||
bunyan: 1.8.15
|
||||
commander: 9.0.0
|
||||
@@ -404,7 +404,7 @@ importers:
|
||||
'@types/semver': 7.3.10
|
||||
'@valuabletouch/winston-seq': 1.3.0
|
||||
bcryptjs: 2.4.3
|
||||
bms-table-loader: 1.0.0
|
||||
bms-table-loader: 1.0.1
|
||||
bullmq: 1.68.4
|
||||
bunyan: 1.8.15
|
||||
commander: 9.0.0
|
||||
@@ -4417,8 +4417,8 @@ packages:
|
||||
/bluebird/3.7.2:
|
||||
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
|
||||
|
||||
/bms-table-loader/1.0.0:
|
||||
resolution: {integrity: sha512-vvwmp8wkhhqyH3T8WlbctxW6MjDXvGuQhnfeNiGQfpo8TgL8eqtKrFWW6WmzLwSqKt7yMvZO02PqCVHaNPD5SA==}
|
||||
/bms-table-loader/1.0.1:
|
||||
resolution: {integrity: sha512-4UtqyeTPcEaHDl+9eeFzU3nf87Lj7uR7SRYzUjHgiXbIZVRj/exrK1g9d/6JFhti87ohnZPp5z31iX/MN5/nxg==}
|
||||
dependencies:
|
||||
node-fetch: 2.6.2
|
||||
zod: 3.19.1
|
||||
|
||||
+5
-4
@@ -30,7 +30,10 @@
|
||||
"tap": "15.1.6"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.49.0",
|
||||
"@aws-sdk/signature-v4-crt": "3.49.0",
|
||||
"@types/bcryptjs": "^2.4.2",
|
||||
"@types/bson": "1.0.11",
|
||||
"@types/bunyan": "^1.8.8",
|
||||
"@types/connect-redis": "0.0.18",
|
||||
"@types/express": "4.17.13",
|
||||
@@ -41,19 +44,17 @@
|
||||
"@types/mkdirp": "1.0.2",
|
||||
"@types/mongodb": "3.6.18",
|
||||
"@types/multer": "1.4.7",
|
||||
"@types/node": "16.11.7",
|
||||
"@types/node-fetch": "2.5.12",
|
||||
"@types/nodemailer": "^6.4.4",
|
||||
"@types/rate-limit-redis": "1.7.4",
|
||||
"@types/redis": "2.8.32",
|
||||
"@types/rimraf": "3.0.2",
|
||||
"@types/safe-json-stringify": "1.1.2",
|
||||
"@aws-sdk/client-s3": "3.49.0",
|
||||
"@aws-sdk/signature-v4-crt": "3.49.0",
|
||||
"@types/bson": "1.0.11",
|
||||
"@types/node": "16.11.7",
|
||||
"@types/semver": "^7.3.9",
|
||||
"@valuabletouch/winston-seq": "1.3.0",
|
||||
"bcryptjs": "2.4.3",
|
||||
"bms-table-loader": "1.0.1",
|
||||
"bullmq": "1.68.4",
|
||||
"bunyan": "1.8.15",
|
||||
"commander": "9.0.0",
|
||||
|
||||
@@ -1,22 +1,16 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import { LoadBMSTable } from "bms-table-loader";
|
||||
import db from "external/mongo/db";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import fetch from "node-fetch";
|
||||
import { BMS_TABLES } from "tachi-common";
|
||||
import { CreateFolderID, InitaliseFolderChartLookup } from "utils/folder";
|
||||
import { FormatBMSTables } from "utils/misc";
|
||||
import type { BMSTableEntry } from "bms-table-loader";
|
||||
import type { BMSTableInfo, ChartDocument } from "tachi-common";
|
||||
|
||||
// this seems to be all we care about
|
||||
interface TableJSONDoc {
|
||||
md5: string;
|
||||
title?: string;
|
||||
level: string;
|
||||
}
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
async function ImportTableLevels(tableJSON: Array<TableJSONDoc>, prefix: string) {
|
||||
async function ImportTableLevels(tableJSON: Array<BMSTableEntry>, prefix: string) {
|
||||
let failures = 0;
|
||||
let success = 0;
|
||||
const total = tableJSON.length;
|
||||
@@ -67,65 +61,19 @@ async function ImportTableLevels(tableJSON: Array<TableJSONDoc>, prefix: string)
|
||||
logger.info(`${success} Success | ${failures} Failures | ${total} Total.`);
|
||||
}
|
||||
|
||||
interface BMSTableChart {
|
||||
title: string;
|
||||
artist: string;
|
||||
url: string;
|
||||
url_diff: string;
|
||||
md5: string;
|
||||
sha256: string;
|
||||
level: string;
|
||||
}
|
||||
export async function UpdateTable(tableInfo: BMSTableInfo) {
|
||||
const table = await LoadBMSTable(tableInfo.url);
|
||||
|
||||
const RETRY_COUNT = 3;
|
||||
|
||||
export async function UpdateTable(table: BMSTableInfo) {
|
||||
const res = await fetch(table.url);
|
||||
|
||||
const statusCode = res.status;
|
||||
|
||||
let retries = 0;
|
||||
|
||||
// instead of using res.json() we have to use res.text() here
|
||||
// so we have proper logging when some of these bms urls
|
||||
// spontaneously return HTML.
|
||||
let tableJSON: Array<BMSTableChart> | undefined;
|
||||
|
||||
// We need to have a retry counter because some of the bms tables
|
||||
// randomly return useless html for no reason.
|
||||
|
||||
// i hate bms.
|
||||
while (!tableJSON) {
|
||||
let text;
|
||||
|
||||
try {
|
||||
text = await res.text();
|
||||
|
||||
// @hack -- this is a force cast.
|
||||
tableJSON = JSON.parse(text) as unknown as Array<BMSTableChart>;
|
||||
} catch (err) {
|
||||
logger.error(`Failed to fetch ${table.url}`, { err, res, statusCode, text });
|
||||
retries++;
|
||||
|
||||
if (retries > RETRY_COUNT) {
|
||||
logger.error(`SKIPPING TABLE!`);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Retrying ${table.url}.`);
|
||||
}
|
||||
}
|
||||
|
||||
const tableID = `bms-${table.playtype}-${table.asciiPrefix}`;
|
||||
const tableID = `bms-${tableInfo.playtype}-${tableInfo.asciiPrefix}`;
|
||||
|
||||
const folderIDs = [];
|
||||
const levels = [...new Set(tableJSON.map((e) => e.level))];
|
||||
const levels = table.getLevelOrder();
|
||||
|
||||
for (const level of levels) {
|
||||
const query = {
|
||||
"data¬tableFolders": {
|
||||
"~elemMatch": {
|
||||
table: table.prefix,
|
||||
table: tableInfo.prefix,
|
||||
|
||||
// just incase some dude tries numbers
|
||||
level: level.toString(),
|
||||
@@ -133,7 +81,7 @@ export async function UpdateTable(table: BMSTableInfo) {
|
||||
},
|
||||
};
|
||||
|
||||
const folderID = CreateFolderID(query, "bms", table.playtype);
|
||||
const folderID = CreateFolderID(query, "bms", tableInfo.playtype);
|
||||
|
||||
folderIDs.push(folderID);
|
||||
|
||||
@@ -146,17 +94,17 @@ export async function UpdateTable(table: BMSTableInfo) {
|
||||
}
|
||||
|
||||
await db.folders.insert({
|
||||
title: `${table.prefix}${level}`,
|
||||
title: `${tableInfo.prefix}${level}`,
|
||||
inactive: false,
|
||||
folderID,
|
||||
game: "bms",
|
||||
playtype: table.playtype,
|
||||
playtype: tableInfo.playtype,
|
||||
type: "charts",
|
||||
data: query,
|
||||
searchTerms: [`${table.name} ${level}`],
|
||||
searchTerms: [`${tableInfo.name} ${level}`],
|
||||
});
|
||||
|
||||
logger.info(`Inserted new folder ${table.prefix}${level}.`);
|
||||
logger.info(`Inserted new folder ${tableInfo.prefix}${level}.`);
|
||||
}
|
||||
|
||||
await db.tables.update(
|
||||
@@ -165,10 +113,10 @@ export async function UpdateTable(table: BMSTableInfo) {
|
||||
$set: {
|
||||
tableID,
|
||||
game: "bms",
|
||||
playtype: table.playtype,
|
||||
playtype: tableInfo.playtype,
|
||||
folders: folderIDs,
|
||||
title: table.name,
|
||||
description: table.description,
|
||||
title: tableInfo.name,
|
||||
description: tableInfo.description,
|
||||
inactive: false,
|
||||
},
|
||||
$setOnInsert: {
|
||||
@@ -180,15 +128,15 @@ export async function UpdateTable(table: BMSTableInfo) {
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Bumped table ${table.name}.`);
|
||||
logger.info(`Bumped table ${tableInfo.name}.`);
|
||||
|
||||
logger.info(`Checking meta-folder...`);
|
||||
|
||||
const metaQuery = {
|
||||
"data¬tableFolders¬table": table.prefix,
|
||||
"data¬tableFolders¬table": tableInfo.prefix,
|
||||
};
|
||||
|
||||
const folderID = CreateFolderID(metaQuery, "bms", table.playtype);
|
||||
const folderID = CreateFolderID(metaQuery, "bms", tableInfo.playtype);
|
||||
|
||||
folderIDs.push(folderID);
|
||||
|
||||
@@ -198,19 +146,19 @@ export async function UpdateTable(table: BMSTableInfo) {
|
||||
|
||||
if (!exists) {
|
||||
await db.folders.insert({
|
||||
title: `${table.name}`,
|
||||
title: `${tableInfo.name}`,
|
||||
inactive: false,
|
||||
folderID,
|
||||
game: "bms",
|
||||
playtype: table.playtype,
|
||||
playtype: tableInfo.playtype,
|
||||
type: "charts",
|
||||
data: metaQuery,
|
||||
searchTerms: [table.asciiPrefix],
|
||||
searchTerms: [tableInfo.asciiPrefix],
|
||||
});
|
||||
|
||||
await db.tables.update(
|
||||
{
|
||||
tableID: `bms-${table.playtype}-meta`,
|
||||
tableID: `bms-${tableInfo.playtype}-meta`,
|
||||
},
|
||||
{
|
||||
$push: {
|
||||
@@ -219,11 +167,11 @@ export async function UpdateTable(table: BMSTableInfo) {
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Inserted meta folder for ${table.name}.`);
|
||||
logger.info(`Inserted meta folder for ${tableInfo.name}.`);
|
||||
}
|
||||
|
||||
logger.info(`Bumping levels...`);
|
||||
await ImportTableLevels(tableJSON, table.prefix);
|
||||
await ImportTableLevels(table.body, tableInfo.prefix);
|
||||
logger.info(`Levels bumped.`);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user