diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 772915e91..4596f7f54 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -336,7 +336,7 @@ importers: '@types/tap': 15.0.5 '@valuabletouch/winston-seq': 1.3.0 bcryptjs: 2.4.3 - bms-table-loader: ^1.0.0 + bms-table-loader: 1.0.1 bullmq: 1.68.4 bunyan: 1.8.15 commander: 9.0.0 @@ -404,7 +404,7 @@ importers: '@types/semver': 7.3.10 '@valuabletouch/winston-seq': 1.3.0 bcryptjs: 2.4.3 - bms-table-loader: 1.0.0 + bms-table-loader: 1.0.1 bullmq: 1.68.4 bunyan: 1.8.15 commander: 9.0.0 @@ -4417,8 +4417,8 @@ packages: /bluebird/3.7.2: resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} - /bms-table-loader/1.0.0: - resolution: {integrity: sha512-vvwmp8wkhhqyH3T8WlbctxW6MjDXvGuQhnfeNiGQfpo8TgL8eqtKrFWW6WmzLwSqKt7yMvZO02PqCVHaNPD5SA==} + /bms-table-loader/1.0.1: + resolution: {integrity: sha512-4UtqyeTPcEaHDl+9eeFzU3nf87Lj7uR7SRYzUjHgiXbIZVRj/exrK1g9d/6JFhti87ohnZPp5z31iX/MN5/nxg==} dependencies: node-fetch: 2.6.2 zod: 3.19.1 diff --git a/server/package.json b/server/package.json index d11bd4fb1..7f1614a73 100644 --- a/server/package.json +++ b/server/package.json @@ -30,7 +30,10 @@ "tap": "15.1.6" }, "dependencies": { + "@aws-sdk/client-s3": "3.49.0", + "@aws-sdk/signature-v4-crt": "3.49.0", "@types/bcryptjs": "^2.4.2", + "@types/bson": "1.0.11", "@types/bunyan": "^1.8.8", "@types/connect-redis": "0.0.18", "@types/express": "4.17.13", @@ -41,19 +44,17 @@ "@types/mkdirp": "1.0.2", "@types/mongodb": "3.6.18", "@types/multer": "1.4.7", + "@types/node": "16.11.7", "@types/node-fetch": "2.5.12", "@types/nodemailer": "^6.4.4", "@types/rate-limit-redis": "1.7.4", "@types/redis": "2.8.32", "@types/rimraf": "3.0.2", "@types/safe-json-stringify": "1.1.2", - "@aws-sdk/client-s3": "3.49.0", - "@aws-sdk/signature-v4-crt": "3.49.0", - "@types/bson": "1.0.11", - "@types/node": "16.11.7", "@types/semver": "^7.3.9", "@valuabletouch/winston-seq": "1.3.0", "bcryptjs": "2.4.3", + "bms-table-loader": "1.0.1", "bullmq": "1.68.4", "bunyan": "1.8.15", "commander": "9.0.0", diff --git a/server/src/lib/jobs/bms-table-sync.ts b/server/src/lib/jobs/bms-table-sync.ts index 167d20b6e..91ac977ac 100644 --- a/server/src/lib/jobs/bms-table-sync.ts +++ b/server/src/lib/jobs/bms-table-sync.ts @@ -1,22 +1,16 @@ /* eslint-disable no-await-in-loop */ +import { LoadBMSTable } from "bms-table-loader"; import db from "external/mongo/db"; import CreateLogCtx from "lib/logger/logger"; -import fetch from "node-fetch"; import { BMS_TABLES } from "tachi-common"; import { CreateFolderID, InitaliseFolderChartLookup } from "utils/folder"; import { FormatBMSTables } from "utils/misc"; +import type { BMSTableEntry } from "bms-table-loader"; import type { BMSTableInfo, ChartDocument } from "tachi-common"; -// this seems to be all we care about -interface TableJSONDoc { - md5: string; - title?: string; - level: string; -} - const logger = CreateLogCtx(__filename); -async function ImportTableLevels(tableJSON: Array, prefix: string) { +async function ImportTableLevels(tableJSON: Array, prefix: string) { let failures = 0; let success = 0; const total = tableJSON.length; @@ -67,65 +61,19 @@ async function ImportTableLevels(tableJSON: Array, prefix: string) logger.info(`${success} Success | ${failures} Failures | ${total} Total.`); } -interface BMSTableChart { - title: string; - artist: string; - url: string; - url_diff: string; - md5: string; - sha256: string; - level: string; -} +export async function UpdateTable(tableInfo: BMSTableInfo) { + const table = await LoadBMSTable(tableInfo.url); -const RETRY_COUNT = 3; - -export async function UpdateTable(table: BMSTableInfo) { - const res = await fetch(table.url); - - const statusCode = res.status; - - let retries = 0; - - // instead of using res.json() we have to use res.text() here - // so we have proper logging when some of these bms urls - // spontaneously return HTML. - let tableJSON: Array | undefined; - - // We need to have a retry counter because some of the bms tables - // randomly return useless html for no reason. - - // i hate bms. - while (!tableJSON) { - let text; - - try { - text = await res.text(); - - // @hack -- this is a force cast. - tableJSON = JSON.parse(text) as unknown as Array; - } catch (err) { - logger.error(`Failed to fetch ${table.url}`, { err, res, statusCode, text }); - retries++; - - if (retries > RETRY_COUNT) { - logger.error(`SKIPPING TABLE!`); - return; - } - - logger.info(`Retrying ${table.url}.`); - } - } - - const tableID = `bms-${table.playtype}-${table.asciiPrefix}`; + const tableID = `bms-${tableInfo.playtype}-${tableInfo.asciiPrefix}`; const folderIDs = []; - const levels = [...new Set(tableJSON.map((e) => e.level))]; + const levels = table.getLevelOrder(); for (const level of levels) { const query = { "data¬tableFolders": { "~elemMatch": { - table: table.prefix, + table: tableInfo.prefix, // just incase some dude tries numbers level: level.toString(), @@ -133,7 +81,7 @@ export async function UpdateTable(table: BMSTableInfo) { }, }; - const folderID = CreateFolderID(query, "bms", table.playtype); + const folderID = CreateFolderID(query, "bms", tableInfo.playtype); folderIDs.push(folderID); @@ -146,17 +94,17 @@ export async function UpdateTable(table: BMSTableInfo) { } await db.folders.insert({ - title: `${table.prefix}${level}`, + title: `${tableInfo.prefix}${level}`, inactive: false, folderID, game: "bms", - playtype: table.playtype, + playtype: tableInfo.playtype, type: "charts", data: query, - searchTerms: [`${table.name} ${level}`], + searchTerms: [`${tableInfo.name} ${level}`], }); - logger.info(`Inserted new folder ${table.prefix}${level}.`); + logger.info(`Inserted new folder ${tableInfo.prefix}${level}.`); } await db.tables.update( @@ -165,10 +113,10 @@ export async function UpdateTable(table: BMSTableInfo) { $set: { tableID, game: "bms", - playtype: table.playtype, + playtype: tableInfo.playtype, folders: folderIDs, - title: table.name, - description: table.description, + title: tableInfo.name, + description: tableInfo.description, inactive: false, }, $setOnInsert: { @@ -180,15 +128,15 @@ export async function UpdateTable(table: BMSTableInfo) { } ); - logger.info(`Bumped table ${table.name}.`); + logger.info(`Bumped table ${tableInfo.name}.`); logger.info(`Checking meta-folder...`); const metaQuery = { - "data¬tableFolders¬table": table.prefix, + "data¬tableFolders¬table": tableInfo.prefix, }; - const folderID = CreateFolderID(metaQuery, "bms", table.playtype); + const folderID = CreateFolderID(metaQuery, "bms", tableInfo.playtype); folderIDs.push(folderID); @@ -198,19 +146,19 @@ export async function UpdateTable(table: BMSTableInfo) { if (!exists) { await db.folders.insert({ - title: `${table.name}`, + title: `${tableInfo.name}`, inactive: false, folderID, game: "bms", - playtype: table.playtype, + playtype: tableInfo.playtype, type: "charts", data: metaQuery, - searchTerms: [table.asciiPrefix], + searchTerms: [tableInfo.asciiPrefix], }); await db.tables.update( { - tableID: `bms-${table.playtype}-meta`, + tableID: `bms-${tableInfo.playtype}-meta`, }, { $push: { @@ -219,11 +167,11 @@ export async function UpdateTable(table: BMSTableInfo) { } ); - logger.info(`Inserted meta folder for ${table.name}.`); + logger.info(`Inserted meta folder for ${tableInfo.name}.`); } logger.info(`Bumping levels...`); - await ImportTableLevels(tableJSON, table.prefix); + await ImportTableLevels(table.body, tableInfo.prefix); logger.info(`Levels bumped.`); }