From 0b57ed5b7573e06d813af0b6a00964905d35fcd5 Mon Sep 17 00:00:00 2001 From: zkldi <20380519+zkldi@users.noreply.github.com> Date: Sun, 26 Jun 2022 17:02:13 +0100 Subject: [PATCH 1/2] feat: autosync ai table and refactor local jobs a bit --- server/package.json | 2 +- server/pnpm-lock.yaml | 8 ++-- server/src/lib/jobs/bms-ai-table-sync.ts | 40 +++++++++++++++++++ server/src/lib/jobs/bms-table-sync.ts | 30 +++++++------- .../lib/jobs/inline-job-runner/job-runner.ts | 19 ++++++++- .../framework/common/string-asserts.ts | 4 +- 6 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 server/src/lib/jobs/bms-ai-table-sync.ts diff --git a/server/package.json b/server/package.json index 149a9800c..927dc7dc1 100644 --- a/server/package.json +++ b/server/package.json @@ -89,7 +89,7 @@ "safe-json-stringify": "1.2.0", "semver": "^7.3.7", "seq-logging": "1.1.1", - "tachi-common": "0.14.9", + "tachi-common": "0.14.15", "ts-node": "10.4.0", "tsconfig-paths": "3.12.0", "typescript": "4.5.5", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 918c1734c..9cf30974b 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -68,7 +68,7 @@ specifiers: semver: ^7.3.7 seq-logging: 1.1.1 supertest: 6.2.2 - tachi-common: 0.14.9 + tachi-common: 0.14.15 tap: 15.1.6 ts-node: 10.4.0 tsconfig-paths: 3.12.0 @@ -117,7 +117,7 @@ dependencies: safe-json-stringify: 1.2.0 semver: 7.3.7 seq-logging: 1.1.1 - tachi-common: 0.14.9 + tachi-common: 0.14.15 ts-node: 10.4.0_2615db9039ce432b4abf2fc39ef336ec tsconfig-paths: 3.12.0 typescript: 4.5.5 @@ -5452,8 +5452,8 @@ packages: engines: {node: '>= 0.4'} dev: true - /tachi-common/0.14.9: - resolution: {integrity: sha512-jW5jNUAIPhTlJUhS69MHirBApP459CTHRFLr2wdetzcLvSf2KPyJjRST3tR9vjRMfu5s9Pvep6AIH+rpHqHK6w==} + /tachi-common/0.14.15: + resolution: {integrity: sha512-lK7jUhfJUAZ8cpukKrZxEO1Cvrn0wwr5YOaHDWlAkvdwFCN4hdf2g+R9SatgDdncAa/uk5angIXkuEdvhJ2Q1g==} dependencies: '@types/express-serve-static-core': 4.17.28 '@types/mongodb': 3.6.18 diff --git a/server/src/lib/jobs/bms-ai-table-sync.ts b/server/src/lib/jobs/bms-ai-table-sync.ts new file mode 100644 index 000000000..e9ba03266 --- /dev/null +++ b/server/src/lib/jobs/bms-ai-table-sync.ts @@ -0,0 +1,40 @@ +/* eslint-disable no-await-in-loop */ +import { PullDatabaseSeeds } from "lib/database-seeds/repo"; +import CreateLogCtx from "lib/logger/logger"; +import fetch from "node-fetch"; +import type { ChartDocument } from "tachi-common"; + +const AI_URL = "https://bms.hexlataia.xyz/tables/json/ai.json"; + +const logger = CreateLogCtx(__filename); + +interface AITableEntry { + level: string; + md5: string; +} + +export async function UpdateAILevels() { + const data = (await fetch(AI_URL).then((r) => r.json())) as Array; + + const map = Object.fromEntries(data.map((e) => [e.md5, e.level])); + + const repo = await PullDatabaseSeeds(); + + await repo.MutateCollection>("charts-bms", (charts) => { + for (const chart of charts) { + chart.data.aiRating = map[chart.data.hashMD5] ?? null; + } + + return charts; + }); +} + +if (require.main === module) { + UpdateAILevels() + .then(() => process.exit(0)) + .catch((err: unknown) => { + logger.error(`Failed to sync BMS AI Table.`, { err }, () => { + process.exit(1); + }); + }); +} diff --git a/server/src/lib/jobs/bms-table-sync.ts b/server/src/lib/jobs/bms-table-sync.ts index 8544f2e65..2a1f2323b 100644 --- a/server/src/lib/jobs/bms-table-sync.ts +++ b/server/src/lib/jobs/bms-table-sync.ts @@ -295,21 +295,23 @@ export async function UpdateTable(table: BMSTablesDataset) { logger.info(`Levels bumped.`); } -if (require.main === module) { - (async () => { - for (const table of registeredTables) { - // eslint-disable-next-line no-await-in-loop - await UpdateTable(table); - } +export async function SyncBMSTables() { + for (const table of registeredTables) { + // eslint-disable-next-line no-await-in-loop + await UpdateTable(table); + } - logger.info(`Re-initialising folder-chart-lookup, since changes may have been made.`); - await InitaliseFolderChartLookup(); - logger.info(`Done.`); + logger.info(`Re-initialising folder-chart-lookup, since changes may have been made.`); + await InitaliseFolderChartLookup(); + logger.info(`Done.`); +} - process.exit(0); - })().catch((err: unknown) => { - logger.error(`Failed to sync BMS Tables.`, { err }, () => { - process.exit(1); +if (require.main === module) { + SyncBMSTables() + .then(() => process.exit(0)) + .catch((err: unknown) => { + logger.error(`Failed to sync BMS Tables.`, { err }, () => { + process.exit(1); + }); }); - }); } diff --git a/server/src/lib/jobs/inline-job-runner/job-runner.ts b/server/src/lib/jobs/inline-job-runner/job-runner.ts index 3e796e253..274a6c6b8 100644 --- a/server/src/lib/jobs/inline-job-runner/job-runner.ts +++ b/server/src/lib/jobs/inline-job-runner/job-runner.ts @@ -1,4 +1,6 @@ import { BacksyncBMSPMSSongsAndCharts } from "../backsync-bms-pms-data"; +import { UpdateAILevels } from "../bms-ai-table-sync"; +import { SyncBMSTables } from "../bms-table-sync"; import { DeoprhanScores } from "../deorphan-scores"; import { UGSSnapshot } from "../ugs-snapshot"; import { UpdatePoyashiData } from "../update-bpi-data"; @@ -37,6 +39,7 @@ if (TachiConfig.TYPE !== "btchi") { cronFormat: "2 0 * * *", run: UpdatePoyashiData, }); + jobs.push({ name: "Update DP Tiers", cronFormat: "3 0 * * *", @@ -44,11 +47,23 @@ if (TachiConfig.TYPE !== "btchi") { }); } -// if bokutachi or omnimitachi +// if bokutachi or omnitachi if (TachiConfig.TYPE !== "ktchi") { jobs.push({ - name: "Backsync BMS + PMS", + name: "Update AI Table", cronFormat: "2 0 * * *", + run: UpdateAILevels, + }); + + jobs.push({ + name: "Update Tables", + cronFormat: "3 0 * * *", + run: SyncBMSTables, + }); + + jobs.push({ + name: "Backsync BMS + PMS", + cronFormat: "4 0 * * *", run: BacksyncBMSPMSSongsAndCharts, }); } diff --git a/server/src/lib/score-import/framework/common/string-asserts.ts b/server/src/lib/score-import/framework/common/string-asserts.ts index e75bd7a11..06c5618f1 100644 --- a/server/src/lib/score-import/framework/common/string-asserts.ts +++ b/server/src/lib/score-import/framework/common/string-asserts.ts @@ -9,7 +9,7 @@ export function AssertStrAsDifficulty( ): Difficulties[IDStrings] { const validDifficulties = GetGamePTConfig(game, playtype).difficulties; - if (!validDifficulties.includes(strVal as Difficulties[IDStrings])) { + if (!validDifficulties.includes(strVal)) { throw new InvalidScoreFailure( `Invalid Difficulty for ${game} ${playtype} - Expected any of ${validDifficulties.join( ", " @@ -17,7 +17,7 @@ export function AssertStrAsDifficulty( ); } - return strVal as Difficulties[IDStrings]; + return strVal; } const isIntegerRegex = /^-?\d+$/u; From 87b8d331e1ffbde5f3cdd80209532b617b2809cc Mon Sep 17 00:00:00 2001 From: zkldi <20380519+zkldi@users.noreply.github.com> Date: Wed, 29 Jun 2022 23:43:29 +0100 Subject: [PATCH 2/2] test: fix noncompiling tests --- server/src/test-utils/mock-db/charts-bms.json | 1 + server/src/test-utils/test-data.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/test-utils/mock-db/charts-bms.json b/server/src/test-utils/mock-db/charts-bms.json index 46914890b..0536b8fa0 100644 --- a/server/src/test-utils/mock-db/charts-bms.json +++ b/server/src/test-utils/mock-db/charts-bms.json @@ -4,6 +4,7 @@ "chartID": "88eb6cc5683e2740cbd07f588a5f3db1db8d467b", "rgcID": null, "data": { + "aiRating": "0", "notecount": 2256, "hashMD5": "38616b85332037cc12924f2ae2840262", "hashSHA256": "195fe1be5c3e74fccd04dc426e05f8a9cfa8a1059c339d0a23e99f63661f0b7d", diff --git a/server/src/test-utils/test-data.ts b/server/src/test-utils/test-data.ts index 3c6952757..b5244d80e 100644 --- a/server/src/test-utils/test-data.ts +++ b/server/src/test-utils/test-data.ts @@ -321,6 +321,7 @@ export const BMSGazerChart: ChartDocument<"bms:7K"> = { chartID: "88eb6cc5683e2740cbd07f588a5f3db1db8d467b", rgcID: null, data: { + aiRating: "0", notecount: 2256, hashMD5: "38616b85332037cc12924f2ae2840262", hashSHA256: "195fe1be5c3e74fccd04dc426e05f8a9cfa8a1059c339d0a23e99f63661f0b7d",