From 4e19023b36af6abaeafc997bd45cfc25f62dcf81 Mon Sep 17 00:00:00 2001 From: zkldi <20380519+zkldi@users.noreply.github.com> Date: Tue, 1 Mar 2022 03:50:47 +0000 Subject: [PATCH] Seeds Update Scripts Fixes #658 --- server/.github/test.conf.json5 | 1 + server/src/external/mongo/sequence-docs.ts | 12 -- server/src/lib/database-seeds/repo.ts | 166 ++++++++++++++++++ server/src/lib/jobs/backsync-bms-data.ts | 42 +++++ server/src/lib/jobs/deorphan-scores.ts | 4 +- .../job-runner.ts | 21 +++ server/src/lib/jobs/update-bpi-data.ts | 151 ++++++++++++++++ server/src/lib/setup/config.ts | 2 + .../{sync-database => }/sync-database.ts | 86 +++++---- server/src/utils/misc.ts | 22 +++ server/tsconfig.json | 7 +- 11 files changed, 450 insertions(+), 64 deletions(-) create mode 100644 server/src/lib/database-seeds/repo.ts create mode 100644 server/src/lib/jobs/backsync-bms-data.ts rename server/src/lib/jobs/{inline-job-runner.ts => inline-job-runner}/job-runner.ts (77%) create mode 100644 server/src/lib/jobs/update-bpi-data.ts rename server/src/scripts/{sync-database => }/sync-database.ts (80%) diff --git a/server/.github/test.conf.json5 b/server/.github/test.conf.json5 index 3f7c251f9..5c854318e 100644 --- a/server/.github/test.conf.json5 +++ b/server/.github/test.conf.json5 @@ -21,6 +21,7 @@ ARC_AUTH_TOKEN: "unused", ENABLE_SERVER_HTTPS: false, OUR_URL: "https://example.com", + SEEDS_URL: null, INVITE_CODE_CONFIG: { BATCH_SIZE: 2, INVITE_CAP: 100, diff --git a/server/src/external/mongo/sequence-docs.ts b/server/src/external/mongo/sequence-docs.ts index 37318525c..e4036bdad 100644 --- a/server/src/external/mongo/sequence-docs.ts +++ b/server/src/external/mongo/sequence-docs.ts @@ -15,14 +15,6 @@ export async function InitSequenceDocs() { } ); - const largestUSCSongID = await db.songs.usc.findOne( - {}, - { - sort: { - id: -1, - }, - } - ); const largestBMSSongID = await db.songs.bms.findOne( {}, { @@ -37,10 +29,6 @@ export async function InitSequenceDocs() { counterName: "users", value: userWithLargestID ? userWithLargestID.id + 1 : 1, }, - { - counterName: "usc-song-id", - value: largestUSCSongID ? largestUSCSongID.id + 1 : 1, - }, { counterName: "bms-song-id", value: largestBMSSongID ? largestBMSSongID.id + 1 : 1, diff --git a/server/src/lib/database-seeds/repo.ts b/server/src/lib/database-seeds/repo.ts new file mode 100644 index 000000000..a8d0b23af --- /dev/null +++ b/server/src/lib/database-seeds/repo.ts @@ -0,0 +1,166 @@ +import path from "path"; +import os from "os"; +import CreateLogCtx from "lib/logger/logger"; +import { Environment, ServerConfig } from "lib/setup/config"; +import { asyncExec } from "utils/misc"; +import { Game } from "tachi-common"; +import fs from "fs/promises"; + +const logger = CreateLogCtx(__filename); + +export type SeedsCollections = + | `charts-${Game}` + | `songs-${Game}` + | "bms-course-lookup" + | "folders" + | "tables"; + +/** + * Class that encapsulates the behaviour of a seeds repo. + */ +export class DatabaseSeedsRepo { + private baseDir: string; + private logger; + + constructor(baseDir: string) { + this.baseDir = baseDir; + this.logger = CreateLogCtx(`DatabaseSeeds:${baseDir}`); + } + + private CollectionNameToPath(collectionName: SeedsCollections) { + return path.join(this.baseDir, "collections", `${collectionName}.json`); + } + + /** + * Reads the data from a collection and returns the parsed JSON. + * + * @returns The data in the requested collection. + */ + async ReadCollection(collectionName: SeedsCollections): Promise { + const data = await fs.readFile(this.CollectionNameToPath(collectionName), { + encoding: "utf-8", + }); + + const parsedData = JSON.parse(data) as D[]; + + return parsedData; + } + + /** + * Writes a new array to the provided collectionName. + * + * @param collectionName - The collection to write to. + * @param content - A new array of objects to write. + */ + async WriteCollection(collectionName: SeedsCollections, content: unknown[]) { + await fs.writeFile(this.CollectionNameToPath(collectionName), JSON.stringify(content)); + + // Deterministically sort whatever content we just wrote. + await asyncExec( + `cd "${this.baseDir}" || exit 1; node scripts/deterministic-collection-sort.js` + ); + } + + async *IterateCollections() { + const collectionNames = (await fs.readdir(path.join(this.baseDir, "collections"))).map( + (e) => path.parse(e).name + ) as SeedsCollections[]; + + for (const collectionName of collectionNames) { + // eslint-disable-next-line no-await-in-loop + yield { collectionName, data: await this.ReadCollection(collectionName) }; + } + } + + /** + * Mutate a collection with a given name. + * + * @param collectionName - The collection to mutate. + * @param mutator - A function that takes the entire collection as an array, then returns a new array. + */ + async MutateCollection(collectionName: SeedsCollections, mutator: (dataset: D[]) => D[]) { + const dataset = await this.ReadCollection(collectionName); + + const newData = mutator(dataset); + + return this.WriteCollection(collectionName, newData); + } + + /** + * Checks for any diffs in the seeds repository we cloned. If there are any, commit them back + * to the repository. + * + * @param commitMsg - The commit message. + * @returns True when a commit has occured, false when it hasn't. Throws on failure. + */ + async CommitChangesBack(commitMsg: string) { + this.logger.verbose(`Received commit-back request.`); + + try { + const { stdout: statusOut } = await asyncExec( + `cd "${this.baseDir}" || exit 1; git status --porcelain` + ); + + if (statusOut === "") { + this.logger.info(`No changes. Not committing any changes back.`); + return false; + } + + this.logger.info(`Changes. Committing changes back.`); + + const { stdout: commitOut, stderr: commitErr } = await asyncExec( + `cd "${this.baseDir}" || exit 2; + git add . || exit 3; + git commit -am "${commitMsg}" || exit 4; + git push` + ); + + this.logger.info(`Commit: ${commitOut}.`); + + if (commitErr) { + this.logger.error(`Commit Err?: ${commitErr}`); + } + + return true; + } catch (err) { + this.logger.error(`Failed to backport commits?`, { err }); + throw err; + } + } +} + +/** + * Pulls the database seeds from github, returns an object that can be used to manipulate them. + */ +export async function PullDatabaseSeeds() { + if (!ServerConfig.SEEDS_URL) { + throw new Error(`SEEDS_URL was null. You cannot pull a seeds repo.`); + } + + const seedsDir = await fs.mkdtemp(path.join(os.tmpdir(), "tachi-database-seeds-")); + + logger.info(`Cloning data to ${seedsDir}.`); + + await fs.rm(seedsDir, { recursive: true, force: true }); + + try { + // stderr in git clone is normal output. + // stdout is for errors. + // there were expletives below this comment, but I have removed them. + const { stdout } = await asyncExec( + `git clone "${ServerConfig.SEEDS_URL}" -b "${ + Environment.nodeEnv === "production" ? "master" : "develop" + }" --depth=1 '${seedsDir}'` + ); + + // isn't that confusing + if (stdout) { + logger.error(stdout); + } + + return new DatabaseSeedsRepo(seedsDir); + } catch ({ err, stdout, stderr }) { + logger.error(`Error cloning database-seeds. ${stderr}.`); + throw err; + } +} diff --git a/server/src/lib/jobs/backsync-bms-data.ts b/server/src/lib/jobs/backsync-bms-data.ts new file mode 100644 index 000000000..dadefbfde --- /dev/null +++ b/server/src/lib/jobs/backsync-bms-data.ts @@ -0,0 +1,42 @@ +import db from "external/mongo/db"; +import { PullDatabaseSeeds } from "lib/database-seeds/repo"; +import CreateLogCtx from "lib/logger/logger"; + +const logger = CreateLogCtx(__filename); + +/** + * The tachi-server may have its BMS database update. It needs to sync this + * information back with the seeds. + */ +export async function BacksyncBMSSongsAndCharts() { + const repo = await PullDatabaseSeeds(); + + logger.info(`Fetching BMS songs from DB.`); + // did you know, this is liable to blow up in my face and OOM one day? + const bmsSongs = await db.songs.bms.find({}); + + logger.info(`Found ${bmsSongs.length} bms songs.`); + + await repo.WriteCollection("songs-bms", bmsSongs); + + // @ts-expect-error This is obviously making something nullable when it shouldn't be. + // but if we don't *force* node to free this damn memory, it kills itself when it + // tries to read even more stuff. + bmsSongs = null; + + logger.info(`Fetching BMS charts from DB.`); + const bmsCharts = await db.charts.bms.find({}); + + logger.info(`Found ${bmsCharts.length} bms charts.`); + + await repo.WriteCollection("charts-bms", bmsCharts); + + // @ts-expect-error See previous expect-error. + bmsCharts = null; + + await repo.CommitChangesBack(`Backsync BMS Songs/Charts ${new Date().toISOString()}`); +} + +if (require.main === module) { + BacksyncBMSSongsAndCharts().then(() => process.exit(0)); +} diff --git a/server/src/lib/jobs/deorphan-scores.ts b/server/src/lib/jobs/deorphan-scores.ts index f3691ba07..199738767 100644 --- a/server/src/lib/jobs/deorphan-scores.ts +++ b/server/src/lib/jobs/deorphan-scores.ts @@ -42,7 +42,5 @@ export async function DeoprhanScores() { } if (require.main === module) { - DeoprhanScores().then(() => { - process.exit(0); - }); + DeoprhanScores().then(() => process.exit(0)); } diff --git a/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts b/server/src/lib/jobs/inline-job-runner/job-runner.ts similarity index 77% rename from server/src/lib/jobs/inline-job-runner.ts/job-runner.ts rename to server/src/lib/jobs/inline-job-runner/job-runner.ts index 9cf6872d3..fc64d3181 100644 --- a/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts +++ b/server/src/lib/jobs/inline-job-runner/job-runner.ts @@ -1,8 +1,11 @@ import { Queue, Worker } from "bullmq"; import CreateLogCtx from "lib/logger/logger"; +import { TachiConfig } from "lib/setup/config"; import { DedupeArr } from "utils/misc"; +import { BacksyncBMSSongsAndCharts } from "../backsync-bms-data"; import { DeoprhanScores } from "../deorphan-scores"; import { UGSSnapshot } from "../ugs-snapshot"; +import { UpdatePoyashiData } from "../update-bpi-data"; interface Job { name: string; @@ -25,6 +28,24 @@ const jobs: Job[] = [ }, ]; +// if kamaitachi or omnitachi +if (TachiConfig.TYPE !== "btchi") { + jobs.push({ + name: "Update BPI", + cronFormat: "2 0 * * *", + run: UpdatePoyashiData, + }); +} + +// if bokutachi or omnimitachi +if (TachiConfig.TYPE !== "ktchi") { + jobs.push({ + name: "Backsync BMS", + cronFormat: "2 0 * * *", + run: BacksyncBMSSongsAndCharts, + }); +} + const logger = CreateLogCtx("JOB_RUNNER"); /** diff --git a/server/src/lib/jobs/update-bpi-data.ts b/server/src/lib/jobs/update-bpi-data.ts new file mode 100644 index 000000000..70a12d637 --- /dev/null +++ b/server/src/lib/jobs/update-bpi-data.ts @@ -0,0 +1,151 @@ +import { PullDatabaseSeeds } from "lib/database-seeds/repo"; +import CreateLogCtx from "lib/logger/logger"; +import { + ChartDocument, + Difficulties, + GPTSupportedVersions, + integer, + Playtypes, + SongDocument, +} from "tachi-common"; + +const logger = CreateLogCtx(__filename); + +const difficultyResolve = { + 3: ["SP", "HYPER"], + 4: ["SP", "ANOTHER"], + 8: ["DP", "HYPER"], + 9: ["DP", "ANOTHER"], + 10: ["SP", "LEGGENDARIA"], + 11: ["DP", "LEGGENDARIA"], +} as Record; + +interface PoyashiProxyBPIInfo { + title: string; + difficulty: string; + wr: integer; + avg: integer; + notes: string; + bpm: string; + textage: string; + difficultyLevel: string; + dpLevel: string; + coef: number | null; + removed?: boolean; +} + +interface PoyashiProxyData { + version: integer; + requireVersion: string; + body: PoyashiProxyBPIInfo[]; +} + +/** + * Fetches Poyashi BPI's latest information and syncs it back to the seeds repository. + * + * @note This function doesn't actually touch or backsync our database at all. Infact, + * the commit that hits the seeds repo will result in a database-sync, meaning this + * will all "just work"(tm) + */ +export async function UpdatePoyashiData() { + const repo = await PullDatabaseSeeds(); + + logger.info("Fetching data from proxy..."); + const data = (await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => + r.json() + )) as PoyashiProxyData; + + logger.info("Fetched data."); + + const iidxSongs = (await repo.ReadCollection("songs-iidx")) as SongDocument<"iidx">[]; + const iidxCharts = (await repo.ReadCollection("charts-iidx")) as ChartDocument< + "iidx:SP" | "iidx:DP" + >[]; + + // Utility functions for finding matching charts. + function FindSongOnTitle(title: string) { + for (const data of iidxSongs) { + if (data.title === title || data.altTitles.includes(title)) { + return data; + } + } + + return null; + } + + function FindChartWithPTDFVersion( + songID: integer, + playtype: Playtypes["iidx"], + diff: Difficulties["iidx:SP" | "iidx:DP"], + version: GPTSupportedVersions["iidx:SP" | "iidx:DP"] + ) { + for (const chart of iidxCharts) { + if ( + chart.songID === songID && + chart.playtype === playtype && + chart.difficulty === diff && + chart.versions.includes(version) + ) { + return chart; + } + } + + return null; + } + + // The actual mutation. + for (const d of data.body) { + const res = difficultyResolve[d.difficulty]; + + if (!res) { + throw new Error(`Unknown difficulty ${d.difficulty}`); + } + + const [playtype, diff] = res; + + const tachiSong = FindSongOnTitle(d.title); + + if (!tachiSong) { + logger.warn(`Cannot find song ${d.title}?`); + continue; + } + + // current poyashi version is 29 + const tachiChart = FindChartWithPTDFVersion(tachiSong.id, playtype, diff, "29"); + + if (!tachiChart) { + logger.warn( + `Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?` + ); + continue; + } + + const kavg = Number(d.avg); + + if (kavg < 0) { + logger.warn( + `${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.` + ); + continue; + } + + if (d.removed) { + logger.info(`Skipping removed chart ${tachiSong.title}.`); + continue; + } + + tachiChart.data.bpiCoefficient = d.coef === -1 || d.coef === undefined ? null : d.coef; + tachiChart.data.kaidenAverage = Number(d.avg); + tachiChart.data.worldRecord = Number(d.wr); + } + + await repo.WriteCollection("charts-iidx", iidxCharts); + + logger.info(`Finished applying BPI changes. Writing back.`); + + await repo.CommitChangesBack(`BPI Update ${new Date().toISOString()}`); +} + +if (require.main === module) { + UpdatePoyashiData(); +} diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index ae1859d95..afbee29fe 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -66,6 +66,7 @@ export interface TachiServerConfig { OPTIONS_ALWAYS_SUCCEEDS?: boolean; USE_EXTERNAL_SCORE_IMPORT_WORKER?: boolean; EXTERNAL_SCORE_IMPORT_WORKER_CONCURRENCY?: integer; + SEEDS_URL: string | null; EMAIL_CONFIG?: { FROM: string; DKIM?: SendMailOptions["dkim"]; @@ -194,6 +195,7 @@ const err = p(config, { } ), }, + SEEDS_URL: "?string", }); if (err) { diff --git a/server/src/scripts/sync-database/sync-database.ts b/server/src/scripts/sync-database.ts similarity index 80% rename from server/src/scripts/sync-database/sync-database.ts rename to server/src/scripts/sync-database.ts index 34460a71c..b55b61603 100644 --- a/server/src/scripts/sync-database/sync-database.ts +++ b/server/src/scripts/sync-database.ts @@ -1,21 +1,19 @@ /* eslint-disable no-await-in-loop */ // This script syncs this tachi instances database up with the tachi-database-seeds. -import { execSync } from "child_process"; -import { monkDB } from "external/mongo/db"; +import db, { monkDB } from "external/mongo/db"; import fjsh from "fast-json-stable-hash"; -import fs from "fs"; +import { PullDatabaseSeeds } from "lib/database-seeds/repo"; import CreateLogCtx, { KtLogger } from "lib/logger/logger"; import UpdateIsPrimaryStatus from "lib/score-mutation/update-isprimary"; -import { Environment, TachiConfig } from "lib/setup/config"; +import { TachiConfig } from "lib/setup/config"; import { BulkWriteOperation } from "mongodb"; import { ICollection } from "monk"; -import os from "os"; -import path from "path"; import { BMSCourseDocument, ChartDocument, FolderDocument, + Game, SongDocument, TableDocument, } from "tachi-common"; @@ -127,26 +125,49 @@ async function GenericUpsert( const syncInstructions: SyncInstructions[] = [ { - pattern: /^charts-(usc|bms)$/u, + pattern: /^charts-bms$/u, handler: async ( charts: ChartDocument[], collection: ICollection, logger ) => { - // Since the USC and BMS databases are managed Bokutachi-side, we - // shouldn't be honoring any sort of updates from tachi-database-seeds - // aside from an initial one. - // - // However, in practice, these syncs are the best way to actually update - // issues in the database. - // We're going to disable this anyway. - // const isInitial = (await collection.findOne()) === null; - const r = await GenericUpsert(charts, collection, "chartID", logger, false); if (r.thingsChanged) { await InitaliseFolderChartLookup(); await UpdateIsPrimaryStatus(); + + const largestBMSSongID = await db.songs.bms.findOne( + {}, + { + sort: { + id: -1, + }, + } + ); + + if (!largestBMSSongID) { + logger.severe( + `No BMS charts loaded, yet BMS sync was attempted? Lost state on bms-song-id counter. Panicking.`, + r + ); + throw new Error(`No BMS charts loaded, yet BMS sync was attempted.`); + } + + await db.counters.update( + { + counterName: "bms-song-id", + }, + { + $set: { + value: largestBMSSongID.id + 1, + }, + } + ); + + await RecalcAllScores({ + chartID: { $in: r.changedFields }, + }); } }, }, @@ -172,7 +193,7 @@ const syncInstructions: SyncInstructions[] = [ { pattern: /^songs-/u, handler: async (songs: SongDocument[], collection: ICollection, logger) => { - const r = await GenericUpsert(songs, collection, "id", logger); + const r = await GenericUpsert(songs, collection, "id", logger, true); if (r.thingsChanged) { await RecalcAllScores({ @@ -213,34 +234,18 @@ const syncInstructions: SyncInstructions[] = [ const logger = CreateLogCtx("Database Sync"); async function SynchroniseDBWithSeeds() { - const seedsDir = fs.mkdtempSync(path.join(os.tmpdir(), "tachi-database-seeds-")); - - logger.info(`Cloning data to ${seedsDir}.`); - - fs.rmSync(seedsDir, { recursive: true, force: true }); - // Wait for mongo to connect first. await monkDB.then(() => void 0); - execSync( - `git clone https://github.com/TNG-dev/tachi-database-seeds -b "${ - Environment.nodeEnv === "production" ? "master" : "develop" - }" --depth=1 "${seedsDir}"`, - { - stdio: "inherit", - } - ); + const databaseSeedsRepo = await PullDatabaseSeeds(); - const collections = fs.readdirSync(path.join(seedsDir, "collections")); - - for (const jsonName of collections) { - const collectionName = path.parse(jsonName).name; + for await (const { collectionName, data } of databaseSeedsRepo.IterateCollections()) { const spawnLogger = CreateLogCtx(`${collectionName} Sync`); if (collectionName.startsWith("songs-") || collectionName.startsWith("charts-")) { const game = collectionName.split("-")[1]; - if (!TachiConfig.GAMES.includes(game as any)) { + if (!TachiConfig.GAMES.includes(game as Game)) { spawnLogger.verbose( `Skipping ${collectionName} (${game}) as it isn't for ${TachiConfig.NAME}.` ); @@ -248,12 +253,6 @@ async function SynchroniseDBWithSeeds() { } } - spawnLogger.verbose(`Getting data.`); - - let data = JSON.parse( - fs.readFileSync(path.join(seedsDir, "collections", jsonName), "utf-8") - ); - spawnLogger.verbose(`Found ${data.length} documents.`); let matchedSomething = false; @@ -266,9 +265,6 @@ async function SynchroniseDBWithSeeds() { } } - // free memory forcefully - data = null; - if (!matchedSomething) { spawnLogger.warn( `Collection ${collectionName} didn't match any sync instructions. Skipping.` diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index c49aa0085..61d3be3a8 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -1,3 +1,4 @@ +import { exec } from "child_process"; import crypto from "crypto"; import { ONE_HOUR } from "lib/constants/time"; import { TachiConfig } from "lib/setup/config"; @@ -161,3 +162,24 @@ export function OmitUndefinedKeys(obj: Partial): Partial { return omittedObj; } + +/** + * Exec shellcode asynchronously. + * + * **DO NOT PASS USER INPUT INTO THIS FUNCTION!** + * **THIS EVALS A STRING AS BASH!** + * @param command A bash command to execute on the system. + * @returns stdout and stderr as strings. + */ +export function asyncExec(command: string) { + return new Promise<{ stdout: string; stderr: string }>((resolve, reject) => { + exec(command, (err, stdout, stderr) => { + if (err) { + // eslint-disable-next-line prefer-promise-reject-errors + return reject({ stdout, stderr, err }); + } + + return resolve({ stdout, stderr }); + }); + }); +} diff --git a/server/tsconfig.json b/server/tsconfig.json index 250264ee0..d51a2a5ac 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -31,11 +31,10 @@ ] }, "include": [ - "src/**/*.ts" + "src/**/*.ts", + "src/lib/jobs/inline-job-runner" ], "exclude": [ - "node_modules", - // "src/test-utils", - // "src/**/*.test.ts" + "node_modules" ] } \ No newline at end of file