diff --git a/server/package.json b/server/package.json index f51f73cf7..dac8cec53 100644 --- a/server/package.json +++ b/server/package.json @@ -51,7 +51,7 @@ "@types/bunyan": "^1.8.7", "@types/deep-equal": "^1.0.1", "bcryptjs": "^2.4.3", - "bull": "^3.29.2", + "bull": "^4.1.0", "bunyan": "^1.8.15", "commander": "7.2.0", "connect-redis": "6.0.0", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 76628b9b4..a33041a2a 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -25,7 +25,7 @@ specifiers: '@typescript-eslint/eslint-plugin': 4.28.0 '@typescript-eslint/parser': 4.28.0 bcryptjs: ^2.4.3 - bull: ^3.29.2 + bull: ^4.1.0 bunyan: ^1.8.15 commander: 7.2.0 connect-redis: 6.0.0 @@ -75,7 +75,7 @@ dependencies: '@types/bunyan': 1.8.7 '@types/deep-equal': 1.0.1 bcryptjs: 2.4.3 - bull: 3.29.2 + bull: 4.1.0 bunyan: 1.8.15 commander: 7.2.0 connect-redis: 6.0.0 @@ -1093,14 +1093,14 @@ packages: /buffer-from/1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - /bull/3.29.2: - resolution: {integrity: sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==} + /bull/4.1.0: + resolution: {integrity: sha512-rQcLuAmzZIv1dHJO/yKrWu497xcTxMpYeTEBfpStrJFZ1UZpBSGHSx+defbtFVeGEeY8Pn0aMGRvRtldUBVUyQ==} engines: {node: '>=10'} dependencies: cron-parser: 2.18.0 debuglog: 1.0.1 get-port: 5.1.1 - ioredis: 4.27.11 + ioredis: 4.28.0 lodash: 4.17.21 p-timeout: 3.2.0 promise.prototype.finally: 3.1.3 @@ -2438,8 +2438,8 @@ packages: has: 1.0.3 side-channel: 1.0.4 - /ioredis/4.27.11: - resolution: {integrity: sha512-qBU2uCHgRs4kJT/qzxeFhbmizmNLQ1HH59EvKt1WnzDPMjjWxZS1+W6ynXdE47TxxqNLKYemJxEXXUb7Y4JJ+w==} + /ioredis/4.28.0: + resolution: {integrity: sha512-I+zkeeWp3XFgPT2CtJKxvaF5FjGBGt4yGYljRjQecdQKteThuAsKqffeF1lgHVlYnuNeozRbPOCDNZ7tDWPeig==} engines: {node: '>=6'} dependencies: cluster-key-slot: 1.1.0 diff --git a/server/src/lib/jobs/jobs/deorphan-scores.ts b/server/src/lib/jobs/deorphan-scores.ts similarity index 85% rename from server/src/lib/jobs/jobs/deorphan-scores.ts rename to server/src/lib/jobs/deorphan-scores.ts index 5facc3993..4a680d043 100644 --- a/server/src/lib/jobs/jobs/deorphan-scores.ts +++ b/server/src/lib/jobs/deorphan-scores.ts @@ -1,13 +1,12 @@ // Attempt to deoprhan lost scores. -import Bull from "bull"; import db from "external/mongo/db"; import CreateLogCtx from "lib/logger/logger"; import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; const logger = CreateLogCtx(__dirname); -export async function DeoprhanScores(job: Bull.Job) { +export async function DeoprhanScores() { const orphans = await db["orphan-scores"].find({}); // ScoreIDs are essentially userID dependent, so this is fine. @@ -15,7 +14,6 @@ export async function DeoprhanScores(job: Bull.Job) { logger.info(`Found ${orphans.length} orphans.`); - let done = 0; let failed = 0; let success = 0; let removed = 0; @@ -23,7 +21,6 @@ export async function DeoprhanScores(job: Bull.Job) { await Promise.all( orphans.map((or) => ReprocessOrphan(or, blacklist, logger).then((r) => { - done++; if (r === null) { removed++; } else if (r === false) { @@ -31,8 +28,6 @@ export async function DeoprhanScores(job: Bull.Job) { } else { success++; } - - job.progress((100 * done) / orphans.length); }) ) ); @@ -41,3 +36,9 @@ export async function DeoprhanScores(job: Bull.Job) { logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`); } + +if (require.main === module) { + DeoprhanScores().then(() => { + process.exit(0); + }); +} diff --git a/server/src/lib/jobs/job-runner.ts b/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts similarity index 93% rename from server/src/lib/jobs/job-runner.ts rename to server/src/lib/jobs/inline-job-runner.ts/job-runner.ts index 63ab7f7c3..f7b642de7 100644 --- a/server/src/lib/jobs/job-runner.ts +++ b/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts @@ -1,8 +1,8 @@ import Queue from "bull"; import CreateLogCtx from "lib/logger/logger"; import { DedupeArr } from "utils/misc"; -import { DeoprhanScores } from "./jobs/deorphan-scores"; -import { UGSSnapshot } from "./jobs/ugs-snapshot"; +import { DeoprhanScores } from "../deorphan-scores"; +import { UGSSnapshot } from "../ugs-snapshot"; interface Job { name: string; diff --git a/server/src/lib/jobs/jobs/ugs-snapshot.ts b/server/src/lib/jobs/ugs-snapshot.ts similarity index 96% rename from server/src/lib/jobs/jobs/ugs-snapshot.ts rename to server/src/lib/jobs/ugs-snapshot.ts index 224381d69..580f53afb 100644 --- a/server/src/lib/jobs/jobs/ugs-snapshot.ts +++ b/server/src/lib/jobs/ugs-snapshot.ts @@ -87,3 +87,9 @@ export async function UGSSnapshot() { throw err; } } + +if (require.main === module) { + UGSSnapshot().then(() => { + process.exit(0); + }); +} diff --git a/server/src/main.ts b/server/src/main.ts index df8367204..df2dda4af 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -63,7 +63,13 @@ if (ServerConfig.ENABLE_SERVER_HTTPS) { } if (process.env.INVOKE_JOB_RUNNER) { - logger.info(`Spawning a tachi-server job runner.`); + logger.info(`Spawning a tachi-server job runner inline.`); + + if (Environment.nodeEnv === "production") { + logger.warn( + `Spawning inline tachi-server job runner in production. Is this actually what you want? You should run a tool like Ofelia to manage this.` + ); + } // Spawn as a separate process to avoid hogging the main thread. const jobProcess = spawn(