Merge pull request #396 from TNG-dev:zkldi/issue-395-Refactor-Job-Runners-to-just-be-single-files-that-can-be-cronned

Fix #395

server/package.json

@@ -51,7 +51,7 @@
 		"@types/bunyan": "^1.8.7",
 		"@types/deep-equal": "^1.0.1",
 		"bcryptjs": "^2.4.3",
-		"bull": "^3.29.2",
+		"bull": "^4.1.0",
 		"bunyan": "^1.8.15",
 		"commander": "7.2.0",
 		"connect-redis": "6.0.0",

server/pnpm-lock.yaml

@@ -25,7 +25,7 @@ specifiers:
   '@typescript-eslint/eslint-plugin': 4.28.0
   '@typescript-eslint/parser': 4.28.0
   bcryptjs: ^2.4.3
-  bull: ^3.29.2
+  bull: ^4.1.0
   bunyan: ^1.8.15
   commander: 7.2.0
   connect-redis: 6.0.0
@@ -75,7 +75,7 @@ dependencies:
   '@types/bunyan': 1.8.7
   '@types/deep-equal': 1.0.1
   bcryptjs: 2.4.3
-  bull: 3.29.2
+  bull: 4.1.0
   bunyan: 1.8.15
   commander: 7.2.0
   connect-redis: 6.0.0
@@ -1093,14 +1093,14 @@ packages:
   /buffer-from/1.1.2:
     resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
 
-  /bull/3.29.2:
-    resolution: {integrity: sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==}
+  /bull/4.1.0:
+    resolution: {integrity: sha512-rQcLuAmzZIv1dHJO/yKrWu497xcTxMpYeTEBfpStrJFZ1UZpBSGHSx+defbtFVeGEeY8Pn0aMGRvRtldUBVUyQ==}
     engines: {node: '>=10'}
     dependencies:
       cron-parser: 2.18.0
       debuglog: 1.0.1
       get-port: 5.1.1
-      ioredis: 4.27.11
+      ioredis: 4.28.0
       lodash: 4.17.21
       p-timeout: 3.2.0
       promise.prototype.finally: 3.1.3
@@ -2438,8 +2438,8 @@ packages:
       has: 1.0.3
       side-channel: 1.0.4
 
-  /ioredis/4.27.11:
-    resolution: {integrity: sha512-qBU2uCHgRs4kJT/qzxeFhbmizmNLQ1HH59EvKt1WnzDPMjjWxZS1+W6ynXdE47TxxqNLKYemJxEXXUb7Y4JJ+w==}
+  /ioredis/4.28.0:
+    resolution: {integrity: sha512-I+zkeeWp3XFgPT2CtJKxvaF5FjGBGt4yGYljRjQecdQKteThuAsKqffeF1lgHVlYnuNeozRbPOCDNZ7tDWPeig==}
     engines: {node: '>=6'}
     dependencies:
       cluster-key-slot: 1.1.0

server/src/lib/jobs/deorphan-scores.ts

@@ -1,13 +1,12 @@
 // Attempt to deoprhan lost scores.
 
-import Bull from "bull";
 import db from "external/mongo/db";
 import CreateLogCtx from "lib/logger/logger";
 import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
 
 const logger = CreateLogCtx(__dirname);
 
-export async function DeoprhanScores(job: Bull.Job) {
+export async function DeoprhanScores() {
 	const orphans = await db["orphan-scores"].find({});
 
 	// ScoreIDs are essentially userID dependent, so this is fine.
@@ -15,7 +14,6 @@ export async function DeoprhanScores(job: Bull.Job) {
 
 	logger.info(`Found ${orphans.length} orphans.`);
 
-	let done = 0;
 	let failed = 0;
 	let success = 0;
 	let removed = 0;
@@ -23,7 +21,6 @@ export async function DeoprhanScores(job: Bull.Job) {
 	await Promise.all(
 		orphans.map((or) =>
 			ReprocessOrphan(or, blacklist, logger).then((r) => {
-				done++;
 				if (r === null) {
 					removed++;
 				} else if (r === false) {
@@ -31,8 +28,6 @@ export async function DeoprhanScores(job: Bull.Job) {
 				} else {
 					success++;
 				}
-
-				job.progress((100 * done) / orphans.length);
 			})
 		)
 	);
@@ -41,3 +36,9 @@ export async function DeoprhanScores(job: Bull.Job) {
 
 	logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`);
 }
+
+if (require.main === module) {
+	DeoprhanScores().then(() => {
+		process.exit(0);
+	});
+}

server/src/lib/jobs/inline-job-runner.ts/job-runner.ts

@@ -1,8 +1,8 @@
 import Queue from "bull";
 import CreateLogCtx from "lib/logger/logger";
 import { DedupeArr } from "utils/misc";
-import { DeoprhanScores } from "./jobs/deorphan-scores";
-import { UGSSnapshot } from "./jobs/ugs-snapshot";
+import { DeoprhanScores } from "../deorphan-scores";
+import { UGSSnapshot } from "../ugs-snapshot";
 
 interface Job {
 	name: string;

server/src/lib/jobs/ugs-snapshot.ts

@@ -87,3 +87,9 @@ export async function UGSSnapshot() {
 		throw err;
 	}
 }
+
+if (require.main === module) {
+	UGSSnapshot().then(() => {
+		process.exit(0);
+	});
+}

server/src/main.ts

@@ -63,7 +63,13 @@ if (ServerConfig.ENABLE_SERVER_HTTPS) {
 }
 
 if (process.env.INVOKE_JOB_RUNNER) {
-	logger.info(`Spawning a tachi-server job runner.`);
+	logger.info(`Spawning a tachi-server job runner inline.`);
+
+	if (Environment.nodeEnv === "production") {
+		logger.warn(
+			`Spawning inline tachi-server job runner in production. Is this actually what you want? You should run a tool like Ofelia to manage this.`
+		);
+	}
 
 	// Spawn as a separate process to avoid hogging the main thread.
 	const jobProcess = spawn(