Merge pull request #366 from TNG-dev:zkldi/issue-250-Job-Config-Runner

Add Job Runner
2026-10-08 06:48:04 +03:00 · 2021-10-13 03:23:14 +01:00
parent 71def93402 abf4f17f16
commit e2dd337b6d
8 changed files with 376 additions and 66 deletions
@@ -43,12 +43,13 @@
 		"prettier": "2.3.1",
 		"supertest": "6.1.3",
 		"tap": "15.0.9",
-		"ts-node": "10.0.0",
 		"tsconfig-paths": "3.10.1",
 		"why-is-node-running": "2.2.0"
 	},
 	"dependencies": {
+		"@types/bull": "^3.15.5",
 		"bcryptjs": "^2.4.3",
+		"bull": "^3.29.2",
 		"commander": "7.2.0",
 		"connect-redis": "6.0.0",
 		"csv-parse": "4.16.0",
@@ -75,6 +76,7 @@
 		"rimraf": "3.0.2",
 		"safe-json-stringify": "1.2.0",
 		"tachi-common": "0.2.26",
+		"ts-node": "10.0.0",
 		"typescript": "4.3.4",
 		"winston": "3.3.3",
 		"winston-transport": "^4.4.0"
@@ -93,4 +95,4 @@
 			"src/external/mongo/schemas.ts"
 		]
 	}
-}
+}
@@ -3,6 +3,7 @@ lockfileVersion: 5.3
 specifiers:
  '@types/bcrypt': 5.0.0
  '@types/bcryptjs': ^2.4.2
+  '@types/bull': ^3.15.5
  '@types/connect-redis': 0.0.16
  '@types/express': 4.17.12
  '@types/express-rate-limit': 5.1.2
@@ -22,6 +23,7 @@ specifiers:
  '@typescript-eslint/eslint-plugin': 4.28.0
  '@typescript-eslint/parser': 4.28.0
  bcryptjs: ^2.4.3
+  bull: ^3.29.2
  commander: 7.2.0
  connect-redis: 6.0.0
  csv-parse: 4.16.0
@@ -65,7 +67,9 @@ specifiers:
  winston-transport: ^4.4.0

 dependencies:
+  '@types/bull': 3.15.5
  bcryptjs: 2.4.3
+  bull: 3.29.2
  commander: 7.2.0
  connect-redis: 6.0.0
  csv-parse: 4.16.0
@@ -92,6 +96,7 @@ dependencies:
  rimraf: 3.0.2
  safe-json-stringify: 1.2.0
  tachi-common: 0.2.26
+  ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b
  typescript: 4.3.4
  winston: 3.3.3
  winston-transport: 4.4.0
@@ -126,7 +131,6 @@ devDependencies:
  prettier: 2.3.1
  supertest: 6.1.3
  tap: 15.0.9_ts-node@10.0.0+typescript@4.3.4
-  ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b
  tsconfig-paths: 3.10.1
  why-is-node-running: 2.2.0

@@ -494,19 +498,19 @@ packages:

  /@tsconfig/node10/1.0.8:
    resolution: {integrity: sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg==}
-    dev: true
+    dev: false

  /@tsconfig/node12/1.0.9:
    resolution: {integrity: sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw==}
-    dev: true
+    dev: false

  /@tsconfig/node14/1.0.1:
    resolution: {integrity: sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==}
-    dev: true
+    dev: false

  /@tsconfig/node16/1.0.2:
    resolution: {integrity: sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==}
-    dev: true
+    dev: false

  /@types/bcrypt/5.0.0:
    resolution: {integrity: sha512-agtcFKaruL8TmcvqbndlqHPSJgsolhf/qPWchFlgnW1gECTN/nKbFcoFnvKAQRFfKbh+BO6A3SWdJu9t+xF3Lw==}
@@ -531,6 +535,12 @@ packages:
      '@types/node': 15.12.4
    dev: false

+  /@types/bull/3.15.5:
+    resolution: {integrity: sha512-XgJQWJ03jyKMfdoL8IAIoHIo7JkkL74kcxuujTONkSJswm0giIJ9kuVgDNHS0OvD+OiPNcFmbBl0H3scj2+A8A==}
+    dependencies:
+      '@types/ioredis': 4.26.7
+    dev: false
+
  /@types/connect-redis/0.0.16:
    resolution: {integrity: sha512-ToZRmwoeqRhzGIqMO9k8Hs1B9pFSGYCjHvMpPlgo+kQ4M7xaPwi2/5HjikJ0MOfPDFcAe9Qwl1SaULYCNE6jYg==}
    dependencies:
@@ -590,7 +600,6 @@ packages:
    resolution: {integrity: sha512-TOGRR+e1to00CihjgPNygD7+G7ruVnMi62YdIvGUBRfj11k/aWq+Fv5Ea8St0Oy56NngTBfA8GvLn1uvHvhX6Q==}
    dependencies:
      '@types/node': 15.12.4
-    dev: true

  /@types/json-schema/7.0.9:
    resolution: {integrity: sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==}
@@ -913,7 +922,7 @@ packages:

  /arg/4.1.3:
    resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
-    dev: true
+    dev: false

  /argparse/1.0.10:
    resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==}
@@ -1066,6 +1075,24 @@ packages:
  /buffer-from/1.1.2:
    resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}

+  /bull/3.29.2:
+    resolution: {integrity: sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==}
+    engines: {node: '>=10'}
+    dependencies:
+      cron-parser: 2.18.0
+      debuglog: 1.0.1
+      get-port: 5.1.1
+      ioredis: 4.27.11
+      lodash: 4.17.21
+      p-timeout: 3.2.0
+      promise.prototype.finally: 3.1.3
+      semver: 7.3.5
+      util.promisify: 1.1.1
+      uuid: 8.3.2
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
  /busboy/0.2.14:
    resolution: {integrity: sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=}
    engines: {node: '>=0.8.0'}
@@ -1094,7 +1121,6 @@ packages:
    dependencies:
      function-bind: 1.1.1
      get-intrinsic: 1.1.1
-    dev: true

  /caller-callsite/2.0.0:
    resolution: {integrity: sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=}
@@ -1196,6 +1222,11 @@ packages:
    engines: {node: '>=0.8'}
    dev: false

+  /cluster-key-slot/1.1.0:
+    resolution: {integrity: sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
  /code-point-at/1.1.0:
    resolution: {integrity: sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=}
    engines: {node: '>=0.10.0'}
@@ -1345,7 +1376,15 @@ packages:

  /create-require/1.1.1:
    resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
-    dev: true
+    dev: false
+
+  /cron-parser/2.18.0:
+    resolution: {integrity: sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==}
+    engines: {node: '>=0.8'}
+    dependencies:
+      is-nan: 1.3.2
+      moment-timezone: 0.5.33
+    dev: false

  /cross-spawn/7.0.3:
    resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
@@ -1389,6 +1428,10 @@ packages:
    dependencies:
      ms: 2.1.2

+  /debuglog/1.0.1:
+    resolution: {integrity: sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=}
+    dev: false
+
  /decamelize/1.2.0:
    resolution: {integrity: sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=}
    engines: {node: '>=0.10.0'}
@@ -1421,7 +1464,6 @@ packages:
    engines: {node: '>= 0.4'}
    dependencies:
      object-keys: 1.1.1
-    dev: true

  /delayed-stream/1.0.0:
    resolution: {integrity: sha1-3zrhmayt+31ECqrgsp4icrJOxhk=}
@@ -1457,7 +1499,6 @@ packages:
  /diff/4.0.2:
    resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}
    engines: {node: '>=0.3.1'}
-    dev: true

  /dir-glob/3.0.1:
    resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==}
@@ -1549,6 +1590,32 @@ packages:
      unbox-primitive: 1.0.1
    dev: true

+  /es-abstract/1.19.1:
+    resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      es-to-primitive: 1.2.1
+      function-bind: 1.1.1
+      get-intrinsic: 1.1.1
+      get-symbol-description: 1.0.0
+      has: 1.0.3
+      has-symbols: 1.0.2
+      internal-slot: 1.0.3
+      is-callable: 1.2.4
+      is-negative-zero: 2.0.1
+      is-regex: 1.1.4
+      is-shared-array-buffer: 1.0.1
+      is-string: 1.0.7
+      is-weakref: 1.0.1
+      object-inspect: 1.11.0
+      object-keys: 1.1.1
+      object.assign: 4.1.2
+      string.prototype.trimend: 1.0.4
+      string.prototype.trimstart: 1.0.4
+      unbox-primitive: 1.0.1
+    dev: false
+
  /es-to-primitive/1.2.1:
    resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==}
    engines: {node: '>= 0.4'}
@@ -1556,7 +1623,6 @@ packages:
      is-callable: 1.2.4
      is-date-object: 1.0.5
      is-symbol: 1.0.4
-    dev: true

  /es6-error/4.1.1:
    resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==}
@@ -1984,6 +2050,12 @@ packages:
    resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==}
    dev: false

+  /for-each/0.3.3:
+    resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==}
+    dependencies:
+      is-callable: 1.2.4
+    dev: false
+
  /foreground-child/2.0.0:
    resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==}
    engines: {node: '>=8.0.0'}
@@ -2048,7 +2120,6 @@ packages:

  /function-bind/1.1.1:
    resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==}
-    dev: true

  /function-loop/2.0.1:
    resolution: {integrity: sha512-ktIR+O6i/4h+j/ZhZJNdzeI4i9lEPeEK6UPR2EVyTVBqOwcU3Za9xYKLH64ZR9HmcROyRrOkizNyjjtWJzDDkQ==}
@@ -2074,13 +2145,25 @@ packages:
      function-bind: 1.1.1
      has: 1.0.3
      has-symbols: 1.0.2
-    dev: true

  /get-package-type/0.1.0:
    resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==}
    engines: {node: '>=8.0.0'}
    dev: true

+  /get-port/5.1.1:
+    resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==}
+    engines: {node: '>=8'}
+    dev: false
+
+  /get-symbol-description/1.0.0:
+    resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      get-intrinsic: 1.1.1
+    dev: false
+
  /getpass/0.1.7:
    resolution: {integrity: sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=}
    dependencies:
@@ -2148,7 +2231,6 @@ packages:

  /has-bigints/1.0.1:
    resolution: {integrity: sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==}
-    dev: true

  /has-flag/3.0.0:
    resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
@@ -2163,21 +2245,18 @@ packages:
  /has-symbols/1.0.2:
    resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==}
    engines: {node: '>= 0.4'}
-    dev: true

  /has-tostringtag/1.0.0:
    resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==}
    engines: {node: '>= 0.4'}
    dependencies:
      has-symbols: 1.0.2
-    dev: true

  /has/1.0.3:
    resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==}
    engines: {node: '>= 0.4.0'}
    dependencies:
      function-bind: 1.1.1
-    dev: true

  /hasha/5.2.2:
    resolution: {integrity: sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==}
@@ -2303,7 +2382,25 @@ packages:
      get-intrinsic: 1.1.1
      has: 1.0.3
      side-channel: 1.0.4
-    dev: true
+
+  /ioredis/4.27.11:
+    resolution: {integrity: sha512-qBU2uCHgRs4kJT/qzxeFhbmizmNLQ1HH59EvKt1WnzDPMjjWxZS1+W6ynXdE47TxxqNLKYemJxEXXUb7Y4JJ+w==}
+    engines: {node: '>=6'}
+    dependencies:
+      cluster-key-slot: 1.1.0
+      debug: 4.3.2
+      denque: 1.5.0
+      lodash.defaults: 4.2.0
+      lodash.flatten: 4.4.0
+      lodash.isarguments: 3.1.0
+      p-map: 2.1.0
+      redis-commands: 1.7.0
+      redis-errors: 1.2.0
+      redis-parser: 3.0.0
+      standard-as-callback: 2.1.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false

  /ipaddr.js/1.9.1:
    resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
@@ -2322,7 +2419,6 @@ packages:
    resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==}
    dependencies:
      has-bigints: 1.0.1
-    dev: true

  /is-binary-path/2.1.0:
    resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
@@ -2337,12 +2433,10 @@ packages:
    dependencies:
      call-bind: 1.0.2
      has-tostringtag: 1.0.0
-    dev: true

  /is-callable/1.2.4:
    resolution: {integrity: sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==}
    engines: {node: '>= 0.4'}
-    dev: true

  /is-core-module/2.5.0:
    resolution: {integrity: sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==}
@@ -2355,7 +2449,6 @@ packages:
    engines: {node: '>= 0.4'}
    dependencies:
      has-tostringtag: 1.0.0
-    dev: true

  /is-extglob/2.1.1:
    resolution: {integrity: sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=}
@@ -2386,17 +2479,23 @@ packages:
      is-extglob: 2.1.1
    dev: true

+  /is-nan/1.3.2:
+    resolution: {integrity: sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+    dev: false
+
  /is-negative-zero/2.0.1:
    resolution: {integrity: sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==}
    engines: {node: '>= 0.4'}
-    dev: true

  /is-number-object/1.0.6:
    resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==}
    engines: {node: '>= 0.4'}
    dependencies:
      has-tostringtag: 1.0.0
-    dev: true

  /is-number/7.0.0:
    resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
@@ -2409,7 +2508,10 @@ packages:
    dependencies:
      call-bind: 1.0.2
      has-tostringtag: 1.0.0
-    dev: true
+
+  /is-shared-array-buffer/1.0.1:
+    resolution: {integrity: sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==}
+    dev: false

  /is-stream/2.0.1:
    resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
@@ -2420,19 +2522,23 @@ packages:
    engines: {node: '>= 0.4'}
    dependencies:
      has-tostringtag: 1.0.0
-    dev: true

  /is-symbol/1.0.4:
    resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==}
    engines: {node: '>= 0.4'}
    dependencies:
      has-symbols: 1.0.2
-    dev: true

  /is-typedarray/1.0.0:
    resolution: {integrity: sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=}
    dev: true

+  /is-weakref/1.0.1:
+    resolution: {integrity: sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==}
+    dependencies:
+      call-bind: 1.0.2
+    dev: false
+
  /is-windows/1.0.2:
    resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==}
    engines: {node: '>=0.10.0'}
@@ -2655,10 +2761,22 @@ packages:
    resolution: {integrity: sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=}
    dev: true

+  /lodash.defaults/4.2.0:
+    resolution: {integrity: sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=}
+    dev: false
+
+  /lodash.flatten/4.4.0:
+    resolution: {integrity: sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=}
+    dev: false
+
  /lodash.flattendeep/4.4.0:
    resolution: {integrity: sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=}
    dev: true

+  /lodash.isarguments/3.1.0:
+    resolution: {integrity: sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=}
+    dev: false
+
  /lodash.merge/4.6.2:
    resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
    dev: true
@@ -2667,6 +2785,10 @@ packages:
    resolution: {integrity: sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=}
    dev: true

+  /lodash/4.17.21:
+    resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
+    dev: false
+
  /log-driver/1.2.7:
    resolution: {integrity: sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==}
    engines: {node: '>=0.8.6'}
@@ -2694,7 +2816,6 @@ packages:
    engines: {node: '>=10'}
    dependencies:
      yallist: 4.0.0
-    dev: true

  /make-dir/3.1.0:
    resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
@@ -2705,7 +2826,7 @@ packages:

  /make-error/1.3.6:
    resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
-    dev: true
+    dev: false

  /media-typer/0.3.0:
    resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=}
@@ -2784,6 +2905,16 @@ packages:
    engines: {node: '>=10'}
    hasBin: true

+  /moment-timezone/0.5.33:
+    resolution: {integrity: sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==}
+    dependencies:
+      moment: 2.29.1
+    dev: false
+
+  /moment/2.29.1:
+    resolution: {integrity: sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==}
+    dev: false
+
  /mongodb/3.6.3:
    resolution: {integrity: sha512-rOZuR0QkodZiM+UbQE5kDsJykBqWi0CL4Ec2i1nrGrUI3KO11r6Fbxskqmq3JK2NH7aW4dcccBuUujAP0ERl5w==}
    engines: {node: '>=4'}
@@ -3007,12 +3138,10 @@ packages:

  /object-inspect/1.11.0:
    resolution: {integrity: sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==}
-    dev: true

  /object-keys/1.1.1:
    resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==}
    engines: {node: '>= 0.4'}
-    dev: true

  /object.assign/4.1.2:
    resolution: {integrity: sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==}
@@ -3022,7 +3151,15 @@ packages:
      define-properties: 1.1.3
      has-symbols: 1.0.2
      object-keys: 1.1.1
-    dev: true
+
+  /object.getownpropertydescriptors/2.1.3:
+    resolution: {integrity: sha512-VdDoCwvJI4QdC6ndjpqFmoL3/+HxffFBbcJzKi5hwLLqqx3mdbedRpfZDdK0SrOSauj8X4GzBvnDZl4vTN7dOw==}
+    engines: {node: '>= 0.8'}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+      es-abstract: 1.19.1
+    dev: false

  /object.values/1.1.4:
    resolution: {integrity: sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==}
@@ -3083,6 +3220,11 @@ packages:
    resolution: {integrity: sha1-Tod/vtqaLsgAD7wLyuOWRe6L+Nw=}
    dev: true

+  /p-finally/1.0.0:
+    resolution: {integrity: sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=}
+    engines: {node: '>=4'}
+    dev: false
+
  /p-limit/1.3.0:
    resolution: {integrity: sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==}
    engines: {node: '>=4'}
@@ -3111,6 +3253,11 @@ packages:
      p-limit: 2.3.0
    dev: true

+  /p-map/2.1.0:
+    resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==}
+    engines: {node: '>=6'}
+    dev: false
+
  /p-map/3.0.0:
    resolution: {integrity: sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==}
    engines: {node: '>=8'}
@@ -3118,6 +3265,13 @@ packages:
      aggregate-error: 3.1.0
    dev: true

+  /p-timeout/3.2.0:
+    resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==}
+    engines: {node: '>=8'}
+    dependencies:
+      p-finally: 1.0.0
+    dev: false
+
  /p-try/1.0.0:
    resolution: {integrity: sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=}
    engines: {node: '>=4'}
@@ -3265,6 +3419,15 @@ packages:
    engines: {node: '>=0.4.0'}
    dev: true

+  /promise.prototype.finally/3.1.3:
+    resolution: {integrity: sha512-EXRF3fC9/0gz4qkt/f5EP5iW4kj9oFpBICNpCNOb/52+8nlHIX07FPLbi/q4qYBQ1xZqivMzTpNQSnArVASolQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+      es-abstract: 1.19.1
+    dev: false
+
  /prop-types/15.7.2:
    resolution: {integrity: sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==}
    dependencies:
@@ -3574,7 +3737,6 @@ packages:
    hasBin: true
    dependencies:
      lru-cache: 6.0.0
-    dev: true

  /send/0.17.1:
    resolution: {integrity: sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==}
@@ -3631,7 +3793,6 @@ packages:
      call-bind: 1.0.2
      get-intrinsic: 1.1.1
      object-inspect: 1.11.0
-    dev: true

  /siginfo/2.0.0:
    resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==}
@@ -3666,7 +3827,6 @@ packages:
    dependencies:
      buffer-from: 1.1.2
      source-map: 0.6.1
-    dev: true

  /source-map/0.5.7:
    resolution: {integrity: sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=}
@@ -3676,7 +3836,6 @@ packages:
  /source-map/0.6.1:
    resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
    engines: {node: '>=0.10.0'}
-    dev: true

  /sparse-bitfield/3.0.3:
    resolution: {integrity: sha1-/0rm5oZWBWuks+eSqzM004JzyhE=}
@@ -3754,6 +3913,10 @@ packages:
    resolution: {integrity: sha1-Gsig2Ug4SNFpXkGLbQMaPDzmjjs=}
    dev: true

+  /standard-as-callback/2.1.0:
+    resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
+    dev: false
+
  /statuses/1.5.0:
    resolution: {integrity: sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=}
    engines: {node: '>= 0.6'}
@@ -3795,14 +3958,12 @@ packages:
    dependencies:
      call-bind: 1.0.2
      define-properties: 1.1.3
-    dev: true

  /string.prototype.trimstart/1.0.4:
    resolution: {integrity: sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==}
    dependencies:
      call-bind: 1.0.2
      define-properties: 1.1.3
-    dev: true

  /string_decoder/0.10.31:
    resolution: {integrity: sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=}
@@ -4091,7 +4252,7 @@ packages:
      source-map-support: 0.5.19
      typescript: 4.3.4
      yn: 3.1.1
-    dev: true
+    dev: false

  /tsconfig-paths/3.10.1:
    resolution: {integrity: sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==}
@@ -4179,7 +4340,6 @@ packages:
      has-bigints: 1.0.1
      has-symbols: 1.0.2
      which-boxed-primitive: 1.0.2
-    dev: true

  /unicode-length/2.0.2:
    resolution: {integrity: sha512-Ph/j1VbS3/r77nhoY2WU0GWGjVYOHL3xpKp0y/Eq2e5r0mT/6b649vm7KFO6RdAdrZkYLdxphYVgvODxPB+Ebg==}
@@ -4202,6 +4362,16 @@ packages:
  /util-deprecate/1.0.2:
    resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=}

+  /util.promisify/1.1.1:
+    resolution: {integrity: sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.1.3
+      for-each: 0.3.3
+      has-symbols: 1.0.2
+      object.getownpropertydescriptors: 2.1.3
+    dev: false
+
  /utils-merge/1.0.1:
    resolution: {integrity: sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=}
    engines: {node: '>= 0.4.0'}
@@ -4213,6 +4383,11 @@ packages:
    hasBin: true
    dev: true

+  /uuid/8.3.2:
+    resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
+    hasBin: true
+    dev: false
+
  /v8-compile-cache/2.3.0:
    resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==}
    dev: true
@@ -4246,7 +4421,6 @@ packages:
      is-number-object: 1.0.6
      is-string: 1.0.7
      is-symbol: 1.0.4
-    dev: true

  /which-module/2.0.0:
    resolution: {integrity: sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=}
@@ -4337,7 +4511,6 @@ packages:

  /yallist/4.0.0:
    resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
-    dev: true

  /yaml/1.10.2:
    resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
@@ -4376,4 +4549,4 @@ packages:
  /yn/3.1.1:
    resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}
    engines: {node: '>=6'}
-    dev: true
+    dev: false
@@ -0,0 +1,74 @@
+import Queue from "bull";
+import CreateLogCtx from "lib/logger/logger";
+import { DedupeArr } from "utils/misc";
+import { DeoprhanScores } from "./jobs/deorphan-scores";
+import { UGSSnapshot } from "./jobs/ugs-snapshot";
+
+interface Job {
+	name: string;
+	cronFormat: string;
+	run: (job: Queue.Job) => Promise<void>;
+}
+
+const jobs: Job[] = [
+	{
+		name: "Snapshot User Game Stats",
+		cronFormat: "0 0 * * *",
+		run: UGSSnapshot,
+	},
+	{
+		name: "De-Orphan Scores",
+		// We run an hour after snapshotting UGS
+		// just to spread load out a bit.
+		cronFormat: "1 * * * * *",
+		run: DeoprhanScores,
+	},
+];
+
+const logger = CreateLogCtx("JOB_RUNNER");
+
+/**
+ * Initalises a tachi-server job runner.
+ * This runs the list of jobs defined in jobConfig.jobs.
+ */
+export function InitialiseJobRunner() {
+	logger.info(`Booting up Job Runner.`);
+
+	const names = jobs.map((e) => e.name);
+
+	if (DedupeArr(names).length !== names.length) {
+		logger.crit(`Jobs has duplicate name fields, refusing to run.`);
+		process.exit(1);
+	}
+
+	const JobQueue = new Queue("Job Runner");
+
+	const jobNameMap = new Map<string, Job>();
+
+	for (const job of jobs) {
+		JobQueue.add({ jobName: job.name }, { repeat: { cron: job.cronFormat } });
+		jobNameMap.set(job.name, job);
+	}
+
+	JobQueue.process(async (j) => {
+		const name = j.data.jobName;
+		logger.info(`Running job ${name}.`);
+
+		const jobInfo = jobNameMap.get(name);
+
+		if (!jobInfo) {
+			logger.severe(`Unknown job name ${name}, couldn't find a run function?`);
+			return false;
+		}
+
+		await jobInfo.run(j);
+
+		return true;
+	});
+
+	logger.info(`Initialised ${jobs.length} jobs.`);
+}
+
+if (require.main === module) {
+	InitialiseJobRunner();
+}
@@ -1,11 +0,0 @@
-{
-	// this format is just an example - @ereti will be designing the jobs system and will *definitely* want something
-	// better. This only exists as a rough example.
-	prefix: "tsnode -r tsconfig-paths/register",
-	jobs: [{
-		path: "ugs-snapshot.ts",
-		name: "Snapshot User Game Stats",
-		// especially this part - i don't expect any machine to parse this text.
-		interval: "daily at midnight"
-	}]
-}
@@ -0,0 +1,43 @@
+// Attempt to deoprhan lost scores.
+
+import Bull from "bull";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
+
+const logger = CreateLogCtx(__dirname);
+
+export async function DeoprhanScores(job: Bull.Job) {
+	const orphans = await db["orphan-scores"].find({});
+
+	// ScoreIDs are essentially userID dependent, so this is fine.
+	const blacklist = (await db["score-blacklist"].find({})).map((e) => e.scoreID);
+
+	logger.info(`Found ${orphans.length} orphans.`);
+
+	let done = 0;
+	let failed = 0;
+	let success = 0;
+	let removed = 0;
+
+	await Promise.all(
+		orphans.map((or) =>
+			ReprocessOrphan(or, blacklist, logger).then((r) => {
+				done++;
+				if (r === null) {
+					removed++;
+				} else if (r === false) {
+					failed++;
+				} else {
+					success++;
+				}
+
+				job.progress((100 * done) / orphans.length);
+			})
+		)
+	);
+
+	logger.info(`Finished attempting deorphaning.`);
+
+	logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`);
+}
@@ -14,16 +14,19 @@ let batchWrite: UserGameStatsSnapshot[] = [];

 // This code is intentionally *very* robust, and handles a lot of unanticipated failures
 // because if it breaks, we brick the database.
-(async () => {
+export async function UGSSnapshot() {
 	const timeStart = process.hrtime.bigint();

 	const alreadyExists = await db["game-stats-snapshots"].findOne({ timestamp: currentTime });

 	if (alreadyExists) {
-		logger.severe(
-			`FATAL IN UGS-SNAPSHOT - There already exists snapshots at this time. Has this script been ran twice?`
+		logger.warn(
+			`There already exists snapshots at this time. Has this script been ran twice on one day? Ignoring request.`
+		);
+
+		throw new Error(
+			`There already exists snapshots at this time. Has this script been ran twice on one day? Ignoring request.`
 		);
-		process.exit(1);
 	}

 	logger.info(`Snapshotting UserGameStats.`);
@@ -69,8 +72,6 @@ let batchWrite: UserGameStatsSnapshot[] = [];
 				currentTime
 			).toString()}. Took ${GetMilisecondsSince(timeStart)} ms.`
 		);
-
-		process.exit(0);
 	} catch (err) {
 		// if we panic, we need to revert whatever we did.
 		logger.severe(`FATAL IN UGS-SNAPSHOT - Possibly failed midway through snapshotting.`, {
@@ -83,6 +84,6 @@ let batchWrite: UserGameStatsSnapshot[] = [];

 		logger.info(`Removed.`);

-		process.exit(1);
+		throw err;
 	}
-})();
+}
@@ -77,6 +77,7 @@ export interface TachiConfig {
 	OUR_URL: string;
 	LOGGER_DISCORD_WEBHOOK?: string;
 	DISCORD_WHO_TO_TAG?: string[];
+	INVOKE_JOB_RUNNER?: boolean;
 }

 const isValidOauth2 = p.optional({
@@ -118,6 +119,7 @@ const err = p(config, {
 	OUR_URL: "string",
 	LOGGER_DISCORD_WEBHOOK: "*string",
 	DISCORD_WHO_TO_TAG: p.optional(["string"]),
+	INVOKE_JOB_RUNNER: "*boolean",
 });

 if (err) {
@@ -3,8 +3,9 @@ import server from "server/server";
 import { ServerTypeInfo, ServerConfig } from "lib/setup/config";
 import https from "https";
 import fs from "fs";
-
 import { FormatVersion } from "./lib/constants/version";
+import { spawn } from "child_process";
+import path from "path";

 const logger = CreateLogCtx(__filename);

@@ -26,3 +27,28 @@ if (ServerConfig.ENABLE_SERVER_HTTPS) {
 	server.listen(ServerConfig.PORT);
 	logger.info(`HTTP Listening on port ${ServerConfig.PORT}`);
 }
+
+if (ServerConfig.INVOKE_JOB_RUNNER || process.env.INVOKE_JOB_RUNNER) {
+	logger.info(`Spawning a tachi-server job runner.`);
+
+	// Spawn as a separate process to avoid hogging the main thread.
+	const jobProcess = spawn(
+		"ts-node",
+		[
+			// Note: Can't use -r tsconfig-paths/register here
+			// because that is rejected by some library called
+			// arg.
+			// I'm not sure why.
+			"--require=tsconfig-paths/register",
+			path.join(__dirname, "../src/lib/jobs/job-runner.ts"),
+		],
+		{
+			stdio: "inherit",
+		}
+	);
+
+	jobProcess.on("error", (err) => {
+		logger.crit(`Failed to spawn job runner. Terminating process.`, { err });
+		process.exit(1);
+	});
+}