From b0603ed7acf86549ea8f437c90b1c7ee1ddeb31e Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 8 Nov 2021 20:08:28 +0000 Subject: [PATCH 1/3] Pull jobs up to lib/jobs and remove job runner. --- server/package.json | 2 - server/pnpm-lock.yaml | 234 +++--------------- .../lib/jobs/{jobs => }/deorphan-scores.ts | 13 +- server/src/lib/jobs/job-runner.ts | 74 ------ .../src/lib/jobs/{jobs => }/ugs-snapshot.ts | 6 + 5 files changed, 44 insertions(+), 285 deletions(-) rename server/src/lib/jobs/{jobs => }/deorphan-scores.ts (85%) delete mode 100644 server/src/lib/jobs/job-runner.ts rename server/src/lib/jobs/{jobs => }/ugs-snapshot.ts (96%) diff --git a/server/package.json b/server/package.json index f51f73cf7..82885461d 100644 --- a/server/package.json +++ b/server/package.json @@ -47,11 +47,9 @@ "why-is-node-running": "2.2.0" }, "dependencies": { - "@types/bull": "^3.15.5", "@types/bunyan": "^1.8.7", "@types/deep-equal": "^1.0.1", "bcryptjs": "^2.4.3", - "bull": "^3.29.2", "bunyan": "^1.8.15", "commander": "7.2.0", "connect-redis": "6.0.0", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 76628b9b4..8becf9645 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -3,7 +3,6 @@ lockfileVersion: 5.3 specifiers: '@types/bcrypt': 5.0.0 '@types/bcryptjs': ^2.4.2 - '@types/bull': ^3.15.5 '@types/bunyan': ^1.8.7 '@types/connect-redis': 0.0.16 '@types/deep-equal': ^1.0.1 @@ -25,7 +24,6 @@ specifiers: '@typescript-eslint/eslint-plugin': 4.28.0 '@typescript-eslint/parser': 4.28.0 bcryptjs: ^2.4.3 - bull: ^3.29.2 bunyan: ^1.8.15 commander: 7.2.0 connect-redis: 6.0.0 @@ -71,11 +69,9 @@ specifiers: winston-transport: ^4.4.0 dependencies: - '@types/bull': 3.15.5 '@types/bunyan': 1.8.7 '@types/deep-equal': 1.0.1 bcryptjs: 2.4.3 - bull: 3.29.2 bunyan: 1.8.15 commander: 7.2.0 connect-redis: 6.0.0 @@ -543,12 +539,6 @@ packages: '@types/node': 15.12.4 dev: false - /@types/bull/3.15.5: - resolution: {integrity: sha512-XgJQWJ03jyKMfdoL8IAIoHIo7JkkL74kcxuujTONkSJswm0giIJ9kuVgDNHS0OvD+OiPNcFmbBl0H3scj2+A8A==} - dependencies: - '@types/ioredis': 4.26.7 - dev: false - /@types/bunyan/1.8.7: resolution: {integrity: sha512-jaNt6xX5poSmXuDAkQrSqx2zkR66OrdRDuVnU8ldvn3k/Ci/7Sf5nooKspQWimDnw337Bzt/yirqSThTjvrHkg==} dependencies: @@ -618,6 +608,7 @@ packages: resolution: {integrity: sha512-TOGRR+e1to00CihjgPNygD7+G7ruVnMi62YdIvGUBRfj11k/aWq+Fv5Ea8St0Oy56NngTBfA8GvLn1uvHvhX6Q==} dependencies: '@types/node': 15.12.4 + dev: true /@types/json-schema/7.0.9: resolution: {integrity: sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==} @@ -1093,24 +1084,6 @@ packages: /buffer-from/1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - /bull/3.29.2: - resolution: {integrity: sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==} - engines: {node: '>=10'} - dependencies: - cron-parser: 2.18.0 - debuglog: 1.0.1 - get-port: 5.1.1 - ioredis: 4.27.11 - lodash: 4.17.21 - p-timeout: 3.2.0 - promise.prototype.finally: 3.1.3 - semver: 7.3.5 - util.promisify: 1.1.1 - uuid: 8.3.2 - transitivePeerDependencies: - - supports-color - dev: false - /bunyan/1.8.15: resolution: {integrity: sha512-0tECWShh6wUysgucJcBAoYegf3JJoZWibxdqhTm7OHPeT42qdjkZ29QCMcKwbgU1kiH+auSIasNRXMLWXafXig==} engines: {'0': node >=0.10.0} @@ -1150,6 +1123,7 @@ packages: dependencies: function-bind: 1.1.1 get-intrinsic: 1.1.1 + dev: true /caller-callsite/2.0.0: resolution: {integrity: sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=} @@ -1251,11 +1225,6 @@ packages: engines: {node: '>=0.8'} dev: false - /cluster-key-slot/1.1.0: - resolution: {integrity: sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==} - engines: {node: '>=0.10.0'} - dev: false - /code-point-at/1.1.0: resolution: {integrity: sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=} engines: {node: '>=0.10.0'} @@ -1407,14 +1376,6 @@ packages: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} dev: false - /cron-parser/2.18.0: - resolution: {integrity: sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==} - engines: {node: '>=0.8'} - dependencies: - is-nan: 1.3.2 - moment-timezone: 0.5.33 - dev: false - /cross-spawn/7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} @@ -1457,10 +1418,6 @@ packages: dependencies: ms: 2.1.2 - /debuglog/1.0.1: - resolution: {integrity: sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=} - dev: false - /decamelize/1.2.0: resolution: {integrity: sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=} engines: {node: '>=0.10.0'} @@ -1493,6 +1450,7 @@ packages: engines: {node: '>= 0.4'} dependencies: object-keys: 1.1.1 + dev: true /delayed-stream/1.0.0: resolution: {integrity: sha1-3zrhmayt+31ECqrgsp4icrJOxhk=} @@ -1628,32 +1586,6 @@ packages: unbox-primitive: 1.0.1 dev: true - /es-abstract/1.19.1: - resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==} - engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.2 - es-to-primitive: 1.2.1 - function-bind: 1.1.1 - get-intrinsic: 1.1.1 - get-symbol-description: 1.0.0 - has: 1.0.3 - has-symbols: 1.0.2 - internal-slot: 1.0.3 - is-callable: 1.2.4 - is-negative-zero: 2.0.1 - is-regex: 1.1.4 - is-shared-array-buffer: 1.0.1 - is-string: 1.0.7 - is-weakref: 1.0.1 - object-inspect: 1.11.0 - object-keys: 1.1.1 - object.assign: 4.1.2 - string.prototype.trimend: 1.0.4 - string.prototype.trimstart: 1.0.4 - unbox-primitive: 1.0.1 - dev: false - /es-to-primitive/1.2.1: resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} engines: {node: '>= 0.4'} @@ -1661,6 +1593,7 @@ packages: is-callable: 1.2.4 is-date-object: 1.0.5 is-symbol: 1.0.4 + dev: true /es6-error/4.1.1: resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} @@ -2094,12 +2027,6 @@ packages: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} dev: false - /for-each/0.3.3: - resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} - dependencies: - is-callable: 1.2.4 - dev: false - /foreground-child/2.0.0: resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} engines: {node: '>=8.0.0'} @@ -2164,6 +2091,7 @@ packages: /function-bind/1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} + dev: true /function-loop/2.0.1: resolution: {integrity: sha512-ktIR+O6i/4h+j/ZhZJNdzeI4i9lEPeEK6UPR2EVyTVBqOwcU3Za9xYKLH64ZR9HmcROyRrOkizNyjjtWJzDDkQ==} @@ -2189,25 +2117,13 @@ packages: function-bind: 1.1.1 has: 1.0.3 has-symbols: 1.0.2 + dev: true /get-package-type/0.1.0: resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} engines: {node: '>=8.0.0'} dev: true - /get-port/5.1.1: - resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} - engines: {node: '>=8'} - dev: false - - /get-symbol-description/1.0.0: - resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} - engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.2 - get-intrinsic: 1.1.1 - dev: false - /getpass/0.1.7: resolution: {integrity: sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=} dependencies: @@ -2286,6 +2202,7 @@ packages: /has-bigints/1.0.1: resolution: {integrity: sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==} + dev: true /has-flag/3.0.0: resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=} @@ -2300,18 +2217,21 @@ packages: /has-symbols/1.0.2: resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==} engines: {node: '>= 0.4'} + dev: true /has-tostringtag/1.0.0: resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.2 + dev: true /has/1.0.3: resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} engines: {node: '>= 0.4.0'} dependencies: function-bind: 1.1.1 + dev: true /hasha/5.2.2: resolution: {integrity: sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==} @@ -2437,25 +2357,7 @@ packages: get-intrinsic: 1.1.1 has: 1.0.3 side-channel: 1.0.4 - - /ioredis/4.27.11: - resolution: {integrity: sha512-qBU2uCHgRs4kJT/qzxeFhbmizmNLQ1HH59EvKt1WnzDPMjjWxZS1+W6ynXdE47TxxqNLKYemJxEXXUb7Y4JJ+w==} - engines: {node: '>=6'} - dependencies: - cluster-key-slot: 1.1.0 - debug: 4.3.2 - denque: 1.5.0 - lodash.defaults: 4.2.0 - lodash.flatten: 4.4.0 - lodash.isarguments: 3.1.0 - p-map: 2.1.0 - redis-commands: 1.7.0 - redis-errors: 1.2.0 - redis-parser: 3.0.0 - standard-as-callback: 2.1.0 - transitivePeerDependencies: - - supports-color - dev: false + dev: true /ipaddr.js/1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} @@ -2474,6 +2376,7 @@ packages: resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==} dependencies: has-bigints: 1.0.1 + dev: true /is-binary-path/2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} @@ -2488,10 +2391,12 @@ packages: dependencies: call-bind: 1.0.2 has-tostringtag: 1.0.0 + dev: true /is-callable/1.2.4: resolution: {integrity: sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==} engines: {node: '>= 0.4'} + dev: true /is-core-module/2.5.0: resolution: {integrity: sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==} @@ -2504,6 +2409,7 @@ packages: engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 + dev: true /is-extglob/2.1.1: resolution: {integrity: sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=} @@ -2534,23 +2440,17 @@ packages: is-extglob: 2.1.1 dev: true - /is-nan/1.3.2: - resolution: {integrity: sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==} - engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.2 - define-properties: 1.1.3 - dev: false - /is-negative-zero/2.0.1: resolution: {integrity: sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==} engines: {node: '>= 0.4'} + dev: true /is-number-object/1.0.6: resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==} engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 + dev: true /is-number/7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} @@ -2563,10 +2463,7 @@ packages: dependencies: call-bind: 1.0.2 has-tostringtag: 1.0.0 - - /is-shared-array-buffer/1.0.1: - resolution: {integrity: sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==} - dev: false + dev: true /is-stream/2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} @@ -2577,23 +2474,19 @@ packages: engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 + dev: true /is-symbol/1.0.4: resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==} engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.2 + dev: true /is-typedarray/1.0.0: resolution: {integrity: sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=} dev: true - /is-weakref/1.0.1: - resolution: {integrity: sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==} - dependencies: - call-bind: 1.0.2 - dev: false - /is-windows/1.0.2: resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} engines: {node: '>=0.10.0'} @@ -2816,22 +2709,10 @@ packages: resolution: {integrity: sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=} dev: true - /lodash.defaults/4.2.0: - resolution: {integrity: sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=} - dev: false - - /lodash.flatten/4.4.0: - resolution: {integrity: sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=} - dev: false - /lodash.flattendeep/4.4.0: resolution: {integrity: sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=} dev: true - /lodash.isarguments/3.1.0: - resolution: {integrity: sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=} - dev: false - /lodash.merge/4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true @@ -2840,10 +2721,6 @@ packages: resolution: {integrity: sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=} dev: true - /lodash/4.17.21: - resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} - dev: false - /log-driver/1.2.7: resolution: {integrity: sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==} engines: {node: '>=0.8.6'} @@ -2871,6 +2748,7 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 + dev: true /make-dir/3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} @@ -2960,12 +2838,6 @@ packages: engines: {node: '>=10'} hasBin: true - /moment-timezone/0.5.33: - resolution: {integrity: sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==} - dependencies: - moment: 2.29.1 - dev: false - /moment/2.29.1: resolution: {integrity: sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==} dev: false @@ -3220,10 +3092,12 @@ packages: /object-inspect/1.11.0: resolution: {integrity: sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==} + dev: true /object-keys/1.1.1: resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} engines: {node: '>= 0.4'} + dev: true /object.assign/4.1.2: resolution: {integrity: sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==} @@ -3233,15 +3107,7 @@ packages: define-properties: 1.1.3 has-symbols: 1.0.2 object-keys: 1.1.1 - - /object.getownpropertydescriptors/2.1.3: - resolution: {integrity: sha512-VdDoCwvJI4QdC6ndjpqFmoL3/+HxffFBbcJzKi5hwLLqqx3mdbedRpfZDdK0SrOSauj8X4GzBvnDZl4vTN7dOw==} - engines: {node: '>= 0.8'} - dependencies: - call-bind: 1.0.2 - define-properties: 1.1.3 - es-abstract: 1.19.1 - dev: false + dev: true /object.values/1.1.4: resolution: {integrity: sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==} @@ -3302,11 +3168,6 @@ packages: resolution: {integrity: sha1-Tod/vtqaLsgAD7wLyuOWRe6L+Nw=} dev: true - /p-finally/1.0.0: - resolution: {integrity: sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=} - engines: {node: '>=4'} - dev: false - /p-limit/1.3.0: resolution: {integrity: sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==} engines: {node: '>=4'} @@ -3335,11 +3196,6 @@ packages: p-limit: 2.3.0 dev: true - /p-map/2.1.0: - resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==} - engines: {node: '>=6'} - dev: false - /p-map/3.0.0: resolution: {integrity: sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==} engines: {node: '>=8'} @@ -3347,13 +3203,6 @@ packages: aggregate-error: 3.1.0 dev: true - /p-timeout/3.2.0: - resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} - engines: {node: '>=8'} - dependencies: - p-finally: 1.0.0 - dev: false - /p-try/1.0.0: resolution: {integrity: sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=} engines: {node: '>=4'} @@ -3501,15 +3350,6 @@ packages: engines: {node: '>=0.4.0'} dev: true - /promise.prototype.finally/3.1.3: - resolution: {integrity: sha512-EXRF3fC9/0gz4qkt/f5EP5iW4kj9oFpBICNpCNOb/52+8nlHIX07FPLbi/q4qYBQ1xZqivMzTpNQSnArVASolQ==} - engines: {node: '>= 0.4'} - dependencies: - call-bind: 1.0.2 - define-properties: 1.1.3 - es-abstract: 1.19.1 - dev: false - /prop-types/15.7.2: resolution: {integrity: sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==} dependencies: @@ -3827,6 +3667,7 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 + dev: true /send/0.17.1: resolution: {integrity: sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==} @@ -3883,6 +3724,7 @@ packages: call-bind: 1.0.2 get-intrinsic: 1.1.1 object-inspect: 1.11.0 + dev: true /siginfo/2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -4003,10 +3845,6 @@ packages: resolution: {integrity: sha1-Gsig2Ug4SNFpXkGLbQMaPDzmjjs=} dev: true - /standard-as-callback/2.1.0: - resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} - dev: false - /statuses/1.5.0: resolution: {integrity: sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=} engines: {node: '>= 0.6'} @@ -4048,12 +3886,14 @@ packages: dependencies: call-bind: 1.0.2 define-properties: 1.1.3 + dev: true /string.prototype.trimstart/1.0.4: resolution: {integrity: sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==} dependencies: call-bind: 1.0.2 define-properties: 1.1.3 + dev: true /string_decoder/0.10.31: resolution: {integrity: sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=} @@ -4430,6 +4270,7 @@ packages: has-bigints: 1.0.1 has-symbols: 1.0.2 which-boxed-primitive: 1.0.2 + dev: true /unicode-length/2.0.2: resolution: {integrity: sha512-Ph/j1VbS3/r77nhoY2WU0GWGjVYOHL3xpKp0y/Eq2e5r0mT/6b649vm7KFO6RdAdrZkYLdxphYVgvODxPB+Ebg==} @@ -4452,16 +4293,6 @@ packages: /util-deprecate/1.0.2: resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=} - /util.promisify/1.1.1: - resolution: {integrity: sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==} - dependencies: - call-bind: 1.0.2 - define-properties: 1.1.3 - for-each: 0.3.3 - has-symbols: 1.0.2 - object.getownpropertydescriptors: 2.1.3 - dev: false - /utils-merge/1.0.1: resolution: {integrity: sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=} engines: {node: '>= 0.4.0'} @@ -4473,11 +4304,6 @@ packages: hasBin: true dev: true - /uuid/8.3.2: - resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} - hasBin: true - dev: false - /v8-compile-cache/2.3.0: resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==} dev: true @@ -4511,6 +4337,7 @@ packages: is-number-object: 1.0.6 is-string: 1.0.7 is-symbol: 1.0.4 + dev: true /which-module/2.0.0: resolution: {integrity: sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=} @@ -4614,6 +4441,7 @@ packages: /yallist/4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: true /yaml/1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} diff --git a/server/src/lib/jobs/jobs/deorphan-scores.ts b/server/src/lib/jobs/deorphan-scores.ts similarity index 85% rename from server/src/lib/jobs/jobs/deorphan-scores.ts rename to server/src/lib/jobs/deorphan-scores.ts index 5facc3993..4a680d043 100644 --- a/server/src/lib/jobs/jobs/deorphan-scores.ts +++ b/server/src/lib/jobs/deorphan-scores.ts @@ -1,13 +1,12 @@ // Attempt to deoprhan lost scores. -import Bull from "bull"; import db from "external/mongo/db"; import CreateLogCtx from "lib/logger/logger"; import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; const logger = CreateLogCtx(__dirname); -export async function DeoprhanScores(job: Bull.Job) { +export async function DeoprhanScores() { const orphans = await db["orphan-scores"].find({}); // ScoreIDs are essentially userID dependent, so this is fine. @@ -15,7 +14,6 @@ export async function DeoprhanScores(job: Bull.Job) { logger.info(`Found ${orphans.length} orphans.`); - let done = 0; let failed = 0; let success = 0; let removed = 0; @@ -23,7 +21,6 @@ export async function DeoprhanScores(job: Bull.Job) { await Promise.all( orphans.map((or) => ReprocessOrphan(or, blacklist, logger).then((r) => { - done++; if (r === null) { removed++; } else if (r === false) { @@ -31,8 +28,6 @@ export async function DeoprhanScores(job: Bull.Job) { } else { success++; } - - job.progress((100 * done) / orphans.length); }) ) ); @@ -41,3 +36,9 @@ export async function DeoprhanScores(job: Bull.Job) { logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`); } + +if (require.main === module) { + DeoprhanScores().then(() => { + process.exit(0); + }); +} diff --git a/server/src/lib/jobs/job-runner.ts b/server/src/lib/jobs/job-runner.ts deleted file mode 100644 index 63ab7f7c3..000000000 --- a/server/src/lib/jobs/job-runner.ts +++ /dev/null @@ -1,74 +0,0 @@ -import Queue from "bull"; -import CreateLogCtx from "lib/logger/logger"; -import { DedupeArr } from "utils/misc"; -import { DeoprhanScores } from "./jobs/deorphan-scores"; -import { UGSSnapshot } from "./jobs/ugs-snapshot"; - -interface Job { - name: string; - cronFormat: string; - run: (job: Queue.Job) => Promise; -} - -const jobs: Job[] = [ - { - name: "Snapshot User Game Stats", - cronFormat: "0 0 * * *", - run: UGSSnapshot, - }, - { - name: "De-Orphan Scores", - // We run an hour after snapshotting UGS - // just to spread load out a bit. - cronFormat: "1 0 * * *", - run: DeoprhanScores, - }, -]; - -const logger = CreateLogCtx("JOB_RUNNER"); - -/** - * Initalises a tachi-server job runner. - * This runs the list of jobs defined in jobConfig.jobs. - */ -export function InitialiseJobRunner() { - logger.info(`Booting up Job Runner.`); - - const names = jobs.map((e) => e.name); - - if (DedupeArr(names).length !== names.length) { - logger.crit(`Jobs has duplicate name fields, refusing to run.`); - process.exit(1); - } - - const JobQueue = new Queue("Job Runner"); - - const jobNameMap = new Map(); - - for (const job of jobs) { - JobQueue.add({ jobName: job.name }, { repeat: { cron: job.cronFormat } }); - jobNameMap.set(job.name, job); - } - - JobQueue.process(async (j) => { - const name = j.data.jobName; - logger.info(`Running job ${name}.`); - - const jobInfo = jobNameMap.get(name); - - if (!jobInfo) { - logger.severe(`Unknown job name ${name}, couldn't find a run function?`); - return false; - } - - await jobInfo.run(j); - - return true; - }); - - logger.info(`Initialised ${jobs.length} jobs (${jobs.map((e) => e.name).join(", ")}).`); -} - -if (require.main === module) { - InitialiseJobRunner(); -} diff --git a/server/src/lib/jobs/jobs/ugs-snapshot.ts b/server/src/lib/jobs/ugs-snapshot.ts similarity index 96% rename from server/src/lib/jobs/jobs/ugs-snapshot.ts rename to server/src/lib/jobs/ugs-snapshot.ts index 224381d69..580f53afb 100644 --- a/server/src/lib/jobs/jobs/ugs-snapshot.ts +++ b/server/src/lib/jobs/ugs-snapshot.ts @@ -87,3 +87,9 @@ export async function UGSSnapshot() { throw err; } } + +if (require.main === module) { + UGSSnapshot().then(() => { + process.exit(0); + }); +} From 17750ec843c1b7b55953aacee91ae69048a6d384 Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 8 Nov 2021 22:19:23 +0000 Subject: [PATCH 2/3] Re add job-runner to inline-job-runner. --- .../jobs/inline-job-runner.ts/job-runner.ts | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 server/src/lib/jobs/inline-job-runner.ts/job-runner.ts diff --git a/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts b/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts new file mode 100644 index 000000000..63ab7f7c3 --- /dev/null +++ b/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts @@ -0,0 +1,74 @@ +import Queue from "bull"; +import CreateLogCtx from "lib/logger/logger"; +import { DedupeArr } from "utils/misc"; +import { DeoprhanScores } from "./jobs/deorphan-scores"; +import { UGSSnapshot } from "./jobs/ugs-snapshot"; + +interface Job { + name: string; + cronFormat: string; + run: (job: Queue.Job) => Promise; +} + +const jobs: Job[] = [ + { + name: "Snapshot User Game Stats", + cronFormat: "0 0 * * *", + run: UGSSnapshot, + }, + { + name: "De-Orphan Scores", + // We run an hour after snapshotting UGS + // just to spread load out a bit. + cronFormat: "1 0 * * *", + run: DeoprhanScores, + }, +]; + +const logger = CreateLogCtx("JOB_RUNNER"); + +/** + * Initalises a tachi-server job runner. + * This runs the list of jobs defined in jobConfig.jobs. + */ +export function InitialiseJobRunner() { + logger.info(`Booting up Job Runner.`); + + const names = jobs.map((e) => e.name); + + if (DedupeArr(names).length !== names.length) { + logger.crit(`Jobs has duplicate name fields, refusing to run.`); + process.exit(1); + } + + const JobQueue = new Queue("Job Runner"); + + const jobNameMap = new Map(); + + for (const job of jobs) { + JobQueue.add({ jobName: job.name }, { repeat: { cron: job.cronFormat } }); + jobNameMap.set(job.name, job); + } + + JobQueue.process(async (j) => { + const name = j.data.jobName; + logger.info(`Running job ${name}.`); + + const jobInfo = jobNameMap.get(name); + + if (!jobInfo) { + logger.severe(`Unknown job name ${name}, couldn't find a run function?`); + return false; + } + + await jobInfo.run(j); + + return true; + }); + + logger.info(`Initialised ${jobs.length} jobs (${jobs.map((e) => e.name).join(", ")}).`); +} + +if (require.main === module) { + InitialiseJobRunner(); +} From af84d24b3778517d68c561d5b38a8c677fbd2518 Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 8 Nov 2021 23:25:41 +0000 Subject: [PATCH 3/3] Re add inline job runner --- server/package.json | 2 + server/pnpm-lock.yaml | 234 +++++++++++++++--- .../jobs/inline-job-runner.ts/job-runner.ts | 4 +- server/src/main.ts | 8 +- 4 files changed, 214 insertions(+), 34 deletions(-) diff --git a/server/package.json b/server/package.json index 82885461d..dac8cec53 100644 --- a/server/package.json +++ b/server/package.json @@ -47,9 +47,11 @@ "why-is-node-running": "2.2.0" }, "dependencies": { + "@types/bull": "^3.15.5", "@types/bunyan": "^1.8.7", "@types/deep-equal": "^1.0.1", "bcryptjs": "^2.4.3", + "bull": "^4.1.0", "bunyan": "^1.8.15", "commander": "7.2.0", "connect-redis": "6.0.0", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 8becf9645..a33041a2a 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -3,6 +3,7 @@ lockfileVersion: 5.3 specifiers: '@types/bcrypt': 5.0.0 '@types/bcryptjs': ^2.4.2 + '@types/bull': ^3.15.5 '@types/bunyan': ^1.8.7 '@types/connect-redis': 0.0.16 '@types/deep-equal': ^1.0.1 @@ -24,6 +25,7 @@ specifiers: '@typescript-eslint/eslint-plugin': 4.28.0 '@typescript-eslint/parser': 4.28.0 bcryptjs: ^2.4.3 + bull: ^4.1.0 bunyan: ^1.8.15 commander: 7.2.0 connect-redis: 6.0.0 @@ -69,9 +71,11 @@ specifiers: winston-transport: ^4.4.0 dependencies: + '@types/bull': 3.15.5 '@types/bunyan': 1.8.7 '@types/deep-equal': 1.0.1 bcryptjs: 2.4.3 + bull: 4.1.0 bunyan: 1.8.15 commander: 7.2.0 connect-redis: 6.0.0 @@ -539,6 +543,12 @@ packages: '@types/node': 15.12.4 dev: false + /@types/bull/3.15.5: + resolution: {integrity: sha512-XgJQWJ03jyKMfdoL8IAIoHIo7JkkL74kcxuujTONkSJswm0giIJ9kuVgDNHS0OvD+OiPNcFmbBl0H3scj2+A8A==} + dependencies: + '@types/ioredis': 4.26.7 + dev: false + /@types/bunyan/1.8.7: resolution: {integrity: sha512-jaNt6xX5poSmXuDAkQrSqx2zkR66OrdRDuVnU8ldvn3k/Ci/7Sf5nooKspQWimDnw337Bzt/yirqSThTjvrHkg==} dependencies: @@ -608,7 +618,6 @@ packages: resolution: {integrity: sha512-TOGRR+e1to00CihjgPNygD7+G7ruVnMi62YdIvGUBRfj11k/aWq+Fv5Ea8St0Oy56NngTBfA8GvLn1uvHvhX6Q==} dependencies: '@types/node': 15.12.4 - dev: true /@types/json-schema/7.0.9: resolution: {integrity: sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==} @@ -1084,6 +1093,24 @@ packages: /buffer-from/1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + /bull/4.1.0: + resolution: {integrity: sha512-rQcLuAmzZIv1dHJO/yKrWu497xcTxMpYeTEBfpStrJFZ1UZpBSGHSx+defbtFVeGEeY8Pn0aMGRvRtldUBVUyQ==} + engines: {node: '>=10'} + dependencies: + cron-parser: 2.18.0 + debuglog: 1.0.1 + get-port: 5.1.1 + ioredis: 4.28.0 + lodash: 4.17.21 + p-timeout: 3.2.0 + promise.prototype.finally: 3.1.3 + semver: 7.3.5 + util.promisify: 1.1.1 + uuid: 8.3.2 + transitivePeerDependencies: + - supports-color + dev: false + /bunyan/1.8.15: resolution: {integrity: sha512-0tECWShh6wUysgucJcBAoYegf3JJoZWibxdqhTm7OHPeT42qdjkZ29QCMcKwbgU1kiH+auSIasNRXMLWXafXig==} engines: {'0': node >=0.10.0} @@ -1123,7 +1150,6 @@ packages: dependencies: function-bind: 1.1.1 get-intrinsic: 1.1.1 - dev: true /caller-callsite/2.0.0: resolution: {integrity: sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=} @@ -1225,6 +1251,11 @@ packages: engines: {node: '>=0.8'} dev: false + /cluster-key-slot/1.1.0: + resolution: {integrity: sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==} + engines: {node: '>=0.10.0'} + dev: false + /code-point-at/1.1.0: resolution: {integrity: sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=} engines: {node: '>=0.10.0'} @@ -1376,6 +1407,14 @@ packages: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} dev: false + /cron-parser/2.18.0: + resolution: {integrity: sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==} + engines: {node: '>=0.8'} + dependencies: + is-nan: 1.3.2 + moment-timezone: 0.5.33 + dev: false + /cross-spawn/7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} @@ -1418,6 +1457,10 @@ packages: dependencies: ms: 2.1.2 + /debuglog/1.0.1: + resolution: {integrity: sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=} + dev: false + /decamelize/1.2.0: resolution: {integrity: sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=} engines: {node: '>=0.10.0'} @@ -1450,7 +1493,6 @@ packages: engines: {node: '>= 0.4'} dependencies: object-keys: 1.1.1 - dev: true /delayed-stream/1.0.0: resolution: {integrity: sha1-3zrhmayt+31ECqrgsp4icrJOxhk=} @@ -1586,6 +1628,32 @@ packages: unbox-primitive: 1.0.1 dev: true + /es-abstract/1.19.1: + resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + es-to-primitive: 1.2.1 + function-bind: 1.1.1 + get-intrinsic: 1.1.1 + get-symbol-description: 1.0.0 + has: 1.0.3 + has-symbols: 1.0.2 + internal-slot: 1.0.3 + is-callable: 1.2.4 + is-negative-zero: 2.0.1 + is-regex: 1.1.4 + is-shared-array-buffer: 1.0.1 + is-string: 1.0.7 + is-weakref: 1.0.1 + object-inspect: 1.11.0 + object-keys: 1.1.1 + object.assign: 4.1.2 + string.prototype.trimend: 1.0.4 + string.prototype.trimstart: 1.0.4 + unbox-primitive: 1.0.1 + dev: false + /es-to-primitive/1.2.1: resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} engines: {node: '>= 0.4'} @@ -1593,7 +1661,6 @@ packages: is-callable: 1.2.4 is-date-object: 1.0.5 is-symbol: 1.0.4 - dev: true /es6-error/4.1.1: resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} @@ -2027,6 +2094,12 @@ packages: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} dev: false + /for-each/0.3.3: + resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} + dependencies: + is-callable: 1.2.4 + dev: false + /foreground-child/2.0.0: resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} engines: {node: '>=8.0.0'} @@ -2091,7 +2164,6 @@ packages: /function-bind/1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - dev: true /function-loop/2.0.1: resolution: {integrity: sha512-ktIR+O6i/4h+j/ZhZJNdzeI4i9lEPeEK6UPR2EVyTVBqOwcU3Za9xYKLH64ZR9HmcROyRrOkizNyjjtWJzDDkQ==} @@ -2117,13 +2189,25 @@ packages: function-bind: 1.1.1 has: 1.0.3 has-symbols: 1.0.2 - dev: true /get-package-type/0.1.0: resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} engines: {node: '>=8.0.0'} dev: true + /get-port/5.1.1: + resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} + engines: {node: '>=8'} + dev: false + + /get-symbol-description/1.0.0: + resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + get-intrinsic: 1.1.1 + dev: false + /getpass/0.1.7: resolution: {integrity: sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=} dependencies: @@ -2202,7 +2286,6 @@ packages: /has-bigints/1.0.1: resolution: {integrity: sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==} - dev: true /has-flag/3.0.0: resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=} @@ -2217,21 +2300,18 @@ packages: /has-symbols/1.0.2: resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==} engines: {node: '>= 0.4'} - dev: true /has-tostringtag/1.0.0: resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.2 - dev: true /has/1.0.3: resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} engines: {node: '>= 0.4.0'} dependencies: function-bind: 1.1.1 - dev: true /hasha/5.2.2: resolution: {integrity: sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==} @@ -2357,7 +2437,25 @@ packages: get-intrinsic: 1.1.1 has: 1.0.3 side-channel: 1.0.4 - dev: true + + /ioredis/4.28.0: + resolution: {integrity: sha512-I+zkeeWp3XFgPT2CtJKxvaF5FjGBGt4yGYljRjQecdQKteThuAsKqffeF1lgHVlYnuNeozRbPOCDNZ7tDWPeig==} + engines: {node: '>=6'} + dependencies: + cluster-key-slot: 1.1.0 + debug: 4.3.2 + denque: 1.5.0 + lodash.defaults: 4.2.0 + lodash.flatten: 4.4.0 + lodash.isarguments: 3.1.0 + p-map: 2.1.0 + redis-commands: 1.7.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false /ipaddr.js/1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} @@ -2376,7 +2474,6 @@ packages: resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==} dependencies: has-bigints: 1.0.1 - dev: true /is-binary-path/2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} @@ -2391,12 +2488,10 @@ packages: dependencies: call-bind: 1.0.2 has-tostringtag: 1.0.0 - dev: true /is-callable/1.2.4: resolution: {integrity: sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==} engines: {node: '>= 0.4'} - dev: true /is-core-module/2.5.0: resolution: {integrity: sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==} @@ -2409,7 +2504,6 @@ packages: engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 - dev: true /is-extglob/2.1.1: resolution: {integrity: sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=} @@ -2440,17 +2534,23 @@ packages: is-extglob: 2.1.1 dev: true + /is-nan/1.3.2: + resolution: {integrity: sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + dev: false + /is-negative-zero/2.0.1: resolution: {integrity: sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==} engines: {node: '>= 0.4'} - dev: true /is-number-object/1.0.6: resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==} engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 - dev: true /is-number/7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} @@ -2463,7 +2563,10 @@ packages: dependencies: call-bind: 1.0.2 has-tostringtag: 1.0.0 - dev: true + + /is-shared-array-buffer/1.0.1: + resolution: {integrity: sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==} + dev: false /is-stream/2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} @@ -2474,19 +2577,23 @@ packages: engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 - dev: true /is-symbol/1.0.4: resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==} engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.2 - dev: true /is-typedarray/1.0.0: resolution: {integrity: sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=} dev: true + /is-weakref/1.0.1: + resolution: {integrity: sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==} + dependencies: + call-bind: 1.0.2 + dev: false + /is-windows/1.0.2: resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} engines: {node: '>=0.10.0'} @@ -2709,10 +2816,22 @@ packages: resolution: {integrity: sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=} dev: true + /lodash.defaults/4.2.0: + resolution: {integrity: sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=} + dev: false + + /lodash.flatten/4.4.0: + resolution: {integrity: sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=} + dev: false + /lodash.flattendeep/4.4.0: resolution: {integrity: sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=} dev: true + /lodash.isarguments/3.1.0: + resolution: {integrity: sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=} + dev: false + /lodash.merge/4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true @@ -2721,6 +2840,10 @@ packages: resolution: {integrity: sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=} dev: true + /lodash/4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + /log-driver/1.2.7: resolution: {integrity: sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==} engines: {node: '>=0.8.6'} @@ -2748,7 +2871,6 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 - dev: true /make-dir/3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} @@ -2838,6 +2960,12 @@ packages: engines: {node: '>=10'} hasBin: true + /moment-timezone/0.5.33: + resolution: {integrity: sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==} + dependencies: + moment: 2.29.1 + dev: false + /moment/2.29.1: resolution: {integrity: sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==} dev: false @@ -3092,12 +3220,10 @@ packages: /object-inspect/1.11.0: resolution: {integrity: sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==} - dev: true /object-keys/1.1.1: resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} engines: {node: '>= 0.4'} - dev: true /object.assign/4.1.2: resolution: {integrity: sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==} @@ -3107,7 +3233,15 @@ packages: define-properties: 1.1.3 has-symbols: 1.0.2 object-keys: 1.1.1 - dev: true + + /object.getownpropertydescriptors/2.1.3: + resolution: {integrity: sha512-VdDoCwvJI4QdC6ndjpqFmoL3/+HxffFBbcJzKi5hwLLqqx3mdbedRpfZDdK0SrOSauj8X4GzBvnDZl4vTN7dOw==} + engines: {node: '>= 0.8'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + es-abstract: 1.19.1 + dev: false /object.values/1.1.4: resolution: {integrity: sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==} @@ -3168,6 +3302,11 @@ packages: resolution: {integrity: sha1-Tod/vtqaLsgAD7wLyuOWRe6L+Nw=} dev: true + /p-finally/1.0.0: + resolution: {integrity: sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=} + engines: {node: '>=4'} + dev: false + /p-limit/1.3.0: resolution: {integrity: sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==} engines: {node: '>=4'} @@ -3196,6 +3335,11 @@ packages: p-limit: 2.3.0 dev: true + /p-map/2.1.0: + resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==} + engines: {node: '>=6'} + dev: false + /p-map/3.0.0: resolution: {integrity: sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==} engines: {node: '>=8'} @@ -3203,6 +3347,13 @@ packages: aggregate-error: 3.1.0 dev: true + /p-timeout/3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + dependencies: + p-finally: 1.0.0 + dev: false + /p-try/1.0.0: resolution: {integrity: sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=} engines: {node: '>=4'} @@ -3350,6 +3501,15 @@ packages: engines: {node: '>=0.4.0'} dev: true + /promise.prototype.finally/3.1.3: + resolution: {integrity: sha512-EXRF3fC9/0gz4qkt/f5EP5iW4kj9oFpBICNpCNOb/52+8nlHIX07FPLbi/q4qYBQ1xZqivMzTpNQSnArVASolQ==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + es-abstract: 1.19.1 + dev: false + /prop-types/15.7.2: resolution: {integrity: sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==} dependencies: @@ -3667,7 +3827,6 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 - dev: true /send/0.17.1: resolution: {integrity: sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==} @@ -3724,7 +3883,6 @@ packages: call-bind: 1.0.2 get-intrinsic: 1.1.1 object-inspect: 1.11.0 - dev: true /siginfo/2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -3845,6 +4003,10 @@ packages: resolution: {integrity: sha1-Gsig2Ug4SNFpXkGLbQMaPDzmjjs=} dev: true + /standard-as-callback/2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false + /statuses/1.5.0: resolution: {integrity: sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=} engines: {node: '>= 0.6'} @@ -3886,14 +4048,12 @@ packages: dependencies: call-bind: 1.0.2 define-properties: 1.1.3 - dev: true /string.prototype.trimstart/1.0.4: resolution: {integrity: sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==} dependencies: call-bind: 1.0.2 define-properties: 1.1.3 - dev: true /string_decoder/0.10.31: resolution: {integrity: sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=} @@ -4270,7 +4430,6 @@ packages: has-bigints: 1.0.1 has-symbols: 1.0.2 which-boxed-primitive: 1.0.2 - dev: true /unicode-length/2.0.2: resolution: {integrity: sha512-Ph/j1VbS3/r77nhoY2WU0GWGjVYOHL3xpKp0y/Eq2e5r0mT/6b649vm7KFO6RdAdrZkYLdxphYVgvODxPB+Ebg==} @@ -4293,6 +4452,16 @@ packages: /util-deprecate/1.0.2: resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=} + /util.promisify/1.1.1: + resolution: {integrity: sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + for-each: 0.3.3 + has-symbols: 1.0.2 + object.getownpropertydescriptors: 2.1.3 + dev: false + /utils-merge/1.0.1: resolution: {integrity: sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=} engines: {node: '>= 0.4.0'} @@ -4304,6 +4473,11 @@ packages: hasBin: true dev: true + /uuid/8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + dev: false + /v8-compile-cache/2.3.0: resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==} dev: true @@ -4337,7 +4511,6 @@ packages: is-number-object: 1.0.6 is-string: 1.0.7 is-symbol: 1.0.4 - dev: true /which-module/2.0.0: resolution: {integrity: sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=} @@ -4441,7 +4614,6 @@ packages: /yallist/4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: true /yaml/1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} diff --git a/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts b/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts index 63ab7f7c3..f7b642de7 100644 --- a/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts +++ b/server/src/lib/jobs/inline-job-runner.ts/job-runner.ts @@ -1,8 +1,8 @@ import Queue from "bull"; import CreateLogCtx from "lib/logger/logger"; import { DedupeArr } from "utils/misc"; -import { DeoprhanScores } from "./jobs/deorphan-scores"; -import { UGSSnapshot } from "./jobs/ugs-snapshot"; +import { DeoprhanScores } from "../deorphan-scores"; +import { UGSSnapshot } from "../ugs-snapshot"; interface Job { name: string; diff --git a/server/src/main.ts b/server/src/main.ts index df8367204..df2dda4af 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -63,7 +63,13 @@ if (ServerConfig.ENABLE_SERVER_HTTPS) { } if (process.env.INVOKE_JOB_RUNNER) { - logger.info(`Spawning a tachi-server job runner.`); + logger.info(`Spawning a tachi-server job runner inline.`); + + if (Environment.nodeEnv === "production") { + logger.warn( + `Spawning inline tachi-server job runner in production. Is this actually what you want? You should run a tool like Ofelia to manage this.` + ); + } // Spawn as a separate process to avoid hogging the main thread. const jobProcess = spawn(