diff --git a/server/package.json b/server/package.json index a8e60fb70..9f48aaf21 100644 --- a/server/package.json +++ b/server/package.json @@ -43,12 +43,13 @@ "prettier": "2.3.1", "supertest": "6.1.3", "tap": "15.0.9", - "ts-node": "10.0.0", "tsconfig-paths": "3.10.1", "why-is-node-running": "2.2.0" }, "dependencies": { + "@types/bull": "^3.15.5", "bcryptjs": "^2.4.3", + "bull": "^3.29.2", "commander": "7.2.0", "connect-redis": "6.0.0", "csv-parse": "4.16.0", @@ -75,6 +76,7 @@ "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", "tachi-common": "0.2.26", + "ts-node": "10.0.0", "typescript": "4.3.4", "winston": "3.3.3", "winston-transport": "^4.4.0" @@ -93,4 +95,4 @@ "src/external/mongo/schemas.ts" ] } -} +} \ No newline at end of file diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 730486181..bf602ad6f 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -3,6 +3,7 @@ lockfileVersion: 5.3 specifiers: '@types/bcrypt': 5.0.0 '@types/bcryptjs': ^2.4.2 + '@types/bull': ^3.15.5 '@types/connect-redis': 0.0.16 '@types/express': 4.17.12 '@types/express-rate-limit': 5.1.2 @@ -22,6 +23,7 @@ specifiers: '@typescript-eslint/eslint-plugin': 4.28.0 '@typescript-eslint/parser': 4.28.0 bcryptjs: ^2.4.3 + bull: ^3.29.2 commander: 7.2.0 connect-redis: 6.0.0 csv-parse: 4.16.0 @@ -65,7 +67,9 @@ specifiers: winston-transport: ^4.4.0 dependencies: + '@types/bull': 3.15.5 bcryptjs: 2.4.3 + bull: 3.29.2 commander: 7.2.0 connect-redis: 6.0.0 csv-parse: 4.16.0 @@ -92,6 +96,7 @@ dependencies: rimraf: 3.0.2 safe-json-stringify: 1.2.0 tachi-common: 0.2.26 + ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b typescript: 4.3.4 winston: 3.3.3 winston-transport: 4.4.0 @@ -126,7 +131,6 @@ devDependencies: prettier: 2.3.1 supertest: 6.1.3 tap: 15.0.9_ts-node@10.0.0+typescript@4.3.4 - ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b tsconfig-paths: 3.10.1 why-is-node-running: 2.2.0 @@ -494,19 +498,19 @@ packages: /@tsconfig/node10/1.0.8: resolution: {integrity: sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg==} - dev: true + dev: false /@tsconfig/node12/1.0.9: resolution: {integrity: sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw==} - dev: true + dev: false /@tsconfig/node14/1.0.1: resolution: {integrity: sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==} - dev: true + dev: false /@tsconfig/node16/1.0.2: resolution: {integrity: sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==} - dev: true + dev: false /@types/bcrypt/5.0.0: resolution: {integrity: sha512-agtcFKaruL8TmcvqbndlqHPSJgsolhf/qPWchFlgnW1gECTN/nKbFcoFnvKAQRFfKbh+BO6A3SWdJu9t+xF3Lw==} @@ -531,6 +535,12 @@ packages: '@types/node': 15.12.4 dev: false + /@types/bull/3.15.5: + resolution: {integrity: sha512-XgJQWJ03jyKMfdoL8IAIoHIo7JkkL74kcxuujTONkSJswm0giIJ9kuVgDNHS0OvD+OiPNcFmbBl0H3scj2+A8A==} + dependencies: + '@types/ioredis': 4.26.7 + dev: false + /@types/connect-redis/0.0.16: resolution: {integrity: sha512-ToZRmwoeqRhzGIqMO9k8Hs1B9pFSGYCjHvMpPlgo+kQ4M7xaPwi2/5HjikJ0MOfPDFcAe9Qwl1SaULYCNE6jYg==} dependencies: @@ -590,7 +600,6 @@ packages: resolution: {integrity: sha512-TOGRR+e1to00CihjgPNygD7+G7ruVnMi62YdIvGUBRfj11k/aWq+Fv5Ea8St0Oy56NngTBfA8GvLn1uvHvhX6Q==} dependencies: '@types/node': 15.12.4 - dev: true /@types/json-schema/7.0.9: resolution: {integrity: sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==} @@ -913,7 +922,7 @@ packages: /arg/4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} - dev: true + dev: false /argparse/1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -1066,6 +1075,24 @@ packages: /buffer-from/1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + /bull/3.29.2: + resolution: {integrity: sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==} + engines: {node: '>=10'} + dependencies: + cron-parser: 2.18.0 + debuglog: 1.0.1 + get-port: 5.1.1 + ioredis: 4.27.11 + lodash: 4.17.21 + p-timeout: 3.2.0 + promise.prototype.finally: 3.1.3 + semver: 7.3.5 + util.promisify: 1.1.1 + uuid: 8.3.2 + transitivePeerDependencies: + - supports-color + dev: false + /busboy/0.2.14: resolution: {integrity: sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=} engines: {node: '>=0.8.0'} @@ -1094,7 +1121,6 @@ packages: dependencies: function-bind: 1.1.1 get-intrinsic: 1.1.1 - dev: true /caller-callsite/2.0.0: resolution: {integrity: sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=} @@ -1196,6 +1222,11 @@ packages: engines: {node: '>=0.8'} dev: false + /cluster-key-slot/1.1.0: + resolution: {integrity: sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==} + engines: {node: '>=0.10.0'} + dev: false + /code-point-at/1.1.0: resolution: {integrity: sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=} engines: {node: '>=0.10.0'} @@ -1345,7 +1376,15 @@ packages: /create-require/1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - dev: true + dev: false + + /cron-parser/2.18.0: + resolution: {integrity: sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==} + engines: {node: '>=0.8'} + dependencies: + is-nan: 1.3.2 + moment-timezone: 0.5.33 + dev: false /cross-spawn/7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} @@ -1389,6 +1428,10 @@ packages: dependencies: ms: 2.1.2 + /debuglog/1.0.1: + resolution: {integrity: sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=} + dev: false + /decamelize/1.2.0: resolution: {integrity: sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=} engines: {node: '>=0.10.0'} @@ -1421,7 +1464,6 @@ packages: engines: {node: '>= 0.4'} dependencies: object-keys: 1.1.1 - dev: true /delayed-stream/1.0.0: resolution: {integrity: sha1-3zrhmayt+31ECqrgsp4icrJOxhk=} @@ -1457,7 +1499,6 @@ packages: /diff/4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} - dev: true /dir-glob/3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} @@ -1549,6 +1590,32 @@ packages: unbox-primitive: 1.0.1 dev: true + /es-abstract/1.19.1: + resolution: {integrity: sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + es-to-primitive: 1.2.1 + function-bind: 1.1.1 + get-intrinsic: 1.1.1 + get-symbol-description: 1.0.0 + has: 1.0.3 + has-symbols: 1.0.2 + internal-slot: 1.0.3 + is-callable: 1.2.4 + is-negative-zero: 2.0.1 + is-regex: 1.1.4 + is-shared-array-buffer: 1.0.1 + is-string: 1.0.7 + is-weakref: 1.0.1 + object-inspect: 1.11.0 + object-keys: 1.1.1 + object.assign: 4.1.2 + string.prototype.trimend: 1.0.4 + string.prototype.trimstart: 1.0.4 + unbox-primitive: 1.0.1 + dev: false + /es-to-primitive/1.2.1: resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} engines: {node: '>= 0.4'} @@ -1556,7 +1623,6 @@ packages: is-callable: 1.2.4 is-date-object: 1.0.5 is-symbol: 1.0.4 - dev: true /es6-error/4.1.1: resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} @@ -1984,6 +2050,12 @@ packages: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} dev: false + /for-each/0.3.3: + resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} + dependencies: + is-callable: 1.2.4 + dev: false + /foreground-child/2.0.0: resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} engines: {node: '>=8.0.0'} @@ -2048,7 +2120,6 @@ packages: /function-bind/1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - dev: true /function-loop/2.0.1: resolution: {integrity: sha512-ktIR+O6i/4h+j/ZhZJNdzeI4i9lEPeEK6UPR2EVyTVBqOwcU3Za9xYKLH64ZR9HmcROyRrOkizNyjjtWJzDDkQ==} @@ -2074,13 +2145,25 @@ packages: function-bind: 1.1.1 has: 1.0.3 has-symbols: 1.0.2 - dev: true /get-package-type/0.1.0: resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} engines: {node: '>=8.0.0'} dev: true + /get-port/5.1.1: + resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} + engines: {node: '>=8'} + dev: false + + /get-symbol-description/1.0.0: + resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + get-intrinsic: 1.1.1 + dev: false + /getpass/0.1.7: resolution: {integrity: sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=} dependencies: @@ -2148,7 +2231,6 @@ packages: /has-bigints/1.0.1: resolution: {integrity: sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==} - dev: true /has-flag/3.0.0: resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=} @@ -2163,21 +2245,18 @@ packages: /has-symbols/1.0.2: resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==} engines: {node: '>= 0.4'} - dev: true /has-tostringtag/1.0.0: resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.2 - dev: true /has/1.0.3: resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} engines: {node: '>= 0.4.0'} dependencies: function-bind: 1.1.1 - dev: true /hasha/5.2.2: resolution: {integrity: sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==} @@ -2303,7 +2382,25 @@ packages: get-intrinsic: 1.1.1 has: 1.0.3 side-channel: 1.0.4 - dev: true + + /ioredis/4.27.11: + resolution: {integrity: sha512-qBU2uCHgRs4kJT/qzxeFhbmizmNLQ1HH59EvKt1WnzDPMjjWxZS1+W6ynXdE47TxxqNLKYemJxEXXUb7Y4JJ+w==} + engines: {node: '>=6'} + dependencies: + cluster-key-slot: 1.1.0 + debug: 4.3.2 + denque: 1.5.0 + lodash.defaults: 4.2.0 + lodash.flatten: 4.4.0 + lodash.isarguments: 3.1.0 + p-map: 2.1.0 + redis-commands: 1.7.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false /ipaddr.js/1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} @@ -2322,7 +2419,6 @@ packages: resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==} dependencies: has-bigints: 1.0.1 - dev: true /is-binary-path/2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} @@ -2337,12 +2433,10 @@ packages: dependencies: call-bind: 1.0.2 has-tostringtag: 1.0.0 - dev: true /is-callable/1.2.4: resolution: {integrity: sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==} engines: {node: '>= 0.4'} - dev: true /is-core-module/2.5.0: resolution: {integrity: sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==} @@ -2355,7 +2449,6 @@ packages: engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 - dev: true /is-extglob/2.1.1: resolution: {integrity: sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=} @@ -2386,17 +2479,23 @@ packages: is-extglob: 2.1.1 dev: true + /is-nan/1.3.2: + resolution: {integrity: sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + dev: false + /is-negative-zero/2.0.1: resolution: {integrity: sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==} engines: {node: '>= 0.4'} - dev: true /is-number-object/1.0.6: resolution: {integrity: sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==} engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 - dev: true /is-number/7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} @@ -2409,7 +2508,10 @@ packages: dependencies: call-bind: 1.0.2 has-tostringtag: 1.0.0 - dev: true + + /is-shared-array-buffer/1.0.1: + resolution: {integrity: sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==} + dev: false /is-stream/2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} @@ -2420,19 +2522,23 @@ packages: engines: {node: '>= 0.4'} dependencies: has-tostringtag: 1.0.0 - dev: true /is-symbol/1.0.4: resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==} engines: {node: '>= 0.4'} dependencies: has-symbols: 1.0.2 - dev: true /is-typedarray/1.0.0: resolution: {integrity: sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=} dev: true + /is-weakref/1.0.1: + resolution: {integrity: sha512-b2jKc2pQZjaeFYWEf7ScFj+Be1I+PXmlu572Q8coTXZ+LD/QQZ7ShPMst8h16riVgyXTQwUsFEl74mDvc/3MHQ==} + dependencies: + call-bind: 1.0.2 + dev: false + /is-windows/1.0.2: resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} engines: {node: '>=0.10.0'} @@ -2655,10 +2761,22 @@ packages: resolution: {integrity: sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=} dev: true + /lodash.defaults/4.2.0: + resolution: {integrity: sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=} + dev: false + + /lodash.flatten/4.4.0: + resolution: {integrity: sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=} + dev: false + /lodash.flattendeep/4.4.0: resolution: {integrity: sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=} dev: true + /lodash.isarguments/3.1.0: + resolution: {integrity: sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=} + dev: false + /lodash.merge/4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true @@ -2667,6 +2785,10 @@ packages: resolution: {integrity: sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=} dev: true + /lodash/4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + /log-driver/1.2.7: resolution: {integrity: sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==} engines: {node: '>=0.8.6'} @@ -2694,7 +2816,6 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 - dev: true /make-dir/3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} @@ -2705,7 +2826,7 @@ packages: /make-error/1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} - dev: true + dev: false /media-typer/0.3.0: resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=} @@ -2784,6 +2905,16 @@ packages: engines: {node: '>=10'} hasBin: true + /moment-timezone/0.5.33: + resolution: {integrity: sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==} + dependencies: + moment: 2.29.1 + dev: false + + /moment/2.29.1: + resolution: {integrity: sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==} + dev: false + /mongodb/3.6.3: resolution: {integrity: sha512-rOZuR0QkodZiM+UbQE5kDsJykBqWi0CL4Ec2i1nrGrUI3KO11r6Fbxskqmq3JK2NH7aW4dcccBuUujAP0ERl5w==} engines: {node: '>=4'} @@ -3007,12 +3138,10 @@ packages: /object-inspect/1.11.0: resolution: {integrity: sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==} - dev: true /object-keys/1.1.1: resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} engines: {node: '>= 0.4'} - dev: true /object.assign/4.1.2: resolution: {integrity: sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==} @@ -3022,7 +3151,15 @@ packages: define-properties: 1.1.3 has-symbols: 1.0.2 object-keys: 1.1.1 - dev: true + + /object.getownpropertydescriptors/2.1.3: + resolution: {integrity: sha512-VdDoCwvJI4QdC6ndjpqFmoL3/+HxffFBbcJzKi5hwLLqqx3mdbedRpfZDdK0SrOSauj8X4GzBvnDZl4vTN7dOw==} + engines: {node: '>= 0.8'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + es-abstract: 1.19.1 + dev: false /object.values/1.1.4: resolution: {integrity: sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==} @@ -3083,6 +3220,11 @@ packages: resolution: {integrity: sha1-Tod/vtqaLsgAD7wLyuOWRe6L+Nw=} dev: true + /p-finally/1.0.0: + resolution: {integrity: sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=} + engines: {node: '>=4'} + dev: false + /p-limit/1.3.0: resolution: {integrity: sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==} engines: {node: '>=4'} @@ -3111,6 +3253,11 @@ packages: p-limit: 2.3.0 dev: true + /p-map/2.1.0: + resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==} + engines: {node: '>=6'} + dev: false + /p-map/3.0.0: resolution: {integrity: sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==} engines: {node: '>=8'} @@ -3118,6 +3265,13 @@ packages: aggregate-error: 3.1.0 dev: true + /p-timeout/3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + dependencies: + p-finally: 1.0.0 + dev: false + /p-try/1.0.0: resolution: {integrity: sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=} engines: {node: '>=4'} @@ -3265,6 +3419,15 @@ packages: engines: {node: '>=0.4.0'} dev: true + /promise.prototype.finally/3.1.3: + resolution: {integrity: sha512-EXRF3fC9/0gz4qkt/f5EP5iW4kj9oFpBICNpCNOb/52+8nlHIX07FPLbi/q4qYBQ1xZqivMzTpNQSnArVASolQ==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + es-abstract: 1.19.1 + dev: false + /prop-types/15.7.2: resolution: {integrity: sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==} dependencies: @@ -3574,7 +3737,6 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 - dev: true /send/0.17.1: resolution: {integrity: sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==} @@ -3631,7 +3793,6 @@ packages: call-bind: 1.0.2 get-intrinsic: 1.1.1 object-inspect: 1.11.0 - dev: true /siginfo/2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -3666,7 +3827,6 @@ packages: dependencies: buffer-from: 1.1.2 source-map: 0.6.1 - dev: true /source-map/0.5.7: resolution: {integrity: sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=} @@ -3676,7 +3836,6 @@ packages: /source-map/0.6.1: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} - dev: true /sparse-bitfield/3.0.3: resolution: {integrity: sha1-/0rm5oZWBWuks+eSqzM004JzyhE=} @@ -3754,6 +3913,10 @@ packages: resolution: {integrity: sha1-Gsig2Ug4SNFpXkGLbQMaPDzmjjs=} dev: true + /standard-as-callback/2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false + /statuses/1.5.0: resolution: {integrity: sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=} engines: {node: '>= 0.6'} @@ -3795,14 +3958,12 @@ packages: dependencies: call-bind: 1.0.2 define-properties: 1.1.3 - dev: true /string.prototype.trimstart/1.0.4: resolution: {integrity: sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==} dependencies: call-bind: 1.0.2 define-properties: 1.1.3 - dev: true /string_decoder/0.10.31: resolution: {integrity: sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=} @@ -4091,7 +4252,7 @@ packages: source-map-support: 0.5.19 typescript: 4.3.4 yn: 3.1.1 - dev: true + dev: false /tsconfig-paths/3.10.1: resolution: {integrity: sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==} @@ -4179,7 +4340,6 @@ packages: has-bigints: 1.0.1 has-symbols: 1.0.2 which-boxed-primitive: 1.0.2 - dev: true /unicode-length/2.0.2: resolution: {integrity: sha512-Ph/j1VbS3/r77nhoY2WU0GWGjVYOHL3xpKp0y/Eq2e5r0mT/6b649vm7KFO6RdAdrZkYLdxphYVgvODxPB+Ebg==} @@ -4202,6 +4362,16 @@ packages: /util-deprecate/1.0.2: resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=} + /util.promisify/1.1.1: + resolution: {integrity: sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==} + dependencies: + call-bind: 1.0.2 + define-properties: 1.1.3 + for-each: 0.3.3 + has-symbols: 1.0.2 + object.getownpropertydescriptors: 2.1.3 + dev: false + /utils-merge/1.0.1: resolution: {integrity: sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=} engines: {node: '>= 0.4.0'} @@ -4213,6 +4383,11 @@ packages: hasBin: true dev: true + /uuid/8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + dev: false + /v8-compile-cache/2.3.0: resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==} dev: true @@ -4246,7 +4421,6 @@ packages: is-number-object: 1.0.6 is-string: 1.0.7 is-symbol: 1.0.4 - dev: true /which-module/2.0.0: resolution: {integrity: sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=} @@ -4337,7 +4511,6 @@ packages: /yallist/4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: true /yaml/1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} @@ -4376,4 +4549,4 @@ packages: /yn/3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} - dev: true + dev: false diff --git a/server/src/lib/jobs/job-runner.ts b/server/src/lib/jobs/job-runner.ts new file mode 100644 index 000000000..dc1f586bb --- /dev/null +++ b/server/src/lib/jobs/job-runner.ts @@ -0,0 +1,74 @@ +import Queue from "bull"; +import CreateLogCtx from "lib/logger/logger"; +import { DedupeArr } from "utils/misc"; +import { DeoprhanScores } from "./jobs/deorphan-scores"; +import { UGSSnapshot } from "./jobs/ugs-snapshot"; + +interface Job { + name: string; + cronFormat: string; + run: (job: Queue.Job) => Promise; +} + +const jobs: Job[] = [ + { + name: "Snapshot User Game Stats", + cronFormat: "0 0 * * *", + run: UGSSnapshot, + }, + { + name: "De-Orphan Scores", + // We run an hour after snapshotting UGS + // just to spread load out a bit. + cronFormat: "1 * * * * *", + run: DeoprhanScores, + }, +]; + +const logger = CreateLogCtx("JOB_RUNNER"); + +/** + * Initalises a tachi-server job runner. + * This runs the list of jobs defined in jobConfig.jobs. + */ +export function InitialiseJobRunner() { + logger.info(`Booting up Job Runner.`); + + const names = jobs.map((e) => e.name); + + if (DedupeArr(names).length !== names.length) { + logger.crit(`Jobs has duplicate name fields, refusing to run.`); + process.exit(1); + } + + const JobQueue = new Queue("Job Runner"); + + const jobNameMap = new Map(); + + for (const job of jobs) { + JobQueue.add({ jobName: job.name }, { repeat: { cron: job.cronFormat } }); + jobNameMap.set(job.name, job); + } + + JobQueue.process(async (j) => { + const name = j.data.jobName; + logger.info(`Running job ${name}.`); + + const jobInfo = jobNameMap.get(name); + + if (!jobInfo) { + logger.severe(`Unknown job name ${name}, couldn't find a run function?`); + return false; + } + + await jobInfo.run(j); + + return true; + }); + + logger.info(`Initialised ${jobs.length} jobs.`); +} + +if (require.main === module) { + InitialiseJobRunner(); +} diff --git a/server/src/lib/jobs/jobconf.json5 b/server/src/lib/jobs/jobconf.json5 deleted file mode 100644 index 58d7374fb..000000000 --- a/server/src/lib/jobs/jobconf.json5 +++ /dev/null @@ -1,11 +0,0 @@ -{ - // this format is just an example - @ereti will be designing the jobs system and will *definitely* want something - // better. This only exists as a rough example. - prefix: "tsnode -r tsconfig-paths/register", - jobs: [{ - path: "ugs-snapshot.ts", - name: "Snapshot User Game Stats", - // especially this part - i don't expect any machine to parse this text. - interval: "daily at midnight" - }] -} \ No newline at end of file diff --git a/server/src/lib/jobs/jobs/deorphan-scores.ts b/server/src/lib/jobs/jobs/deorphan-scores.ts new file mode 100644 index 000000000..5facc3993 --- /dev/null +++ b/server/src/lib/jobs/jobs/deorphan-scores.ts @@ -0,0 +1,43 @@ +// Attempt to deoprhan lost scores. + +import Bull from "bull"; +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans"; + +const logger = CreateLogCtx(__dirname); + +export async function DeoprhanScores(job: Bull.Job) { + const orphans = await db["orphan-scores"].find({}); + + // ScoreIDs are essentially userID dependent, so this is fine. + const blacklist = (await db["score-blacklist"].find({})).map((e) => e.scoreID); + + logger.info(`Found ${orphans.length} orphans.`); + + let done = 0; + let failed = 0; + let success = 0; + let removed = 0; + + await Promise.all( + orphans.map((or) => + ReprocessOrphan(or, blacklist, logger).then((r) => { + done++; + if (r === null) { + removed++; + } else if (r === false) { + failed++; + } else { + success++; + } + + job.progress((100 * done) / orphans.length); + }) + ) + ); + + logger.info(`Finished attempting deorphaning.`); + + logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`); +} diff --git a/server/src/lib/jobs/ugs-snapshot.ts b/server/src/lib/jobs/jobs/ugs-snapshot.ts similarity index 87% rename from server/src/lib/jobs/ugs-snapshot.ts rename to server/src/lib/jobs/jobs/ugs-snapshot.ts index 7f5628c4b..224381d69 100644 --- a/server/src/lib/jobs/ugs-snapshot.ts +++ b/server/src/lib/jobs/jobs/ugs-snapshot.ts @@ -14,16 +14,19 @@ let batchWrite: UserGameStatsSnapshot[] = []; // This code is intentionally *very* robust, and handles a lot of unanticipated failures // because if it breaks, we brick the database. -(async () => { +export async function UGSSnapshot() { const timeStart = process.hrtime.bigint(); const alreadyExists = await db["game-stats-snapshots"].findOne({ timestamp: currentTime }); if (alreadyExists) { - logger.severe( - `FATAL IN UGS-SNAPSHOT - There already exists snapshots at this time. Has this script been ran twice?` + logger.warn( + `There already exists snapshots at this time. Has this script been ran twice on one day? Ignoring request.` + ); + + throw new Error( + `There already exists snapshots at this time. Has this script been ran twice on one day? Ignoring request.` ); - process.exit(1); } logger.info(`Snapshotting UserGameStats.`); @@ -69,8 +72,6 @@ let batchWrite: UserGameStatsSnapshot[] = []; currentTime ).toString()}. Took ${GetMilisecondsSince(timeStart)} ms.` ); - - process.exit(0); } catch (err) { // if we panic, we need to revert whatever we did. logger.severe(`FATAL IN UGS-SNAPSHOT - Possibly failed midway through snapshotting.`, { @@ -83,6 +84,6 @@ let batchWrite: UserGameStatsSnapshot[] = []; logger.info(`Removed.`); - process.exit(1); + throw err; } -})(); +} diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index ffbec3ab4..2032b85fd 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -77,6 +77,7 @@ export interface TachiConfig { OUR_URL: string; LOGGER_DISCORD_WEBHOOK?: string; DISCORD_WHO_TO_TAG?: string[]; + INVOKE_JOB_RUNNER?: boolean; } const isValidOauth2 = p.optional({ @@ -118,6 +119,7 @@ const err = p(config, { OUR_URL: "string", LOGGER_DISCORD_WEBHOOK: "*string", DISCORD_WHO_TO_TAG: p.optional(["string"]), + INVOKE_JOB_RUNNER: "*boolean", }); if (err) { diff --git a/server/src/main.ts b/server/src/main.ts index bb6b98668..04e803f6c 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -3,8 +3,9 @@ import server from "server/server"; import { ServerTypeInfo, ServerConfig } from "lib/setup/config"; import https from "https"; import fs from "fs"; - import { FormatVersion } from "./lib/constants/version"; +import { spawn } from "child_process"; +import path from "path"; const logger = CreateLogCtx(__filename); @@ -26,3 +27,28 @@ if (ServerConfig.ENABLE_SERVER_HTTPS) { server.listen(ServerConfig.PORT); logger.info(`HTTP Listening on port ${ServerConfig.PORT}`); } + +if (ServerConfig.INVOKE_JOB_RUNNER || process.env.INVOKE_JOB_RUNNER) { + logger.info(`Spawning a tachi-server job runner.`); + + // Spawn as a separate process to avoid hogging the main thread. + const jobProcess = spawn( + "ts-node", + [ + // Note: Can't use -r tsconfig-paths/register here + // because that is rejected by some library called + // arg. + // I'm not sure why. + "--require=tsconfig-paths/register", + path.join(__dirname, "../src/lib/jobs/job-runner.ts"), + ], + { + stdio: "inherit", + } + ); + + jobProcess.on("error", (err) => { + logger.crit(`Failed to spawn job runner. Terminating process.`, { err }); + process.exit(1); + }); +}