From a16a3035ccd4ebd743714548ed7f82b18eebd343 Mon Sep 17 00:00:00 2001 From: zkldi Date: Wed, 10 Apr 2024 02:02:31 +0100 Subject: [PATCH] feat: redo logging (#1032) file/whatever logging options are silly --- pnpm-lock.yaml | 256 ++++++++++++++++++-- server/example/conf.json5 | 2 - server/package.json | 3 +- server/src/lib/logger/colours.ts | 29 --- server/src/lib/logger/discord-transport.ts | 269 --------------------- server/src/lib/logger/logger.ts | 84 ++----- server/src/lib/setup/config.ts | 28 +-- server/test.conf.json5 | 4 +- 8 files changed, 261 insertions(+), 414 deletions(-) delete mode 100644 server/src/lib/logger/colours.ts delete mode 100644 server/src/lib/logger/discord-transport.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d370b3077..851d10f6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -648,9 +648,6 @@ importers: semver: specifier: ^7.3.7 version: 7.3.7 - seq-logging: - specifier: 1.1.1 - version: 1.1.1 tachi-common: specifier: workspace:../common version: link:../common @@ -663,9 +660,9 @@ importers: winston: specifier: 3.5.1 version: 3.5.1 - winston-daily-rotate-file: - specifier: 4.6.0 - version: 4.6.0(winston@3.5.1) + winston-loki: + specifier: ^6.1.0 + version: 6.1.0 winston-transport: specifier: 4.5.0 version: 4.5.0 @@ -2590,6 +2587,123 @@ packages: dev: false optional: true + /@napi-rs/snappy-android-arm-eabi@7.2.2: + resolution: {integrity: sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-android-arm64@7.2.2: + resolution: {integrity: sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-darwin-arm64@7.2.2: + resolution: {integrity: sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-darwin-x64@7.2.2: + resolution: {integrity: sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-freebsd-x64@7.2.2: + resolution: {integrity: sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-arm-gnueabihf@7.2.2: + resolution: {integrity: sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-arm64-gnu@7.2.2: + resolution: {integrity: sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-arm64-musl@7.2.2: + resolution: {integrity: sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-x64-gnu@7.2.2: + resolution: {integrity: sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-linux-x64-musl@7.2.2: + resolution: {integrity: sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-win32-arm64-msvc@7.2.2: + resolution: {integrity: sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-win32-ia32-msvc@7.2.2: + resolution: {integrity: sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@napi-rs/snappy-win32-x64-msvc@7.2.2: + resolution: {integrity: sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@nivo/annotations@0.79.1(@nivo/core@0.79.0)(prop-types@15.6.0)(react-dom@17.0.2)(react@17.0.2): resolution: {integrity: sha512-lYso9Luu0maSDtIufwvyVt2+Wue7R9Fh3CIjuRDmNR72UjAgAVEcCar27Fy865UXGsj2hRJZ7KY/1s6kT3gu/w==} peerDependencies: @@ -3017,6 +3131,49 @@ packages: resolution: {integrity: sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==} dev: false + /@protobufjs/aspromise@1.1.2: + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + dev: false + + /@protobufjs/base64@1.1.2: + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + dev: false + + /@protobufjs/codegen@2.0.4: + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + dev: false + + /@protobufjs/eventemitter@1.1.0: + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + dev: false + + /@protobufjs/fetch@1.1.0: + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + dev: false + + /@protobufjs/float@1.0.2: + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + dev: false + + /@protobufjs/inquire@1.1.0: + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + dev: false + + /@protobufjs/path@1.1.2: + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + dev: false + + /@protobufjs/pool@1.1.0: + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + dev: false + + /@protobufjs/utf8@1.1.0: + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + dev: false + /@react-aria/ssr@3.7.0(react@17.0.2): resolution: {integrity: sha512-bfufjg4ESE5giN+Fxj1XIzS5f/YIhqcGc+Ve+vUUKU8xZ8t/Xtjlv8F3kjqDBQdk//n3mluFY7xG1wQVB9rMLQ==} peerDependencies: @@ -3864,6 +4021,11 @@ packages: engines: {node: '>=0.8'} dev: true + /async-exit-hook@2.0.1: + resolution: {integrity: sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw==} + engines: {node: '>=0.12.0'} + dev: false + /async-hook-domain@2.0.4: resolution: {integrity: sha512-14LjCmlK1PK8eDtTezR6WX8TMaYNIzBIsd2D1sGoGjgx0BuNMMoSdk7i/drlbtamy0AWv9yv2tkB+ASdmeqFIw==} engines: {node: '>=10'} @@ -4139,6 +4301,12 @@ packages: resolution: {integrity: sha512-gvW7InbIyF8AicrqWoptdW08pUxuhq8BEgowNajy9RhiE86fmGAGl+bLKo6oB8QP0CkqHLowfN0oJdKC/J6LbA==} dev: false + /btoa@1.2.1: + resolution: {integrity: sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==} + engines: {node: '>= 0.4.0'} + hasBin: true + dev: false + /buffer-equal-constant-time@1.0.1: resolution: {integrity: sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=} dev: false @@ -6031,12 +6199,6 @@ packages: flat-cache: 3.0.4 dev: true - /file-stream-rotator@0.6.1: - resolution: {integrity: sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==} - dependencies: - moment: 2.29.4 - dev: false - /file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} requiresBuild: true @@ -7405,6 +7567,10 @@ packages: triple-beam: 1.3.0 dev: false + /long@5.2.3: + resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} + dev: false + /loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -8292,11 +8458,6 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} - /object-hash@2.2.0: - resolution: {integrity: sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==} - engines: {node: '>= 6'} - dev: false - /object-inspect@1.12.3: resolution: {integrity: sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==} @@ -8645,6 +8806,25 @@ packages: resolution: {integrity: sha512-hrzC564QIl0r0vy4l6MvRLhafmUowhO/O3KgVSoXIbbA2Sz4j8HGpJc6T2cubRVwMwpdiG/vKGfhT4IixmKN9w==} dev: false + /protobufjs@7.2.6: + resolution: {integrity: sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==} + engines: {node: '>=12.0.0'} + requiresBuild: true + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 18.11.18 + long: 5.2.3 + dev: false + /proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -9570,6 +9750,27 @@ packages: engines: {node: '>=8'} dev: true + /snappy@7.2.2: + resolution: {integrity: sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA==} + engines: {node: '>= 10'} + requiresBuild: true + optionalDependencies: + '@napi-rs/snappy-android-arm-eabi': 7.2.2 + '@napi-rs/snappy-android-arm64': 7.2.2 + '@napi-rs/snappy-darwin-arm64': 7.2.2 + '@napi-rs/snappy-darwin-x64': 7.2.2 + '@napi-rs/snappy-freebsd-x64': 7.2.2 + '@napi-rs/snappy-linux-arm-gnueabihf': 7.2.2 + '@napi-rs/snappy-linux-arm64-gnu': 7.2.2 + '@napi-rs/snappy-linux-arm64-musl': 7.2.2 + '@napi-rs/snappy-linux-x64-gnu': 7.2.2 + '@napi-rs/snappy-linux-x64-musl': 7.2.2 + '@napi-rs/snappy-win32-arm64-msvc': 7.2.2 + '@napi-rs/snappy-win32-ia32-msvc': 7.2.2 + '@napi-rs/snappy-win32-x64-msvc': 7.2.2 + dev: false + optional: true + /source-map-js@1.0.2: resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} engines: {node: '>=0.10.0'} @@ -10470,6 +10671,10 @@ packages: resolution: {integrity: sha512-H6dnQ/yPAAVzMQRvEvyz01hhfQL5qRWSEt7BX8t9DqnPw9BjMb64fjIRq76Uvf1hkHp+mTZvEVJ5guXOT0Xqaw==} dev: false + /url-polyfill@1.1.12: + resolution: {integrity: sha512-mYFmBHCapZjtcNHW0MDq9967t+z4Dmg5CJ0KqysK3+ZbyoNOWQHksGCTWwDhxGXllkWlOc10Xfko6v4a3ucM6A==} + dev: false + /use-isomorphic-layout-effect@1.1.2(@types/react@17.0.13)(react@17.0.2): resolution: {integrity: sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==} peerDependencies: @@ -10687,17 +10892,16 @@ packages: hasBin: true dev: false - /winston-daily-rotate-file@4.6.0(winston@3.5.1): - resolution: {integrity: sha512-mvpFb1LYmTvh/vz0dIS/aDCwEm0cvDa8D/tE4xWwdUYolD250wf+n0y1PZ2xr7fbvTLF/PQYqXtFIFrmog03Ow==} - engines: {node: '>=12'} - peerDependencies: - winston: ^3 + /winston-loki@6.1.0: + resolution: {integrity: sha512-XX0Wni8eFEktUeO/YmexOw8OXLjmHjW3yuF8dr9/O+zQsG+6/b22g/ZKh2c8rJjOOQIK0h5BdVNPOtivGdV/2Q==} dependencies: - file-stream-rotator: 0.6.1 - object-hash: 2.2.0 - triple-beam: 1.3.0 - winston: 3.5.1 + async-exit-hook: 2.0.1 + btoa: 1.2.1 + protobufjs: 7.2.6 + url-polyfill: 1.1.12 winston-transport: 4.5.0 + optionalDependencies: + snappy: 7.2.2 dev: false /winston-transport@4.5.0: diff --git a/server/example/conf.json5 b/server/example/conf.json5 index e810cee1d..31e781b33 100644 --- a/server/example/conf.json5 +++ b/server/example/conf.json5 @@ -81,8 +81,6 @@ ], }, LOGGER_CONFIG: { - FILE: false, - CONSOLE: true, LOG_LEVEL: "info", }, SEEDS_CONFIG: { diff --git a/server/package.json b/server/package.json index 72d76cc41..35c6b4ec9 100644 --- a/server/package.json +++ b/server/package.json @@ -96,12 +96,11 @@ "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", "semver": "^7.3.7", - "seq-logging": "1.1.1", "tachi-common": "workspace:../common", "ts-node": "10.9.1", "typescript": "4.9.4", "winston": "3.5.1", - "winston-daily-rotate-file": "4.6.0", + "winston-loki": "^6.1.0", "winston-transport": "4.5.0" }, "nyc": { diff --git a/server/src/lib/logger/colours.ts b/server/src/lib/logger/colours.ts deleted file mode 100644 index 26da457ba..000000000 --- a/server/src/lib/logger/colours.ts +++ /dev/null @@ -1,29 +0,0 @@ -const Colours = { - crit: ["bgRed", "black"], - severe: ["bgBrightRed", "black"], - error: ["red"], - warn: ["yellow"], - info: ["blue"], - verbose: ["cyan"], - debug: ["white"], -}; - -// Discord ONLY accepts decimal colours, so. -// lets run with that. -export const DiscordColours = { - // #ff0000 - crit: 16711680, - - // #ff3333 - severe: 16719904, - - // #cc0000 - error: 11730944, - - // #ffcc00 - warn: 16703232, - - // other log colours don't matter -- they can all default to gray -}; - -export default Colours; diff --git a/server/src/lib/logger/discord-transport.ts b/server/src/lib/logger/discord-transport.ts deleted file mode 100644 index c68a68253..000000000 --- a/server/src/lib/logger/discord-transport.ts +++ /dev/null @@ -1,269 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { DiscordColours } from "./colours"; -import { ONE_HOUR, ONE_MINUTE } from "lib/constants/time"; -import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; -import SafeJSONStringify from "safe-json-stringify"; -import fetch from "utils/fetch"; -import { IsRecord, Sleep } from "utils/misc"; -import Transport from "winston-transport"; -import type { Response } from "node-fetch"; -import type { TransportStreamOptions } from "winston-transport"; - -interface DiscordTransportOptions extends TransportStreamOptions { - webhook: string; -} - -interface LogLevelCountState { - warn: Array; - error: Array; -} - -interface LoggerInfo { - level: "crit" | "debug" | "info" | "severe" | "verbose" | "warn"; - message: string; - noDiscord?: boolean; - meta?: Record; -} - -/** - * Creates a discord winston transport. This is a slightly adapted version - * of sidhantpanda's winston-discord-transport, modified for our use case. - * - * @param webhookUrl The webhook to connect to. - * @returns A winston tranport. - */ -export default class DiscordTransport extends Transport { - private webhookUrl = ""; - - /** Initialization promise resolved after retrieving discord id and token */ - private readonly initalised: Promise; - - private bucketData: LogLevelCountState = { - warn: [], - error: [], - }; - - private isBucketing = false; - private bucketStart: Date | null = null; - - constructor(opts: DiscordTransportOptions) { - super(opts); - - this.resetBucketData(); - - this.initalised = this.connect(opts); - } - - private async connect(opts: DiscordTransportOptions) { - let connected = false; - - let outerData: Response | undefined; - - while (!connected) { - // eslint-disable-next-line no-await-in-loop - const data = await fetch(opts.webhook); - - if (!data.ok) { - if (data.status === 429) { - // Any amount of miliseconds between 6 minutes and 60 minutes. - const time = Math.max(Math.random(), 0.1) * ONE_HOUR; - - // eslint-disable-next-line no-console - console.error( - `Couldn't connect to discord transport (${ - data.status - }). Going to try again in ${(time / ONE_MINUTE).toFixed(2)} minutes.` - ); - - // eslint-disable-next-line no-await-in-loop - await Sleep(time); - } else { - throw new Error( - `Couldn't connect to discord transport (${data.status}). This is not retryable.` - ); - } - } else { - connected = true; - outerData = data; - } - } - - if (!outerData) { - throw Error( - `Unreachable state. Connected with no outerData. This is not even remotely possible, so good luck figuring this one out.` - ); - } - - const content = (await outerData.json()) as unknown; - - if (!IsRecord(content)) { - throw new Error( - `Discord returned non-object data as JSON? Can't get id or token ${content}.` - ); - } - - this.webhookUrl = `https://discordapp.com/api/v6/webhooks/${content.id}/${content.token}`; - } - - private resetBucketData() { - this.bucketData = { - warn: [], - error: [], - }; - } - - log(info: LoggerInfo, cb: () => void) { - if (info.noDiscord !== false) { - try { - setImmediate(() => { - void this.initalised.then(() => { - void this.handleSendToDiscord(info); - }); - }); - } catch (err) { - // eslint-disable-next-line no-console - console.error(`Failed to send content to discord transport`, err); - } - } - - // don't bother waiting around. - cb(); - } - - private handleSendToDiscord(info: LoggerInfo) { - if (info.level === "crit" || info.level === "severe") { - return this.sendLogDirectlyToDiscord(info); - } - - if (!["warn", "error", "severe"].includes(info.level)) { - // Don't need to send notifications about these. - return; - } - - this.bucketData[info.level as "error" | "warn"].push(info.message); - - if (!this.isBucketing) { - this.isBucketing = true; - this.bucketStart = new Date(); - setTimeout(() => { - this.sendBucketData(); - }, ONE_MINUTE); - } - } - - private getWhoToTag() { - return ServerConfig.LOGGER_CONFIG.DISCORD?.WHO_TO_TAG - ? ServerConfig.LOGGER_CONFIG.DISCORD.WHO_TO_TAG.map((e) => `<@${e}>`).join(" ") - : "Nobody configured to tag, but this is bad, get someone!"; - } - - private sendBucketData() { - let color = 0; - - const fields = []; - - for (const key of ["warn", "error"] as const) { - if (this.bucketData[key].length !== 0) { - color = DiscordColours[key]; - fields.push({ - // uppercase first char - name: `${key[0]!.toUpperCase()}${key.slice(1)}s`, - value: this.bucketData[key].length, - }); - } - } - - const logSnippet = [ - ...this.bucketData.error.map((e) => `[ERROR] ${e}`), - ...this.bucketData.warn.map((e) => `[WARN] ${e}`), - ].join("\n"); - - const postBody = { - content: ` - \`\`\`${logSnippet.length > 1500 ? `${logSnippet.slice(0, 1500 - 3)}...` : logSnippet}\`\`\``, - embeds: [ - { - title: `${TachiConfig.NAME} Log Summary${ - Environment.replicaIdentity - ? ` (Replica: ${Environment.replicaIdentity})` - : "" - }`, - fields, - description: `Log summary for ${this.bucketStart?.toISOString()} to ${new Date().toISOString()}.`, - color, - timestamp: new Date().toISOString(), - }, - ], - }; - - void this.POSTData(postBody); - this.resetBucketData(); - this.isBucketing = false; - this.bucketStart = null; - } - - private async sendLogDirectlyToDiscord(info: LoggerInfo) { - const postBody = { - content: "", - embeds: [ - { - description: `[${info.level}] ${info.message}`, - - // it's Colour!! - color: DiscordColours[info.level as keyof typeof DiscordColours], - timestamp: new Date().toISOString(), - }, - ], - }; - - if (info.meta) { - postBody.content = `\`\`\`${SafeJSONStringify(info.meta, null, 4)}\`\`\``; - } - - // These two levels are bad, and require near-immediate attention. - - if (info.level === "severe") { - postBody.content = `SEVERE ERROR: ${this.getWhoToTag()}\n${postBody.content}`; - } - - if (info.level === "crit") { - postBody.content = `CRITICAL ERROR: ${this.getWhoToTag()}\n${postBody.content}`; - } - - await this.POSTData(postBody); - } - - private async POSTData(postBody: unknown, scaleRetryDebounce = 2) { - const res = await fetch(this.webhookUrl, { - method: "POST", - body: JSON.stringify(postBody), - headers: { - "Content-Type": "application/json", - }, - }); - - if (res.status === 429) { - // being rate limited. - const content = (await res.json()) as unknown; - - // Try and retry when they say so. The issue is that our logging - // is very async, and this is a billion race conditions. - // It's possible that messages here could all get stuck in an - // awful loop, so we have a tuning off parameter. - // The scaleRetryDebouncer will get squared every call, - // so the initial request takes 2 * (generally 1milliseconds), - // then following requests will take even longer... - // It's possible this might blow up in our face. - // We'll have to see. - zkldi 2021/09/17 - - if (IsRecord(content) && typeof content.retry_after === "number") { - setTimeout(() => { - void this.POSTData(postBody, scaleRetryDebounce ** 2); - }, content.retry_after * scaleRetryDebounce); - } - } else if (!res.ok) { - // eslint-disable-next-line no-console - console.error(`Failed to send to discord ${res.status} ${await res.text()}`); - } - } -} diff --git a/server/src/lib/logger/logger.ts b/server/src/lib/logger/logger.ts index 8cd210bcb..c7f80b206 100644 --- a/server/src/lib/logger/logger.ts +++ b/server/src/lib/logger/logger.ts @@ -1,12 +1,10 @@ -import DiscordWinstonTransport from "./discord-transport"; import { Transport as SeqTransport } from "@valuabletouch/winston-seq"; import { Environment, ServerConfig, TachiConfig } from "lib/setup/config"; import SafeJSONStringify from "safe-json-stringify"; import { EscapeStringRegexp } from "utils/misc"; import winston, { format, transports } from "winston"; -import type { SeqLogLevel } from "seq-logging"; +import LokiTransport from "winston-loki"; import type { LeveledLogMethod, Logger } from "winston"; -import "winston-daily-rotate-file"; export type KtLogger = Logger & { severe: LeveledLogMethod }; @@ -123,61 +121,27 @@ const consoleFormatRoute = format.combine( }) ); -const tports: Array = []; +const tports: Array = [ + new transports.Console({ + format: consoleFormatRoute, + }), +]; -if (ServerConfig.LOGGER_CONFIG.FILE) { - tports.push( - new transports.DailyRotateFile({ - filename: "logs/tachi-%DATE%.log", - datePattern: "YYYY-MM-DD-HH", - zippedArchive: true, - maxSize: "20m", - maxFiles: "14d", - createSymlink: true, - symlinkName: "tachi.log", - format: defaultFormatRoute, - }) - ); -} - -// If this is a job, force stdout writing no matter what. -if (ServerConfig.LOGGER_CONFIG.CONSOLE || process.env.IS_JOB || process.env.FORCE_CONSOLE_LOG) { - tports.push( - new transports.Console({ - format: consoleFormatRoute, - }) - ); -} - -if (ServerConfig.LOGGER_CONFIG.DISCORD) { - tports.push( - new DiscordWinstonTransport({ - webhook: ServerConfig.LOGGER_CONFIG.DISCORD.WEBHOOK_URL, - level: "warn", - }) - ); -} - -if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) { - // Turns winston log levels into seq format. - const levelMap: Record = { +if (ServerConfig.LOGGER_CONFIG.SEQ) { + const levelMap: Record = { crit: "Fatal", severe: "Error", error: "Error", warn: "Warning", info: "Information", - - // Note that Seq interprets these in reverse, - // however, it's easier to read this code if I just - // use the same levels, instead of the right ones. verbose: "Verbose", debug: "Debug", }; tports.push( new SeqTransport({ - apiKey: ServerConfig.LOGGER_CONFIG.SEQ_API_KEY, - serverUrl: Environment.seqUrl, + apiKey: ServerConfig.LOGGER_CONFIG.SEQ.API_KEY, + serverUrl: ServerConfig.LOGGER_CONFIG.SEQ.URL, onError: (err) => { // eslint-disable-next-line no-console console.error(`Failed to send seq message: ${err.message}.`); @@ -189,6 +153,15 @@ if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) { ); } +if (ServerConfig.LOGGER_CONFIG.LOKI) { + tports.push( + new LokiTransport({ + host: ServerConfig.LOGGER_CONFIG.LOKI.URL, + json: true, + }) + ); +} + export const rootLogger = winston.createLogger({ levels: { // entire process termination is necessary @@ -222,19 +195,6 @@ export const rootLogger = winston.createLogger({ }, }) as KtLogger; -if (!!ServerConfig.LOGGER_CONFIG.SEQ_API_KEY !== !!Environment.seqUrl) { - rootLogger.warn( - `Only one of SEQ_API_KEY (conf.json5) and SEQ_URL (Environment) were set. Not sending logs to Seq, as both must be provided.` - ); -} - -if (tports.length === 0) { - // eslint-disable-next-line no-console - console.warn( - "You have no transports set. Absolutely no logs will be saved. This is a terrible idea!" - ); -} - function CreateLogCtx(filename: string, lg = rootLogger): KtLogger { const replacedFilename = filename.replace( new RegExp(`^${EscapeStringRegexp(process.cwd())}/((js|src)/)?`, "u"), @@ -264,18 +224,18 @@ export function ChangeRootLogLevel( ) { rootLogger.info(`Changing log level to ${level}.`); - for (const tp of tports) { + for (const tp of rootLogger.transports) { tp.level = level; } } export function GetLogLevel() { return ( - tports.map((e) => e.level).find((e) => typeof e === "string") ?? + rootLogger.transports.map((e) => e.level).find((e) => typeof e === "string") ?? ServerConfig.LOGGER_CONFIG.LOG_LEVEL ); } -export const Transports = tports; +export const Transports = rootLogger.transports; export default CreateLogCtx; diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index 2909601c8..37da04fe0 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -115,12 +115,12 @@ export interface TachiServerConfig { TACHI_CONFIG: TachiServerCoreConfig; LOGGER_CONFIG: { LOG_LEVEL: "crit" | "debug" | "error" | "info" | "severe" | "verbose" | "warn"; - CONSOLE: boolean; - FILE: boolean; - SEQ_API_KEY: string | undefined; - DISCORD?: { - WEBHOOK_URL: string; - WHO_TO_TAG: Array; + SEQ?: { + API_KEY: string; + URL: string; + }; + LOKI?: { + URL: string; }; }; CDN_CONFIG: { @@ -220,13 +220,8 @@ const err = p(config, { LOG_LEVEL: p.optional( p.isIn("debug", "verbose", "info", "warn", "error", "severe", "crit") ), - CONSOLE: "*boolean", - FILE: "*boolean", SEQ_API_KEY: "*string", - DISCORD: p.optional({ - WEBHOOK_URL: "string", - WHO_TO_TAG: ["string"], - }), + LOKI_URL: "*string", }, CDN_CONFIG: { WEB_LOCATION: "string", @@ -300,14 +295,6 @@ if (!mongoUrl) { process.exit(1); } -const seqUrl = process.env.SEQ_URL ?? ""; - -if (!seqUrl && tachiServerConfig.LOGGER_CONFIG.SEQ_API_KEY) { - logger.warn( - `No SEQ_URL specified in environment, yet LOGGER_CONFIG.SEQ_API_KEY was defined. No logs will be sent to Seq!` - ); -} - const nodeEnv = process.env.NODE_ENV ?? ""; if (!nodeEnv) { @@ -339,6 +326,5 @@ export const Environment = { mongoUrl, nodeEnv: nodeEnv as "dev" | "production" | "staging" | "test", replicaIdentity, - seqUrl, commitHash: process.env.COMMIT_HASH, }; diff --git a/server/test.conf.json5 b/server/test.conf.json5 index 4a6c152c1..0d5e63b18 100644 --- a/server/test.conf.json5 +++ b/server/test.conf.json5 @@ -49,7 +49,7 @@ "pms", "maimai", "maimaidx", - "arcaea" + "arcaea", ], IMPORT_TYPES: [ "file/eamusement-iidx-csv", @@ -71,8 +71,6 @@ ], }, LOGGER_CONFIG: { - FILE: false, - CONSOLE: true, LOG_LEVEL: "info", }, }