diff --git a/server/package.json b/server/package.json index 1f2733109..a1f724c34 100644 --- a/server/package.json +++ b/server/package.json @@ -75,7 +75,8 @@ "safe-json-stringify": "1.2.0", "tachi-common": "0.2.18", "typescript": "4.3.4", - "winston": "3.3.3" + "winston": "3.3.3", + "winston-transport": "^4.4.0" }, "nyc": { "reporter": [ diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index b13177bc4..b5cbd4224 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -61,6 +61,7 @@ specifiers: typescript: 4.3.4 why-is-node-running: 2.2.0 winston: 3.3.3 + winston-transport: ^4.4.0 dependencies: bcryptjs: 2.4.3 @@ -91,6 +92,7 @@ dependencies: tachi-common: 0.2.18 typescript: 4.3.4 winston: 3.3.3 + winston-transport: 4.4.0 devDependencies: '@types/bcrypt': 5.0.0 diff --git a/server/src/lib/logger/colours.ts b/server/src/lib/logger/colours.ts new file mode 100644 index 000000000..ac4c84ef4 --- /dev/null +++ b/server/src/lib/logger/colours.ts @@ -0,0 +1,20 @@ +const Colours = { + crit: ["bgRed", "black"], + severe: ["bgBrightRed", "black"], + error: ["red"], + warn: ["yellow"], + info: ["blue"], + verbose: ["cyan"], + debug: ["white"], +}; + +// Discord ONLY accepts decimal colours, so. +// lets run with that. +export const DiscordColours = { + crit: 16711680, // #ff0000 + severe: 16719904, // #ff3333 + error: 11730944, // #cc0000 + warn: 16703232, // #ffcc00 + // other ones dont matter +}; +export default Colours; diff --git a/server/src/lib/logger/discord-transport.ts b/server/src/lib/logger/discord-transport.ts new file mode 100644 index 000000000..e60e146d7 --- /dev/null +++ b/server/src/lib/logger/discord-transport.ts @@ -0,0 +1,128 @@ +import { ServerConfig } from "lib/setup/config"; +import SafeJSONStringify from "safe-json-stringify"; +import fetch from "utils/fetch"; +import Transport, { TransportStreamOptions } from "winston-transport"; +import { DiscordColours } from "./colours"; + +interface DiscordTransportOptions extends TransportStreamOptions { + /** Webhook obtained from Discord */ + webhook: string; +} + +/** + * Creates a discord winston transport. This is a slightly adapted version + * of sidhantpanda's winston-discord-transport, modified for our use case. + * + * @param webhookUrl The webhook to connect to. + * @returns A winston tranport. + */ +export default class DiscordTransport extends Transport { + private webhookUrl = ""; + + /** Initialization promise resolved after retrieving discord id and token */ + private initalised: Promise; + + constructor(opts: DiscordTransportOptions) { + super(opts); + + this.initalised = fetch(opts.webhook) + .then((r) => { + if (!r.ok) { + throw new Error(`Couldn't connect to discord transport. ${r.status}.`); + } + + return r.json(); + }) + .then((content) => { + this.webhookUrl = `https://discordapp.com/api/v6/webhooks/${content.id}/${content.token}`; + }); + } + + log(info: any, cb: () => void) { + if (info.noDiscord !== false) { + try { + setImmediate(() => { + this.initalised.then(() => { + this.SendToDiscord(this.webhookUrl, info); + }); + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`Failed to send content to discord transport`, err); + } + } + + // don't bother waiting around. + cb(); + } + + private GetWhoToTag() { + return ServerConfig.DISCORD_WHO_TO_TAG + ? ServerConfig.DISCORD_WHO_TO_TAG.map((e) => `<@${e}>`).join(" ") + : "Nobody configured to tag, but this is bad, get someone!"; + } + + private async SendToDiscord(webhookLocation: string, info: any) { + const postBody = { + content: "", + embeds: [ + { + description: `[${info.level}] ${info.message}`, + // it's Colour!! + color: DiscordColours[info.level as keyof typeof DiscordColours], + timestamp: new Date().toISOString(), + }, + ], + }; + + if (info.meta) { + postBody.content = `\`\`\`${SafeJSONStringify(info.meta, null, 4)}\`\`\``; + } + + // These two levels are bad, and require near-immediate attention. + + if (info.level === "severe") { + postBody.content = `SEVERE ERROR: ${this.GetWhoToTag()}\n${postBody.content}`; + } + + if (info.level === "crit") { + postBody.content = `CRITICAL ERROR: ${this.GetWhoToTag()}\n${postBody.content}`; + } + + await this.POSTData(postBody); + } + + private async POSTData(postBody: unknown, scaleRetryDebounce = 2) { + const res = await fetch(this.webhookUrl, { + method: "POST", + body: JSON.stringify(postBody), + headers: { + "Content-Type": "application/json", + }, + }); + + if (res.status === 429) { + // being rate limited. + const content = await res.json(); + + // Try and retry when they say so. The issue is that our logging + // is very async, and this is a billion race conditions. + // It's possible that messages here could all get stuck in an + // awful loop, so we have a tuning off parameter. + // The scaleRetryDebouncer will get squared every call, + // so the initial request takes 2 * (generally 1milisecond), + // then following requests will take even longer... + // It's possible this might blow up in our face. + // We'll have to see. - zkldi 2021/09/17 + + if (content.retry_after) { + setTimeout(() => { + this.POSTData(postBody, scaleRetryDebounce ** 2); + }, content.retry_after * scaleRetryDebounce); + } + } else if (!res.ok) { + // eslint-disable-next-line no-console + console.error(`Failed to send to discord ${res.status} ${await res.text()}`); + } + } +} diff --git a/server/src/lib/logger/logger.test.ts b/server/src/lib/logger/logger.test.ts index 58c16ff5f..aebb17a6a 100644 --- a/server/src/lib/logger/logger.test.ts +++ b/server/src/lib/logger/logger.test.ts @@ -32,6 +32,7 @@ t.test("#GetLogLevel", (t) => { t.equal(GetLogLevel(), "crit"); ChangeRootLogLevel(LOG_LEVEL); + t.equal(GetLogLevel(), LOG_LEVEL); t.end(); }); diff --git a/server/src/lib/logger/logger.ts b/server/src/lib/logger/logger.ts index bb89fbf12..eb73f86c1 100644 --- a/server/src/lib/logger/logger.ts +++ b/server/src/lib/logger/logger.ts @@ -2,13 +2,12 @@ import winston, { format, transports, Logger, LeveledLogMethod } from "winston"; import { EscapeStringRegexp } from "utils/misc"; import SafeJSONStringify from "safe-json-stringify"; import { ServerConfig } from "lib/setup/config"; +import CreateDiscordWinstonTransport from "./discord-transport"; export type KtLogger = Logger & { severe: LeveledLogMethod }; const level = process.env.LOG_LEVEL ?? ServerConfig.LOG_LEVEL; -const IN_TESTING = process.env.NODE_ENV === "test"; - const formatExcessProperties = (meta: Record) => { let i = 0; for (const key in meta) { @@ -98,10 +97,7 @@ const consoleFormatRoute = format.combine( }) ); -const tports: ( - | winston.transports.ConsoleTransportInstance - | winston.transports.FileTransportInstance -)[] = [ +const tports: winston.transport[] = [ new transports.File({ filename: "logs/tachi-error.log", level: "error", @@ -118,6 +114,15 @@ if (!ServerConfig.NO_CONSOLE) { ); } +if (ServerConfig.LOGGER_DISCORD_WEBHOOK) { + tports.push( + new CreateDiscordWinstonTransport({ + webhook: ServerConfig.LOGGER_DISCORD_WEBHOOK, + level: "warn", + }) + ); +} + export const rootLogger = winston.createLogger({ levels: { crit: 0, // entire process termination is necessary @@ -133,6 +138,10 @@ export const rootLogger = winston.createLogger({ transports: tports, }); +if (ServerConfig.LOGGER_DISCORD_WEBHOOK) { + rootLogger.info(`Discord logging enabled.`); +} + function CreateLogCtx(filename: string, lg = rootLogger): KtLogger { const replacedFilename = filename.replace( new RegExp(`^${EscapeStringRegexp(process.cwd())}/((js|src)/)?`, "u"), diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index 7491a2060..19c41e06c 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -75,6 +75,8 @@ export interface TachiConfig { USC_QUEUE_SIZE: integer; BEATORAJA_QUEUE_SIZE: integer; OUR_URL: string; + LOGGER_DISCORD_WEBHOOK?: string; + DISCORD_WHO_TO_TAG?: string[]; } const isValidOauth2 = p.optional({ @@ -113,6 +115,8 @@ const err = p(config, { USC_QUEUE_SIZE: p.optional(p.gteInt(2)), BEATORAJA_QUEUE_SIZE: p.optional(p.gteInt(2)), OUR_URL: "string", + LOGGER_DISCORD_WEBHOOK: "*string", + DISCORD_WHO_TO_TAG: p.optional(["string"]), }); if (err) {