Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-09-28 18:08:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix: defer discord transport submissions if we get 429d

Browse Source 
This was giving us brownout problems. When we tried to restart all of
our things, only some of them would connect to discord. The rest
would be 429d, and not boot.

This stopped us from firing up score workers, amusingly.
This commit is contained in:
zkldi
2022-08-24 09:25:25 +01:00
parent d3e6611a33
commit d7900638ba
1 changed files with 50 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/lib/logger/discord-transport.ts
+50 -14
View File
@@ -1,11 +1,12 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { DiscordColours } from "./colours";
import { ONE_MINUTE } from "lib/constants/time";
import { ONE_HOUR, ONE_MINUTE } from "lib/constants/time";
import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
import SafeJSONStringify from "safe-json-stringify";
import fetch from "utils/fetch";
import { IsRecord } from "utils/misc";
import { IsRecord, Sleep } from "utils/misc";
import Transport from "winston-transport";
import type { Response } from "node-fetch";
import type { TransportStreamOptions } from "winston-transport";
interface DiscordTransportOptions extends TransportStreamOptions {
@@ -50,23 +51,58 @@ export default class DiscordTransport extends Transport {
this.resetBucketData();
this.initalised = fetch(opts.webhook)
.then((r) => {
if (!r.ok) {
throw new Error(`Couldn't connect to discord transport. ${r.status}.`);
}
this.initalised = this.connect(opts);
}
return r.json() as unknown;
})
.then((content) => {
if (!IsRecord(content)) {
private async connect(opts: DiscordTransportOptions) {
let connected = false;
let outerData: Response | undefined;
while (!connected) {
// eslint-disable-next-line no-await-in-loop
const data = await fetch(opts.webhook);
if (!data.ok) {
if (data.status === 429) {
// Any amount of miliseconds between 6 minutes and 60 minutes.
const time = Math.max(Math.random(), 0.1) * ONE_HOUR;
// eslint-disable-next-line no-console
console.error(
`Couldn't connect to discord transport (${
data.status
}). Going to try again in ${(time / ONE_MINUTE).toFixed(2)} minutes.`
);
// eslint-disable-next-line no-await-in-loop
await Sleep(time);
} else {
throw new Error(
`Discord returned non-object data as JSON? Can't get id or token ${content}.`
`Couldn't connect to discord transport (${data.status}). This is not retryable.`
);
}
} else {
connected = true;
outerData = data;
}
}
this.webhookUrl = `https://discordapp.com/api/v6/webhooks/${content.id}/${content.token}`;
});
if (!outerData) {
throw Error(
`Unreachable state. Connected with no outerData. This is not even remotely possible, so good luck figuring this one out.`
);
}
const content = (await outerData.json()) as unknown;
if (!IsRecord(content)) {
throw new Error(
`Discord returned non-object data as JSON? Can't get id or token ${content}.`
);
}
this.webhookUrl = `https://discordapp.com/api/v6/webhooks/${content.id}/${content.token}`;
}
private resetBucketData() {