From bbd8e686aa69291e9db784a94ecda2c00a21b734 Mon Sep 17 00:00:00 2001 From: zkldi Date: Sat, 23 Oct 2021 01:11:36 +0100 Subject: [PATCH] Move certain properties to environment variables from conf.json5 --- server/.github/test.conf.json5 | 2 -- server/.github/workflows/tests.yml | 33 +++--------------- .../sync-database/tachi-database-seeds | 1 + server/src/external/mongo/db.ts | 6 ++-- server/src/external/redis/redis.ts | 5 ++- server/src/lib/setup/config.ts | 34 +++++++++++++++---- server/src/main.ts | 12 +++---- server/src/test-utils/resets.ts | 4 +-- 8 files changed, 47 insertions(+), 50 deletions(-) create mode 160000 server/scripts/sync-database/tachi-database-seeds diff --git a/server/.github/test.conf.json5 b/server/.github/test.conf.json5 index a3d13e92e..3bf6f03b1 100644 --- a/server/.github/test.conf.json5 +++ b/server/.github/test.conf.json5 @@ -1,7 +1,6 @@ // Config file that the CI deployment uses. This is moved to $pwd/conf.json5 before CI tests are ran. { - MONGO_CONNECTION_URL: "127.0.0.1:27017", MONGO_DATABASE_NAME: "testingdb", LOG_LEVEL: "info", CAPTCHA_SECRET_KEY: "unused", @@ -22,7 +21,6 @@ }, ARC_AUTH_TOKEN: "unused", CDN_FILE_ROOT: "./test-cdn", - PORT: 8080, TYPE: "omni", ENABLE_SERVER_HTTPS: false, RUN_OWN_CDN: true, diff --git a/server/.github/workflows/tests.yml b/server/.github/workflows/tests.yml index 70fb127b6..4ebf723ad 100644 --- a/server/.github/workflows/tests.yml +++ b/server/.github/workflows/tests.yml @@ -40,6 +40,10 @@ jobs: - name: Run Tests run: pnpm test + env: + PORT: 8080 + MONGO_URL: "127.0.0.1" + REDIS_URL: "127.0.0.1" - name: Lint Code run: pnpm lint @@ -49,32 +53,3 @@ jobs: run: | pnpm install -g codecov cat coverage/lcov.info | codecov - # deploy: - # if: ${{ github.ref == 'master' }} - # needs: test - # name: Deploy - # runs-on: ubuntu-latest - # steps: - # - name: Checkout Code - # uses: actions/checkout@v2 - - # - name: Install Node.js 15 - # uses: actions/setup-node@v2 - # with: - # node-version: 15.x - - # - name: Install PNPM - # uses: pnpm/action-setup@v2.0.1 - # with: - # version: 6.0.2 - - # # configure external dbs - # - uses: supercharge/mongodb-github-action@1.4.1 - # - uses: supercharge/redis-github-action@1.1.0 - - # - name: Install Dependencies - # run: pnpm i - - # # for the time being, disable lint checking. - # # - name: Lint Code - # # run: pnpm lint diff --git a/server/scripts/sync-database/tachi-database-seeds b/server/scripts/sync-database/tachi-database-seeds new file mode 160000 index 000000000..619dd5719 --- /dev/null +++ b/server/scripts/sync-database/tachi-database-seeds @@ -0,0 +1 @@ +Subproject commit 619dd5719ded0a195f9f72585b508c0d33dc65ab diff --git a/server/src/external/mongo/db.ts b/server/src/external/mongo/db.ts index 9621bbe3d..bfde9ce49 100644 --- a/server/src/external/mongo/db.ts +++ b/server/src/external/mongo/db.ts @@ -1,7 +1,7 @@ import { ONE_MINUTE } from "lib/constants/time"; import CreateLogCtx from "lib/logger/logger"; import { OrphanScoreDocument } from "lib/score-import/import-types/common/types"; -import { ServerConfig } from "lib/setup/config"; +import { Environment, ServerConfig } from "lib/setup/config"; import monk, { TMiddleware } from "monk"; import { APITokenDocument, @@ -51,12 +51,12 @@ if (process.env.NODE_ENV === "test") { dbName = `testingdb`; } -logger.info(`Connecting to database ${ServerConfig.MONGO_CONNECTION_URL}/${dbName}...`); +logger.info(`Connecting to database ${Environment.mongoUrl}/${dbName}...`); const dbtime = process.hrtime.bigint(); // By default the connectTimeoutMS is 30 seconds. This has been upped to 2 minutes, due to poor performance // inside githubs test runners. -export const monkDB = monk(`${ServerConfig.MONGO_CONNECTION_URL}/${dbName}`, { +export const monkDB = monk(`${Environment.mongoUrl}/${dbName}`, { serverSelectionTimeoutMS: ONE_MINUTE * 2, }); diff --git a/server/src/external/redis/redis.ts b/server/src/external/redis/redis.ts index 9c6ac5188..df6d7dff1 100644 --- a/server/src/external/redis/redis.ts +++ b/server/src/external/redis/redis.ts @@ -1,11 +1,14 @@ import CreateLogCtx from "lib/logger/logger"; +import { Environment } from "lib/setup/config"; import redis from "redis"; const logger = CreateLogCtx(__filename); logger.verbose("Instantiated Redis Store"); -export const RedisClient = redis.createClient(); +export const RedisClient = redis.createClient({ + url: `redis://${Environment.redisUrl}`, +}); logger.verbose("Instantiated Redis Client"); function EmitCritical() { diff --git a/server/src/lib/setup/config.ts b/server/src/lib/setup/config.ts index 2032b85fd..40fb4c3fc 100644 --- a/server/src/lib/setup/config.ts +++ b/server/src/lib/setup/config.ts @@ -7,7 +7,7 @@ import { FormatPrError } from "utils/prudence"; import { integer, StaticConfig } from "tachi-common"; dotenv.config(); // imports things like NODE_ENV from a local .env file if one is present. -// stub - having a real logger here creates a circular dependency.r +// stub - having a real logger here creates a circular dependency. const logger = console; // CreateLogCtx(__filename); const confLocation = process.env.TCHIS_CONF_LOCATION ?? "./conf.json5"; @@ -44,7 +44,6 @@ export interface OAuth2Info { } export interface TachiConfig { - MONGO_CONNECTION_URL: string; MONGO_DATABASE_NAME: string; LOG_LEVEL: "debug" | "verbose" | "info" | "warn" | "error" | "severe" | "crit"; CAPTCHA_SECRET_KEY: string; @@ -59,7 +58,6 @@ export interface TachiConfig { ARC_AUTH_TOKEN: string; CDN_FILE_ROOT: string; TYPE: "ktchi" | "btchi" | "omni"; - PORT: integer; ENABLE_SERVER_HTTPS?: boolean; RUN_OWN_CDN?: boolean; CLIENT_DEV_SERVER?: string | null; @@ -77,7 +75,6 @@ export interface TachiConfig { OUR_URL: string; LOGGER_DISCORD_WEBHOOK?: string; DISCORD_WHO_TO_TAG?: string[]; - INVOKE_JOB_RUNNER?: boolean; } const isValidOauth2 = p.optional({ @@ -87,7 +84,6 @@ const isValidOauth2 = p.optional({ }); const err = p(config, { - MONGO_CONNECTION_URL: "string", MONGO_DATABASE_NAME: "string", LOG_LEVEL: p.isIn("debug", "verbose", "info", "warn", "error", "severe", "crit"), CAPTCHA_SECRET_KEY: "string", @@ -101,7 +97,6 @@ const err = p(config, { MIN_OAUTH2_INFO: isValidOauth2, ARC_AUTH_TOKEN: "string", CDN_FILE_ROOT: "string", - PORT: p.isPositiveInteger, ENABLE_SERVER_HTTPS: "*boolean", RUN_OWN_CDN: "*boolean", CLIENT_DEV_SERVER: "*?string", @@ -119,7 +114,6 @@ const err = p(config, { OUR_URL: "string", LOGGER_DISCORD_WEBHOOK: "*string", DISCORD_WHO_TO_TAG: p.optional(["string"]), - INVOKE_JOB_RUNNER: "*boolean", }); if (err) { @@ -148,3 +142,29 @@ if (tachiConfig.EMAIL_CONFIG) { export const ServerTypeInfo = tachiConfig.SERVER_TYPE_INFO; export const ServerConfig = tachiConfig; + +// Environment Variable Validation + +let port = Number(process.env.PORT); +if (Number.isNaN(port)) { + logger.warn(`No/invalid PORT specified in environment, defaulting to 8080.`); + port = 8080; +} + +const redisUrl = process.env.REDIS_URL; +if (!redisUrl) { + logger.error(`No REDIS_URL specified in environment. Terminating.`); + process.exit(1); +} + +const mongoUrl = process.env.MONGO_URL; +if (!mongoUrl) { + logger.error(`No MONGO_URL specified in environment. Terminating.`); + process.exit(1); +} + +export const Environment = { + port, + redisUrl, + mongoUrl, +}; diff --git a/server/src/main.ts b/server/src/main.ts index 51334790d..6853c133e 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -1,6 +1,6 @@ import CreateLogCtx from "lib/logger/logger"; import server from "server/server"; -import { ServerTypeInfo, ServerConfig } from "lib/setup/config"; +import { ServerTypeInfo, ServerConfig, Environment } from "lib/setup/config"; import https from "https"; import fs from "fs"; import { FormatVersion } from "./lib/constants/version"; @@ -21,14 +21,14 @@ if (ServerConfig.ENABLE_SERVER_HTTPS) { const httpsServer = https.createServer({ key: privateKey, cert: certificate }, server); - httpsServer.listen(ServerConfig.PORT); - logger.info(`HTTPS Listening on port ${ServerConfig.PORT}`); + httpsServer.listen(Environment.port); + logger.info(`HTTPS Listening on port ${Environment.port}`); } else { - server.listen(ServerConfig.PORT); - logger.info(`HTTP Listening on port ${ServerConfig.PORT}`); + server.listen(Environment.port); + logger.info(`HTTP Listening on port ${Environment.port}`); } -if (ServerConfig.INVOKE_JOB_RUNNER || process.env.INVOKE_JOB_RUNNER) { +if (process.env.INVOKE_JOB_RUNNER) { logger.info(`Spawning a tachi-server job runner.`); // Spawn as a separate process to avoid hogging the main thread. diff --git a/server/src/test-utils/resets.ts b/server/src/test-utils/resets.ts index 9fc101284..d1db07a6d 100644 --- a/server/src/test-utils/resets.ts +++ b/server/src/test-utils/resets.ts @@ -8,7 +8,7 @@ import CreateLogCtx from "lib/logger/logger"; // im installing an entire library for rm rf... import rimraf from "rimraf"; import { SetIndexes } from "external/mongo/indexes"; -import { ServerConfig } from "lib/setup/config"; +import { Environment, ServerConfig } from "lib/setup/config"; const logger = CreateLogCtx(__filename); @@ -90,7 +90,7 @@ export function ResetCDN() { export async function SetIndexesForDB() { await ResetDBState(); - const url = `${ServerConfig.MONGO_CONNECTION_URL}/testingdb`; + const url = `${Environment.mongoUrl}/testingdb`; logger.info(`Setting indexes for ${url}`);