Merge branch 'server-develop'

server/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

server/.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = tab
+trim_trailing_whitespace = true

server/.eslintignore

@@ -0,0 +1,7 @@
+# Things we don't want ESLint to lint. This is just compiled output
+# and other people's code.
+# Note that an editor or eslint implementation not honouring this file
+# may cause crashes, as it will try and lint a couple million lines of JS.
+
+node_modules
+js

server/.eslintrc

@@ -0,0 +1,33 @@
+{
+	"plugins": [
+		"@typescript-eslint",
+		"import",
+		"cadence"
+	],
+	"extends": [
+		"plugin:cadence/recommended"
+	],
+	"rules": {
+		"@typescript-eslint/no-non-null-assertion": "off"
+	},
+	"overrides": [
+		{
+			"files": [
+				"**/*.test.ts"
+			],
+			"rules": {
+				// Our test files break these rules *all* the time, and there's
+				// no point refactoring. Our tests deliberately play with the
+				// dynamic nature of TS to more accurately test arbitrary input.
+				"@typescript-eslint/no-floating-promises": "off",
+				"@typescript-eslint/no-unsafe-member-access": "off",
+				"@typescript-eslint/no-unsafe-assignment": "off",
+				"@typescript-eslint/no-unsafe-return": "off",
+				"@typescript-eslint/no-unsafe-argument": "off",
+				"@typescript-eslint/no-unsafe-call": "off",
+				// we abuse this one for faking data, but we really shouldn't.
+				"@typescript-eslint/consistent-type-assertions": "warn"
+			}
+		}
+	]
+}

server/.git-blame-ignore-revs

@@ -0,0 +1,13 @@
+# You can enable this as an ignored revision file with
+# git config blame.ignoreRevsFile .git-blame-ignore-revs.
+# We have this to ignore some slightly annoying changelogs from screwing
+# with git blame.
+
+#188 - Spaces to tabs
+47c981f4b3189e9cf995ed03256609f3bf56b930
+
+#219 - Refactor entire repo to use baseUrl "/src"
+897e832b01fafade554f99c4fba629c42a50468f
+
+#435 - Standardise line breaks to LF.
+7d477970c610b223d42fbd0a3fccffd53498271e

server/.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # zkldi
+open_collective: # Replace with a single Open Collective username
+ko_fi: zkldi
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

server/.github/codecov.yml

@@ -0,0 +1,26 @@
+codecov:
+  require_ci_to_pass: yes
+  branch: develop
+
+coverage:
+  precision: 2
+  round: down
+  range: "50...100"
+  status:
+    patch:
+      default:
+        informational: true
+    project:
+      default:
+        target: 80
+        threshold: 2
+        base: auto
+        informational: true
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: yes
+      loop: yes
+      method: no
+      macro: no

server/.github/test.conf.json5

@@ -0,0 +1,81 @@
+// Config file that the CI deployment uses. This is moved to $pwd/conf.json5 before CI tests are ran.
+
+{
+	MONGO_DATABASE_NAME: "testingdb",
+	CAPTCHA_SECRET_KEY: "unused",
+	SESSION_SECRET: "unused",
+	FLO_API_URL: "https://flo.example.com",
+	EAG_API_URL: "https://eag.example.com",
+	MIN_API_URL: "https://min.example.com",
+	ARC_API_URL: "https://arc.example.com",
+	FLO_OAUTH2_INFO: {
+		CLIENT_ID: "DUMMY_CLIENT_ID",
+		CLIENT_SECRET: "DUMMY_CLIENT_SECRET",
+		REDIRECT_URI: "https://example.com",
+	},
+	EAG_OAUTH2_INFO: {
+		CLIENT_ID: "DUMMY_CLIENT_ID",
+		CLIENT_SECRET: "DUMMY_CLIENT_SECRET",
+		REDIRECT_URI: "https://example.com",
+	},
+	ARC_AUTH_TOKEN: "unused",
+	ENABLE_SERVER_HTTPS: false,
+	OUR_URL: "https://example.com",
+	INVITE_CODE_CONFIG: {
+		BATCH_SIZE: 2,
+		INVITE_CAP: 100,
+		BETA_USER_BONUS: 5,
+	},
+	CDN_CONFIG: {
+		WEB_LOCATION: "/cdn",
+		SAVE_LOCATION: {
+			TYPE: "LOCAL_FILESYSTEM",
+			LOCATION: "./test-cdn",
+			SERVE_OWN_CDN: true,
+		},
+	},
+	TACHI_CONFIG: {
+		TYPE: "omni",
+		NAME: "Tachi Testing",
+		GAMES: [
+			"iidx",
+			"museca",
+			"maimai",
+			"sdvx",
+			"ddr",
+			"bms",
+			"chunithm",
+			"usc",
+			"wacca",
+			"popn",
+			"jubeat",
+			"pms",
+		],
+		IMPORT_TYPES: [
+			"file/eamusement-iidx-csv",
+			"file/batch-manual",
+			"file/solid-state-squad",
+			"file/mer-iidx",
+			"file/pli-iidx-csv",
+			"ir/direct-manual",
+			"ir/barbatos",
+			"ir/fervidex",
+			"ir/fervidex-static",
+			"ir/beatoraja",
+			"ir/usc",
+			"ir/kshook-sv6c",
+			"api/arc-iidx",
+			"api/arc-sdvx",
+			"api/eag-iidx",
+			"api/eag-sdvx",
+			"api/flo-iidx",
+			"api/flo-sdvx",
+			"api/min-sdvx",
+		],
+	},
+	LOGGER_CONFIG: {
+		FILE: false,
+		CONSOLE: true,
+		LOG_LEVEL: "info",
+	},
+}

server/.github/workflows/test-build-deploy.yml

@@ -0,0 +1,119 @@
+name: Test, Build, Deploy
+
+on:
+  push:
+    branches: [master, develop]
+  pull_request:
+    branches: [master, develop]
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      NODE_ENV: "test"
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Node.js 16
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16.x
+
+      - name: Install PNPM
+        uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.0.2
+
+      - name: Install Dependencies
+        run: pnpm i
+
+        # configure external dbs
+      - uses: supercharge/mongodb-github-action@1.4.1
+      - uses: supercharge/redis-github-action@1.1.0
+
+      - name: Move Config File
+        run: mv ./.github/test.conf.json5 ./test.conf.json5
+
+      - name: Run Tests
+        run: pnpm test
+        env:
+          NODE_ENV: "test"
+          PORT: 8080
+          MONGO_URL: "127.0.0.1"
+          REDIS_URL: "127.0.0.1"
+
+      - name: Lint Code
+        run: pnpm lint
+
+      - name: Typecheck Code
+        run: pnpm typecheck
+
+      - name: Upload Coverage
+        if: always()
+        run: |
+          pnpm install -g codecov
+          cat coverage/lcov.info | codecov
+  docker-push:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: github.event_name == 'push'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Docker Hub login
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Develop tag
+        if: github.ref == 'refs/heads/develop'
+        run: echo "BRANCH_TAG=develop" >> $GITHUB_ENV
+
+      - name: Stable tag
+        if: github.ref == 'refs/heads/master'
+        run: echo "BRANCH_TAG=stable" >> $GITHUB_ENV
+
+      - name: Version tag
+        run: 'echo "VERSION_TAG=$(cat package.json | grep version | head -1 | awk -F: ''{ print $2 }'' | sed ''s/[", ]//g'')" >> $GITHUB_ENV'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKER_USERNAME }}/tachi-server:${{ env.BRANCH_TAG }},${{ secrets.DOCKER_USERNAME }}/tachi-server:${{ env.VERSION_TAG }},
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [docker-push]
+    if: github.event_name == 'push'
+    steps:
+      - name: Deploy to staging
+        if: github.ref == 'refs/heads/develop'
+        uses: appleboy/ssh-action@1a8b3784eaa665f677fa114edd5683bb6a6bfaa8
+        with:
+          host: ${{ secrets.TACHI_HOST }}
+          username: tachi
+          key: ${{ secrets.TACHI_KEY }}
+          script: tachi-server-deploy -s
+
+      - name: Deploy to live
+        if: github.ref == 'refs/heads/master'
+        uses: appleboy/ssh-action@1a8b3784eaa665f677fa114edd5683bb6a6bfaa8
+        with:
+          host: ${{ secrets.TACHI_HOST }}
+          username: tachi
+          key: ${{ secrets.TACHI_KEY }}
+          script: tachi-server-deploy

server/.gitignore

@@ -0,0 +1,39 @@
+# General node stuffs
+node_modules
+js
+logs
+.pnpm-debug.log
+
+# Test stuffs
+.nyc_output
+coverage
+.env
+.setup_env
+failed-tests.txt
+
+# Config files
+conf.json5
+*.conf.json5
+
+# HTTPS local certs
+cert
+
+# Error Files
+scripts/validate-database-errs/*
+
+# IDE Specific files
+.vscode
+.idea
+
+# Inline CDN
+local-cdn
+test-cdn
+
+# Weird Redis Dumps
+*.rdb
+
+scripts/quick-insert/insert.ts
+scripts/bpi-poyashi-data/output.json
+
+scripts/sync-database/tachi-database-seeds
+scripts/arc-sync/cache.json

server/.taprc

@@ -0,0 +1,15 @@
+files: "src/test-utils/single-process-tap.ts"
+test-env: [NODE_ENV=test, TCHIS_CONF_LOCATION=test.conf.json5]
+# On a local machine, these tests run in under 2-3 minutes.
+# On github actions, these tests can take an inordinate amount of time.
+# The timeout is set to 16 minutes, which is ridiculous.
+# Doesn't matter for real machines.
+timeout: 960
+ts: true
+before: "./src/test-utils/hack-setup.js"
+check-coverage: true
+branches: 60
+functions: 80
+lines: 80
+statements: 80
+reporter: terse

server/@types/express-request-mock/index.d.ts

@@ -0,0 +1,18 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+// this is a hack fix for express-request-mock's lack of typing.
+
+declare module "express-request-mock" {
+	import { Request, RequestHandler, Response } from "express";
+	import { RequestOptions, MockResponse } from "node-mocks-http";
+
+	export default function (
+		callback: RequestHandler,
+		options?: RequestOptions,
+		decorators?: Record<string, unknown>
+	): Promise<{
+		req: Request;
+		res: MockResponse<Response<any, Record<string, any>>>;
+		request: Request;
+		response: MockResponse<Response<any, Record<string, any>>>;
+	}>;
+}

server/@types/express/index.d.ts

@@ -0,0 +1,27 @@
+import type { SYMBOL_TACHI_API_AUTH, SYMBOL_TACHI_DATA } from "../../src/lib/constants/tachi";
+import type { TachiRequestData, TachiSessionData } from "../../src/utils/types";
+import type { Session, SessionData } from "express-session";
+import type { APITokenDocument } from "tachi-common";
+
+// this only exists for testing context - it doesn't seem to work properly otherwise.
+declare module "express-session" {
+
+	// Inject additional properties on express-session
+	interface SessionData {
+		tachi: TachiSessionData;
+	}
+}
+
+declare global {
+	namespace Express {
+		interface Request {
+			session: Partial<SessionData> & Session;
+
+			[SYMBOL_TACHI_DATA]?: Partial<TachiRequestData>;
+
+			// even though this is technically *not* present on every request
+			// it's always assigned in the main router, so its functionally equivalent.
+			[SYMBOL_TACHI_API_AUTH]: APITokenDocument;
+		}
+	}
+}

server/@types/mongodb/index.d.ts

@@ -0,0 +1,8 @@
+declare module "mongodb" {
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	interface FindOneOptions<T> {
+		projectID?: boolean;
+	}
+}
+
+export {};

server/@types/why-is-node-running/index.d.ts

@@ -0,0 +1,3 @@
+declare module "why-is-node-running" {
+	export default function log(): void;
+}

server/CHANGELOG.md

@@ -0,0 +1,168 @@
+<!-- This file is not automatically generated. We write this by hand. -->
+<!-- You must update it by hand too! -->
+
+# Syntax
+
+|  Type  |                            Description                            |
+| :----: | :---------------------------------------------------------------: |
+|  BUG   |                        This was a bug fix.                        |
+|  ENH   | This was an enhancement, An improvement to an existing behaviour. |
+|  FEAT  |                          A new feature.                           |
+| REWORK |             An existing feature was reworked/changed.             |
+
+Anything prefixed with `INTERNAL_` has absolutely no bearing on end users, and should
+not generally be shown to end users.
+
+# Changelog
+
+# Current
+
+- [INTERNAL_REWORK] The entire codebase has been moved to a new eslint config, called Cadence. This is in preparation for an even larger refactor, but that likely won't happen for a while.
+
+*****
+
+- [FEAT] Add an endpoint for reverting imports.
+- [FEAT] Rivals have been added.
+- [FEAT] Goals have been added.
+- [FEAT] Milestones have been added.
+- [FEAT] Milestone Sets have been added.
+- [FEAT] Batch Manual imports can now pass `classes`, to hand-declare certain classes, such as IIDX Dans, WACCA Stage Ups, or similar.
+- [FEAT] Added support for an IIDX DP tierlist.
+- [FEAT] Added support for exporting all of a users scores.
+- [FEAT] Added support for reverting an import.
+- [FEAT] Added support for retrieving a users imports, including by user-intent.
+- [FEAT] Notifications can now be sent to users.
+- [FEAT] Search implementation is massively improved (#713)
+- [FEAT] BMS and PMS database changes are automatically backsynced to their parent seeds repository (#732)
+- [FEAT] Support for the BMS AI Table.
+- [INTERNAL_FEAT] Schemas have been moved into tachi-common and properly refactored.
+- [INTERNAL_FEAT] Tachi-Server now has a migration engine, for applying database updates automagically.
+
+*****
+
+- [ENH] Force Static Import now disables itself after doing one static import. This is a safety feature to prevent you from spamming your profile with pb imports.
+- [ENH] Milestones will automatically update and resync off of database-seed changes.
+- [ENH] The beatoraja IR now says how many people have played a chart on its way to de-orphaning.
+- [ENH] Server administrators can now delete other's scores.
+- [ENH] Server administrators can now revert other's imports.
+- [ENH] You can now set Jubility targets.
+- [ENH] Folder Showcase Stats are now removed from a player in the case of the folder being removed.
+- [ENH] ARC SDVX Support has been bumped to EXCEED GEAR.
+- [ENH] The AI table now automatically updates itself.
+- [INTERNAL_ENH] Server version info is now read from package.json, removing redundancy.
+- [INTERNAL_ENH] The commit the server is running under is now exposed publically in version info.
+# 2.1.18
+
+- [BUG] BMS charts are moved into the correct folders after an automatic table synchronisation.
+
+# 2.1.17
+
+- [BUG] `.59` is no longer interpreted as a decimal instead of a song title in SOLID STATE SQUAD imports.
+
+# 2.1.16
+
+- [BUG] EXCEED GEAR scores can now be imported for real for real.
+
+# 2.1.15
+
+- [BUG] EXCEED GEAR scores can now be imported for real.
+
+# 2.1.14
+
+- [BUG] Fixed bug where folders/tables wouldn't sync automatically from the seeds, again.
+
+# 2.1.13
+
+- [BUG] Fixed bug where folders/tables wouldn't sync automatically from the seeds.
+
+# 2.1.12
+
+- [ENH] EXCEED GEAR scores can now be imported.
+
+# 2.1.11
+
+- [ENH] The LR2Hook now saves courses.
+- [ENH] The LR2Hook now saves gauge information.
+
+*****
+
+- [BUG] Having less than 20 valid scores for BPI is no longer advantageous due to regression to 0 profile BPI.
+- [BUG] Timelines now prefer scores with timestamps if present, instead of preferring scores with null timestamps.
+
+# 2.1.10
+
+- [BUG] Users can no longer permanently lock themselves out of an import by making an import while one is ongoing.
+
+# 2.1.9
+
+- [ENH] Add an endpoint on UGPT/folder/:folderID/stats. This returns lamp and grade distributions for this ugpt:folder combo.
+- [ENH] The UGPT Folder timeline endpoint now also returns the folder it worked on.
+
+*****
+
+- [BUG] OAuth2 tokens are now properly expired.
+- [BUG] ForceStaticImport is no longer always on, even if off.
+
+# 2.1.4
+
+- [BUG] IIDX, BMS and PMS scores can no longer get an invalid grade if they are exactly 77.7777...%
+
+# 2.1.3
+
+- [INTERNAL_FEAT] Database seeds can now be automatically updated from a controlling server (#658, et. al.).
+
+*****
+
+- [ENH] Batch Manual can now update your game classes (Thanks, cg505).
+
+*****
+
+- [BUG] PMS charts now automatically insert themselves into the database instead of crashing.
+
+## 2.1.2
+
+- [ENH] Properly capitalise some improperly capitalised error messages.
+
+## 2.1.1
+
+- [INTERNAL_REWORK] Scripts are now included inside `src/` instead of outside it.
+
+## 2.1.0
+
+- [FEAT] Added Jubeat Support. (#169)
+- [FEAT] Added Pop'n Music Support. (#69)
+- [FEAT] Added WACCA Support. (#598)
+- [FEAT] Added support for Skye's pop'n hook. (#563)
+- [FEAT] Added PMS Support. (#45)
+- [FEAT] Users can now be banned. (#640)
+- [FEAT] A new endpoint has been added for searching on BMS, PMS and USC chart hashes. (#630)
+- [FEAT] A new endpoint has been added for seeing your recently used imports. (#642)
+- [FEAT] Added support for the SDVX EXCEED GEAR Cloud hook. (#636)
+- [INTERNAL_FEAT] Config is now served from the server instead of being cloned to the client.
+- [INTERNAL_FEAT] Database seeds is now split for staging and production usage.
+
+*****
+
+- [REWORK] KtRating is no longer the default rating algorithm for IIDX. It has been removed, and the new default is ktLampRating. (#597)
+- [REWORK] USC no longer applies 3-person Automatic Score Reification. Charts must now be manually whitelisted. Note that scores are still saved for charts that might not be in the database at that time.
+
+*****
+
+- [ENH] Added a CHANGELOG.md file. (#627)
+- [ENH] BATCH-MANUAL now supports the `percent` field for jubeat only. (#620)
+- [ENH] Add support for jubility. (#163)
+- [ENH] Move BPI related data (Kaiden Average and BPI) to IIDX charts, instead of being in a separate collection. This lets us display more stats on the UI. (#599)
+- [ENH] Add game specific indexes. (#606)
+- [ENH] Ratings are nullable now. (#611)
+- [ENH] Pre-HV Leggendaria songIDs are now properly resolved. (#487)
+- [ENH] IIDX Imports are now significantly faster.
+- [ENH] GITADORA now supports `naiveSkill` -- Although GITADORA support isn't actually on yet. (#624)
+- [ENH] Emails now go to spam significantly less often, and are more likely to actually go to your mailbox. (#460)
+- [INTERNAL_ENH] Most dependencies have been updated.
+
+*****
+
+- [BUG] UpdateClass now records the class as being recently achieved (i.e. display Chuuden -> Kaiden on the UI) (#589)
+- [BUG] CHUNITHM's rating calculator no longer over-rewards SS rank scores. (#615)
+- [BUG] Folder timelines now pull the oldest score for a given chart, rather than the newest. (#610)
+- [BUG] Fixed bug where good plays on 3y3s long would get dropped due to a misunderstanding of a field. (#602)

server/Dockerfile

@@ -0,0 +1,13 @@
+# This dockerfile spins up an instance of tachi-server and *tachi-server* alone.
+# It does not spin up mongodb instances or redis instances, which the server
+# does need to boot. You should consider using docker-compose for this.
+
+FROM node:16 as build
+WORKDIR /app
+COPY --chown=node:node . /app
+RUN npm install --silent -g pnpm@6 && groupmod -g 1003 node && chown node:node /app
+USER node
+RUN pnpm install --silent --frozen-lockfile && pnpm build && pnpm prune --silent --production && pnpm store prune --silent
+HEALTHCHECK --interval=15s --timeout=5s CMD curl -f http://localhost:8080/api/v1/status || exit 1
+ENV NODE_PATH=js/
+CMD ["node", "js/main.js"]

server/LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

server/README.md

@@ -0,0 +1,36 @@
+# Tachi-Server
+
+[![codecov](https://codecov.io/gh/zkldi/tachi-server/branch/develop/graph/badge.svg?token=RAZSDSH1Y9)](https://codecov.io/gh/zkldi/tachi-server)
+
+Tachi-Server is the backend for Tachi.
+
+Tachi itself is a supercharged Rhythm Game Score Tracker, powering multiple games!
+
+## What's it do?
+
+- Hosts an express backend server.
+- Opens a public API on `/api`.
+- Supports various IR frameworks under `/ir`
+- Powers a ridiculously overengineered score import framework.
+
+## Other Repos
+
+`tachi-server` is one part of the larger Tachi framework. Other repositories include:
+
+[tachi-client](https://github.com/tng-dev/tachi-client): A React frontend for an instance of `tachi-server`.
+
+[tachi-common](https://github.com/tng-dev/tachi-common): Common Types and Utilities for Tachi. (Stuff that's shared between the client and the server, basically.)
+
+[tachi-docs](https://github.com/tng-dev/tachi-docs): Documentation for Tachi.
+
+[tachi-import-scripts](https://github.com/tng-dev/tachi-import-scripts): A UI for converting local files into Batch-Manual format, then uploading to Tachi.
+
+[tachi-beatoraja-ir](https://github.com/tng-dev/tachi-beatoraja-ir): A beatoraja IR client for integrating with Tachi.
+
+[tachi-database-seeds](https://github.com/tng-dev/tachi-database-seeds): Backbone data for Tachi, such as song and chart information. Free to use.
+
+[tachi-bot](https://github.com/tng-dev/tachi-bot): A discord bot that integrates with the Tachi API.
+
+## Documentation
+
+[Over here!](https://tachi.rtfd.io)

server/package.json

@@ -0,0 +1,114 @@
+{
+	"name": "tachi-server",
+	"version": "2.2.0-dev",
+	"description": "A score tracking server.",
+	"main": "js/index.js",
+	"private": true,
+	"scripts": {
+		"test": "tap",
+		"snap": "TAP_SNAPSHOT=1 tap",
+		"watchtest": "tap --watch",
+		"build": "tsc --project tsconfig.build.json",
+		"typecheck": "tsc --project tsconfig.build.json --noEmit",
+		"lint": "eslint ./src --ext .ts --fix",
+		"start": "tsc --project tsconfig.build.json && export NODE_PATH=js/ && node js/main.js",
+		"run": "export NODE_PATH=js/ && node js/main.js",
+		"runscoreworker": "export NODE_PATH=js/ && node js/lib/score-import/worker/worker.js"
+	},
+	"author": "zkldi",
+	"license": "AGPL3",
+	"devDependencies": {
+		"@types/bcrypt": "5.0.0",
+		"@types/bcryptjs": "^2.4.2",
+		"@types/bunyan": "^1.8.8",
+		"@types/connect-redis": "0.0.18",
+		"@types/express": "4.17.13",
+		"@types/express-session": "1.17.4",
+		"@types/mkdirp": "1.0.2",
+		"@types/multer": "1.4.7",
+		"@types/node": "16.11.7",
+		"@types/node-fetch": "2.5.12",
+		"@types/nodemailer": "^6.4.4",
+		"@types/range-parser": "1.2.4",
+		"@types/rate-limit-redis": "1.7.4",
+		"@types/rimraf": "3.0.2",
+		"@types/safe-json-stringify": "1.1.2",
+		"@types/supertest": "2.0.11",
+		"@types/tap": "15.0.5",
+		"@types/uuid": "8.3.4",
+		"@typescript-eslint/eslint-plugin": "5.22.0",
+		"@typescript-eslint/parser": "5.22.0",
+		"eslint": "8.8.0",
+		"eslint-config-prettier": "8.3.0",
+		"eslint-plugin-cadence": "0.0.2",
+		"eslint-plugin-import": "2.26.0",
+		"eslint-plugin-prettier": "4.0.0",
+		"express-request-mock": "3.1.0",
+		"node-mocks-http": "1.11.0",
+		"prettier": "2.5.1",
+		"supertest": "6.2.2",
+		"tap": "15.1.6",
+		"why-is-node-running": "2.2.1"
+	},
+	"dependencies": {
+		"@aws-sdk/client-s3": "3.49.0",
+		"@aws-sdk/signature-v4-crt": "3.49.0",
+		"@types/bson": "1.0.11",
+		"@types/semver": "^7.3.9",
+		"@valuabletouch/winston-seq": "1.3.0",
+		"bcryptjs": "2.4.3",
+		"bullmq": "1.68.4",
+		"bunyan": "1.8.15",
+		"commander": "9.0.0",
+		"connect-redis": "6.1.1",
+		"csv-parse": "5.0.4",
+		"deepmerge": "4.2.2",
+		"dotenv": "16.0.0",
+		"express": "4.17.2",
+		"express-async-errors": "3.1.1",
+		"express-rate-limit": "5.5.1",
+		"express-session": "1.17.2",
+		"fast-json-stable-hash": "1.0.3",
+		"fast-xml-parser": "4.0.2",
+		"helmet": "5.0.2",
+		"html-entities": "^2.3.2",
+		"json5": "2.2.0",
+		"mkdirp": "1.0.4",
+		"mongodb": "3.6.3",
+		"monk": "7.3.4",
+		"multer": "1.4.4",
+		"node-cache": "5.1.2",
+		"node-fetch": "2.6.7",
+		"node-html-parser": "^5.3.3",
+		"nodemailer": "6.7.2",
+		"prudence": "0.9.7",
+		"rate-limit-redis": "2.1.0",
+		"redis": "3.1.2",
+		"rg-stats": "^0.3.0",
+		"rimraf": "3.0.2",
+		"safe-json-stringify": "1.2.0",
+		"semver": "^7.3.7",
+		"seq-logging": "1.1.1",
+		"tachi-common": "0.14.15",
+		"ts-node": "10.4.0",
+		"tsconfig-paths": "3.12.0",
+		"typescript": "4.5.5",
+		"winston": "3.5.1",
+		"winston-daily-rotate-file": "4.6.0",
+		"winston-transport": "4.5.0"
+	},
+	"nyc": {
+		"reporter": [
+			"html",
+			"lcov"
+		],
+		"all": true,
+		"include": [
+			"src/*/!(test-utils)/**/*.ts"
+		],
+		"exclude": [
+			"src/**/*.test.ts",
+			"src/external/mongo/schemas.ts"
+		]
+	}
+}

server/scripts/make-selfsigned-https.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+# This command is for dev servers only! You should use this to
+# create a self-signed HTTPS certificate for local dev.
+# That is it. This is not secure.
+mkdir -p cert
+
+yes "" | openssl req -x509 -newkey rsa:4096 -keyout ./cert/key.pem -out ./cert/cert.pem -sha256 -days 365 --nodes &> /dev/null
+
+echo "Created HTTPS Certificates!"

server/src/external/mongo/db.test.ts

@@ -0,0 +1,33 @@
+import db from "./db";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+
+t.test("ID field autoprojection", async (t) => {
+	await ResetDBState();
+
+	const res = await db.scores.findOne();
+
+	t.equal(res!._id, undefined);
+
+	const res2 = await db.scores.findOne({}, { projectID: true });
+
+	t.not(res2!._id, undefined);
+
+	const res3 = await db.scores.findOne({}, { projection: { scoreID: 1 } });
+
+	t.equal(res3!._id, undefined);
+
+	const res4 = await db.scores.findOne({}, { projection: { scoreID: 1 }, projectID: true });
+
+	t.not(res4!._id, undefined);
+
+	const res5 = await db.scores.findOne({}, { projection: { scoreID: 0 } });
+
+	t.equal(res5!._id, undefined);
+
+	const res6 = await db.scores.findOne({}, { projection: { scoreID: 0 }, projectID: true });
+
+	t.not(res6!._id, undefined);
+
+	t.end();
+});

server/src/external/mongo/db.ts

@@ -0,0 +1,193 @@
+/* eslint-disable @typescript-eslint/strict-boolean-expressions */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+// ^ These rules are disabled for good reason. We have to deal with some very nonsensical types here
+// so we just disable these rules. I know, it sucks, but we'll live.
+import { ONE_MINUTE } from "lib/constants/time";
+import CreateLogCtx from "lib/logger/logger";
+import { Environment, ServerConfig } from "lib/setup/config";
+import monk from "monk";
+import { allSupportedGames } from "tachi-common/js/config/static-config";
+import { GetMillisecondsSince } from "utils/misc";
+import type { OrphanScoreDocument } from "lib/score-import/import-types/common/types";
+import type { TMiddleware, ICollection } from "monk";
+import type {
+	APITokenDocument,
+	ARCSavedProfileDocument,
+	BMSCourseDocument,
+	ChartDocument,
+	ClassAchievementDocument,
+	CounterDocument,
+	FervidexSettingsDocument,
+	FolderChartLookup,
+	FolderDocument,
+	Game,
+	GoalDocument,
+	GoalSubscriptionDocument,
+	ImportDocument,
+	ImportTimingsDocument,
+	integer,
+	InviteCodeDocument,
+	KaiAuthDocument,
+	MilestoneDocument,
+	MilestoneSetDocument,
+	MilestoneSubscriptionDocument,
+	NotificationDocument,
+	OrphanChart,
+	PBScoreDocument,
+	PublicUserDocument,
+	RecentlyViewedFolderDocument,
+	ScoreDocument,
+	SessionDocument,
+	SessionViewDocument,
+	SongDocument,
+	TableDocument,
+	TachiAPIClientDocument,
+	UGPTSettings,
+	UserGameStats,
+	UserGameStatsSnapshot,
+	UserSettings,
+} from "tachi-common";
+import type { MigrationDocument, PrivateUserInfoDocument } from "utils/types";
+
+const logger = CreateLogCtx(__filename);
+
+let dbName = ServerConfig.MONGO_DATABASE_NAME;
+
+/* istanbul ignore next */
+if (Environment.nodeEnv === "test") {
+	dbName = `testingdb`;
+}
+
+logger.info(`Connecting to database ${Environment.mongoUrl}/${dbName}...`, { bootInfo: true });
+const dbtime = process.hrtime.bigint();
+
+// By default the connectTimeoutMS is 30 seconds. This has been upped to 2 minutes, due to poor performance
+// inside githubs test runners.
+export const monkDB = monk(`${Environment.mongoUrl}/${dbName}`, {
+	serverSelectionTimeoutMS: ONE_MINUTE * 2,
+});
+
+/* istanbul ignore next */
+monkDB
+	.then(() => {
+		logger.info(`Database connection successful: took ${GetMillisecondsSince(dbtime)}ms`, {
+			bootInfo: true,
+		});
+	})
+	.catch((err) => {
+		logger.crit(`Failed to connect to database: ${err}`, () => {
+			process.exit(1);
+		});
+	});
+
+/**
+ * Removes _id from the returns of find/findOne. Note that this function is littered with eslint-disables
+ * due to it working with some pretty wobbly types.
+ */
+const RemoveIDFromFindReturnsMiddleware: TMiddleware = () => (next) => (args: any, method) => {
+	if ((method === "find" || method === "findOne") && !args.options.projectID) {
+		if (args.options.projection) {
+			args.options.projection._id = 0;
+		} else {
+			args.options.projection = { _id: 0 };
+		}
+	}
+
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+	return next(args, method);
+};
+
+// a bug in monks types means that :any has to be used here. Maybe we'll make a PR for this?
+const StripIDFromDBInsertsMiddleware: TMiddleware = () => (next) => (args: any, method) => {
+	if (method === "insert") {
+		if (Array.isArray(args.data)) {
+			for (const d of args.data) {
+				delete d._id;
+			}
+		} else {
+			delete args.data._id;
+		}
+	}
+
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+	return next(args, method);
+};
+
+monkDB.addMiddleware(StripIDFromDBInsertsMiddleware);
+monkDB.addMiddleware(RemoveIDFromFindReturnsMiddleware);
+
+export async function CloseMongoConnection() {
+	await monkDB.close();
+}
+
+// Typescript incorrectly casts this into songs[string] => songdocument,
+// Force cast it out.
+const songs = Object.fromEntries(
+	allSupportedGames.map((e) => [e, monkDB.get<SongDocument>(`songs-${e}`)])
+) as unknown as Record<Game, ICollection<SongDocument>>;
+
+const charts = Object.fromEntries(
+	allSupportedGames.map((e) => [e, monkDB.get<ChartDocument>(`charts-${e}`)])
+) as unknown as Record<Game, ICollection<ChartDocument>>;
+
+const db = {
+	// i have to handwrite this out for TS... :(
+	// dont worry, it was all macro'd.
+	songs,
+	charts,
+	scores: monkDB.get<ScoreDocument>("scores"),
+	"personal-bests": monkDB.get<PBScoreDocument>("personal-bests"),
+	folders: monkDB.get<FolderDocument>("folders"),
+	"folder-chart-lookup": monkDB.get<FolderChartLookup>("folder-chart-lookup"),
+	goals: monkDB.get<GoalDocument>("goals"),
+	"goal-subs": monkDB.get<GoalSubscriptionDocument>("goal-subs"),
+	milestones: monkDB.get<MilestoneDocument>("milestones"),
+	"milestone-subs": monkDB.get<MilestoneSubscriptionDocument>("milestone-subs"),
+	users: monkDB.get<PublicUserDocument>("users"),
+	imports: monkDB.get<ImportDocument>("imports"),
+	"import-timings": monkDB.get<ImportTimingsDocument>("import-timings"),
+	sessions: monkDB.get<SessionDocument>("sessions"),
+	invites: monkDB.get<InviteCodeDocument>("invites"),
+	counters: monkDB.get<CounterDocument>("counters"),
+	"game-stats": monkDB.get<UserGameStats>("game-stats"),
+	"kai-auth-tokens": monkDB.get<KaiAuthDocument>("kai-auth-tokens"),
+	"bms-course-lookup": monkDB.get<BMSCourseDocument>("bms-course-lookup"),
+	"api-tokens": monkDB.get<APITokenDocument>("api-tokens"),
+	"orphan-scores": monkDB.get<OrphanScoreDocument>("orphan-scores"),
+	"import-locks": monkDB.get<{ userID: integer; locked: boolean }>("import-locks"),
+	tables: monkDB.get<TableDocument>("tables"),
+	"game-settings": monkDB.get<UGPTSettings>("game-settings"),
+	"game-stats-snapshots": monkDB.get<UserGameStatsSnapshot>("game-stats-snapshots"),
+	"session-view-cache": monkDB.get<SessionViewDocument>("session-view-cache"),
+	"arc-saved-profiles": monkDB.get<ARCSavedProfileDocument>("arc-saved-profiles"),
+	"user-settings": monkDB.get<UserSettings>("user-settings"),
+	"user-private-information": monkDB.get<PrivateUserInfoDocument>("user-private-information"),
+	"api-clients": monkDB.get<TachiAPIClientDocument>("api-clients"),
+
+	// i've inlined this one because i don't see it appearing anywhere else.
+	"oauth2-auth-codes":
+		monkDB.get<{ code: string; userID: integer; createdOn: number }>("oauth2-auth-codes"),
+
+	"fer-settings": monkDB.get<FervidexSettingsDocument>("fer-settings"),
+	"orphan-chart-queue": monkDB.get<OrphanChart>("orphan-chart-queue"),
+	"password-reset-codes": monkDB.get<{
+		code: string;
+		userID: integer;
+		createdOn: number;
+	}>("password-reset-codes"),
+	"class-achievements": monkDB.get<ClassAchievementDocument>("class-achievements"),
+	"score-blacklist":
+		monkDB.get<{ scoreID: string; userID: integer; score: ScoreDocument }>("score-blacklist"),
+	"verify-email-codes":
+		monkDB.get<{ userID: integer; code: string; email: string }>("verify-email-codes"),
+	"recent-folder-views": monkDB.get<RecentlyViewedFolderDocument>("recent-folder-views"),
+	"milestone-sets": monkDB.get<MilestoneSetDocument>("milestone-sets"),
+	migrations: monkDB.get<MigrationDocument>("migrations"),
+	notifications: monkDB.get<NotificationDocument>("notifications"),
+};
+
+export type StaticDatabases = Exclude<keyof typeof db, "charts" | "songs">;
+
+export type Databases = StaticDatabases | `charts-${Game}` | `songs-${Game}`;
+
+export default db;

server/src/external/mongo/indexes.ts

@@ -0,0 +1,228 @@
+/* eslint-disable no-await-in-loop */
+import db, { monkDB } from "./db";
+import { ONE_DAY } from "lib/constants/time";
+import CreateLogCtx from "lib/logger/logger";
+import { TachiConfig } from "lib/setup/config";
+import monk from "monk";
+import { Random20Hex } from "utils/misc";
+import type { Databases } from "./db";
+import type { IndexOptions } from "mongodb";
+import type { IMonkManager } from "monk";
+
+const logger = CreateLogCtx(__filename);
+
+interface Index {
+	fields: Record<string, unknown>;
+	options?: IndexOptions;
+}
+
+function index(fields: Record<string, unknown>, options?: IndexOptions) {
+	return { fields, options };
+}
+
+const UNIQUE = { unique: true };
+
+const staticIndexes: Partial<Record<Databases, Array<Index>>> = {
+	scores: [
+		index({ scoreID: 1 }, UNIQUE),
+		index({ chartID: 1, userID: 1 }),
+		index({ userID: 1, game: 1, playtype: 1 }),
+	],
+	"personal-bests": [
+		index({ chartID: 1, userID: 1 }, UNIQUE),
+		index({ chartID: 1, "scoreData.percent": 1 }),
+		index({ userID: 1, game: 1, playtype: 1 }),
+	],
+	sessions: [
+		index({ userID: 1, game: 1, playtype: 1, timeStarted: 1, timeEnded: 1 }),
+
+		// Optimises score modification, since sessions need to be repointed.
+		// also, just generally useful.
+		index({ "scoreInfo.scoreID": 1 }),
+		index({ name: "text" }),
+	],
+	"game-stats": [index({ userID: 1, game: 1, playtype: 1 }, UNIQUE)],
+	"game-settings": [index({ userID: 1, game: 1, playtype: 1 }, UNIQUE), index({ rivals: 1 })],
+	"folder-chart-lookup": [index({ chartID: 1, folderID: 1 }, UNIQUE)],
+	goals: [index({ goalID: 1 }, UNIQUE), index({ game: 1, playtype: 1 }), index({ name: "text" })],
+	"goal-subs": [index({ goalID: 1, userID: 1 }, UNIQUE), index({ goalID: 1 })],
+	milestones: [
+		index({ milestoneID: 1 }, UNIQUE),
+		index({ game: 1, playtype: 1 }),
+		index({ name: "text" }),
+	],
+	"milestone-subs": [
+		index({ milestoneID: 1, userID: 1 }, UNIQUE),
+		index({ userID: 1, game: 1, playtype: 1 }),
+	],
+	"milestone-sets": [
+		index({ setID: 1 }, UNIQUE),
+		index({ game: 1, playtype: 1 }),
+		index({ milestones: 1 }),
+		index({ name: "text" }),
+	],
+	imports: [index({ importID: 1 }, UNIQUE)],
+	"import-timings": [
+		index({ importID: 1 }, UNIQUE),
+		index({ timestamp: 1 }),
+		index({ total: 1 }),
+	],
+	users: [
+		index({ id: 1 }, UNIQUE),
+		index({ id: 1, authLevel: 1 }, UNIQUE),
+		index({ username: 1 }, UNIQUE),
+		index({ usernameLowercase: 1 }, UNIQUE),
+	],
+	folders: [
+		index({ folderID: 1 }, UNIQUE),
+		index({ game: 1, playtype: 1, table: 1, tableIndex: 1 }),
+		index({ title: "text", searchTerms: "text" }),
+	],
+	"kai-auth-tokens": [index({ userID: 1, service: 1 }, UNIQUE)],
+	"bms-course-lookup": [index({ md5sums: 1 }, UNIQUE)],
+	"api-tokens": [index({ token: 1 }, UNIQUE), index({ userID: 1 })],
+	tables: [index({ tableID: 1, game: 1, playtype: 1 }, UNIQUE)],
+	"game-stats-snapshots": [index({ timestamp: 1, userID: 1, game: 1, playtype: 1 }, UNIQUE)],
+	"session-view-cache": [
+		index({ sessionID: 1, ip: 1 }, UNIQUE),
+		index({ timestamp: 1 }, { expireAfterSeconds: ONE_DAY / 1000 }),
+	],
+	"user-settings": [index({ userID: 1 }, UNIQUE)],
+	"user-private-information": [index({ userID: 1 }, UNIQUE), index({ email: 1 }, UNIQUE)],
+	"fer-settings": [index({ userID: 1 }, UNIQUE)],
+	counters: [index({ counterName: 1 }, UNIQUE)],
+	"class-achievements": [index({ game: 1, playtype: 1, timeAchieved: 1 })],
+	"api-clients": [index({ clientID: 1 }, UNIQUE)],
+	"charts-iidx": [
+		index(
+			{ "data.arcChartID": 1 },
+			{ unique: true, partialFilterExpression: { "data.arcChartID": { $type: "string" } } }
+		),
+		index({ "data.hashSHA256": 1 }),
+		index({ "data.inGameID": 1, playtype: 1, difficulty: 1 }),
+	],
+	"charts-bms": [index({ "data.hashMD5": 1 }, UNIQUE), index({ "data.hashSHA256": 1 }, UNIQUE)],
+	"charts-popn": [index({ "data.hashSHA256": 1 }, UNIQUE)],
+	"charts-sdvx": [index({ "data.inGameID": 1, difficulty: 1 })],
+	"charts-maimai": [
+		index({ "data.inGameID": 1, difficulty: 1 }, UNIQUE),
+		index({ "data.inGameStrID": 1, difficulty: 1 }, UNIQUE),
+	],
+	"charts-museca": [index({ "data.inGameID": 1, difficulty: 1 }, UNIQUE)],
+	"charts-chunithm": [index({ "data.inGameID": 1, difficulty: 1 }, UNIQUE)],
+	"charts-gitadora": [index({ "data.inGameID": 1, difficulty: 1 }, UNIQUE)],
+	"charts-wacca": [index({ isHot: 1 })],
+	"charts-usc": [index({ "data.hashSHA1": 1, playtype: 1 }, UNIQUE)],
+	"charts-jubeat": [index({ "data.inGameID": 1, difficulty: 1 }, UNIQUE)],
+	"charts-pms": [
+		index({ "data.hashSHA256": 1, playtype: 1 }, UNIQUE),
+		index({ "data.hashMD5": 1, playtype: 1 }, UNIQUE),
+	],
+	"import-locks": [index({ userID: 1 }, UNIQUE)],
+	"arc-saved-profiles": [index({ userID: 1, forImportType: 1 }, UNIQUE)],
+	"score-blacklist": [index({ scoreID: 1 }, UNIQUE)],
+	migrations: [index({ migrationID: 1 }, UNIQUE)],
+	notifications: [index({ notifID: 1 }, UNIQUE), index({ sentTo: 1, sentAt: 1 })],
+};
+
+const indexes: Partial<Record<Databases, Array<Index>>> = staticIndexes;
+
+for (const game of TachiConfig.GAMES) {
+	if (indexes[`charts-${game}` as Databases]) {
+		indexes[`charts-${game}` as Databases]!.push(
+			index({ chartID: 1 }, UNIQUE),
+			index(
+				{ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 },
+				{ unique: true, partialFilterExpression: { isPrimary: { $eq: true } } }
+			),
+			index({ songID: 1 }),
+			index({ playtype: 1 })
+		);
+	} else {
+		indexes[`charts-${game}` as Databases] = [
+			index({ chartID: 1 }, UNIQUE),
+			index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE),
+			index({ songID: 1 }),
+			index({ playtype: 1 }),
+		];
+	}
+
+	if (indexes[`songs-${game}` as Databases]) {
+		indexes[`songs-${game}` as Databases]!.push(
+			index({ id: 1 }, UNIQUE),
+			index({ title: "text", artist: "text", altTitles: "text", searchTerms: "text" })
+		);
+	} else {
+		indexes[`songs-${game}` as Databases] = [
+			index({ id: 1 }, UNIQUE),
+			index({ title: 1 }),
+			index({ title: "text", artist: "text", altTitles: "text", searchTerms: "text" }),
+		];
+	}
+}
+
+export async function SetIndexesWithDB(db: IMonkManager, reset: boolean) {
+	const collections = (await db.listCollections()).map((e) => e.name);
+
+	for (const [collection, values] of Object.entries(indexes)) {
+		if (!collections.includes(collection)) {
+			// this creates a collection, i cant find the createCollection
+			// call.
+			const tmp = Random20Hex();
+
+			await db.get(collection).insert({ __tmp: tmp });
+			await db.get(collection).remove({ __tmp: tmp });
+		}
+
+		if (reset) {
+			await db.get(collection).dropIndexes();
+			logger.debug(`Reset ${collection}.`);
+		}
+
+		for (const index of values) {
+			// @ts-expect-error Type-mismatch here. our index.fileds are just boring records. I know
+			// that this sucks...
+			const r = await db.get(collection).createIndex(index.fields, index.options);
+
+			logger.debug(r);
+		}
+	}
+
+	logger.debug("Done.");
+}
+
+export async function SetIndexes(mongoUrl: string, reset: boolean) {
+	const monkDb = monk(mongoUrl);
+
+	await SetIndexesWithDB(monkDb, reset);
+
+	await monkDb.close();
+}
+
+export function SetIndexesIfNoneSet() {
+	// If no indexes are set, then we need to load mongo indexes.
+	return db.users
+		.indexes()
+		.then((r) => {
+			// If there's only one index on users
+			// that means that only _id has indexes.
+			// This means that there are likely to be no indexes
+			// configured in the database.
+			if (Object.keys(r).length === 1) {
+				logger.info(
+					`No indexes on users, First-time Tachi-Server startup assumed. Running SetIndexes.`
+				);
+				return SetIndexesWithDB(monkDB, true);
+			}
+		})
+		.catch((err: unknown) => {
+			logger.info(
+				`Error in finding users collection. First time startup likely. Running SetIndexes.`,
+				{ err }
+			);
+			return SetIndexesWithDB(monkDB, true);
+		})
+		.catch((err: unknown) => {
+			logger.error(`Failed to set indexes on assumed first-time-setup?`, { err });
+		});
+}

server/src/external/mongo/schemas.ts

@@ -0,0 +1,79 @@
+import { TachiConfig } from "lib/setup/config";
+import p from "prudence";
+import { SCHEMAS } from "tachi-common/js/lib/schemas";
+import type { Databases } from "./db";
+import type { PrudenceSchema } from "prudence";
+import type { SchemaValidatorFunction } from "tachi-common/js/lib/schemas";
+
+// The idea of this file is to export a function for every database
+// that will validate that collection.
+
+// If it returns true, that document is valid. If it throws any sort
+// of error, it is invalid.
+
+function prSchemaify(schema: PrudenceSchema) {
+	return (s: unknown): true => {
+		const err = p(s, schema);
+
+		if (err) {
+			throw err;
+		}
+
+		return true;
+	};
+}
+
+export const DatabaseSchemas: Record<Databases, SchemaValidatorFunction> = {
+	"import-locks": prSchemaify({
+		userID: p.isPositiveNonZeroInteger,
+	}),
+	"kai-auth-tokens": prSchemaify({
+		userID: p.isPositiveNonZeroInteger,
+		token: "string",
+		refreshToken: "string",
+		service: p.isIn("FLO", "EAG", "MIN"),
+	}),
+	"oauth2-auth-codes": prSchemaify({
+		code: "string",
+		userID: p.isPositiveNonZeroInteger,
+		createdOn: p.isPositive,
+	}),
+	"password-reset-codes": prSchemaify({
+		code: "string",
+		userID: p.isPositiveNonZeroInteger,
+		createdOn: p.isPositive,
+	}),
+	"verify-email-codes": prSchemaify({
+		userID: p.isPositiveNonZeroInteger,
+		code: "string",
+		email: "string",
+	}),
+	migrations: prSchemaify({
+		migrationID: "string",
+		status: p.isIn("applied", "pending"),
+		appliedOn: (self, parent) => {
+			if (parent.status === "pending") {
+				return (
+					self === undefined ||
+					`Expected appliedOn to be undefined since status was pending.`
+				);
+			}
+
+			return p.isPositiveInteger(self);
+		},
+	}),
+	"orphan-scores": prSchemaify({
+		importType: p.isIn(TachiConfig.IMPORT_TYPES),
+		game: p.isIn(TachiConfig.GAMES),
+		userID: p.isPositiveNonZeroInteger,
+		orphanID: "string",
+		timeInserted: p.isPositive,
+		errMsg: "?string",
+
+		// @todo Type these properly? This will take a lot of effort, as it will require
+		// formal types for all parser data:context duos.
+		data: p.any,
+		context: p.any,
+	}),
+	...SCHEMAS,
+};

server/src/external/mongo/sequence-docs.ts

@@ -0,0 +1,56 @@
+import db from "./db";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+export async function InitSequenceDocs() {
+	await db.counters.remove({});
+
+	const userWithLargestID = await db.users.findOne(
+		{},
+		{
+			sort: {
+				id: -1,
+			},
+		}
+	);
+
+	const largestBMSSongID = await db.songs.bms.findOne(
+		{},
+		{
+			sort: {
+				id: -1,
+			},
+		}
+	);
+
+	const largestPMSSongID = await db.songs.pms.findOne(
+		{},
+		{
+			sort: {
+				id: -1,
+			},
+		}
+	);
+
+	const Counters = [
+		{
+			counterName: "users",
+			value: userWithLargestID ? userWithLargestID.id + 1 : 1,
+		},
+		{
+			counterName: "bms-song-id",
+			value: largestBMSSongID ? largestBMSSongID.id + 1 : 1,
+		},
+		{
+			counterName: "pms-song-id",
+			value: largestPMSSongID ? largestPMSSongID.id + 1 : 1,
+		},
+	];
+
+	logger.verbose(
+		`Setting Counters -> ${Counters.map((e) => `${e.counterName}: ${e.value}`).join(", ")}`
+	);
+
+	await db.counters.insert(Counters);
+}

server/src/external/redis/redis-IPC.test.ts

@@ -0,0 +1,19 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { RedisPub, RedisSub } from "./redis-IPC";
+import t from "tap";
+
+/**
+ * Note: The functions these things use have been stubbed out. However, the tests still run and
+ * they also still work. We'll skip them though, anyway.
+ */
+
+t.skip("Basic PUB/SUB testing", (t) => {
+	t.setTimeout(2000);
+	RedisSub("test" as any, (d) => {
+		t.equal(d.userID, 1);
+		t.pass();
+		t.end();
+	});
+
+	RedisPub("test" as any, { userID: 1 } as any);
+});

server/src/external/redis/redis-IPC.ts

@@ -0,0 +1,101 @@
+import CreateLogCtx from "lib/logger/logger";
+import { TachiConfig } from "lib/setup/config";
+import redis from "redis";
+
+/**
+ * This code has been stubbed out! It doesn't really have a use at the moment.
+ * At some point, though, we might actually need this. (Maybe for a logging framework)
+ *
+ * This was used to communicate with other processes running on the same host as this server.
+ * The idea was the use it for the discord bot, however, we have webhooks now that do this (better).
+ * Thanks!
+ */
+
+const logger = CreateLogCtx(__filename);
+
+// Stub types. In the future, we may actually use this.
+export type RedisIPCChannels = "";
+export interface RedisIPCData {
+	"": unknown;
+}
+
+type RedisSubCallback<T extends RedisIPCChannels> = (data: RedisIPCData[T]) => void;
+
+type SubCallbacksType = {
+	[K in RedisIPCChannels]: Array<RedisSubCallback<K>>;
+};
+
+const SubCallbacks: Partial<SubCallbacksType> = {};
+
+// Redis doesn't allow one client to subscribe and publish
+// There's little to no overhead to having these three clients,
+// so - that's why its like this.
+const SubClient = redis.createClient();
+const PubClient = redis.createClient();
+
+const PREFIX = TachiConfig.NAME.toUpperCase();
+
+export function RedisPub<T extends RedisIPCChannels>(channel: T, data: RedisIPCData[T]) {
+	PubClient.publish(`${PREFIX}-${channel}`, JSON.stringify(data));
+}
+
+export function RedisSub<T extends RedisIPCChannels>(channel: T, callback: RedisSubCallback<T>) {
+	if (SubCallbacks[channel]) {
+		// @ts-expect-error It's complaining that the T in channel might be a different T to the T in callback
+		// this is obviously nonsense.
+		SubCallbacks[channel]!.push(callback);
+		logger.debug(`Pushed callback ${callback.name} to channel ${channel}.`);
+	} else {
+		// @ts-expect-error see above.
+		SubCallbacks[channel] = [callback];
+		SubClient.subscribe(`${PREFIX}-${channel}`);
+		logger.debug(`Added first callback ${callback.name} to channel ${channel}.`);
+	}
+}
+
+SubClient.on("message", (channel, strData) => {
+	if (!channel.startsWith(`${PREFIX}-`)) {
+		// not our business
+		return;
+	}
+
+	const ktChannel = channel.slice(`${PREFIX}-`.length) as RedisIPCChannels;
+
+	if (!Object.prototype.hasOwnProperty.call(SubCallbacks, ktChannel)) {
+		// no callbacks to call
+		return;
+	}
+
+	const jsData = JSON.parse(strData) as unknown;
+
+	for (const cb of SubCallbacks[ktChannel]!) {
+		try {
+			cb(jsData);
+		} catch (err) {
+			logger.error(`Error calling callback ${cb.name} for channel ${ktChannel}`, { err });
+		}
+	}
+});
+
+// Awful...
+// Function is near-impossible to test.
+/* istanbul ignore next */
+export function CloseRedisPubSub() {
+	return new Promise<void>((resolve, reject) => {
+		PubClient.quit((err: unknown) => {
+			if (err !== null && err !== undefined) {
+				logger.crit(`PubClient QUIT error: ${err}`, { err });
+				reject(err);
+			}
+
+			SubClient.quit((err: unknown) => {
+				if (err !== null && err !== undefined) {
+					logger.crit(`SubClient QUIT error: ${err}`, { err });
+					reject(err);
+				}
+
+				resolve();
+			});
+		});
+	});
+}

server/src/external/redis/redis.ts

@@ -0,0 +1,47 @@
+import { ONE_MINUTE } from "lib/constants/time";
+import CreateLogCtx from "lib/logger/logger";
+import { Environment } from "lib/setup/config";
+import redis from "redis";
+import { GetMillisecondsSince } from "utils/misc";
+
+const logger = CreateLogCtx(__filename);
+
+logger.verbose("Instantiated Redis Store", { bootInfo: true });
+
+export const RedisClient = redis.createClient({
+	url: `redis://${Environment.redisUrl}`,
+});
+const startConnect = process.hrtime.bigint();
+
+logger.verbose("Instantiated Redis Client", { bootInfo: true });
+
+function EmitCritical() {
+	/* istanbul ignore next */
+	if (!RedisClient.connected) {
+		logger.crit(`Could not connect to redis in time. No more information is available.`, () => {
+			process.exit(1);
+		});
+	}
+}
+
+const ref = setTimeout(EmitCritical, ONE_MINUTE * 2);
+
+RedisClient.on("connect", () => {
+	logger.info(`Connected to Redis. Took ${GetMillisecondsSince(startConnect)}ms`, {
+		bootInfo: true,
+	});
+
+	clearTimeout(ref);
+});
+
+export function CloseRedisConnection() {
+	return new Promise((resolve, reject) => {
+		RedisClient.quit((err, reply) => {
+			if (err) {
+				reject(err);
+			} else {
+				resolve(reply);
+			}
+		});
+	});
+}

server/src/lib/builtin-clients/builtin-clients.ts

@@ -0,0 +1,231 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import { DatabaseSchemas } from "external/mongo/schemas";
+import fjsh from "fast-json-stable-hash";
+import CreateLogCtx from "lib/logger/logger";
+import { ServerConfig, TachiConfig } from "lib/setup/config";
+import { UserAuthLevels } from "tachi-common";
+import { Random20Hex } from "utils/misc";
+import { FormatPrError } from "utils/prudence";
+import type { PrudenceError } from "prudence";
+import type { TachiAPIClientDocument } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+type DefaultClients = Array<Omit<TachiAPIClientDocument, "author" | "clientSecret">>;
+
+// Defines some Tachi API Clients that should come default with a Tachi
+// environment.
+// These use the special Client ID prefix "CX" instead of "CI", which
+// means they cannot possibly be collided.1
+const KtchiDefaultClients: DefaultClients = [
+	{
+		name: "Fervidex",
+		webhookUri: null,
+		redirectUri: null,
+		requestedPermissions: ["submit_score"],
+		clientID: "CXFervidex",
+		apiKeyFilename: "kamaitachi.fervidex.json",
+		apiKeyTemplate: JSON.stringify(
+			{
+				url: `${ServerConfig.OUR_URL}/ir/fervidex`,
+				token: "%%TACHI_KEY%%",
+			},
+			null,
+			"\t"
+		),
+	},
+	{
+		name: "Barbatos",
+		webhookUri: null,
+		redirectUri: null,
+		requestedPermissions: ["submit_score"],
+		clientID: "CXBarbatos",
+		apiKeyFilename: "barbatos.json",
+		apiKeyTemplate: JSON.stringify(
+			{
+				api_key: "%%TACHI_KEY%%",
+			},
+			null,
+			"\t"
+		),
+	},
+	{
+		name: "Konaste Hook",
+		webhookUri: null,
+		redirectUri: null,
+		requestedPermissions: ["submit_score"],
+		clientID: "CXKsHook",
+		apiKeyFilename: "kamaitachi.kshook.json",
+		apiKeyTemplate: JSON.stringify(
+			{
+				url: `${ServerConfig.OUR_URL}/ir/kshook`,
+				token: "%%TACHI_KEY%%",
+				games: ["sv6c"],
+			},
+			null,
+			"\t"
+		),
+	},
+	{
+		clientID: "CXChunitachi",
+		requestedPermissions: ["submit_score"],
+		name: "ChunItachi",
+		redirectUri: null,
+		webhookUri: null,
+		apiKeyFilename: "ChunItachi.ini",
+		apiKeyTemplate: `[general]
+; Show information for debugging. Useful incase things go wrong.
+showDebug = true
+
+; Set this to an extID if multiple people play on your setup
+; and you only want to import scores from a specific user.
+; You can find your extID inside your database...
+; ...or you can set it to junk like 123 and check the debug logs.
+extID = 0
+
+; The current version of the game you're playing on.
+; This should be any of the following:
+; game = amazon
+; game = amazonplus
+; game = crystal
+; game = paradise
+; game = paradiselost
+
+game = PUT_YOUR_GAME_HERE
+
+; If true, Fails will take more priority than things like FULL COMBOs.
+failOverLamp = false
+
+[kamaitachi]
+; What to use to check whether we're online or not.
+apiStatus = ${ServerConfig.OUR_URL}/api/v1/status
+
+; Where to POST scores to.
+apiEndpoint = ${ServerConfig.OUR_URL}/ir/direct-manual/import
+
+; This thing is secret! Keep it so. You can manage your API Keys at
+; ${ServerConfig.OUR_URL}/users/me -> Integrations -> API Keys!
+apikey = %%TACHI_KEY%%`,
+	},
+	{
+		name: "SilentHook",
+		apiKeyFilename: "silent-config.json",
+		apiKeyTemplate: JSON.stringify(
+			{
+				key: "%%TACHI_KEY%%",
+				url: `${ServerConfig.OUR_URL}/ir/direct-manual/import`,
+			},
+			null,
+			"\t"
+		),
+		clientID: "CXSilentHook",
+		redirectUri: null,
+		requestedPermissions: ["submit_score"],
+		webhookUri: null,
+	},
+];
+
+const BtchiDefaultClients: DefaultClients = [
+	{
+		name: "Beatoraja IR",
+		webhookUri: null,
+		redirectUri: null,
+		requestedPermissions: ["submit_score"],
+		clientID: "CXBeatorajaIR",
+		apiKeyTemplate: null,
+		apiKeyFilename: null,
+	},
+	{
+		name: "USC IR",
+		webhookUri: null,
+		redirectUri: null,
+		requestedPermissions: ["submit_score"],
+		clientID: "CXUSCIR",
+		apiKeyTemplate: null,
+		apiKeyFilename: null,
+	},
+	{
+		name: "LR2 Hook",
+		webhookUri: null,
+		redirectUri: null,
+		requestedPermissions: ["submit_score"],
+		clientID: "CXLR2Hook",
+		apiKeyFilename: "BokutachiAuth.json",
+		apiKeyTemplate: JSON.stringify(
+			{
+				url: `${ServerConfig.OUR_URL}/ir/lr2hook/import`,
+				apiKey: "%%TACHI_KEY%%",
+			},
+			null,
+			"\t"
+		),
+	},
+];
+
+export async function LoadDefaultClients() {
+	if (TachiConfig.TYPE === "ktchi") {
+		await LoadClients(KtchiDefaultClients);
+	} else if (TachiConfig.TYPE === "btchi") {
+		await LoadClients(BtchiDefaultClients);
+	} else {
+		await LoadClients(KtchiDefaultClients);
+		await LoadClients(BtchiDefaultClients);
+	}
+}
+
+async function LoadClients(clients: DefaultClients) {
+	const firstAdmin = await db.users.findOne({
+		authLevel: UserAuthLevels.ADMIN,
+	});
+
+	if (!firstAdmin) {
+		logger.error(
+			`There are no admins on this instance of tachi-server. We cannot create default API Clients!`
+		);
+		return;
+	}
+
+	for (const client of clients) {
+		const exists = await db["api-clients"].findOne(
+			{
+				clientID: client.clientID,
+			},
+			{
+				projection: {
+					clientSecret: 0,
+					author: 0,
+				},
+			}
+		);
+
+		// Skip if nothing has changed.
+		if (fjsh.hash(exists, "sha256") === fjsh.hash(client, "sha256")) {
+			continue;
+		}
+
+		const realClient: TachiAPIClientDocument = {
+			...client,
+			clientSecret: `CS${Random20Hex()}`,
+			author: 1,
+		};
+
+		try {
+			DatabaseSchemas["api-clients"](realClient);
+		} catch (err) {
+			logger.error(
+				`Invalid API Client ${client.name}: ${FormatPrError(err as PrudenceError)}.`
+			);
+			continue;
+		}
+
+		// No replaceOne support in monk -- have to do this.
+		await db["api-clients"].remove({
+			clientID: client.clientID,
+		});
+
+		await db["api-clients"].insert(realClient);
+
+		logger.info(`Loaded/Modified new built-in client ${client.name}.`);
+	}
+}

server/src/lib/cdn/cdn.test.ts

@@ -0,0 +1,139 @@
+import { CDNDelete, CDNRedirect, CDNRetrieve, CDNStoreOrOverwrite } from "./cdn";
+import expressRequestMock from "express-request-mock";
+import { Environment, ServerConfig } from "lib/setup/config";
+import t from "tap";
+import fs from "fs";
+import path from "path";
+
+if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") {
+	throw new Error(`Tests must run against LOCAL_FILESYSTEM.`);
+}
+
+const CDN_FILE_ROOT = ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION;
+
+function getTestTxt() {
+	return fs.readFileSync(path.join(CDN_FILE_ROOT, "test.txt"), "utf-8");
+}
+
+const ResetFileRoot = () => {
+	if (Environment.nodeEnv !== "test") {
+		throw new Error(
+			`Not in test, yet CDN.test.ts was triggered, which could rm -rf something important.`
+		);
+	}
+
+	if (CDN_FILE_ROOT !== "./test-cdn") {
+		throw new Error(
+			`Unexpected CDN_FILE_ROOT of ${CDN_FILE_ROOT}. This is a security precaution, so that tests do not unexpectedly rm -rf important directories.`
+		);
+	}
+
+	// isn't this terrifyingly risky?
+	fs.rmSync(CDN_FILE_ROOT, { recursive: true, force: true });
+};
+
+t.test("#CDNRetrieve", (t) => {
+	t.beforeEach(ResetFileRoot);
+
+	t.test("Should retrieve the file at the given location.", async (t) => {
+		await CDNStoreOrOverwrite("test.txt", "1");
+
+		const data = await CDNRetrieve("test.txt");
+
+		t.equal(data.toString(), "1", "Should contain the contents of test.txt");
+
+		t.end();
+	});
+
+	t.test("Should throw if the file does not exist.", (t) => {
+		t.rejects(() => CDNRetrieve("fake-file.txt"));
+
+		t.end();
+	});
+
+	t.end();
+});
+
+t.test("#CDNStoreOrOverwrite", (t) => {
+	t.beforeEach(ResetFileRoot);
+
+	t.test("Should store a value.", async (t) => {
+		await CDNStoreOrOverwrite("test.txt", "hello world");
+
+		const data = fs.readFileSync(path.join(CDN_FILE_ROOT, "test.txt"), "utf8");
+
+		t.equal(data, "hello world", "Should store the data at the CDN_FILE_ROOT.");
+
+		t.end();
+	});
+
+	t.test("Should generate paths on the way to the file if they do not exist.", async (t) => {
+		await CDNStoreOrOverwrite("a/b/c/d/e/f/g.txt", "hello");
+
+		const data = fs.readFileSync(path.join(CDN_FILE_ROOT, "a/b/c/d/e/f/g.txt"), "utf8");
+
+		t.equal(data, "hello", "Should store the data at the deeply nested location.");
+
+		t.end();
+	});
+
+	t.test("Should store a file if one doesn't exist", async (t) => {
+		await CDNStoreOrOverwrite("test.txt", "1");
+
+		t.equal(getTestTxt(), "1");
+
+		t.end();
+	});
+
+	t.test("Should overwrite the file if it exists.", async (t) => {
+		await CDNStoreOrOverwrite("test.txt", "1");
+		t.equal(getTestTxt(), "1");
+
+		await CDNStoreOrOverwrite("test.txt", "2");
+		t.equal(getTestTxt(), "2");
+
+		t.end();
+	});
+
+	t.end();
+});
+
+t.test("#CDNDelete", (t) => {
+	t.beforeEach(ResetFileRoot);
+
+	t.test("Should delete the file at the given location.", async (t) => {
+		await CDNStoreOrOverwrite("test.txt", "1");
+
+		const data = getTestTxt();
+
+		t.equal(data, "1");
+
+		await CDNDelete("test.txt");
+
+		t.not(fs.existsSync(path.join(CDN_FILE_ROOT, "test.txt")), "File should not exist.");
+
+		t.end();
+	});
+
+	t.end();
+});
+
+t.test("#CDNRedirect", (t) => {
+	t.beforeEach(ResetFileRoot);
+
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	const mockMW = (req: any, res: any) => {
+		CDNRedirect(res, "/test.txt");
+	};
+
+	t.test("Should redirect a user to the CDN Url", async (t) => {
+		const { res } = await expressRequestMock(mockMW, {});
+
+		t.equal(res.statusCode, 302);
+		t.equal(res._getRedirectUrl(), "/cdn/test.txt");
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/cdn/cdn.ts

@@ -0,0 +1,93 @@
+import { DeleteFromS3, PushToS3 } from "./s3";
+import CreateLogCtx from "lib/logger/logger";
+import { ServerConfig } from "lib/setup/config";
+import mkdirp from "mkdirp";
+import fs from "fs";
+import path from "path";
+import { promisify } from "util";
+import type { Response } from "express";
+
+const readFilePromise = promisify(fs.readFile);
+const writeFilePromise = promisify(fs.writeFile);
+const rmFilePromise = promisify(fs.rm);
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * Joins a file location against the location of the CDN static store.
+ *
+ * @danger - This function should **NEVER** be called with unsanitised user input!
+ * Path directory traversal *is* possible, and *will* ruin your day.
+ */
+export function CDNFileSystemRoot(fileLoc: string) {
+	if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "LOCAL_FILESYSTEM") {
+		logger.severe(
+			`Attempted to run CDNFileSystemRoot, but was not using LOCAL_FILESYSTEM as a CDN.`,
+			{ fileLoc, conf: ServerConfig.CDN_CONFIG }
+		);
+
+		throw new Error(
+			`Attempted to run CDNFileSystemRoot, but was not using LOCAL_FILESYSTEM as a CDN.`
+		);
+	}
+
+	return path.join(ServerConfig.CDN_CONFIG.SAVE_LOCATION.LOCATION, fileLoc);
+}
+
+/**
+ * Retrieves the data of the file at the given CDN location.
+ *
+ * This is used for quick development setups, where a cdn server isn't available.
+ * As in, this ruins the purpose of a CDN! make sure you have one running.
+ */
+export function CDNRetrieve(fileLoc: string) {
+	logger.debug(`Retrieving path ${fileLoc} locally.`);
+
+	return readFilePromise(CDNFileSystemRoot(fileLoc));
+}
+
+/**
+ * Redirects the response to the CDN server at the given path.
+ */
+export function CDNRedirect(res: Response, fileLoc: string) {
+	if (!fileLoc.startsWith("/")) {
+		throw new Error(`Invalid fileLoc - did not start with /.`);
+	}
+
+	logger.debug(`CDN Redirecting to ${ServerConfig.CDN_CONFIG.WEB_LOCATION}${fileLoc}.`);
+
+	res.redirect(`${ServerConfig.CDN_CONFIG.WEB_LOCATION}${fileLoc}`);
+}
+
+/**
+ * Stores a file at fileLoc. If it already exists, overwrite it.
+ */
+export async function CDNStoreOrOverwrite(fileLoc: string, data: Buffer | string): Promise<void> {
+	logger.debug(`Storing or overwriting path ${fileLoc}.`);
+
+	if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM") {
+		const loc = CDNFileSystemRoot(fileLoc);
+
+		// make the parent folders if they dont exist. else, mkdirp is a no-op.
+		await mkdirp(path.dirname(loc));
+
+		return writeFilePromise(loc, data);
+	}
+
+	await PushToS3(fileLoc, data);
+}
+
+/**
+ * Removes a file at this CDN location.
+ */
+export async function CDNDelete(fileLoc: string) {
+	logger.verbose(`Deleting path ${fileLoc}.`);
+
+	if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "LOCAL_FILESYSTEM") {
+		const loc = CDNFileSystemRoot(fileLoc);
+
+		return rmFilePromise(loc, { force: true });
+	}
+
+	await DeleteFromS3(fileLoc);
+}

server/src/lib/cdn/s3.ts

@@ -0,0 +1,69 @@
+import { DeleteObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import CreateLogCtx from "lib/logger/logger";
+import { ServerConfig } from "lib/setup/config";
+
+const logger = CreateLogCtx("S3 Client");
+
+let s3: S3Client | null = null;
+
+if (ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE === "S3_BUCKET") {
+	const data = ServerConfig.CDN_CONFIG.SAVE_LOCATION;
+
+	logger.info(`Using S3_BUCKET as CDN location.`, { bootInfo: true });
+	s3 = new S3Client({
+		endpoint: data.ENDPOINT,
+		region:
+			data.REGION ?? "NO_REGION_BUT_YOU_HAVE_TO_SPECIFY_ONE_OTHERWISE_THIS_LIBRARY_ERRORS",
+		credentials: {
+			accessKeyId: data.ACCESS_KEY_ID,
+			secretAccessKey: data.SECRET_ACCESS_KEY,
+		},
+	});
+}
+
+/**
+ * Pushes a file to the configured S3 Bucket. Overwrites if already exists.
+ */
+export function PushToS3(path: string, content: Buffer | string) {
+	logger.debug(`Saving content on S3 at ${path}.`);
+
+	if (!s3 || ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "S3_BUCKET") {
+		logger.severe(
+			`Attempted to push to S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?`,
+			ServerConfig.CDN_CONFIG
+		);
+		throw new Error(
+			`Attempted to push to S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?`
+		);
+	}
+
+	return s3.send(
+		new PutObjectCommand({
+			Bucket: ServerConfig.CDN_CONFIG.SAVE_LOCATION.BUCKET,
+			Key: (ServerConfig.CDN_CONFIG.SAVE_LOCATION.KEY_PREFIX ?? "") + path,
+			Body: content,
+		})
+	);
+}
+
+/**
+ * Deletes the provided file from the configured S3 bucket.
+ */
+export function DeleteFromS3(path: string) {
+	if (!s3 || ServerConfig.CDN_CONFIG.SAVE_LOCATION.TYPE !== "S3_BUCKET") {
+		logger.severe(
+			`Attempted to delete from S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?`,
+			ServerConfig.CDN_CONFIG
+		);
+		throw new Error(
+			`Attempted to delete from S3, but CDN_CONFIG.SAVE_LOCATION.TYPE was not S3_BUCKET?`
+		);
+	}
+
+	return s3.send(
+		new DeleteObjectCommand({
+			Bucket: ServerConfig.CDN_CONFIG.SAVE_LOCATION.BUCKET,
+			Key: path,
+		})
+	);
+}

server/src/lib/cdn/url-format.ts

@@ -0,0 +1,13 @@
+import type { integer } from "tachi-common";
+
+export function GetUSCIRReplayURL(scoreID: string) {
+	return `/uscir/replays/${scoreID}`;
+}
+
+export function GetProfilePictureURL(userID: integer, contentHash: string) {
+	return `/users/${userID}/pfp-${contentHash}`;
+}
+
+export function GetProfileBannerURL(userID: integer, contentHash: string) {
+	return `/users/${userID}/banner-${contentHash}`;
+}

server/src/lib/constants/bms-tables.ts

@@ -0,0 +1,13 @@
+export const BMS_TABLES = {
+	insane: "★",
+	overjoy: "★★",
+	normal: "☆",
+	normal2: "▽",
+	insane2: "▼",
+	stella: "st",
+	satellite: "sl",
+	dpNormal: "δ",
+	dpInsane: "★",
+	scratch: "h◎",
+	ln: "◆",
+};

server/src/lib/constants/classes.ts

@@ -0,0 +1,243 @@
+export enum IIDXDans {
+	KYU_7 = 0,
+	KYU_6 = 1,
+	KYU_5 = 2,
+	KYU_4 = 3,
+	KYU_3 = 4,
+	KYU_2 = 5,
+	KYU_1 = 6,
+
+	DAN_1 = 7,
+	DAN_2 = 8,
+	DAN_3 = 9,
+	DAN_4 = 10,
+	DAN_5 = 11,
+	DAN_6 = 12,
+	DAN_7 = 13,
+	DAN_8 = 14,
+	DAN_9 = 15,
+	DAN_10 = 16,
+
+	CHUUDEN = 17,
+	KAIDEN = 18,
+}
+
+export enum GitadoraColours {
+	WHITE = 0,
+
+	ORANGE = 1,
+	ORANGE_GRADIENT = 2,
+	YELLOW = 3,
+	YELLOW_GRADIENT = 4,
+	GREEN = 5,
+	GREEN_GRADIENT = 6,
+	BLUE = 7,
+	BLUE_GRADIENT = 8,
+	PURPLE = 9,
+	PURPLE_GRADIENT = 10,
+	RED = 11,
+	RED_GRADIENT = 12,
+
+	BRONZE = 13,
+	SILVER = 14,
+	GOLD = 15,
+	RAINBOW = 16,
+}
+
+export enum BMSGenocideDans {
+	NORMAL_1 = 0,
+	NORMAL_2 = 1,
+	NORMAL_3 = 2,
+	NORMAL_4 = 3,
+	NORMAL_5 = 4,
+	NORMAL_6 = 5,
+	NORMAL_7 = 6,
+	NORMAL_8 = 7,
+	NORMAL_9 = 8,
+	NORMAL_10 = 9,
+
+	INSANE_1 = 10,
+	INSANE_2 = 11,
+	INSANE_3 = 12,
+	INSANE_4 = 13,
+	INSANE_5 = 14,
+	INSANE_6 = 15,
+	INSANE_7 = 16,
+	INSANE_8 = 17,
+	INSANE_9 = 18,
+	INSANE_10 = 19,
+	INSANE_KAIDEN = 20,
+
+	OVERJOY = 21,
+}
+
+export enum BMSStSlDans {
+	SL_0 = 0,
+	SL_1 = 1,
+	SL_2 = 2,
+	SL_3 = 3,
+	SL_4 = 4,
+	SL_5 = 5,
+	SL_6 = 6,
+	SL_7 = 7,
+	SL_8 = 8,
+	SL_9 = 9,
+	SL_10 = 10,
+	SL_11 = 11,
+	SL_12 = 12,
+
+	ST_0 = 13,
+	ST_1 = 14,
+	ST_2 = 15,
+	ST_3 = 16,
+	ST_4 = 17,
+	ST_5 = 18,
+	ST_6 = 19,
+	ST_7 = 20,
+	ST_8 = 21,
+	ST_9 = 22,
+	ST_10 = 23,
+	ST_11 = 24,
+	ST_12 = 25,
+}
+
+export enum PMSInsaneDans {
+	DAN_1 = 0,
+	DAN_2 = 1,
+	DAN_3 = 2,
+	DAN_4 = 3,
+	DAN_5 = 4,
+	DAN_6 = 5,
+	DAN_7 = 6,
+	DAN_8 = 7,
+	DAN_9 = 8,
+	DAN_10 = 9,
+	KAIDEN = 10,
+	OVERJOY = 11,
+	UNDEFINED = 12,
+}
+
+export enum SDVXDans {
+	DAN_1 = 0,
+	DAN_2 = 1,
+	DAN_3 = 2,
+	DAN_4 = 3,
+	DAN_5 = 4,
+	DAN_6 = 5,
+	DAN_7 = 6,
+	DAN_8 = 7,
+	DAN_9 = 8,
+	DAN_10 = 9,
+	DAN_11 = 10,
+	INF = 11,
+}
+
+// ewww
+export enum SDVXVFClasses {
+	SIENNA_I = 0,
+	SIENNA_II = 1,
+	SIENNA_III = 2,
+	SIENNA_IV = 3,
+	COBALT_I = 4,
+	COBALT_II = 5,
+	COBALT_III = 6,
+	COBALT_IV = 7,
+	DANDELION_I = 8,
+	DANDELION_II = 9,
+	DANDELION_III = 10,
+	DANDELION_IV = 11,
+	CYAN_I = 12,
+	CYAN_II = 13,
+	CYAN_III = 14,
+	CYAN_IV = 15,
+	SCARLET_I = 16,
+	SCARLET_II = 17,
+	SCARLET_III = 18,
+	SCARLET_IV = 19,
+	CORAL_I = 20,
+	CORAL_II = 21,
+	CORAL_III = 22,
+	CORAL_IV = 23,
+	ARGENTO_I = 24,
+	ARGENTO_II = 25,
+	ARGENTO_III = 26,
+	ARGENTO_IV = 27,
+	ELDORA_I = 28,
+	ELDORA_II = 29,
+	ELDORA_III = 30,
+	ELDORA_IV = 31,
+	CRIMSON_I = 32,
+	CRIMSON_II = 33,
+	CRIMSON_III = 34,
+	CRIMSON_IV = 35,
+	IMPERIAL_I = 36,
+	IMPERIAL_II = 37,
+	IMPERIAL_III = 38,
+	IMPERIAL_IV = 39,
+}
+
+export enum CHUNITHM_COLOURS {
+	BLUE = 0,
+	GREEN = 1,
+	ORANGE = 2,
+	RED = 3,
+	PURPLE = 4,
+	BRONZE = 5,
+	SILVER = 6,
+	GOLD = 7,
+	PLATINUM = 8,
+	RAINBOW = 9,
+}
+
+export enum WACCA_STAGEUPS {
+	I = 0,
+	II = 1,
+	III = 2,
+	IV = 3,
+	V = 4,
+	VI = 5,
+	VII = 6,
+	VIII = 7,
+	IX = 8,
+	X = 9,
+	XI = 10,
+	XII = 11,
+	XIII = 12,
+	XIV = 13,
+}
+
+export enum WACCA_COLOURS {
+	ASH = 0,
+	NAVY = 1,
+	YELLOW = 2,
+	RED = 3,
+	PURPLE = 4,
+	BLUE = 5,
+	SILVER = 6,
+	GOLD = 7,
+	RAINBOW = 8,
+}
+
+export enum POPN_CLASSES {
+	KITTY = 0,
+	GRADE_SCHOOL = 1,
+	DELINQUENT = 2,
+	DETECTIVE = 3,
+	IDOL = 4,
+	GENERAL = 5,
+	HERMIT = 6,
+	GOD = 7,
+}
+
+export enum JUBEAT_COLOURS {
+	BLACK = 0,
+	YELLOW_GREEN = 1,
+	GREEN = 2,
+	LIGHT_BLUE = 3,
+	BLUE = 4,
+	VIOLET = 5,
+	PURPLE = 6,
+	PINK = 7,
+	ORANGE = 8,
+	GOLD = 9,
+}

server/src/lib/constants/ea3id.ts

@@ -0,0 +1,14 @@
+// Various EA3-SoftID constants to make code more readable.
+export const MODEL_INFINITAS_2 = "P2D";
+export const MODEL_IIDX = "LDJ";
+export const MODEL_IIDX_LIGHTNING = "TDJ";
+
+export const REV_2DXTRA = "E";
+export const REV_2DXBMS = "Z";
+export const REV_OMNIMIX = "X";
+export const REV_NORMAL = "A";
+
+export const EXT_HEROIC_VERSE = "2020092900";
+export const EXT_BISTROVER = "2021091500";
+
+export const MODEL_SDVX3_KONASTE = "QCV";

server/src/lib/constants/err-codes.ts

@@ -0,0 +1,10 @@
+export enum SubscribeFailReasons {
+	ALREADY_SUBSCRIBED = 0,
+	ALREADY_ACHIEVED = 1,
+}
+
+export enum SetRivalsFailReasons {
+	TOO_MANY = 0,
+	RIVALED_SELF = 1,
+	RIVALS_HAVENT_PLAYED_GPT = 2,
+}

server/src/lib/constants/filesize.ts

@@ -0,0 +1,2 @@
+export const ONE_MEGABYTE = 1024 * 1024;
+export const SIXTEEN_MEGABTYES = ONE_MEGABYTE * 16;

server/src/lib/constants/tachi.ts

@@ -0,0 +1,3 @@
+export const SYMBOL_TACHI_DATA = Symbol("tachi-request-patch");
+export const SYMBOL_TACHI_API_AUTH = Symbol("tachi-api-auth-info");
+export const JOB_RETRY_COUNT = 7;

server/src/lib/constants/time.ts

@@ -0,0 +1,7 @@
+export const ONE_SECOND = 1000;
+export const ONE_MINUTE = ONE_SECOND * 60;
+export const ONE_HOUR = ONE_MINUTE * 60;
+export const ONE_DAY = ONE_HOUR * 24;
+export const ONE_WEEK = ONE_DAY * 7;
+export const ONE_MONTH = ONE_WEEK * 4;
+export const ONE_YEAR = ONE_MONTH * 12;

server/src/lib/constants/usc-ir.ts

@@ -0,0 +1,10 @@
+// we'll go for 3, why not.
+export const USCIR_ADJACENT_SCORE_N = 3;
+
+export const USCIR_MAX_LEADERBOARD_N = 10;
+
+export const USC_DEFAULT_PERFECT = 46;
+export const USC_DEFAULT_NEAR = 150;
+export const USC_DEFAULT_HOLD = 150;
+export const USC_DEFAULT_MISS = 300;
+export const USC_DEFAULT_SLAM = 84;

server/src/lib/constants/version.ts

@@ -0,0 +1,87 @@
+// Note that Version Info is intentionally identical between all variants of tachi
+// viz. btchi and ktchi share versioning. This is because they
+// essentially share codebases.
+import CreateLogCtx from "lib/logger/logger";
+import semver from "semver";
+import { execSync } from "child_process";
+import fs from "fs";
+import path from "path";
+
+// As is with all front-facing zkldi projects, the version names for tachi-server
+// are from an album I like. In this case, the al(bum is Portishead - Dummy.
+// The fact that this album is very similar to Mezzanine
+// - the album used for Kamaitachi v1 - is deliberate. :) - zkldi
+const PORTISHEAD_DUMMY = [
+	// v2.0.0
+	"Mysterons",
+
+	// v2.1.0
+	"Sour Times",
+
+	// v2.2.0
+	"Strangers",
+
+	// v2.3.0
+	"It Could Be Sweet",
+
+	// v2.4.0
+	"Wandering Star",
+
+	// v2.5.0
+	"It's a Fire",
+
+	// v2.6.0
+	"Numb",
+
+	// v2.7.0
+	"Roads",
+
+	// v2.8.0
+	"Pedestal",
+
+	// v2.9.0
+	"Biscuit",
+
+	// v2.10.0
+	"Glory Box",
+
+	// v2.11+ onwards are not expected to occur, but if they do, we'll figure something out.
+];
+
+const logger = CreateLogCtx(__filename);
+
+const packageJSON = JSON.parse(
+	fs.readFileSync(path.join(__dirname, "../../../package.json"), "utf-8")
+) as { version: string };
+
+const semverInfo = semver.parse(packageJSON.version);
+
+if (!semverInfo) {
+	logger.warn(
+		`Couldn't parse semverInfo out of package.json? Got a string of '${packageJSON.version}', but couldn't parse. Defaulting to v0.0.0-unknown.`
+	);
+}
+
+let commit: string | null = null;
+
+try {
+	// This fetches the HEAD of our current running branch. This is useful for debugging.
+	// Note that git returns this with a trailing newline, so we have to trim that off.
+	commit = execSync("git rev-parse --short HEAD").toString("utf-8").trim();
+} catch (err) {
+	logger.warn(`Failed to read what commit we're on -- Using null as our commit version.`, {
+		err,
+	});
+}
+
+export const VERSION_INFO = {
+	major: semverInfo?.major ?? 0,
+	minor: semverInfo?.minor ?? 0,
+	patch: semverInfo?.patch ?? 0,
+	commit,
+	name: PORTISHEAD_DUMMY[semverInfo?.minor ?? 0] ?? "No Version Name",
+};
+
+export const VERSION_STR = semverInfo?.raw ?? "0.0.0-unknown";
+
+export const VERSION_PRETTY = `v${VERSION_STR} (${VERSION_INFO.name}) [${commit}]`;

server/src/lib/database-seeds/repo.ts

@@ -0,0 +1,205 @@
+import CreateLogCtx from "lib/logger/logger";
+import { Environment, ServerConfig } from "lib/setup/config";
+import { asyncExec } from "utils/misc";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import type { Game } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+export type SeedsCollections =
+	| "bms-course-lookup"
+	| "folders"
+	| "tables"
+	| `charts-${Game}`
+	| `songs-${Game}`;
+
+/**
+ * Class that encapsulates the behaviour of a seeds repo.
+ */
+export class DatabaseSeedsRepo {
+	private readonly baseDir: string;
+	private readonly logger;
+
+	constructor(baseDir: string) {
+		this.baseDir = baseDir;
+		this.logger = CreateLogCtx(`DatabaseSeeds:${baseDir}`);
+	}
+
+	private CollectionNameToPath(collectionName: SeedsCollections) {
+		return path.join(this.baseDir, "collections", `${collectionName}.json`);
+	}
+
+	/**
+	 * Reads the data from a collection and returns the parsed JSON.
+	 *
+	 * @returns The data in the requested collection.
+	 */
+	async ReadCollection<D>(collectionName: SeedsCollections): Promise<Array<D>> {
+		const data = await fs.readFile(this.CollectionNameToPath(collectionName), {
+			encoding: "utf-8",
+		});
+
+		const parsedData = JSON.parse(data) as Array<D>;
+
+		return parsedData;
+	}
+
+	/**
+	 * Writes a new array to the provided collectionName.
+	 *
+	 * @param collectionName - The collection to write to.
+	 * @param content - A new array of objects to write.
+	 */
+	async WriteCollection(collectionName: SeedsCollections, content: Array<unknown>) {
+		await fs.writeFile(this.CollectionNameToPath(collectionName), JSON.stringify(content));
+
+		// Deterministically sort whatever content we just wrote.
+		await asyncExec(
+			`cd "${this.baseDir}" || exit 1; node scripts/deterministic-collection-sort.js`
+		);
+	}
+
+	async *IterateCollections() {
+		const collectionNames = (await fs.readdir(path.join(this.baseDir, "collections"))).map(
+			(e) => path.parse(e).name
+		) as Array<SeedsCollections>;
+
+		for (const collectionName of collectionNames) {
+			// eslint-disable-next-line no-await-in-loop
+			yield { collectionName, data: await this.ReadCollection(collectionName) };
+		}
+	}
+
+	/**
+	 * Mutate a collection with a given name.
+	 *
+	 * @param collectionName - The collection to mutate.
+	 * @param mutator - A function that takes the entire collection as an array, then returns a new array.
+	 */
+	async MutateCollection<D>(
+		collectionName: SeedsCollections,
+		mutator: (dataset: Array<D>) => Array<D>
+	) {
+		const dataset = await this.ReadCollection<D>(collectionName);
+
+		const newData = mutator(dataset);
+
+		return this.WriteCollection(collectionName, newData);
+	}
+
+	/**
+	 * Provide authentication so that CommitChangesBack can do its job.
+	 */
+	#AuthenticateWithGitServer() {
+		if (!ServerConfig.SEEDS_CONFIG) {
+			// Shouldn't be possible. Ever, since SEEDS_CONFIG must be deffed in order
+			// to run PullDBSeeds
+			throw new Error(`Cannot commit changes back. SEEDS_CONFIG is not set.`);
+		}
+
+		if (!ServerConfig.SEEDS_CONFIG.USER_NAME || !ServerConfig.SEEDS_CONFIG.USER_EMAIL) {
+			throw new Error(
+				`Cannot commit changes back if SEEDS_CONFIG.USER_NAME/SEEDS_CONFIG.USER_EMAIL aren't defined.`
+			);
+		}
+
+		// @ereti is insistent that this sleep 1 is fine, so, whatever.
+		return asyncExec(
+			`git config user.name "${ServerConfig.SEEDS_CONFIG.USER_NAME}" || exit 3;
+			git config user.email "${ServerConfig.SEEDS_CONFIG.USER_EMAIL}" || exit 4;
+			git config credential.helper '!f() { sleep 1; echo "username=\${GIT_USER}"; echo "password=\${GIT_PASSWORD}"; }; f' || exit 5;`
+		);
+	}
+
+	/**
+	 * Checks for any diffs in the seeds repository we cloned. If there are any, commit them back
+	 * to the repository.
+	 *
+	 * @param commitMsg - The commit message.
+	 * @returns True when a commit has occured, false when it hasn't. Throws on failure.
+	 */
+	async CommitChangesBack(commitMsg: string) {
+		this.logger.verbose(`Received commit-back request.`);
+
+		try {
+			const { stdout: statusOut } = await asyncExec(
+				`cd "${this.baseDir}" || exit 1; git status --porcelain`
+			);
+
+			if (statusOut === "") {
+				this.logger.info(`No changes. Not committing any changes back.`);
+				return false;
+			}
+
+			// Ok, Testing this is actually a bad idea. Hear me out.
+			// It's exceptionally difficult to actually store and look at the test output. (potentially huge)
+			// especially when filesystem size is our biggest constraint at the moment.
+			//
+			// It's better for us to commit straight away, and have the tests on our github CI fail
+			// (and subsequently yell at us.)
+			//
+			// try {
+			// 	await asyncExec(`cd "${this.baseDir}/scripts" || exit 2; pnpm install; pnpm test`);
+			// } catch ({ err, stdout, stderr }) {
+			// 	logger.error(`Testing the changes failed. ${err}. Not committing back!`, { err });
+			// 	throw err;
+			// }
+
+			this.logger.info(`Changes detected. Authenticating with Github.`);
+
+			await this.#AuthenticateWithGitServer();
+
+			const { stdout: commitOut } = await asyncExec(
+				`cd "${this.baseDir}" || exit 2;
+				git add . || exit 3;
+				git commit -am "${commitMsg}" || exit 4;
+				git push`
+			);
+
+			this.logger.info(`Commit: ${commitOut}.`);
+
+			return true;
+		} catch (err) {
+			this.logger.error(`Failed to backport commits?`, { err });
+			throw err;
+		}
+	}
+}
+
+/**
+ * Pulls the database seeds from github, returns an object that can be used to manipulate them.
+ */
+export async function PullDatabaseSeeds() {
+	if (!ServerConfig.SEEDS_CONFIG) {
+		throw new Error(`SEEDS_CONFIG was not defined. You cannot pull a seeds repo.`);
+	}
+
+	const seedsDir = await fs.mkdtemp(path.join(os.tmpdir(), "tachi-database-seeds-"));
+
+	logger.info(`Cloning data to ${seedsDir}.`);
+
+	await fs.rm(seedsDir, { recursive: true, force: true });
+
+	try {
+		// stderr in git clone is normal output.
+		// stdout is for errors.
+		// there were expletives below this comment, but I have removed them.
+		const { stdout } = await asyncExec(
+			`git clone "${ServerConfig.SEEDS_CONFIG.REPO_URL}" -b "${
+				Environment.nodeEnv === "production" ? "production" : "develop"
+			}" --depth=1 '${seedsDir}'`
+		);
+
+		// isn't that confusing
+		if (stdout) {
+			logger.error(stdout);
+		}
+
+		return new DatabaseSeedsRepo(seedsDir);
+	} catch ({ err, stderr }) {
+		logger.error(`Error cloning database-seeds. ${stderr}.`);
+		throw err;
+	}
+}

server/src/lib/email/client.ts

@@ -0,0 +1,79 @@
+import bunyan from "bunyan";
+import CreateLogCtx from "lib/logger/logger";
+import { Environment, ServerConfig } from "lib/setup/config";
+import nodemailer from "nodemailer";
+import type { SentMessageInfo, Transporter } from "nodemailer";
+
+const logger = CreateLogCtx(__filename);
+
+let transporter: Transporter | undefined;
+
+if (ServerConfig.EMAIL_CONFIG) {
+	logger.info(`Connecting to email server...`, { bootInfo: true });
+	const conf = ServerConfig.EMAIL_CONFIG;
+
+	try {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+		transporter = nodemailer.createTransport({
+			newline: "unix",
+			// eslint-disable-next-line @typescript-eslint/strict-boolean-expressions, @typescript-eslint/no-unsafe-member-access
+			logger: conf.TRANSPORT_OPS?.debug
+				? bunyan.createLogger({ name: "Email Logger" })
+				: undefined,
+			...(conf.TRANSPORT_OPS ?? {}),
+		});
+
+		transporter.verify((err) => {
+			if (err) {
+				logger.crit(`Could not connect to email server.`, { err });
+				throw err;
+			} else {
+				logger.info(`Successfully connected to email server.`, { bootInfo: true });
+			}
+		});
+	} catch (err) {
+		logger.crit(`Failed to create email client.`, { err });
+		throw err;
+	}
+} else {
+	logger.warn(`No EMAIL_CONFIG present in conf, emails will not be sent from the server.`, {
+		bootInfo: true,
+	});
+}
+
+export function SendEmail(
+	to: string,
+	subject: string,
+	htmlContent: string,
+	textContent: string
+): Promise<SentMessageInfo> | undefined {
+	if (Environment.nodeEnv === "test") {
+		logger.debug(`Stubbed out SendEmail as env was test.`);
+		return;
+	}
+
+	if (!transporter || !ServerConfig.EMAIL_CONFIG) {
+		logger.debug(`Stubbed out SendEmail as no EMAIL_CONFIG was set.`);
+		return;
+	}
+
+	logger.verbose(`Sending email to ${to}.`);
+
+	return transporter
+		.sendMail({
+			from: ServerConfig.EMAIL_CONFIG.FROM,
+			to,
+			subject,
+			html: htmlContent,
+			text: textContent,
+			dkim: ServerConfig.EMAIL_CONFIG.DKIM,
+		})
+		.catch((err: unknown) => {
+			logger.info(`Failed to send email to ${to}.`, {
+				err,
+				subject,
+				textContent,
+				htmlContent,
+			});
+		});
+}

server/src/lib/email/formats.ts

@@ -0,0 +1,45 @@
+// Email HTML is a hellish mess of IE5 era nonsense.
+// Good luck.
+
+import { ServerConfig, TachiConfig } from "lib/setup/config";
+
+export function EmailFormatResetPassword(username: string, resetCode: string, ipAddr: string) {
+	return {
+		html: MainHTMLWrapper(
+			`Hey ${username}, you've received a password reset request.<br/><a href="${ServerConfig.OUR_URL}/reset-password?code=${resetCode}">Click here</a> to perform the reset.<br/>If you did not request this reset, report this! This reset request was made by ${ipAddr}.`
+		),
+		text: `Hey ${username}, you've received a password reset request. This link
+${ServerConfig.OUR_URL}/reset-password?code=${resetCode} will perform the reset.
+If you did not request this reset, report this! This reset request was made by ${ipAddr}.`,
+	};
+}
+
+export function EmailFormatVerifyEmail(username: string, code: string) {
+	return {
+		html: MainHTMLWrapper(
+			`Hey ${username}, You need to verify your email before you can use the site.<br/><a href="${ServerConfig.OUR_URL}/verify-email?code=${code}">Click here</a> to verify your email.`
+		),
+		text: `Hey ${username}, You need to verify your email before you can use the site.
+Please go to ${ServerConfig.OUR_URL}/verify-email?code=${code} to verify!`,
+	};
+}
+
+export function MainHTMLWrapper(innerHTML: string) {
+	return `<!DOCTYPE html>
+	<html lang="en">
+	<head>
+		<meta charset="UTF-8">
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<meta http-equiv="Content-Type" content="text/html charset=UTF-8" />
+		<title>${TachiConfig.NAME}</title>
+	</head>
+	<body>
+		<img src="${ServerConfig.CDN_CONFIG.WEB_LOCATION}/logos/logo-wordmark.png" />
+		<hr />
+		<div style="text-align: left;">
+		${innerHTML}
+		</div>
+	</body>
+	</html>`;
+}

server/src/lib/game-settings/create-game-settings.test.ts

@@ -0,0 +1,32 @@
+import { CreateGameSettings } from "./create-game-settings";
+import db from "external/mongo/db";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+
+t.test("#CreateGameSettings", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.test("Should create a new user's settings.", async (t) => {
+		await CreateGameSettings(1, "bms", "7K");
+
+		const data = await db["game-settings"].findOne({
+			userID: 1,
+			game: "bms",
+			playtype: "7K",
+		});
+
+		t.not(data, null);
+
+		t.end();
+	});
+
+	t.test("Should throw an error if the user already has game-settings.", async (t) => {
+		await CreateGameSettings(1, "bms", "7K");
+
+		t.rejects(() => CreateGameSettings(1, "bms", "7K"));
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/game-settings/create-game-settings.ts

@@ -0,0 +1,53 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import type { Game, integer, Playtype } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * Create GameSettings for a UGPT (which contains their preferences).
+ */
+export async function CreateGameSettings(userID: integer, game: Game, playtype: Playtype) {
+	const exists = await db["game-settings"].findOne({
+		userID,
+		game,
+		playtype,
+	});
+
+	if (exists) {
+		logger.error(
+			`Cannot create ${userID} ${game} ${playtype} game-settings as one already exists?`
+		);
+
+		throw new Error(
+			`Cannot create ${userID} ${game} ${playtype} game-settings as one already exists?`
+		);
+	}
+
+	let gameSpecific = {};
+
+	if (game === "iidx") {
+		gameSpecific = {
+			display2DXTra: false,
+			bpiTarget: 0,
+		};
+	}
+
+	await db["game-settings"].insert({
+		userID,
+		game,
+		playtype,
+		preferences: {
+			preferredProfileAlg: null,
+			preferredSessionAlg: null,
+			preferredScoreAlg: null,
+			scoreBucket: null,
+			defaultTable: null,
+			stats: [],
+			gameSpecific,
+		},
+		rivals: [],
+	});
+
+	logger.info(`Created game settings for ${userID} (${game} ${playtype}).`);
+}

server/src/lib/handlers/sigterm.ts

@@ -0,0 +1,33 @@
+import { monkDB } from "external/mongo/db";
+import { CloseRedisConnection } from "external/redis/redis";
+import { rootLogger } from "lib/logger/logger";
+import { CloseScoreImportQueue } from "lib/score-import/worker/queue";
+import type http from "http";
+import type https from "https";
+
+const logger = rootLogger;
+
+export function HandleSIGTERMGracefully(instance?: http.Server | https.Server) {
+	logger.info("SIGTERM Received, closing program.", { shutdownInfo: true });
+
+	if (instance) {
+		instance.close(() => CloseEverythingElse());
+	} else {
+		return CloseEverythingElse();
+	}
+}
+
+async function CloseEverythingElse() {
+	logger.info("Closing Mongo Database.", { shutdownInfo: true });
+	await monkDB.close();
+
+	logger.info("Closing Redis Connection.", { shutdownInfo: true });
+	await CloseRedisConnection();
+
+	logger.info("Closing Score Import Queue.", { shutdownInfo: true });
+	await CloseScoreImportQueue();
+
+	logger.info("Everything closed. Waiting for process to exit naturally.", {
+		shutdownInfo: true,
+	});
+}

server/src/lib/imports/imports.test.ts

@@ -0,0 +1,59 @@
+import { RevertImport } from "./imports";
+import db from "external/mongo/db";
+import t from "tap";
+import { dmf, mkFakeImport } from "test-utils/misc";
+import ResetDBState from "test-utils/resets";
+import { TestingIIDXSPScore } from "test-utils/test-data";
+
+t.test("#RevertImport", (t) => {
+	t.beforeEach(ResetDBState);
+	t.beforeEach(() =>
+		db.scores.insert([
+			dmf(TestingIIDXSPScore, { scoreID: "score_1" }),
+			dmf(TestingIIDXSPScore, { scoreID: "score_2" }),
+			dmf(TestingIIDXSPScore, { scoreID: "score_3" }),
+		])
+	);
+
+	t.test("Should revert an import and all of its contained scores.", async (t) => {
+		const importDoc = mkFakeImport({
+			scoreIDs: ["score_1", "score_2"],
+		});
+
+		await db.imports.insert(importDoc);
+
+		await RevertImport(importDoc);
+
+		const dbRes = await db.imports.findOne({ importID: "fake_import" });
+
+		t.equal(dbRes, null, "Should have removed the import from the DB.");
+
+		t.resolveMatch(
+			db.scores.findOne({ userID: 1, scoreID: "score_1" }),
+
+			// @ts-expect-error https://github.com/DefinitelyTyped/DefinitelyTyped/pull/60020
+			null,
+			"Score_1 should be removed from the database."
+		);
+
+		t.resolveMatch(
+			db.scores.findOne({ userID: 1, scoreID: "score_2" }),
+
+			// @ts-expect-error see above
+			null,
+			"Score_2 should be removed from the database."
+		);
+
+		t.resolveMatch(
+			db.scores.findOne({ userID: 1, scoreID: "score_3" }),
+
+			// @ts-expect-error see above
+			{ scoreID: "score_3" },
+			"Score_2 should NOT be removed from the database."
+		);
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/imports/imports.ts

@@ -0,0 +1,44 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { DeleteMultipleScores } from "lib/score-mutation/delete-scores";
+import type { ImportDocument } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * Given an importDocument, undo it. This will remove all of the scores inside the import.
+ *
+ * It will *not* undo things like classes that were set, but it will invoke a profile recalculation.
+ *
+ * If this results in sessions being deleted, it will delete them.
+ */
+export async function RevertImport(importDoc: ImportDocument) {
+	logger.info(`Received revert-import request for import '${importDoc.importID}'`, { importDoc });
+
+	const scores = await GetImportScores(importDoc);
+
+	await DeleteMultipleScores(scores);
+
+	logger.info(
+		`Deleted ${scores.length} scores as part of reverting import '${importDoc.importID}'.`,
+		{ importDoc }
+	);
+
+	try {
+		await db.imports.remove({ importID: importDoc.importID });
+
+		logger.info(`Reverted and deleted import '${importDoc.importID}'.`);
+	} catch (err) {
+		logger.severe(
+			`Deleted scores that were part of import, but failed to remove the actual import? There is a stale import with ID '${importDoc.importID}', which must be removed manually.`,
+			{ importDoc, err }
+		);
+	}
+}
+
+/**
+ * Retrieve the scores inside this import.
+ */
+export function GetImportScores(importDoc: ImportDocument) {
+	return db.scores.find({ scoreID: { $in: importDoc.scoreIDs } });
+}

server/src/lib/invites/invites.ts

@@ -0,0 +1,38 @@
+import { ONE_MONTH } from "lib/constants/time";
+import CreateLogCtx from "lib/logger/logger";
+import { ServerConfig } from "lib/setup/config";
+import type { PublicUserDocument } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * Users are only allowed to invite so many users, and their invites are
+ * trickled out in bursts of INVITE_BATCH_SIZE.
+ *
+ * Users get those N additional invites every month since they join.
+ * This is capped at INVITE_CAP, which defaults to 100.
+ */
+export function GetTotalAllowedInvites(user: PublicUserDocument) {
+	if (!ServerConfig.INVITE_CODE_CONFIG) {
+		logger.warn(
+			`No INVITE_CODE_CONFIG set, but tried to get total allowed invites? Returning 0.`
+		);
+		return 0;
+	}
+
+	const joinedSince = Date.now() - user.joinDate;
+
+	const monthsSinceJoin = Math.floor(joinedSince / ONE_MONTH);
+
+	let invites = monthsSinceJoin * ServerConfig.INVITE_CODE_CONFIG.BATCH_SIZE;
+
+	if (user.badges.includes("alpha") || user.badges.includes("beta")) {
+		invites = invites + ServerConfig.INVITE_CODE_CONFIG.BETA_USER_BONUS;
+	}
+
+	if (invites > ServerConfig.INVITE_CODE_CONFIG.INVITE_CAP) {
+		return ServerConfig.INVITE_CODE_CONFIG.INVITE_CAP;
+	}
+
+	return invites;
+}

server/src/lib/jobs/backsync-bms-pms-data.ts

@@ -0,0 +1,52 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
+import CreateLogCtx from "lib/logger/logger";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * The tachi-server may have its BMS or PMS database update. It needs to sync this
+ * information back with the seeds.
+ */
+export async function BacksyncBMSPMSSongsAndCharts() {
+	const repo = await PullDatabaseSeeds();
+
+	for (const game of ["bms", "pms"] as const) {
+		logger.info(`Fetching BMS songs from DB.`);
+
+		// did you know, this code is liable to blow up in my face and OOM one day?
+		let songs = await db.songs[game].find({});
+
+		logger.info(`Found ${songs.length} ${game} songs.`);
+
+		await repo.WriteCollection(`songs-${game}`, songs);
+
+		// @ts-expect-error This is obviously making something nullable when it shouldn't be.
+		// but if we don't *force* node to free this damn memory, it kills itself when it
+		// tries to read even more stuff.
+		songs = null;
+
+		logger.info(`Fetching ${game} charts from DB.`);
+		let charts = await db.charts.bms.find({});
+
+		logger.info(`Found ${charts.length} bms charts.`);
+
+		await repo.WriteCollection(`charts-${game}`, charts);
+
+		// @ts-expect-error See previous expect-error.
+		charts = null;
+	}
+
+	await repo.CommitChangesBack(`Backsync BMS+PMS Songs/Charts ${new Date().toISOString()}`);
+}
+
+if (require.main === module) {
+	BacksyncBMSPMSSongsAndCharts()
+		.then(() => process.exit(0))
+		.catch((err: unknown) => {
+			logger.error(`Failed to backsync bms/pms songs and charts.`, { err }, () => {
+				process.exit(1);
+			});
+		});
+}

server/src/lib/jobs/bms-ai-table-sync.ts

@@ -0,0 +1,40 @@
+/* eslint-disable no-await-in-loop */
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
+import CreateLogCtx from "lib/logger/logger";
+import fetch from "node-fetch";
+import type { ChartDocument } from "tachi-common";
+
+const AI_URL = "https://bms.hexlataia.xyz/tables/json/ai.json";
+
+const logger = CreateLogCtx(__filename);
+
+interface AITableEntry {
+	level: string;
+	md5: string;
+}
+
+export async function UpdateAILevels() {
+	const data = (await fetch(AI_URL).then((r) => r.json())) as Array<AITableEntry>;
+
+	const map = Object.fromEntries(data.map((e) => [e.md5, e.level]));
+
+	const repo = await PullDatabaseSeeds();
+
+	await repo.MutateCollection<ChartDocument<"bms:7K">>("charts-bms", (charts) => {
+		for (const chart of charts) {
+			chart.data.aiRating = map[chart.data.hashMD5] ?? null;
+		}
+
+		return charts;
+	});
+}
+
+if (require.main === module) {
+	UpdateAILevels()
+		.then(() => process.exit(0))
+		.catch((err: unknown) => {
+			logger.error(`Failed to sync BMS AI Table.`, { err }, () => {
+				process.exit(1);
+			});
+		});
+}

server/src/lib/jobs/bms-table-sync.ts

@@ -0,0 +1,317 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import { BMS_TABLES } from "lib/constants/bms-tables";
+import CreateLogCtx from "lib/logger/logger";
+import fetch from "node-fetch";
+import { CreateFolderID, InitaliseFolderChartLookup } from "utils/folder";
+import { FormatBMSTables } from "utils/misc";
+import type { ChartDocument } from "tachi-common";
+
+// I have no confidence in half of these links surviving.
+// Eventually, some of these are going to disappear, and we'll have to find
+// something else. probably.
+const registeredTables: Array<BMSTablesDataset> = [
+	{
+		name: "Insane",
+		humanisedPrefix: "insane",
+		playtype: "7K",
+		prefix: BMS_TABLES.insane,
+		description: "The 7K GENOSIDE insane table.",
+		url: "http://www.ribbit.xyz/bms/tables/insane_body.json",
+	},
+	{
+		name: "Normal",
+		humanisedPrefix: "normal",
+		playtype: "7K",
+		prefix: BMS_TABLES.normal,
+		description: "The 7K GENOSIDE normal table.",
+		url: "http://www.ribbit.xyz/bms/tables/normal_body.json",
+	},
+	{
+		name: "Stella",
+		humanisedPrefix: "st",
+		playtype: "7K",
+		prefix: BMS_TABLES.stella,
+		description: "The stella table, from Stellaverse.",
+		url: "https://stellabms.xyz/st/score.json",
+	},
+	{
+		name: "Satellite",
+		humanisedPrefix: "sl",
+		playtype: "7K",
+		prefix: BMS_TABLES.satellite,
+		description: "The satellite table, from Stellaverse.",
+		url: "https://stellabms.xyz/sl/score.json",
+	},
+	{
+		name: "Insane2",
+		humanisedPrefix: "insane2",
+		playtype: "7K",
+		prefix: BMS_TABLES.insane2,
+		description: "A successor to the original insane table, but is generally less consistent.",
+		url: "http://rattoto10.jounin.jp/js/insane_data.json",
+	},
+	{
+		name: "Normal2",
+		humanisedPrefix: "normal2",
+		playtype: "7K",
+		prefix: BMS_TABLES.normal2,
+		description: "A successor to the original normal table.",
+		url: "http://rattoto10.jounin.jp/js/score.json",
+	},
+	{
+		name: "Overjoy",
+		description: "The overjoy table. Level 1 is roughly equivalent to Insane 20.",
+		humanisedPrefix: "overjoy",
+		playtype: "7K",
+		prefix: BMS_TABLES.overjoy,
+		url: "http://lr2.sakura.ne.jp/data/score.json",
+	},
+	{
+		name: "DP Insane",
+		description: "The 14K Insane table.",
+		humanisedPrefix: "dpInsane",
+		prefix: BMS_TABLES.dpInsane,
+		playtype: "14K",
+		url: "http://dpbmsdelta.web.fc2.com/table/data/insane_data.json",
+	},
+	{
+		name: "DP Normal",
+		description: "The 14K Normal table, Sometimes called the delta table.",
+		humanisedPrefix: "dpNormal",
+		prefix: BMS_TABLES.dpNormal,
+		playtype: "14K",
+		url: "http://dpbmsdelta.web.fc2.com/table/data/dpdelta_data.json",
+	},
+	{
+		name: "DP Satellite",
+		description: "The 14K Satellite table.",
+		humanisedPrefix: "dpSatellite",
+		prefix: BMS_TABLES.satellite,
+		playtype: "14K",
+		url: "https://stellabms.xyz/dp/score.json",
+	},
+	{
+		name: "Scratch 3rd",
+		description: "The 7K Sara 3 table.",
+		humanisedPrefix: "scr",
+		prefix: BMS_TABLES.scratch,
+		playtype: "7K",
+		url: "http://minddnim.web.fc2.com/sara/3rd_hard/json/data.json",
+	},
+	{
+		name: "LN",
+		description: "The 7K LN table.",
+		humanisedPrefix: "ln",
+		prefix: BMS_TABLES.ln,
+		playtype: "7K",
+		url: "http://flowermaster.web.fc2.com/lrnanido/gla/score.json",
+	},
+];
+
+// this seems to be all we care about
+interface TableJSONDoc {
+	md5: string;
+	title?: string;
+	level: string;
+}
+
+const logger = CreateLogCtx(__filename);
+
+async function ImportTableLevels(tableJSON: Array<TableJSONDoc>, prefix: string) {
+	let failures = 0;
+	let success = 0;
+	const total = tableJSON.length;
+
+	for (const td of tableJSON) {
+		const chart = (await db.charts.bms.findOne({ "data.hashMD5": td.md5 })) as ChartDocument<
+			"bms:7K" | "bms:14K"
+		> | null;
+
+		if (!chart) {
+			logger.warn(
+				`No chart exists in table for ${td.md5} Possible title: ${td.title} ${prefix}${td.level}`
+			);
+			failures++;
+			continue;
+		}
+
+		const tableFolders = chart.data.tableFolders.filter((e) => e.table !== prefix);
+
+		tableFolders.push({ table: prefix, level: td.level.toString() });
+
+		await db.charts.bms.update(
+			{
+				chartID: chart.chartID,
+			},
+			{
+				$set: {
+					"data.tableFolders": tableFolders,
+				},
+			}
+		);
+
+		await db.songs.bms.update(
+			{
+				id: chart.songID,
+			},
+			{
+				$set: {
+					"data.tableString": FormatBMSTables(tableFolders),
+				},
+			}
+		);
+
+		success++;
+	}
+
+	logger.info(`Finished updating table ${prefix}.`);
+	logger.info(`${success} Success | ${failures} Failures | ${total} Total.`);
+}
+
+export interface BMSTablesDataset {
+	url: string;
+	name: string;
+	description: string;
+	humanisedPrefix: string;
+	prefix: string;
+	playtype: "7K" | "14K";
+}
+
+interface BMSTableChart {
+	title: string;
+	artist: string;
+	url: string;
+	url_diff: string;
+	md5: string;
+	sha256: string;
+	level: string;
+}
+
+const RETRY_COUNT = 3;
+
+export async function UpdateTable(table: BMSTablesDataset) {
+	const res = await fetch(table.url);
+
+	const statusCode = res.status;
+
+	let retries = 0;
+
+	// instead of using res.json() we have to use res.text() here
+	// so we have proper logging when some of these bms urls
+	// spontaneously return HTML.
+	let tableJSON: Array<BMSTableChart> | undefined;
+
+	// We need to have a retry counter because some of the bms tables
+	// randomly return useless html for no reason.
+
+	// i hate bms.
+	while (!tableJSON) {
+		let text;
+
+		try {
+			text = await res.text();
+
+			// @hack -- this is a force cast.
+			tableJSON = JSON.parse(text) as unknown as Array<BMSTableChart>;
+		} catch (err) {
+			logger.error(`Failed to fetch ${table.url}`, { err, res, statusCode, text });
+			retries++;
+
+			if (retries > RETRY_COUNT) {
+				logger.error(`SKIPPING TABLE!`);
+				return;
+			}
+
+			logger.info(`Retrying ${table.url}.`);
+		}
+	}
+
+	const tableID = `bms-${table.playtype}-${table.humanisedPrefix}`;
+
+	const folderIDs = [];
+	const levels = [...new Set(tableJSON.map((e) => e.level))];
+
+	for (const level of levels) {
+		const query = {
+			"data¬tableFolders": {
+				"~elemMatch": {
+					table: table.prefix,
+
+					// just incase some dude tries numbers
+					level: level.toString(),
+				},
+			},
+		};
+
+		const folderID = CreateFolderID(query, "bms", table.playtype);
+
+		folderIDs.push(folderID);
+
+		const exists = await db.folders.findOne({
+			folderID,
+		});
+
+		if (exists) {
+			continue;
+		}
+
+		await db.folders.insert({
+			title: `${table.prefix}${level}`,
+			inactive: false,
+			folderID,
+			game: "bms",
+			playtype: table.playtype,
+			type: "charts",
+			data: query,
+			searchTerms: [`${table.name} ${level}`],
+		});
+
+		logger.info(`Inserted new table ${table.prefix}${level}.`);
+	}
+
+	await db.tables.update(
+		{ tableID },
+		{
+			$set: {
+				tableID,
+				game: "bms",
+				playtype: table.playtype,
+				folders: folderIDs,
+				title: table.name,
+				description: table.description,
+				inactive: false,
+			},
+		},
+		{
+			upsert: true,
+		}
+	);
+
+	logger.info(`Bumped table ${table.name}.`);
+
+	logger.info(`Bumping levels...`);
+	await ImportTableLevels(tableJSON, table.prefix);
+	logger.info(`Levels bumped.`);
+}
+
+export async function SyncBMSTables() {
+	for (const table of registeredTables) {
+		// eslint-disable-next-line no-await-in-loop
+		await UpdateTable(table);
+	}
+
+	logger.info(`Re-initialising folder-chart-lookup, since changes may have been made.`);
+	await InitaliseFolderChartLookup();
+	logger.info(`Done.`);
+}
+
+if (require.main === module) {
+	SyncBMSTables()
+		.then(() => process.exit(0))
+		.catch((err: unknown) => {
+			logger.error(`Failed to sync BMS Tables.`, { err }, () => {
+				process.exit(1);
+			});
+		});
+}

server/src/lib/jobs/deorphan-scores.ts

@@ -0,0 +1,52 @@
+// Attempt to deoprhan lost scores.
+
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { ReprocessOrphan } from "lib/score-import/framework/orphans/orphans";
+
+const logger = CreateLogCtx(__dirname);
+
+export async function DeoprhanScores() {
+	const orphans = await db["orphan-scores"].find({});
+
+	// ScoreIDs are essentially userID dependent, so this is fine.
+	const blacklist = (await db["score-blacklist"].find({})).map((e) => e.scoreID);
+
+	logger.info(`Found ${orphans.length} orphans.`);
+
+	let failed = 0;
+	let success = 0;
+	let removed = 0;
+
+	for (const or of orphans) {
+		// We have to await like this to avoid mid-air race conditions,
+		// where two orphans attempt to deorphan to the same scoreID
+		// at the same time.
+		// See #511.
+
+		// eslint-disable-next-line no-await-in-loop
+		const r = await ReprocessOrphan(or, blacklist, logger);
+
+		if (r === null) {
+			removed++;
+		} else if (r === false) {
+			failed++;
+		} else {
+			success++;
+		}
+	}
+
+	logger.info(`Finished attempting deorphaning.`);
+
+	logger.info(`Success: ${success} | Failed ${failed} | Removed ${removed}.`);
+}
+
+if (require.main === module) {
+	DeoprhanScores()
+		.then(() => process.exit(0))
+		.catch((err: unknown) => {
+			logger.error(`Failed to de-orphan scores.`, { err }, () => {
+				process.exit(1);
+			});
+		});
+}

server/src/lib/jobs/inline-job-runner/job-runner.ts

@@ -0,0 +1,121 @@
+import { BacksyncBMSPMSSongsAndCharts } from "../backsync-bms-pms-data";
+import { UpdateAILevels } from "../bms-ai-table-sync";
+import { SyncBMSTables } from "../bms-table-sync";
+import { DeoprhanScores } from "../deorphan-scores";
+import { UGSSnapshot } from "../ugs-snapshot";
+import { UpdatePoyashiData } from "../update-bpi-data";
+import { UpdateDPTiers } from "../update-dp-tiers";
+import { Queue, Worker } from "bullmq";
+import CreateLogCtx from "lib/logger/logger";
+import { TachiConfig } from "lib/setup/config";
+import { DedupeArr } from "utils/misc";
+
+interface Job {
+	name: string;
+	cronFormat: string;
+	run: () => Promise<void>;
+}
+
+const jobs: Array<Job> = [
+	{
+		name: "Snapshot User Game Stats",
+		cronFormat: "0 0 * * *",
+		run: UGSSnapshot,
+	},
+	{
+		name: "De-Orphan Scores",
+
+		// We run an hour after snapshotting UGS
+		// just to spread load out a bit.
+		cronFormat: "1 0 * * *",
+		run: DeoprhanScores,
+	},
+];
+
+// if kamaitachi or omnitachi
+if (TachiConfig.TYPE !== "btchi") {
+	jobs.push({
+		name: "Update BPI",
+		cronFormat: "2 0 * * *",
+		run: UpdatePoyashiData,
+	});
+
+	jobs.push({
+		name: "Update DP Tiers",
+		cronFormat: "3 0 * * *",
+		run: UpdateDPTiers,
+	});
+}
+
+// if bokutachi or omnitachi
+if (TachiConfig.TYPE !== "ktchi") {
+	jobs.push({
+		name: "Update AI Table",
+		cronFormat: "2 0 * * *",
+		run: UpdateAILevels,
+	});
+
+	jobs.push({
+		name: "Update Tables",
+		cronFormat: "3 0 * * *",
+		run: SyncBMSTables,
+	});
+
+	jobs.push({
+		name: "Backsync BMS + PMS",
+		cronFormat: "4 0 * * *",
+		run: BacksyncBMSPMSSongsAndCharts,
+	});
+}
+
+const logger = CreateLogCtx("JOB_RUNNER");
+
+/**
+ * Initalises a tachi-server job runner.
+ * This runs the list of jobs defined in jobConfig.jobs.
+ */
+export function InitialiseJobRunner() {
+	logger.info(`Booting up Job Runner.`);
+
+	const names = jobs.map((e) => e.name);
+
+	if (DedupeArr(names).length !== names.length) {
+		logger.crit(`Jobs has duplicate name fields, refusing to run.`, () => {
+			process.exit(1);
+		});
+	}
+
+	const JobQueue = new Queue("Job Runner");
+
+	const jobNameMap = new Map<string, Job>();
+
+	for (const job of jobs) {
+		void JobQueue.add(job.name, { jobName: job.name }, { repeat: { cron: job.cronFormat } });
+		jobNameMap.set(job.name, job);
+	}
+
+	const worker = new Worker("Job Runner", async (j) => {
+		const { jobName } = j.data as { jobName: string };
+
+		logger.info(`Running job ${jobName}.`);
+
+		const jobInfo = jobNameMap.get(jobName);
+
+		if (!jobInfo) {
+			logger.severe(`Unknown job name ${jobName}, couldn't find a run function?`);
+			return false;
+		}
+
+		await jobInfo.run();
+
+		return true;
+	});
+
+	logger.info(`Initialised ${jobs.length} jobs (${jobs.map((e) => e.name).join(", ")}).`);
+
+	return worker;
+}
+
+if (require.main === module) {
+	InitialiseJobRunner();
+}

server/src/lib/jobs/ugs-snapshot.ts

@@ -0,0 +1,107 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { GetMillisecondsSince } from "utils/misc";
+import { GetAllRankings } from "utils/user";
+import type { UserGameStats, UserGameStatsSnapshot } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+// get the time of this midnight. it's possible this script eclipses itself when weird timezone
+// nonsense happens. we'll have to see.
+const currentTime = new Date().setUTCHours(0, 0, 0, 0);
+
+let batchWrite: Array<UserGameStatsSnapshot> = [];
+
+// This code is intentionally *very* robust, and handles a lot of unanticipated failures
+// because if it breaks, we brick the database.
+export async function UGSSnapshot() {
+	const timeStart = process.hrtime.bigint();
+
+	const alreadyExists = await db["game-stats-snapshots"].findOne({ timestamp: currentTime });
+
+	if (alreadyExists) {
+		logger.warn(
+			`There already exists snapshots at this time. Has this script been ran twice on one day? Ignoring request.`
+		);
+
+		throw new Error(
+			`There already exists snapshots at this time. Has this script been ran twice on one day? Ignoring request.`
+		);
+	}
+
+	logger.info(`Snapshotting UserGameStats.`);
+
+	try {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-call
+		await db["game-stats"]
+			.find({})
+
+			// @ts-expect-error faulty TS types
+			.each(async (ugs: UserGameStats, { pause, resume }) => {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-call
+				pause();
+
+				logger.debug(`Snapshotting ${ugs.userID} ${ugs.playtype} ${ugs.game}.`);
+
+				const [playcount, rankings] = await Promise.all([
+					db.scores.count({ userID: ugs.userID, playtype: ugs.playtype, game: ugs.game }),
+					GetAllRankings(ugs),
+				]);
+
+				const ugsSnapshot: UserGameStatsSnapshot = {
+					...ugs,
+					playcount,
+					rankings,
+					timestamp: currentTime,
+				};
+
+				batchWrite.push(ugsSnapshot);
+
+				if (batchWrite.length >= 500) {
+					logger.verbose(`Flushed batch.`);
+					await db["game-stats-snapshots"].insert(batchWrite);
+					// eslint-disable-next-line require-atomic-updates
+					batchWrite = [];
+				}
+
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-call
+				resume();
+			});
+
+		if (batchWrite.length) {
+			await db["game-stats-snapshots"].insert(batchWrite);
+		}
+
+		logger.info(
+			`Successfully snapshotted all data as of ${new Date(
+				currentTime
+			).toString()}. Took ${GetMillisecondsSince(timeStart)} ms.`
+		);
+	} catch (err) {
+		// if we panic, we need to revert whatever we did.
+		logger.severe(`FATAL IN UGS-SNAPSHOT - Possibly failed midway through snapshotting.`, {
+			err,
+		});
+
+		logger.info(`Removing all snapshots at this timestamp (${currentTime}).`);
+
+		await db["game-stats-snapshots"].remove({ timestamp: currentTime });
+
+		logger.info(`Removed.`);
+
+		throw err;
+	}
+}
+
+if (require.main === module) {
+	UGSSnapshot()
+		.then(() => {
+			process.exit(0);
+		})
+		.catch((err: unknown) => {
+			// This is a severe error, not an error. Running the UGS snapshot every day is necessary.
+			logger.severe(`Failed to snapshot user game stats.`, { err });
+
+			process.exit(1);
+		});
+}

server/src/lib/jobs/update-bpi-data.ts

@@ -0,0 +1,187 @@
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
+import CreateLogCtx from "lib/logger/logger";
+import { RecalcAllScores } from "utils/calculations/recalc-scores";
+import fetch from "utils/fetch";
+import type {
+	ChartDocument,
+	Difficulties,
+	GPTSupportedVersions,
+	integer,
+	Playtypes,
+	SongDocument,
+} from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+const difficultyResolve: Record<string, [Playtypes["iidx"], Difficulties["iidx:DP" | "iidx:SP"]]> =
+	{
+		3: ["SP", "HYPER"],
+		4: ["SP", "ANOTHER"],
+		8: ["DP", "HYPER"],
+		9: ["DP", "ANOTHER"],
+		10: ["SP", "LEGGENDARIA"],
+		11: ["DP", "LEGGENDARIA"],
+	};
+
+interface PoyashiProxyBPIInfo {
+	title: string;
+	difficulty: string;
+	wr: integer;
+	avg: integer;
+	notes: string;
+	bpm: string;
+	textage: string;
+	difficultyLevel: string;
+	dpLevel: string;
+	coef?: number | null;
+	removed?: boolean;
+}
+
+interface PoyashiProxyData {
+	version: integer;
+	requireVersion: string;
+	body: Array<PoyashiProxyBPIInfo>;
+}
+
+/**
+ * Fetches Poyashi BPI's latest information and syncs it back to the seeds repository.
+ *
+ * @note This function doesn't actually touch or backsync our database at all. Infact,
+ * the commit that hits the seeds repo will result in a database-sync, meaning this
+ * will all "just work"(tm)
+ */
+export async function UpdatePoyashiData() {
+	const repo = await PullDatabaseSeeds();
+
+	logger.info("Fetching data from proxy...");
+	const data = (await fetch("https://proxy.poyashi.me/?type=bpi").then((r) =>
+		r.json()
+	)) as PoyashiProxyData;
+
+	logger.info("Fetched data.");
+
+	const iidxSongs: Array<SongDocument<"iidx">> = await repo.ReadCollection("songs-iidx");
+	const iidxCharts: Array<ChartDocument<"iidx:DP" | "iidx:SP">> = await repo.ReadCollection(
+		"charts-iidx"
+	);
+
+	// Utility functions for finding matching charts.
+	function FindSongOnTitle(title: string) {
+		for (const data of iidxSongs) {
+			if (data.title === title || data.altTitles.includes(title)) {
+				return data;
+			}
+		}
+
+		return null;
+	}
+
+	function FindChartWithPTDFVersion(
+		songID: integer,
+		playtype: Playtypes["iidx"],
+		diff: Difficulties["iidx:DP" | "iidx:SP"],
+		version: GPTSupportedVersions["iidx:DP" | "iidx:SP"]
+	) {
+		for (const chart of iidxCharts) {
+			if (
+				chart.songID === songID &&
+				chart.playtype === playtype &&
+				chart.difficulty === diff &&
+				chart.versions.includes(version)
+			) {
+				return chart;
+			}
+		}
+
+		return null;
+	}
+
+	const updatedChartIDs = [];
+
+	// The actual mutation.
+	for (const d of data.body) {
+		const res: ["DP" | "SP", Difficulties["iidx:DP" | "iidx:SP"]] | undefined =
+			difficultyResolve[d.difficulty];
+
+		if (!res) {
+			throw new Error(`Unknown difficulty ${d.difficulty}`);
+		}
+
+		const [playtype, diff] = res;
+
+		const tachiSong = FindSongOnTitle(d.title);
+
+		if (!tachiSong) {
+			logger.warn(`Cannot find song ${d.title}?`);
+			continue;
+		}
+
+		// current poyashi version is 29
+		const tachiChart = FindChartWithPTDFVersion(tachiSong.id, playtype, diff, "29");
+
+		if (!tachiChart) {
+			logger.warn(
+				`Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?`
+			);
+			continue;
+		}
+
+		const kavg = Number(d.avg);
+
+		if (kavg < 0) {
+			logger.warn(
+				`${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.`
+			);
+			continue;
+		}
+
+		if (d.removed === true) {
+			logger.info(`Skipping removed chart ${tachiSong.title}.`);
+			continue;
+		}
+
+		const newCoef = d.coef === -1 || d.coef === undefined ? null : d.coef;
+		const newKavg = Number(d.avg);
+		const newWR = Number(d.wr);
+
+		if (
+			tachiChart.data.bpiCoefficient !== newCoef ||
+			tachiChart.data.kaidenAverage !== newKavg ||
+			tachiChart.data.worldRecord !== newWR
+		) {
+			updatedChartIDs.push(tachiChart.chartID);
+
+			tachiChart.data.bpiCoefficient = newCoef;
+			tachiChart.data.kaidenAverage = newKavg;
+			tachiChart.data.worldRecord = newWR;
+		}
+	}
+
+	await repo.WriteCollection("charts-iidx", iidxCharts);
+
+	if (updatedChartIDs.length !== 0) {
+		logger.info(`Finished applying BPI changes. Writing back.`);
+
+		await repo.CommitChangesBack(`BPI Update ${new Date().toISOString()}`);
+
+		logger.info(`Recalcing scores.`);
+		await RecalcAllScores({
+			game: "iidx",
+			chartID: { $in: updatedChartIDs },
+		});
+
+		logger.info(`Finished recalcing scores.`);
+	}
+}
+
+if (require.main === module) {
+	UpdatePoyashiData()
+		.then(() => {
+			process.exit(0);
+		})
+		.catch((err: unknown) => {
+			logger.error("Failed to update poyashi data.", { err }, () => {
+				process.exit(1);
+			});
+		});
+}

server/src/lib/jobs/update-dp-tiers.ts

@@ -0,0 +1,140 @@
+/* eslint-disable no-await-in-loop */
+import db from "external/mongo/db";
+import { decode } from "html-entities";
+import { PullDatabaseSeeds } from "lib/database-seeds/repo";
+import CreateLogCtx from "lib/logger/logger";
+import { parse } from "node-html-parser";
+import { RecalcAllScores } from "utils/calculations/recalc-scores";
+import fetch from "utils/fetch";
+import { FindSongOnTitle } from "utils/queries/songs";
+
+const logger = CreateLogCtx(__filename);
+
+export async function UpdateDPTiers() {
+	const rawHTML = await fetch("https://zasa.sakura.ne.jp/dp/run.php").then((r) => r.text());
+
+	// time for some fun html parsing.
+
+	const data = parse(rawHTML);
+
+	const rows = data.querySelectorAll("tr");
+
+	const parsedData = [];
+
+	for (const row of rows) {
+		// We only care about rows with 4 children.
+		// that's because the data is formatted like
+		// [HYPER, ANOTHER, LEGGENDARIA, SONG_TITLE]
+		// but has interspersed headers for game versions.
+		if (row.childNodes.length !== 4) {
+			continue;
+		}
+
+		parsedData.push({
+			// length is asserted above -- all of these are guaranteeably non-null
+			HYPER: ParseTierStr(row.childNodes[0]!.innerText),
+			ANOTHER: ParseTierStr(row.childNodes[1]!.innerText),
+			LEGGENDARIA: ParseTierStr(row.childNodes[2]!.innerText),
+			songTitle: decode(row.childNodes[3]!.innerText),
+		});
+	}
+
+	logger.info(`Got DP tier data. Applying it.`);
+
+	const updatedSongIDs = new Set();
+
+	await Promise.all(
+		parsedData.map(async (d) => {
+			const song = await FindSongOnTitle("iidx", d.songTitle);
+
+			if (!song) {
+				logger.warn(`Couldn't find song with title ${d.songTitle}.`);
+				return;
+			}
+
+			for (const difficulty of ["HYPER", "ANOTHER", "LEGGENDARIA"] as const) {
+				if (d[difficulty] === null) {
+					continue;
+				}
+
+				const result = await db.charts.iidx.update(
+					{
+						versions: "29",
+						playtype: "DP",
+						songID: song.id,
+						difficulty,
+					},
+					{
+						$set: {
+							"tierlistInfo.dp-tier": {
+								text: d[difficulty]!.toString(),
+								value: d[difficulty]!,
+							},
+						},
+					}
+				);
+
+				if (result.nModified) {
+					updatedSongIDs.add(song.id);
+				}
+			}
+		})
+	);
+
+	if (updatedSongIDs.size !== 0) {
+		logger.info(
+			`${updatedSongIDs.size} songs were changed. Recalcing the relevant scores now.`
+		);
+
+		await RecalcAllScores({
+			game: "iidx",
+			playtype: "DP",
+			songID: { $in: [...updatedSongIDs.values()] },
+		});
+
+		logger.info(`Recalced those scores.`);
+
+		const repo = await PullDatabaseSeeds();
+
+		let charts = await db.charts.iidx.find({});
+
+		await repo.WriteCollection("charts-iidx", charts);
+
+		// @ts-expect-error Force node to free the memory.
+		charts = null;
+
+		await repo.CommitChangesBack(`Update DP Tierlist ${new Date().toISOString()}`);
+	}
+
+	logger.info("Done.");
+}
+
+/**
+ * Parses a formatted string on the zasa website into a number. Returns null
+ * if the string is "-", which indicates nothing is present.
+ */
+function ParseTierStr(tierStr: string) {
+	if (tierStr === "-") {
+		return null;
+	}
+
+	const result = /\((.*)\)$/u.exec(tierStr);
+
+	if (result?.[1]) {
+		return Number(result[1]);
+	}
+
+	throw new Error(`Can't parse tierStr ${tierStr}.`);
+}
+
+if (require.main === module) {
+	UpdateDPTiers()
+		.then(() => {
+			process.exit(0);
+		})
+		.catch((err: unknown) => {
+			logger.error(`Failed to update DP Tiers.`, { err }, () => {
+				process.exit(1);
+			});
+		});
+}

server/src/lib/logger/colours.ts

@@ -0,0 +1,29 @@
+const Colours = {
+	crit: ["bgRed", "black"],
+	severe: ["bgBrightRed", "black"],
+	error: ["red"],
+	warn: ["yellow"],
+	info: ["blue"],
+	verbose: ["cyan"],
+	debug: ["white"],
+};
+
+// Discord ONLY accepts decimal colours, so.
+// lets run with that.
+export const DiscordColours = {
+	// #ff0000
+	crit: 16711680,
+
+	// #ff3333
+	severe: 16719904,
+
+	// #cc0000
+	error: 11730944,
+
+	// #ffcc00
+	warn: 16703232,
+
+	// other log colours don't matter -- they can all default to gray
+};
+
+export default Colours;

server/src/lib/logger/discord-transport.ts

@@ -0,0 +1,233 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { DiscordColours } from "./colours";
+import { ONE_MINUTE } from "lib/constants/time";
+import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
+import SafeJSONStringify from "safe-json-stringify";
+import fetch from "utils/fetch";
+import { IsRecord } from "utils/misc";
+import Transport from "winston-transport";
+import type { TransportStreamOptions } from "winston-transport";
+
+interface DiscordTransportOptions extends TransportStreamOptions {
+	webhook: string;
+}
+
+interface LogLevelCountState {
+	warn: Array<string>;
+	error: Array<string>;
+}
+
+interface LoggerInfo {
+	level: "crit" | "debug" | "info" | "severe" | "verbose" | "warn";
+	message: string;
+	noDiscord?: boolean;
+	meta?: Record<string, any>;
+}
+
+/**
+ * Creates a discord winston transport. This is a slightly adapted version
+ * of sidhantpanda's winston-discord-transport, modified for our use case.
+ *
+ * @param webhookUrl The webhook to connect to.
+ * @returns A winston tranport.
+ */
+export default class DiscordTransport extends Transport {
+	private webhookUrl = "";
+
+	/** Initialization promise resolved after retrieving discord id and token */
+	private readonly initalised: Promise<void>;
+
+	private bucketData: LogLevelCountState = {
+		warn: [],
+		error: [],
+	};
+
+	private isBucketing = false;
+	private bucketStart: Date | null = null;
+
+	constructor(opts: DiscordTransportOptions) {
+		super(opts);
+
+		this.resetBucketData();
+
+		this.initalised = fetch(opts.webhook)
+			.then((r) => {
+				if (!r.ok) {
+					throw new Error(`Couldn't connect to discord transport. ${r.status}.`);
+				}
+
+				return r.json() as unknown;
+			})
+			.then((content) => {
+				if (!IsRecord(content)) {
+					throw new Error(
+						`Discord returned non-object data as JSON? Can't get id or token ${content}.`
+					);
+				}
+
+				this.webhookUrl = `https://discordapp.com/api/v6/webhooks/${content.id}/${content.token}`;
+			});
+	}
+
+	private resetBucketData() {
+		this.bucketData = {
+			warn: [],
+			error: [],
+		};
+	}
+
+	log(info: LoggerInfo, cb: () => void) {
+		if (info.noDiscord !== false) {
+			try {
+				setImmediate(() => {
+					void this.initalised.then(() => {
+						void this.handleSendToDiscord(info);
+					});
+				});
+			} catch (err) {
+				// eslint-disable-next-line no-console
+				console.error(`Failed to send content to discord transport`, err);
+			}
+		}
+
+		// don't bother waiting around.
+		cb();
+	}
+
+	private handleSendToDiscord(info: LoggerInfo) {
+		if (info.level === "crit" || info.level === "severe") {
+			return this.sendLogDirectlyToDiscord(info);
+		}
+
+		if (!["warn", "error", "severe"].includes(info.level)) {
+			// Don't need to send notifications about these.
+			return;
+		}
+
+		this.bucketData[info.level as "error" | "warn"].push(info.message);
+
+		if (!this.isBucketing) {
+			this.isBucketing = true;
+			this.bucketStart = new Date();
+			setTimeout(() => {
+				this.sendBucketData();
+			}, ONE_MINUTE);
+		}
+	}
+
+	private getWhoToTag() {
+		return ServerConfig.LOGGER_CONFIG.DISCORD?.WHO_TO_TAG
+			? ServerConfig.LOGGER_CONFIG.DISCORD.WHO_TO_TAG.map((e) => `<@${e}>`).join(" ")
+			: "Nobody configured to tag, but this is bad, get someone!";
+	}
+
+	private sendBucketData() {
+		let color = 0;
+
+		const fields = [];
+
+		for (const key of ["warn", "error"] as const) {
+			if (this.bucketData[key].length !== 0) {
+				color = DiscordColours[key];
+				fields.push({
+					// uppercase first char
+					name: `${key[0]!.toUpperCase()}${key.slice(1)}s`,
+					value: this.bucketData[key].length,
+				});
+			}
+		}
+
+		const logSnippet = [
+			...this.bucketData.error.map((e) => `[ERROR] ${e}`),
+			...this.bucketData.warn.map((e) => `[WARN] ${e}`),
+		].join("\n");
+
+		const postBody = {
+			content: `
+			\`\`\`${logSnippet.length > 1500 ? `${logSnippet.slice(0, 1500 - 3)}...` : logSnippet}\`\`\``,
+			embeds: [
+				{
+					title: `${TachiConfig.NAME} Log Summary${
+						Environment.replicaIdentity
+							? ` (Replica: ${Environment.replicaIdentity})`
+							: ""
+					}`,
+					fields,
+					description: `Log summary for ${this.bucketStart?.toISOString()} to ${new Date().toISOString()}.`,
+					color,
+					timestamp: new Date().toISOString(),
+				},
+			],
+		};
+
+		void this.POSTData(postBody);
+		this.resetBucketData();
+		this.isBucketing = false;
+		this.bucketStart = null;
+	}
+
+	private async sendLogDirectlyToDiscord(info: LoggerInfo) {
+		const postBody = {
+			content: "",
+			embeds: [
+				{
+					description: `[${info.level}] ${info.message}`,
+
+					// it's Colour!!
+					color: DiscordColours[info.level as keyof typeof DiscordColours],
+					timestamp: new Date().toISOString(),
+				},
+			],
+		};
+
+		if (info.meta) {
+			postBody.content = `\`\`\`${SafeJSONStringify(info.meta, null, 4)}\`\`\``;
+		}
+
+		// These two levels are bad, and require near-immediate attention.
+
+		if (info.level === "severe") {
+			postBody.content = `SEVERE ERROR: ${this.getWhoToTag()}\n${postBody.content}`;
+		}
+
+		if (info.level === "crit") {
+			postBody.content = `CRITICAL ERROR: ${this.getWhoToTag()}\n${postBody.content}`;
+		}
+
+		await this.POSTData(postBody);
+	}
+
+	private async POSTData(postBody: unknown, scaleRetryDebounce = 2) {
+		const res = await fetch(this.webhookUrl, {
+			method: "POST",
+			body: JSON.stringify(postBody),
+			headers: {
+				"Content-Type": "application/json",
+			},
+		});
+
+		if (res.status === 429) {
+			// being rate limited.
+			const content = (await res.json()) as unknown;
+
+			// Try and retry when they say so. The issue is that our logging
+			// is very async, and this is a billion race conditions.
+			// It's possible that messages here could all get stuck in an
+			// awful loop, so we have a tuning off parameter.
+			// The scaleRetryDebouncer will get squared every call,
+			// so the initial request takes 2 * (generally 1milliseconds),
+			// then following requests will take even longer...
+			// It's possible this might blow up in our face.
+			// We'll have to see. - zkldi 2021/09/17
+
+			if (IsRecord(content) && typeof content.retry_after === "number") {
+				setTimeout(() => {
+					void this.POSTData(postBody, scaleRetryDebounce ** 2);
+				}, content.retry_after * scaleRetryDebounce);
+			}
+		} else if (!res.ok) {
+			// eslint-disable-next-line no-console
+			console.error(`Failed to send to discord ${res.status} ${await res.text()}`);
+		}
+	}
+}

server/src/lib/logger/logger.test.ts

@@ -0,0 +1,61 @@
+import CreateLogCtx, { ChangeRootLogLevel, GetLogLevel, rootLogger, Transports } from "./logger";
+import { ServerConfig } from "lib/setup/config";
+import t from "tap";
+
+const LOG_LEVEL = ServerConfig.LOGGER_CONFIG.LOG_LEVEL;
+
+t.test("Logger Tests", (t) => {
+	const logger = CreateLogCtx(__filename);
+
+	if (!Transports[0]) {
+		throw new Error(`No transports were defined? Can't perform logger tests.`);
+	}
+
+	// lol
+	Transports[0].level = "debug";
+
+	logger.debug("Debug Message Test");
+	logger.verbose("Verbose Message Test");
+	logger.info("Info Message Test");
+	logger.warn("Warning Message Test");
+	logger.error("Error Message Test");
+	logger.severe("Severe Message Test");
+	logger.crit("Critical Message Test");
+
+	Transports[0].level = process.env.LOG_LEVEL ?? "info";
+
+	logger.debug("This message shouldn't appear.");
+
+	t.end();
+});
+
+t.test("#GetLogLevel", (t) => {
+	t.equal(GetLogLevel(), LOG_LEVEL);
+
+	ChangeRootLogLevel("crit");
+	t.equal(GetLogLevel(), "crit");
+
+	ChangeRootLogLevel(LOG_LEVEL);
+	t.equal(GetLogLevel(), LOG_LEVEL);
+
+	t.end();
+});
+
+t.test("#ChangeRootLogLevel", (t) => {
+	t.test("Should work with the root logger.", (t) => {
+		rootLogger.info("The below message should appear.");
+
+		ChangeRootLogLevel("verbose");
+		rootLogger.verbose("== SHOULD APPEAR ==");
+
+		ChangeRootLogLevel("info");
+
+		rootLogger.info("The below message SHOULD NOT appear.");
+
+		rootLogger.verbose("== SHOULD NOT APPEAR ==");
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/logger/logger.ts

@@ -0,0 +1,277 @@
+import DiscordWinstonTransport from "./discord-transport";
+import { Transport as SeqTransport } from "@valuabletouch/winston-seq";
+import { Environment, ServerConfig, TachiConfig } from "lib/setup/config";
+import SafeJSONStringify from "safe-json-stringify";
+import { EscapeStringRegexp } from "utils/misc";
+import winston, { format, transports } from "winston";
+import type { SeqLogLevel } from "seq-logging";
+import type { LeveledLogMethod, Logger } from "winston";
+import "winston-daily-rotate-file";
+
+export type KtLogger = Logger & { severe: LeveledLogMethod };
+
+const level = process.env.LOG_LEVEL ?? ServerConfig.LOGGER_CONFIG.LOG_LEVEL;
+
+const formatExcessProperties = (meta: Record<string, unknown>, limit = false) => {
+	let i = 0;
+
+	for (const [key, val] of Object.entries(meta)) {
+		if (val instanceof Error) {
+			meta[key] = { message: val.message, stack: val.stack };
+		}
+
+		i++;
+	}
+
+	if (!i) {
+		return "";
+	}
+
+	const content = SafeJSONStringify(meta);
+
+	return ` ${limit ? StrCap(content) : content}`;
+};
+
+function StrCap(string: string) {
+	if (string.length > 300) {
+		return `${string.slice(0, 297)}...`;
+	}
+
+	return string;
+}
+
+const formatExcessPropertiesNoStack = (
+	meta: Record<string, unknown>,
+	omitKeys: Array<string> = [],
+	limit = false
+) => {
+	const realMeta: Record<string, unknown> = {};
+
+	for (const [key, val] of Object.entries(meta)) {
+		if (omitKeys.includes(key)) {
+			continue;
+		}
+
+		if (val instanceof Error) {
+			realMeta[key] = { message: val.message };
+		} else if (!key.startsWith("__") && !key.startsWith("!")) {
+			realMeta[key] = val;
+		}
+	}
+
+	if (Object.keys(realMeta).length === 0) {
+		return "";
+	}
+
+	const content = SafeJSONStringify(realMeta);
+
+	return ` ${limit ? StrCap(content) : content}`;
+};
+
+const replicaInfo = Environment.replicaIdentity ? ` (${Environment.replicaIdentity})` : "";
+
+const tachiPrintf = format.printf(
+	({ level, message, context = "tachi-root", timestamp, ...meta }) =>
+		`${timestamp}${replicaInfo} [${
+			Array.isArray(context) ? context.join(" | ") : context
+		}] ${level}: ${message}${formatExcessProperties(meta)}`
+);
+
+const tachiConsolePrintf = format.printf(
+	({ level, message, context = "tachi-root", timestamp, hideFromConsole, ...meta }) =>
+		`${timestamp}${replicaInfo} [${
+			Array.isArray(context) ? context.join(" | ") : context
+		}] ${level}: ${message}${formatExcessPropertiesNoStack(
+			meta,
+			hideFromConsole as Array<string>,
+			true
+		)}`
+);
+
+winston.addColors({
+	crit: ["bgRed", "black"],
+	severe: ["bgBrightRed", "black"],
+	error: ["red"],
+	warn: ["yellow"],
+	info: ["blue"],
+	verbose: ["cyan"],
+	debug: ["white"],
+});
+
+const baseFormatRoute = format.combine(
+	format.timestamp({
+		format: "YYYY-MM-DD HH:mm:ss",
+	})
+);
+
+const defaultFormatRoute = format.combine(
+	baseFormatRoute,
+	format.errors({ stack: false }),
+	tachiPrintf
+);
+
+const consoleFormatRoute = format.combine(
+	baseFormatRoute,
+	format.errors({ stack: false }),
+	tachiConsolePrintf,
+	format.colorize({
+		all: true,
+	})
+);
+
+const tports: Array<winston.transport> = [];
+
+if (ServerConfig.LOGGER_CONFIG.FILE) {
+	tports.push(
+		new transports.DailyRotateFile({
+			filename: "logs/tachi-%DATE%.log",
+			datePattern: "YYYY-MM-DD-HH",
+			zippedArchive: true,
+			maxSize: "20m",
+			maxFiles: "14d",
+			createSymlink: true,
+			symlinkName: "tachi.log",
+			format: defaultFormatRoute,
+		})
+	);
+}
+
+// If this is a job, force stdout writing no matter what.
+if (ServerConfig.LOGGER_CONFIG.CONSOLE || process.env.IS_JOB || process.env.FORCE_CONSOLE_LOG) {
+	tports.push(
+		new transports.Console({
+			format: consoleFormatRoute,
+		})
+	);
+}
+
+if (ServerConfig.LOGGER_CONFIG.DISCORD) {
+	tports.push(
+		new DiscordWinstonTransport({
+			webhook: ServerConfig.LOGGER_CONFIG.DISCORD.WEBHOOK_URL,
+			level: "warn",
+		})
+	);
+}
+
+if (ServerConfig.LOGGER_CONFIG.SEQ_API_KEY && Environment.seqUrl) {
+	// Turns winston log levels into seq format.
+	const levelMap: Record<string, SeqLogLevel> = {
+		crit: "Fatal",
+		severe: "Error",
+		error: "Error",
+		warn: "Warning",
+		info: "Information",
+
+		// Note that Seq interprets these in reverse,
+		// however, it's easier to read this code if I just
+		// use the same levels, instead of the right ones.
+		verbose: "Verbose",
+		debug: "Debug",
+	};
+
+	tports.push(
+		new SeqTransport({
+			apiKey: ServerConfig.LOGGER_CONFIG.SEQ_API_KEY,
+			serverUrl: Environment.seqUrl,
+			onError: (err) => {
+				// eslint-disable-next-line no-console
+				console.error(`Failed to send seq message: ${err.message}.`);
+			},
+			levelMapper(level = "") {
+				return levelMap[level] ?? "Information";
+			},
+		})
+	);
+}
+
+export const rootLogger = winston.createLogger({
+	levels: {
+		// entire process termination is necessary
+		crit: 0,
+
+		// something is wrong, and more than one function is affected (such as a failed assertion that is definitely expected to be true).
+		severe: 1,
+
+		// function call (or related process) has failed unexpectedly
+		error: 2,
+
+		// function call has hit something it didn't want, but can recover
+		warn: 3,
+
+		// something has happened that is expected, but worth logging
+		info: 4,
+
+		// something has happened
+		verbose: 5,
+
+		// glorified console.log debugging
+		debug: 6,
+	},
+	level,
+	format: defaultFormatRoute,
+	transports: tports,
+	defaultMeta: {
+		__ServerName: TachiConfig.NAME,
+		__Worker: !!process.env.IS_WORKER,
+		__ReplicaID: Environment.replicaIdentity,
+	},
+}) as KtLogger;
+
+if (!!ServerConfig.LOGGER_CONFIG.SEQ_API_KEY !== !!Environment.seqUrl) {
+	rootLogger.warn(
+		`Only one of SEQ_API_KEY (conf.json5) and SEQ_URL (Environment) were set. Not sending logs to Seq, as both must be provided.`
+	);
+}
+
+if (tports.length === 0) {
+	// eslint-disable-next-line no-console
+	console.warn(
+		"You have no transports set. Absolutely no logs will be saved. This is a terrible idea!"
+	);
+}
+
+function CreateLogCtx(filename: string, lg = rootLogger): KtLogger {
+	const replacedFilename = filename.replace(
+		new RegExp(`^${EscapeStringRegexp(process.cwd())}/((js|src)/)?`, "u"),
+		""
+	);
+
+	const logger = lg.child({
+		context: [replacedFilename],
+	}) as KtLogger;
+
+	// @hack, defaultMeta isn't reactive -- won't be updated unless we do this.
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+	logger.defaultMeta = { ...(logger.defaultMeta ?? {}), context: [replacedFilename] };
+
+	return logger;
+}
+
+export function AppendLogCtx(context: string, lg: KtLogger): KtLogger {
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
+	const newContext = [...lg.defaultMeta.context, context];
+
+	return lg.child({ context: newContext }) as KtLogger;
+}
+
+export function ChangeRootLogLevel(
+	level: "crit" | "debug" | "error" | "info" | "severe" | "verbose" | "warn"
+) {
+	rootLogger.info(`Changing log level to ${level}.`);
+
+	for (const tp of tports) {
+		tp.level = level;
+	}
+}
+
+export function GetLogLevel() {
+	return (
+		tports.map((e) => e.level).find((e) => typeof e === "string") ??
+		ServerConfig.LOGGER_CONFIG.LOG_LEVEL
+	);
+}
+
+export const Transports = tports;
+
+export default CreateLogCtx;

server/src/lib/migration/migrations.test.ts

@@ -0,0 +1,128 @@
+import { ApplyMigration, FAKE_MIGRATION, FindUnappliedMigrations } from "./migrations";
+import db from "external/mongo/db";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+
+t.test("#FindUnappliedMigrations", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.test("Should find all unapplied migrations", async (t) => {
+		const res = await FindUnappliedMigrations();
+
+		t.equal(
+			res.includes("fake-migration"),
+			true,
+			"Should contain fake-migration since it isn't applied."
+		);
+
+		t.end();
+	});
+
+	t.test("Shouldn't return pending migrations", async (t) => {
+		await db.migrations.insert({
+			migrationID: "fake-migration",
+			status: "pending",
+		});
+
+		const res = await FindUnappliedMigrations();
+
+		t.equal(res.includes("fake-migration"), false);
+
+		t.end();
+	});
+
+	t.test("Shouldn't return applied migrations", async (t) => {
+		await db.migrations.insert({
+			migrationID: "fake-migration",
+			status: "applied",
+			appliedOn: 1000,
+		});
+
+		const res = await FindUnappliedMigrations();
+
+		t.equal(res.includes("fake-migration"), false);
+
+		t.end();
+	});
+
+	t.end();
+});
+
+t.test("#ApplyMigration", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.test("Should apply a migration.", async (t) => {
+		await ApplyMigration(FAKE_MIGRATION);
+
+		const dbRes = await db.users.findOne({
+			id: 1,
+		});
+
+		t.equal(
+			// eslint-disable-next-line lines-around-comment
+			// @ts-expect-error Our migration is a fake one that adds __wasMigrated
+			// to all user documents. There's nothing special about this property name.
+			dbRes?.__wasMigrated,
+			true,
+			"Migration should be applied on the users."
+		);
+
+		t.end();
+	});
+
+	t.test("Should panic and reset if something goes wrong.", async (t) => {
+		try {
+			await ApplyMigration({
+				id: "bad-migration",
+				up: () => {
+					throw new Error("Failed to migrate.");
+				},
+				down: () =>
+					db.users.update({}, { $set: { __wasUnmigrated: true } }, { multi: true }),
+			});
+		} catch (e) {
+			const err = e as Error;
+
+			t.equal(err.message, "Was going to exit with statusCode 1, but we're in testing.");
+		}
+
+		const dbRes = await db.users.findOne({
+			id: 1,
+		});
+
+		t.equal(
+			// eslint-disable-next-line lines-around-comment
+			// @ts-expect-error Our un-migration adds this property to check if it worked.
+			dbRes?.__wasUnmigrated,
+			true,
+			"Migration should be applied on the users."
+		);
+
+		t.end();
+	});
+
+	t.test(
+		"Should panic and panic again if un-migration failed, with a scarier log message.",
+		async (t) => {
+			try {
+				await ApplyMigration({
+					id: "bad-migration",
+					up: () => {
+						throw new Error("Failed to migrate.");
+					},
+					down: () => {
+						throw new Error("Failed to undo migration.");
+					},
+				});
+			} catch (e) {
+				const err = e as Error;
+
+				t.equal(err.message, "Was going to exit with statusCode 1, but we're in testing.");
+			}
+
+			t.end();
+		}
+	);
+
+	t.end();
+});

server/src/lib/migration/migrations.ts

@@ -0,0 +1,207 @@
+import UGPTRivalsMigration from "./migrations/add-rivals-to-ugpt";
+import RemoveMultifolderStats from "./migrations/remove-multifolder-stats";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { Environment } from "lib/setup/config";
+import type { Migration } from "utils/types";
+
+const logger = CreateLogCtx(__filename);
+
+// NOTE: This is declared here, and NOT in test-utils. If any runtime file attempts to depend
+// on test-utils, the server will not boot.
+
+export const FAKE_MIGRATION: Migration = {
+	id: "fake-migration",
+	up: () => db.users.update({}, { $set: { __wasMigrated: true } }, { multi: true }),
+	down: () => db.users.update({}, { $unset: { __wasMigrated: 1 } }, { multi: true }),
+};
+
+// Migrations are stored in an array because they have some concept of order
+// That is, migrations should ideally be applied in a fixed order just to avoid
+// any potential unsavoury interactions.
+// If we're testing, we should pull fake migrations instead to ensure the tests
+// stay consistent
+const REGISTERED_MIGRATIONS: Array<Migration> =
+	Environment.nodeEnv === "test"
+		? [FAKE_MIGRATION]
+		: [UGPTRivalsMigration, RemoveMultifolderStats];
+
+function CreateMigrationLookupMap(migrations: Array<Migration>) {
+	const map = new Map<string, Migration>();
+
+	for (const mig of migrations) {
+		if (map.get(mig.id)) {
+			logger.crit(
+				`Multiple migrations are registered for ${mig.id}. Cannot safely apply migrations.`,
+				() => {
+					// note, we want to exit in testing here, this is fine.
+					process.exit(1);
+				}
+			);
+		}
+
+		map.set(mig.id, mig);
+	}
+
+	return map;
+}
+
+// That said, we still make a lookup map so we can efficiently find registered
+// migrations.
+const MIGRATION_LOOKUP = CreateMigrationLookupMap(REGISTERED_MIGRATIONS);
+
+/**
+ * Return the migrationID of all migrations that have not been applied.
+ */
+export async function FindUnappliedMigrations() {
+	// An unapplied migration is one that is acknowledged in the codebase
+	// (see REGISTERED_MIGRATIONS), but not stored inside the mongodb instance.
+	const migrations = await db.migrations.find(
+		{},
+		{
+			projection: {
+				migrationID: 1,
+			},
+		}
+	);
+
+	const appliedIDs = new Set(migrations.map((e) => e.migrationID));
+
+	const notApplied = [];
+
+	for (const migration of REGISTERED_MIGRATIONS) {
+		if (!appliedIDs.has(migration.id)) {
+			notApplied.push(migration.id);
+		}
+	}
+
+	return notApplied;
+}
+
+export async function ApplyUnappliedMigrations() {
+	const unapplied = await FindUnappliedMigrations();
+
+	if (unapplied.length === 0) {
+		logger.verbose(`Found no migrations to apply. Not bothering.`);
+		return;
+	}
+
+	logger.info(
+		`Found ${
+			unapplied.length
+		} migration(s) that have not been applied yet. These are '${unapplied.join("', '")}'.`,
+		{ unapplied }
+	);
+
+	for (const migrationID of unapplied) {
+		// These things need to apply in lockstep.
+		// Note that if any migration fails, this will exit at CRIT level
+		// and not continue.
+		// eslint-disable-next-line no-await-in-loop
+		await ApplyMigrationByID(migrationID);
+	}
+}
+
+export function ApplyMigrationByID(migrationID: string) {
+	const migration = MIGRATION_LOOKUP.get(migrationID);
+
+	if (!migration) {
+		logger.error(
+			`Attempted to apply migration ${migrationID}, but that migration doesn't exist?`
+		);
+		throw new Error(
+			`Attempted to apply migration ${migrationID}, but that migration doesn't exist?`
+		);
+	}
+
+	return ApplyMigration(migration);
+}
+
+/**
+ * Applies a migration. Will **EXIT TACHI** if a migration fails to apply, as this implies
+ * database level inconsistencies which could cause severe problems.
+ */
+export async function ApplyMigration(migration: Migration) {
+	const migrationID = migration.id;
+
+	logger.info(`Received request to apply migration '${migrationID}'.`);
+
+	// Lock the migration here:
+	// If the migration is already applied (i.e. migrationID is set and pending/applied)
+	// don't bother applying again
+	// If the migration isn't, set it as pending and continue. This is race-condition
+	// safe.
+	const wasApplied = await db.migrations.findOneAndUpdate(
+		{ migrationID },
+		{
+			$setOnInsert: {
+				migrationID,
+				status: "pending",
+			},
+		},
+		{
+			returnOriginal: true,
+			upsert: true,
+		}
+	);
+
+	if (wasApplied) {
+		logger.error(`Tried to apply migration ${migrationID}, but it was already applied.`);
+		throw new Error(`Tried to apply migration ${migrationID}, but it was already applied.`);
+	}
+
+	try {
+		await migration.up();
+
+		logger.info(`Successfully applied migration '${migrationID}'.`);
+
+		await db.migrations.update(
+			{
+				migrationID,
+			},
+			{
+				$set: {
+					status: "applied",
+					appliedOn: Date.now(),
+				},
+			}
+		);
+
+		logger.info(`Successfully set migration state for '${migrationID}' to applied.`);
+	} catch (err) {
+		logger.severe(
+			`Failed to apply migration '${migrationID}'. Attempting to revert migration. Tachi will exit after attempting to revert this migration.`,
+			{ err }
+		);
+
+		try {
+			await migration.down();
+
+			logger.info(
+				`Successfully reverted migration. Database state is okay, but Tachi will not boot.`
+			);
+
+			logger.crit(
+				`Failed to apply migration '${migrationID}'. Although successfully reverted, Tachi will not boot if a migration fails.`
+			);
+		} catch (err) {
+			logger.crit(
+				`FAILED TO REVERT PARTIAL MIGRATION '${migrationID}'. THE DATABASE IS ALMOST CERTAINLY TAINTED. MANUAL INTERVENTION/BACKUP RESTORE NECESSARY.`,
+				{
+					err,
+				}
+			);
+		}
+
+		// remove the stale migration so it can be re-ran in the future.
+		await db.migrations.findOneAndDelete({ migrationID });
+
+		if (Environment.nodeEnv === "test") {
+			throw new Error("Was going to exit with statusCode 1, but we're in testing.");
+		}
+
+		logger.crit(`Exiting.`, () => {
+			process.exit(1);
+		});
+	}
+}

server/src/lib/migration/migrations/add-rivals-to-ugpt.ts

@@ -0,0 +1,28 @@
+import db from "external/mongo/db";
+import type { Migration } from "utils/types";
+
+const migration: Migration = {
+	id: "add-rivals-to-ugpt",
+	up: async () => {
+		await db["game-settings"].update(
+			{},
+			{
+				$set: {
+					rivals: [],
+				},
+			},
+			{ multi: true }
+		);
+	},
+	down: async () => {
+		await db["game-settings"].update(
+			{},
+			{
+				$unset: { rivals: 1 },
+			},
+			{ multi: true }
+		);
+	},
+};
+
+export default migration;

server/src/lib/migration/migrations/remove-multifolder-stats.ts

@@ -0,0 +1,27 @@
+import db from "external/mongo/db";
+import type { Migration } from "utils/types";
+
+const migration: Migration = {
+	id: "remove-multifolder-stats",
+	up: async () => {
+		await db["game-settings"].update(
+			{
+				"preferences.stats.mode": "folder",
+				"preferences.stats.folderID": { $type: "array" },
+			},
+			{
+				$pull: {
+					"preferences.stats": {
+						folderID: { $type: "array" },
+					},
+				},
+			},
+			{ multi: true }
+		);
+	},
+	down: () => {
+		throw new Error(`Reverting this change is not possible.`);
+	},
+};
+
+export default migration;

server/src/lib/notifications/notification-wrappers.ts

@@ -0,0 +1,46 @@
+import { SendNotification } from "./notifications";
+import db from "external/mongo/db";
+import { FormatGame } from "tachi-common";
+import type { Game, integer, Playtype, PublicUserDocument } from "tachi-common";
+
+/**
+ * A utility wrapper for sending the SET_RIVAL notification.
+ *
+ * @param toUserID - The user to send this notification to.
+ * @param fromUser - The user who rivalled them.
+ * @param game - The game they rivalled them on.
+ * @param playtype - The playtype they rivalled them on.
+ */
+export async function SendSetRivalNotification(
+	toUserID: integer,
+	fromUser: PublicUserDocument,
+	game: Game,
+	playtype: Playtype
+) {
+	const alreadyBeenPinged = await db.notifications.findOne({
+		sentTo: toUserID,
+		"body.type": "RIVALED_BY",
+		"body.content": {
+			userID: fromUser.id,
+			game,
+			playtype,
+		},
+	});
+
+	if (alreadyBeenPinged) {
+		return;
+	}
+
+	return SendNotification(
+		`${fromUser.username} just added you as a rival for ${FormatGame(game, playtype)}`,
+		toUserID,
+		{
+			type: "RIVALED_BY",
+			content: {
+				userID: fromUser.id,
+				game,
+				playtype,
+			},
+		}
+	);
+}

server/src/lib/notifications/notifications.test.ts

@@ -0,0 +1,76 @@
+import { BulkSendNotification, SendNotification } from "./notifications";
+import db from "external/mongo/db";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+
+t.test("#SendNotification", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.test("Should send a notification.", async (t) => {
+		await SendNotification("title", 1, {
+			type: "MILESTONE_CHANGED",
+			content: { milestoneID: "foo" },
+		});
+
+		const dbRes = await db.notifications.findOne({
+			sentTo: 1,
+		});
+
+		t.hasStrict(dbRes, {
+			sentTo: 1,
+			title: "title",
+			read: false,
+			body: {
+				type: "MILESTONE_CHANGED",
+				content: { milestoneID: "foo" },
+			},
+		});
+
+		t.end();
+	});
+
+	t.end();
+});
+
+t.test("#BulkSendNotification", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.test("Should send multiple notifications.", async (t) => {
+		await BulkSendNotification("title", [1, 2], {
+			type: "MILESTONE_CHANGED",
+			content: { milestoneID: "foo" },
+		});
+
+		const dbRes = await db.notifications.findOne({
+			sentTo: 1,
+		});
+
+		t.hasStrict(dbRes, {
+			sentTo: 1,
+			title: "title",
+			read: false,
+			body: {
+				type: "MILESTONE_CHANGED",
+				content: { milestoneID: "foo" },
+			},
+		});
+
+		const dbRes2 = await db.notifications.findOne({
+			sentTo: 2,
+		});
+
+		t.hasStrict(dbRes2, {
+			sentTo: 2,
+			title: "title",
+			read: false,
+			body: {
+				type: "MILESTONE_CHANGED",
+				content: { milestoneID: "foo" },
+			},
+		});
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/notifications/notifications.ts

@@ -0,0 +1,45 @@
+import db from "external/mongo/db";
+import { Random20Hex } from "utils/misc";
+import type { integer, NotificationBody, NotificationDocument } from "tachi-common";
+
+function ConstructNotificationDoc(
+	title: string,
+	toUserID: integer,
+	body: NotificationBody
+): NotificationDocument {
+	return {
+		title,
+		sentTo: toUserID,
+		read: false,
+		sentAt: Date.now(),
+		notifID: `N${Random20Hex()}`,
+		body,
+	};
+}
+
+/**
+ * Send a notification to a user.
+ *
+ * @param title - A human friendly title for this notification.
+ * @param toUserID - The user to send the notification to.
+ * @param body - The body of the notification.
+ */
+export function SendNotification(title: string, toUserID: integer, body: NotificationBody) {
+	const notification = ConstructNotificationDoc(title, toUserID, body);
+
+	return db.notifications.insert(notification);
+}
+
+/**
+ * Send notifications to multiple users at once. This is more efficient than calling
+ * send notification in parallel.
+ */
+export function BulkSendNotification(
+	title: string,
+	toUserIDs: Array<integer>,
+	body: NotificationBody
+) {
+	const notifications = toUserIDs.map((e) => ConstructNotificationDoc(title, e, body));
+
+	return db.notifications.insert(notifications);
+}

server/src/lib/orphan-queue/orphan-queue.ts

@@ -0,0 +1,105 @@
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { GetNextCounterValue } from "utils/db";
+import { DedupeArr } from "utils/misc";
+import type { FilterQuery } from "mongodb";
+import type {
+	ChartDocument,
+	IDStrings,
+	IDStringToGame,
+	integer,
+	OrphanChart,
+	SongDocument,
+} from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * Handles an orphan queue request.
+ *
+ * If the chart has never been seen before, add it to the orphan queue
+ * and return null.
+ *
+ * If the chart has been seen before, and has less than N unique players
+ * who have played it, return null.
+ *
+ * If the chart has been seen before, and has >= N unique players who have
+ * played it, unorphan the chart, and return it.
+ */
+export async function HandleOrphanQueue<I extends IDStrings>(
+	idString: I,
+	game: IDStringToGame[I],
+	chartDoc: ChartDocument<I>,
+	songDoc: SongDocument<IDStringToGame[I]>,
+	orphanMatchCriteria: FilterQuery<OrphanChart<I>>,
+	queueSize: integer,
+	userID: integer,
+	chartName: string
+) {
+	logger.debug(`Received orphanqueue request for ${chartName}.`);
+
+	const orphanChart = await db["orphan-chart-queue"].findOne(
+		{ idString, ...orphanMatchCriteria },
+		{
+			projectID: true,
+		}
+	);
+
+	if (!orphanChart) {
+		logger.verbose(`Received unknown chart ${chartName}, orphaning.`);
+
+		await db["orphan-chart-queue"].insert({
+			idString,
+			chartDoc,
+			songDoc,
+			userIDs: [userID],
+		});
+
+		return null;
+	}
+
+	orphanChart.userIDs.push(userID);
+
+	const uniqueUsersArr = DedupeArr(orphanChart.userIDs);
+
+	const playcount = uniqueUsersArr.length;
+
+	// If N or more people have played this chart while orphaned, unorphan
+	// it.
+
+	if (playcount >= queueSize) {
+		logger.info(`Song ${chartName} was unorphaned by userIDs ${uniqueUsersArr.join(", ")}.`);
+		const songID = await GetNextCounterValue(`${game}-song-id`);
+
+		logger.verbose(`${chartName} has been assigned songID ${songID}.`);
+
+		const { songDoc, chartDoc } = orphanChart;
+
+		songDoc.id = songID;
+		chartDoc.songID = songID;
+
+		await db.songs[game].insert(songDoc);
+		await db.charts[game].insert(chartDoc);
+		await db["orphan-chart-queue"].remove({
+			_id: orphanChart._id,
+		});
+
+		return chartDoc as ChartDocument<I>;
+	}
+
+	// otherwise, update the state of this orphan.
+
+	logger.verbose(`UserID ${userID} played ${chartName}, which is now at ${playcount} plays.`);
+	await db["orphan-chart-queue"].update(
+		{
+			_id: orphanChart._id,
+		},
+		{
+			$set: {
+				userIDs: uniqueUsersArr,
+			},
+		}
+	);
+
+	return null;
+}

server/src/lib/rivals/rivals.ts

@@ -0,0 +1,188 @@
+import db from "external/mongo/db";
+import { SetRivalsFailReasons } from "lib/constants/err-codes";
+import CreateLogCtx from "lib/logger/logger";
+import { SendSetRivalNotification } from "lib/notifications/notification-wrappers";
+import { FormatGame } from "tachi-common";
+import { ArrayDiff } from "utils/misc";
+import { GetUsersWithIDs, GetUserWithIDGuaranteed } from "utils/user";
+import type { Game, integer, Playtype } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * Retrieve all of a user's set rival IDs.
+ * Throws if the user hasn't played the GPT in question.
+ */
+export async function GetRivalIDs(userID: integer, game: Game, playtype: Playtype) {
+	const gameSettings = await db["game-settings"].findOne(
+		{
+			userID,
+			game,
+			playtype,
+		},
+		{
+			projection: {
+				rivals: 1,
+			},
+		}
+	);
+
+	if (!gameSettings) {
+		throw new Error(
+			`User ${userID} has not played ${FormatGame(game, playtype)}. Cannot retrieve rivals.`
+		);
+	}
+
+	return gameSettings.rivals;
+}
+
+/**
+ * Get the user documents of the rivals for this UGPT.
+ * Throws if the user hasn't played the GPT in question.
+ */
+export async function GetRivalUsers(userID: integer, game: Game, playtype: Playtype) {
+	const rivalIDs = await GetRivalIDs(userID, game, playtype);
+
+	const rivals = await GetUsersWithIDs(rivalIDs);
+
+	return rivals;
+}
+
+/**
+ * Sets an array of userIDs to be this user's rivals. Performs validation on all of the
+ * rivals being players of the game, and not being duplicates. The maximum amount of rivals
+ * a player can have is 5.
+ */
+export async function SetRivals(
+	userID: integer,
+	game: Game,
+	playtype: Playtype,
+	newRivals: Array<integer>
+) {
+	if (newRivals.length > 5) {
+		return SetRivalsFailReasons.TOO_MANY;
+	}
+
+	if (newRivals.some((e) => e === userID)) {
+		return SetRivalsFailReasons.RIVALED_SELF;
+	}
+
+	const playedGPTCount = await db["game-settings"].count({
+		userID: { $in: newRivals },
+		game,
+		playtype,
+	});
+
+	// note: this check also checks that nothing provided is a duplicate.
+	if (playedGPTCount !== newRivals.length) {
+		return SetRivalsFailReasons.RIVALS_HAVENT_PLAYED_GPT;
+	}
+
+	const currentGameSettings = await db["game-settings"].findOne({
+		userID,
+		game,
+		playtype,
+	});
+
+	if (!currentGameSettings) {
+		logger.severe(
+			`User ${userID} attempted to set rivals for ${FormatGame(
+				game,
+				playtype
+			)}, but doesn't have game settings. Was their account deleted in midair?`
+		);
+
+		throw new Error(
+			`User ${userID} attempted to set rivals for ${FormatGame(
+				game,
+				playtype
+			)}, but doesn't have game settings. Was their account deleted in midair?`
+		);
+	}
+
+	const newSubs = ArrayDiff(currentGameSettings.rivals, newRivals);
+
+	const user = await GetUserWithIDGuaranteed(userID);
+
+	await Promise.all(
+		newSubs.map((toUserID) => SendSetRivalNotification(toUserID, user, game, playtype))
+	);
+
+	return db["game-settings"].update(
+		{
+			userID,
+			game,
+			playtype,
+		},
+		{
+			$set: {
+				rivals: newRivals,
+			},
+		}
+	);
+}
+
+/**
+ * Add a single new rival by their userID.
+ */
+export async function AddRival(userID: integer, game: Game, playtype: Playtype, newRival: integer) {
+	const rivalIDs = await GetRivalIDs(userID, game, playtype);
+
+	rivalIDs.push(newRival);
+
+	// We use set rivals because its race condition safe. We don't have to check > 5
+	// or anything.
+	return SetRivals(userID, game, playtype, rivalIDs);
+}
+
+/**
+ * Remove a single rival by their userID.
+ *
+ * @returns null if this UGPT is not rivals with the user, and therefore there is
+ * nothing to change.
+ */
+export async function RemoveRival(
+	userID: integer,
+	game: Game,
+	playtype: Playtype,
+	toRemove: integer
+) {
+	const rivalIDs = await GetRivalIDs(userID, game, playtype);
+
+	const filteredRivals = rivalIDs.filter((e) => e !== toRemove);
+
+	if (filteredRivals.length === rivalIDs.length) {
+		return null;
+	}
+
+	return SetRivals(userID, game, playtype, rivalIDs);
+}
+
+/**
+ * Get all of the userIDs of people who rival the userID for this GPT.
+ */
+export async function GetChallengerIDs(userID: integer, game: Game, playtype: Playtype) {
+	const result = await db["game-settings"].find(
+		{
+			game,
+			playtype,
+			rivals: userID,
+		},
+		{
+			projection: {
+				userID: 1,
+			},
+		}
+	);
+
+	return result.map((e) => e.userID);
+}
+
+/**
+ * Get the user documents of everyone who is rivalling this userID for this GPT.
+ */
+export async function GetChallengerUsers(userID: integer, game: Game, playtype: Playtype) {
+	const challengerIDs = await GetChallengerIDs(userID, game, playtype);
+
+	return GetUsersWithIDs(challengerIDs);
+}

server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts

@@ -0,0 +1,391 @@
+import { CalculateDataForGamePT, CreateCalculatedData } from "./calculated-data";
+import CreateLogCtx from "lib/logger/logger";
+import p from "prudence";
+import t from "tap";
+import { prAssert } from "test-utils/asserts";
+import {
+	Testing511SPA,
+	TestingDoraChart,
+	TestingGITADORADoraDryScore,
+	TestingIIDXSPDryScore,
+	TestingSDVXSingleDryScore,
+} from "test-utils/test-data";
+import type { ChartDocument, ScoreDocument } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+t.test("#CreateCalculatedData", async (t) => {
+	const res = await CreateCalculatedData(TestingIIDXSPDryScore, Testing511SPA, 30, logger);
+
+	prAssert(
+		t,
+		res,
+		{
+			ktLampRating: p.equalTo(10),
+			BPI: "?number",
+		},
+		"Should correctly produce calculatedData"
+	);
+
+	const gitadoraRes = await CreateCalculatedData(
+		TestingGITADORADoraDryScore,
+		TestingDoraChart,
+		30,
+		logger
+	);
+
+	prAssert(
+		t,
+		gitadoraRes,
+		{
+			skill: p.isPositiveNonZero,
+		},
+		"Should correctly call rating function overrides for different games"
+	);
+
+	const uscRes = await CreateCalculatedData(
+		{ game: "usc", playtype: "Controller" } as ScoreDocument,
+		{ data: { isOfficial: false }, playtype: "Controller" } as ChartDocument,
+		null,
+		logger
+	);
+
+	t.strictSame(uscRes, { VF6: null }, "Should return null if chart was not an official.");
+
+	const uscKbRes = await CreateCalculatedData(
+		{ game: "usc", playtype: "Keyboard" } as ScoreDocument,
+		{ data: { isOfficial: false }, playtype: "Keyboard" } as ChartDocument,
+		null,
+		logger
+	);
+
+	t.strictSame(
+		uscKbRes,
+		{ VF6: null },
+		"Should return null if chart was not an official (Keyboard)."
+	);
+
+	t.end();
+});
+
+/**
+ * These tests only check that the right properties are assigned.
+ */
+t.test("#CalculateDataForGamePT", (t) => {
+	t.test("IIDX:SP", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"iidx",
+			"SP",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			30,
+
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				ktLampRating: "?number",
+				BPI: "?number",
+			},
+			"Response should contain keys for IIDX:SP"
+		);
+
+		t.end();
+	});
+
+	t.test("IIDX:DP", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"iidx",
+			"DP",
+			Testing511SPA,
+
+			TestingIIDXSPDryScore,
+			30,
+
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				ktLampRating: "?number",
+				BPI: "?number",
+			},
+			"Response should contain keys for IIDX:DP"
+		);
+
+		t.end();
+	});
+
+	t.test("SDVX:Single", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"sdvx",
+			"Single",
+			Testing511SPA,
+			TestingSDVXSingleDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				VF6: p.nullable(p.isPositive),
+			},
+			"Response should contain keys for SDVX:Single"
+		);
+
+		t.end();
+	});
+
+	t.test("DDR:SP", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"ddr",
+			"SP",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				MFCP: p.nullable(p.isPositiveInteger),
+				ktRating: "?number",
+			},
+			"Response should contain keys for DDR:SP"
+		);
+
+		t.end();
+	});
+
+	t.test("DDR:DP", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"ddr",
+			"DP",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				MFCP: "null",
+				ktRating: "?number",
+			},
+			"Response should contain nulled keys for DDR:DP"
+		);
+
+		t.end();
+	});
+
+	t.test("chunithm:Single", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"chunithm",
+			"Single",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				rating: "?number",
+			},
+			"Response should contain nulled keys for chunithm:Single"
+		);
+
+		t.end();
+	});
+
+	t.test("maimai:Single", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"maimai",
+			"Single",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				ktRating: "?number",
+			},
+			"Response should contain nulled keys for maimai:Single"
+		);
+
+		t.end();
+	});
+
+	t.test("museca:Single", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"museca",
+			"Single",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				ktRating: "?number",
+			},
+			"Response should contain nulled keys for museca:Single"
+		);
+
+		t.end();
+	});
+
+	t.test("bms:7K", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"bms",
+			"7K",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				sieglinde: "?number",
+			},
+			"Response should contain nulled keys for bms:7K"
+		);
+
+		t.end();
+	});
+
+	t.test("bms:14K", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"bms",
+			"14K",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				sieglinde: "?number",
+			},
+			"Response should contain nulled keys for bms:14K"
+		);
+
+		t.end();
+	});
+
+	t.test("gitadora:Gita", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"gitadora",
+			"Gita",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				skill: "?number",
+			},
+			"Response should contain nulled keys for gitadora:Gita"
+		);
+
+		t.end();
+	});
+
+	t.test("gitadora:Dora", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"gitadora",
+			"Dora",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				skill: "?number",
+			},
+			"Response should contain nulled keys for gitadora:Dora"
+		);
+
+		t.end();
+	});
+
+	t.test("usc:Controller", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"usc",
+			"Controller",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				VF6: "?number",
+			},
+			"Response should contain nulled keys for usc:Controller"
+		);
+
+		t.end();
+	});
+
+	t.test("usc:Keyboard", async (t) => {
+		const res = await CalculateDataForGamePT(
+			"usc",
+			"Keyboard",
+			Testing511SPA,
+			TestingIIDXSPDryScore,
+			null,
+			logger
+		);
+
+		prAssert(
+			t,
+			res,
+			{
+				VF6: "?number",
+			},
+			"Response should contain nulled keys for usc:Keyboard"
+		);
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/score-import/framework/calculated-data/calculated-data.ts

@@ -0,0 +1,291 @@
+import {
+	CalculateKTLampRatingIIDXDP,
+	CalculateKTLampRatingIIDXSP,
+	CalculateKTRating,
+	CalculateMFCP,
+	CalculateSieglinde,
+} from "./stats";
+import {
+	CHUNITHMRating,
+	GITADORASkill,
+	Jubility,
+	PopnClassPoints,
+	PoyashiBPI,
+	Volforce,
+	WACCARate,
+	ITGHighestUnbroken,
+} from "rg-stats";
+import { GetGamePTConfig } from "tachi-common";
+import { IsNullish } from "utils/misc";
+import type { DryScore } from "../common/types";
+import type { KtLogger } from "lib/logger/logger";
+import type { ChartDocument, Game, IDStrings, Lamps, Playtypes, ScoreDocument } from "tachi-common";
+
+export async function CreateCalculatedData(
+	dryScore: DryScore,
+	chart: ChartDocument,
+	esd: number | null,
+	logger: KtLogger
+): Promise<ScoreDocument["calculatedData"]> {
+	const game = dryScore.game;
+	const playtype = chart.playtype;
+
+	const calculatedData = await CalculateDataForGamePT(
+		game,
+		playtype,
+		chart,
+		dryScore,
+		esd,
+		logger
+	);
+
+	return calculatedData;
+}
+
+type CalculatedDataFunctionsType = {
+	[I in IDStrings]: (
+		dryScore: DryScore<I>,
+		chart: ChartDocument<I>,
+		logger: KtLogger
+	) => Promise<ScoreDocument<I>["calculatedData"]> | ScoreDocument<I>["calculatedData"];
+};
+
+const CalculatedDataFunctions: CalculatedDataFunctionsType = {
+	"iidx:SP": CalculateDataIIDX,
+	"iidx:DP": CalculateDataIIDX,
+	"sdvx:Single": CalculateDataSDVXorUSC,
+	"popn:9B": CalculateDataPopn,
+	"museca:Single": CalculateDataMuseca,
+	"chunithm:Single": CalculateDataCHUNITHM,
+	"maimai:Single": () => ({ ktRating: null }),
+	"gitadora:Gita": CalculateDataGitadora,
+	"gitadora:Dora": CalculateDataGitadora,
+	"bms:7K": CalculateDataPMSorBMS,
+	"bms:14K": CalculateDataPMSorBMS,
+	"ddr:SP": CalculateDataDDR,
+	"ddr:DP": CalculateDataDDR,
+	"usc:Controller": CalculateDataSDVXorUSC,
+	"usc:Keyboard": CalculateDataSDVXorUSC,
+	"wacca:Single": CalculateDataWACCA,
+	"jubeat:Single": CalculateDataJubeat,
+	"pms:Keyboard": CalculateDataPMSorBMS,
+	"pms:Controller": CalculateDataPMSorBMS,
+	"itg:Stamina": CalculateDataITGStamina,
+};
+
+// Creates Game-Specific calculatedData for the provided game & playtype.
+// eslint-disable-next-line require-await
+export async function CalculateDataForGamePT<G extends Game>(
+	game: G,
+	playtype: Playtypes[G],
+	chart: ChartDocument,
+	dryScore: DryScore,
+
+	// ESD gets specially passed through because it's not part of the DryScore, but
+	// can be used for statistics anyway.
+	esd: number | null,
+	logger: KtLogger
+): Promise<ScoreDocument["calculatedData"]> {
+	// @ts-expect-error too many minor complains here...
+	return CalculatedDataFunctions[`${game}:${playtype}` as IDStrings](dryScore, chart, logger);
+}
+
+type CalculatedData<I extends IDStrings> = Required<ScoreDocument<I>["calculatedData"]>;
+
+function CalculateDataIIDX(
+	dryScore: DryScore,
+	chart: ChartDocument<"iidx:DP" | "iidx:SP">,
+	logger: KtLogger
+): CalculatedData<"iidx:SP"> {
+	let bpi = null;
+
+	// Legacy check! This can't be undefined now, but it used to be before BPI was
+	// migrated to chart.data.
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+	if (chart.data.kaidenAverage === undefined || chart.data.worldRecord === undefined) {
+		logger.warn(
+			`Chart ${chart.chartID}'s kaidenAverage/worldRecord was undefined. This is not legal, and it should be set to null.`,
+			{ chart }
+		);
+	} else if (chart.data.kaidenAverage !== null && chart.data.worldRecord !== null) {
+		bpi = PoyashiBPI.calculate(
+			dryScore.scoreData.score,
+			chart.data.kaidenAverage,
+			chart.data.worldRecord,
+			(chart as ChartDocument<"iidx:SP">).data.notecount * 2,
+			chart.data.bpiCoefficient
+		);
+
+		// kesdc = esd === null ? null : CalculateKESDC(BPIData.kesd, esd); disabled
+	}
+
+	const ktLampRating =
+		chart.playtype === "SP"
+			? CalculateKTLampRatingIIDXSP(dryScore, chart as ChartDocument<"iidx:SP">)
+			: CalculateKTLampRatingIIDXDP(dryScore, chart as ChartDocument<"iidx:DP">);
+
+	return {
+		BPI: bpi,
+		ktLampRating,
+	};
+}
+
+function CalculateDataSDVXorUSC(
+	dryScore: DryScore,
+	chart: ChartDocument,
+	_logger: KtLogger
+): CalculatedData<"sdvx:Single" | "usc:Controller" | "usc:Keyboard"> {
+	// for usc, unofficial charts currently have no VF6 value.
+	if (
+		dryScore.game === "usc" &&
+		!(chart as ChartDocument<"usc:Controller" | "usc:Keyboard">).data.isOfficial
+	) {
+		return { VF6: null };
+	}
+
+	const VF6 = Volforce.calculateVF6(
+		dryScore.scoreData.score,
+		dryScore.scoreData.lamp as Lamps["sdvx:Single"],
+		chart.levelNum
+	);
+
+	return { VF6 };
+}
+
+function CalculateDataMuseca(
+	dryScore: DryScore,
+	chart: ChartDocument,
+	logger: KtLogger
+): CalculatedData<"museca:Single"> {
+	return {
+		ktRating: CalculateKTRating(dryScore, "museca", "Single", chart, logger),
+	};
+}
+
+function CalculateDataJubeat(
+	dryScore: DryScore,
+	chart: ChartDocument,
+	_logger: KtLogger
+): CalculatedData<"jubeat:Single"> {
+	return {
+		jubility: Jubility.calculate(
+			dryScore.scoreData.score,
+			dryScore.scoreData.percent,
+			chart.levelNum
+		),
+	};
+}
+
+function CalculateDataCHUNITHM(
+	dryScore: DryScore,
+	chart: ChartDocument
+): CalculatedData<"chunithm:Single"> {
+	return {
+		rating: CHUNITHMRating.calculate(dryScore.scoreData.score, chart.levelNum),
+	};
+}
+
+function CalculateDataGitadora(
+	dryScore: DryScore,
+	chart: ChartDocument
+): CalculatedData<"gitadora:Dora" | "gitadora:Gita"> {
+	return {
+		skill: GITADORASkill.calculate(dryScore.scoreData.percent, chart.levelNum),
+	};
+}
+
+function CalculateDataDDR(
+	dryScore: DryScore,
+	chart: ChartDocument,
+	logger: KtLogger
+): CalculatedData<"ddr:DP" | "ddr:SP"> {
+	return {
+		MFCP: CalculateMFCP(dryScore, chart, logger),
+		ktRating: CalculateKTRating(dryScore, "ddr", chart.playtype, chart, logger),
+	};
+}
+
+export function CalculateDataPMSorBMS(
+	dryScore: DryScore,
+	chart: ChartDocument,
+	_logger: KtLogger
+): CalculatedData<"pms:Controller" | "pms:Keyboard"> {
+	const gptConfig = GetGamePTConfig(dryScore.game, chart.playtype);
+
+	const lampIndex = gptConfig.lamps.indexOf(dryScore.scoreData.lamp);
+
+	return {
+		sieglinde: CalculateSieglinde(chart, lampIndex),
+	};
+}
+
+function CalculateDataWACCA(
+	dryScore: DryScore,
+	chart: ChartDocument,
+	_logger: KtLogger
+): CalculatedData<"wacca:Single"> {
+	const rate = WACCARate.calculate(dryScore.scoreData.score, chart.levelNum);
+
+	return { rate };
+}
+
+function CalculateDataPopn(
+	dryScore: DryScore<"popn:9B">,
+	chart: ChartDocument<"popn:9B">,
+	_logger: KtLogger
+): CalculatedData<"popn:9B"> {
+	return {
+		classPoints: PopnClassPoints.calculate(
+			dryScore.scoreData.score,
+			dryScore.scoreData.lamp,
+			chart.levelNum
+		),
+	};
+}
+
+function CalculateDataITGStamina(
+	dryScore: DryScore<"itg:Stamina">,
+	chart: ChartDocument<"itg:Stamina">
+): CalculatedData<"itg:Stamina"> {
+	// If the user failed -- and we don't know when, return null for everything.
+	if (dryScore.scoreData.lamp === "FAILED" && IsNullish(dryScore.scoreData.hitMeta.diedAt)) {
+		return {
+			blockRating: null,
+			highest32: null,
+			highest256: null,
+		};
+	}
+
+	let h32 = ITGHighestUnbroken.calculateFromNPSPerMeasure(
+		chart.data.breakdown.npsPerMeasure,
+		chart.data.breakdown.notesPerMeasure,
+		dryScore.scoreData.hitMeta.diedAt,
+		32
+	);
+
+	let h256 = ITGHighestUnbroken.calculateFromNPSPerMeasure(
+		chart.data.breakdown.npsPerMeasure,
+		chart.data.breakdown.notesPerMeasure,
+		dryScore.scoreData.hitMeta.diedAt,
+		256
+	);
+
+	// To avoid confusing players, we reject highest 32s less than
+	// 100bpm. Due to how highest32 is calculated, it correctly comes
+	// to the conclusion that sometimes you technically just hit 32 unbroken
+	// measures at like 14 BPM. This is confusing to end users, so we should
+	// hide it.
+	if (h32 !== null && h32 < 100) {
+		h32 = null;
+	}
+
+	if (h256 !== null && h256 < 100) {
+		h256 = null;
+	}
+
+	return {
+		blockRating: chart.levelNum,
+		highest32: h32,
+		highest256: h256,
+	};
+}

server/src/lib/score-import/framework/calculated-data/stats.test.ts

@@ -0,0 +1,357 @@
+import { CalculateKTLampRatingIIDXDP, CalculateKTLampRatingIIDXSP, CalculateMFCP } from "./stats";
+import deepmerge from "deepmerge";
+import CreateLogCtx from "lib/logger/logger";
+import t from "tap";
+import { Testing511SPA, TestingIIDXSPDryScore } from "test-utils/test-data";
+import type { DryScore } from "../common/types";
+import type { ChartDocument, Difficulties, Lamps, ScoreDocument } from "tachi-common";
+
+t.test("#CalculateKTLampRatingIIDXSP", (t) => {
+	function c(
+		nc: number | null,
+		hc: number | null,
+		exhc: number | null
+	): ChartDocument<"iidx:SP"> {
+		const tierlistInfo: ChartDocument<"iidx:SP">["tierlistInfo"] = {};
+
+		if (exhc !== null) {
+			tierlistInfo["kt-EXHC"] = {
+				value: exhc,
+				text: `EXHC ${exhc}`,
+			};
+		}
+
+		if (hc !== null) {
+			tierlistInfo["kt-HC"] = {
+				value: hc,
+				text: `HC ${hc}`,
+			};
+		}
+
+		if (nc !== null) {
+			tierlistInfo["kt-NC"] = {
+				value: nc,
+				text: `HC ${nc}`,
+			};
+		}
+
+		return {
+			...Testing511SPA,
+			tierlistInfo,
+		};
+	}
+
+	function s(lamp: Lamps["iidx:DP" | "iidx:SP"]): DryScore<"iidx:DP" | "iidx:SP"> {
+		return deepmerge(TestingIIDXSPDryScore, {
+			scoreData: {
+				lamp,
+			},
+		});
+	}
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("CLEAR"), c(10.5, 10.6, 10.7)),
+		10.5,
+		"Should return NC if the score was NC."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(10.5, 10.6, 10.7)),
+		10.6,
+		"Should return HC if the score was HC."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("EX HARD CLEAR"), c(10.5, 10.6, 10.7)),
+		10.7,
+		"Should return EXHC if the score was EXHC."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("CLEAR"), c(null, 10.6, 10.7)),
+		0,
+		"Should return 0 if the score was NC but no NC was available."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(null, null, 10.7)),
+		0,
+		"Should return 0 if the score was HC but no HC or NC was available."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(10.5, null, 10.7)),
+		10.5,
+		"Should return NC if the score was HC but no HC was available."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("HARD CLEAR"), c(10.9, 10.5, 10.7)),
+		10.9,
+		"Should return NC if the score was HC but NC was worth more."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("EX HARD CLEAR"), c(10.9, 10.5, 10.7)),
+		10.9,
+		"Should return NC if the score was EXHC but NC was worth more."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("EX HARD CLEAR"), c(10.4, 10.9, 10.7)),
+		10.9,
+		"Should return HC if the score was EXHC but HC was worth more."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXSP(s("CLEAR"), c(null, null, null)),
+		10,
+		"Should return chart level if the chart has no tierlist data."
+	);
+
+	t.end();
+});
+
+t.test("#CalculateKTLampRatingIIDXDP", (t) => {
+	function c(dpTier: number | null): ChartDocument<"iidx:DP"> {
+		return {
+			...Testing511SPA,
+			playtype: "DP",
+			tierlistInfo: {
+				"dp-tier": {
+					value: dpTier,
+				},
+			},
+
+			// hack over-assert this type, since we're merging the SPA into being
+			// a DPA.
+		} as unknown as ChartDocument<"iidx:DP">;
+	}
+
+	function s(lamp: Lamps["iidx:DP" | "iidx:SP"]): DryScore<"iidx:DP" | "iidx:SP"> {
+		return deepmerge(TestingIIDXSPDryScore, {
+			scoreData: {
+				lamp,
+			},
+		});
+	}
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("CLEAR"), c(10.4)),
+		10.4,
+		"Should return 10.4 if chart has tierlist value of 10.4 and was clear."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("FAILED"), c(10.4)),
+		0,
+		"Should return 0 if chart has tierlist value of 10.4 and was not cleared."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("CLEAR"), c(null)),
+		10,
+		"Should return chart level if the chart has no tierlist data."
+	);
+
+	t.equal(
+		CalculateKTLampRatingIIDXDP(s("FAILED"), c(null)),
+		0,
+		"Should return 0 if the chart has no tierlist data and was failed."
+	);
+
+	t.end();
+});
+
+const logger = CreateLogCtx(__filename);
+
+t.test("#CalculateMFCP", (t) => {
+	function TestMFCP(
+		lamp: Lamps["ddr:DP" | "ddr:SP"],
+		levelNum: number,
+		difficulty: Difficulties["ddr:DP" | "ddr:SP"]
+	) {
+		return CalculateMFCP(
+			{
+				scoreData: {
+					lamp,
+				},
+			} as ScoreDocument,
+			{
+				difficulty,
+				levelNum,
+			} as ChartDocument,
+			logger
+		);
+	}
+
+	t.equal(TestMFCP("FAILED", 10, "EXPERT"), null, "Should return null for non-mfcs");
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 10, "BASIC"),
+		null,
+		"Should reject charts on BASIC difficulty."
+	);
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 10, "BEGINNER"),
+		null,
+		"Should reject charts on BEGINNER difficulty."
+	);
+
+	t.test("Should return null for charts with level less than 8", (t) => {
+		for (let i = 1; i <= 7; i++) {
+			t.equal(
+				TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"),
+				null,
+				`Should return null for charts with level ${i}`
+			);
+		}
+
+		t.end();
+	});
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 8, "EXPERT"),
+		1,
+		"Should return 1 for charts with level 8"
+	);
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 9, "EXPERT"),
+		1,
+		"Should return 1 for charts with level 9"
+	);
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 10, "EXPERT"),
+		1,
+		"Should return 1 for charts with level 10"
+	);
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 11, "EXPERT"),
+		2,
+		"Should return 2 for charts with level 11"
+	);
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 12, "EXPERT"),
+		2,
+		"Should return 2 for charts with level 12"
+	);
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 13, "EXPERT"),
+		4,
+		"Should return 4 for charts with level 13"
+	);
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 14, "EXPERT"),
+		8,
+		"Should return 8 for charts with level 14"
+	);
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", 15, "EXPERT"),
+		15,
+		"Should return 15 for charts with level 15"
+	);
+
+	t.test("Should return 25 for charts with level 16-20", (t) => {
+		for (let i = 16; i <= 20; i++) {
+			t.equal(
+				TestMFCP("MARVELOUS FULL COMBO", i, "EXPERT"),
+				25,
+				`Should return 25 for charts with level ${i}`
+			);
+		}
+
+		t.end();
+	});
+
+	t.equal(
+		TestMFCP("MARVELOUS FULL COMBO", NaN, "EXPERT"),
+		null,
+		"Invalid level triggers failsafe."
+	);
+
+	t.end();
+});
+
+t.skip("#CalculateKTRating", (t) => {
+	// t.test("Should call the success calculator if percent > pivotPercent", async (t) => {
+	// 	const r = await CalculateKTRating(
+	// 		deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 80 } }),
+	// 		"iidx",
+	// 		"SP",
+	// 		Testing511SPA,
+	// 		logger
+	// 	);
+
+	// 	t.ok(r > 10, "Should return rating greater than the levelNum of the chart.");
+
+	// 	t.end();
+	// });
+
+	// t.test("Should call the fail calculator if percent > pivotPercent", async (t) => {
+	// 	const r = await CalculateKTRating(
+	// 		TestingIIDXSPDryScore,
+	// 		"iidx",
+	// 		"SP",
+	// 		Testing511SPA,
+	// 		logger
+	// 	);
+
+	// 	t.ok(r < 10, "Should return rating less than the levelNum of the chart.");
+
+	// 	t.end();
+	// });
+
+	// t.test("Should call levelNum if percent === pivotPercent", async (t) => {
+	// 	const r = await CalculateKTRating(
+	// 		deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 77.7777 } }),
+	// 		"iidx",
+	// 		"SP",
+	// 		Testing511SPA,
+	// 		logger
+	// 	);
+
+	// 	t.equal(
+	// 		// hack for approximate tests
+	// 		parseFloat(r.toFixed(2)),
+	// 		10,
+	// 		"Should return rating exactly that of the levelNum of the chart."
+	// 	);
+
+	// 	t.end();
+	// });
+
+	// t.test(
+	// 	"Should trigger safety if completely invalid percent somehow gets through",
+	// 	async (t) => {
+	// 		let r = await CalculateKTRating(
+	// 			deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 1000000000 } }),
+	// 			"iidx",
+	// 			"SP",
+	// 			Testing511SPA,
+	// 			logger
+	// 		);
+
+	// 		t.equal(r, 0, "Should safely return 0 and log a warning.");
+
+	// 		r = await CalculateKTRating(
+	// 			// not high enough to be non-finite but high enough to be > 1000
+	// 			deepmerge(TestingIIDXSPDryScore, { scoreData: { percent: 200 } }),
+	// 			"iidx",
+	// 			"SP",
+	// 			Testing511SPA,
+	// 			logger
+	// 		);
+
+	// 		t.equal(r, 0, "Should safely return 0 and log a warning.");
+
+	// 		t.end();
+	// 	}
+	// );
+
+	t.end();
+});

server/src/lib/score-import/framework/calculated-data/stats.ts

@@ -0,0 +1,216 @@
+import { GetGamePTConfig, IIDX_LAMPS } from "tachi-common";
+import type { DryScore } from "../common/types";
+import type { KtLogger } from "lib/logger/logger";
+import type { ChartDocument, Game, integer, Playtype } from "tachi-common";
+
+/**
+ * Calculate Marvelous Full Combo Points. This algorithm
+ * is used in LIFE4, and described here:
+ * https://life4ddr.com/requirements/#mfcpoints
+ * @returns Null if this score was not eligible, a number otherwise.
+ */
+export function CalculateMFCP(dryScore: DryScore, chartData: ChartDocument, logger: KtLogger) {
+	if (dryScore.scoreData.lamp !== "MARVELOUS FULL COMBO") {
+		return null;
+	}
+
+	// Beginner and BASIC scores are explicitly excluded.
+	if (chartData.difficulty === "BEGINNER" || chartData.difficulty === "BASIC") {
+		return null;
+	}
+
+	if (chartData.levelNum < 8) {
+		return null;
+	} else if (chartData.levelNum <= 10) {
+		return 1;
+	} else if (chartData.levelNum <= 12) {
+		return 2;
+	} else if (chartData.levelNum === 13) {
+		return 4;
+	} else if (chartData.levelNum === 14) {
+		return 8;
+	} else if (chartData.levelNum === 15) {
+		return 15;
+	} else if (chartData.levelNum >= 16) {
+		return 25;
+	}
+
+	logger.warn(
+		`Invalid levelNum passed to MFCP ${chartData.levelNum}. ChartID ${chartData.chartID}.`
+	);
+
+	// failsafe
+	return null;
+}
+
+interface RatingParameters {
+	failHarshnessMultiplier: number;
+	pivotPercent: number;
+	clearExpMultiplier: number;
+}
+
+// Generic Rating Calc that is guaranteed to work for everything. This is unspecialised, and not great.
+function KTRatingCalcV1(
+	percent: number,
+	levelNum: number,
+	parameters: RatingParameters,
+	logger: KtLogger
+) {
+	const percentDiv100 = percent / 100;
+
+	if (percentDiv100 < parameters.pivotPercent) {
+		return RatingCalcV0Fail(percentDiv100, levelNum, parameters);
+	}
+
+	return RatingCalcV1Clear(percentDiv100, levelNum, parameters, logger);
+}
+
+function RatingCalcV1Clear(
+	percentDiv100: number,
+	levelNum: number,
+	parameters: RatingParameters,
+	logger: KtLogger
+) {
+	// https://www.desmos.com/calculator/hn7uxjmjkc
+
+	const rating =
+		Math.cosh(
+			parameters.clearExpMultiplier * levelNum * (percentDiv100 - parameters.pivotPercent)
+		) +
+		(levelNum - 1);
+
+	// checks for Infinity or NaN. I'm not sure how this would happen, but it's a failsafe.
+	if (!Number.isFinite(rating)) {
+		logger.warn(
+			`Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid. Defaulting to 0.`
+		);
+		return 0;
+	} else if (rating > 1000) {
+		logger.warn(
+			`Percent: ${percentDiv100}, Level: ${levelNum} resulted in rating of ${rating}, which is invalid (> 1000). Defaulting to 0.`
+		);
+		return 0;
+	}
+
+	return rating;
+}
+
+function RatingCalcV0Fail(percentDiv100: number, levelNum: number, parameters: RatingParameters) {
+	// https://www.desmos.com/calculator/hn7uxjmjkc
+	return (
+		percentDiv100 ** (parameters.failHarshnessMultiplier * levelNum) *
+		(levelNum / parameters.pivotPercent ** (parameters.failHarshnessMultiplier * levelNum))
+	);
+}
+
+const ratingParameters = {
+	museca: {
+		failHarshnessMultiplier: 1,
+
+		// grade: not fail
+		pivotPercent: 0.8,
+
+		// no real reason
+		clearExpMultiplier: 1,
+	},
+	ddr: {
+		failHarshnessMultiplier: 0.9,
+		pivotPercent: 0.9,
+		clearExpMultiplier: 1,
+	},
+};
+
+export function CalculateKTRating(
+	dryScore: DryScore,
+	game: "ddr" | "museca",
+	playtype: Playtype,
+	chart: ChartDocument,
+	logger: KtLogger
+) {
+	const parameters = ratingParameters[game];
+
+	const levelNum = chart.levelNum;
+
+	return KTRatingCalcV1(dryScore.scoreData.percent, levelNum, parameters, logger);
+}
+
+export function CalculateKTLampRatingIIDXSP(dryScore: DryScore, chart: ChartDocument<"iidx:SP">) {
+	const ncValue = chart.tierlistInfo["kt-NC"]?.value ?? 0;
+	const hcValue = Math.max(chart.tierlistInfo["kt-HC"]?.value ?? 0, ncValue);
+	const exhcValue = Math.max(chart.tierlistInfo["kt-EXHC"]?.value ?? 0, hcValue);
+
+	if (!exhcValue && !hcValue && !ncValue) {
+		return LampRatingNoTierlistInfo(dryScore, "iidx", chart.playtype, chart);
+	}
+
+	const lamp = dryScore.scoreData.lamp;
+
+	const gptConfig = GetGamePTConfig("iidx", chart.playtype);
+
+	const lampIndex = gptConfig.lamps.indexOf(lamp);
+
+	if (exhcValue && lampIndex >= IIDX_LAMPS.EX_HARD_CLEAR) {
+		return exhcValue;
+	} else if (hcValue && lampIndex >= IIDX_LAMPS.HARD_CLEAR) {
+		return hcValue;
+	} else if (ncValue && lampIndex >= IIDX_LAMPS.CLEAR) {
+		return ncValue;
+	}
+
+	return 0;
+}
+
+export function CalculateKTLampRatingIIDXDP(dryScore: DryScore, chart: ChartDocument<"iidx:DP">) {
+	const tierlistValue = chart.tierlistInfo["dp-tier"]?.value ?? 0;
+
+	if (!tierlistValue) {
+		return LampRatingNoTierlistInfo(dryScore, "iidx", chart.playtype, chart);
+	}
+
+	const lamp = dryScore.scoreData.lamp;
+
+	const gptConfig = GetGamePTConfig("iidx", chart.playtype);
+
+	const lampIndex = gptConfig.lamps.indexOf(lamp);
+
+	if (lampIndex >= IIDX_LAMPS.CLEAR) {
+		return tierlistValue;
+	}
+
+	return 0;
+}
+
+function LampRatingNoTierlistInfo(
+	dryScore: DryScore,
+	game: Game,
+	playtype: Playtype,
+	chart: ChartDocument
+) {
+	const gptConfig = GetGamePTConfig(game, playtype);
+
+	const lamps = gptConfig.lamps;
+
+	const CLEAR_LAMP_INDEX = lamps.indexOf(gptConfig.clearLamp);
+
+	// if this is a clear
+	if (lamps.indexOf(dryScore.scoreData.lamp) >= CLEAR_LAMP_INDEX) {
+		// return this chart's numeric level as the lamp rating
+		return chart.levelNum;
+	}
+
+	// else, this score is worth 0.
+	return 0;
+}
+
+export function CalculateSieglinde(chart: ChartDocument, lampIndex: integer) {
+	const ecValue = chart.tierlistInfo["sgl-EC"]?.value ?? 0;
+	const hcValue = chart.tierlistInfo["sgl-HC"]?.value ?? 0;
+
+	if (lampIndex >= IIDX_LAMPS.HARD_CLEAR) {
+		return Math.max(hcValue, ecValue);
+	} else if (lampIndex >= IIDX_LAMPS.EASY_CLEAR) {
+		return ecValue;
+	}
+
+	return 0;
+}

server/src/lib/score-import/framework/common/converter-failures.ts

@@ -0,0 +1,56 @@
+// Various Errors that can occur during processing.
+
+import type { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types";
+import type { ImportTypes } from "tachi-common";
+
+export class ConverterFailure extends Error {
+	message: string;
+
+	constructor(message: string) {
+		super();
+		this.message = message;
+	}
+}
+
+/**
+ * SkipScoreFailure - This score could not be processed because we do not
+ * support it. This is not an error, but is not a success either.
+ * An example scenario would be something like a 5KEY score being imported from SSS.
+ */
+export class SkipScoreFailure extends ConverterFailure {}
+
+/**
+ * KTDataNotFoundError - We could not find the song or chart this score
+ * belongs to. The identifier used to try and match this chart is stored here.
+ * Alongside any other fields used in the query.
+ */
+export class KTDataNotFoundFailure<T extends ImportTypes> extends ConverterFailure {
+	data: ImportTypeDataMap[T];
+	converterContext: ImportTypeContextMap[T];
+	importType: T;
+
+	constructor(
+		message: string,
+		importType: T,
+		data: ImportTypeDataMap[T],
+		context: ImportTypeContextMap[T]
+	) {
+		super(message);
+
+		this.importType = importType;
+		this.data = data;
+		this.converterContext = context;
+	}
+}
+
+/**
+ * InvalidScoreError - This score provided invalid data that we
+ * can not accept.
+ */
+export class InvalidScoreFailure extends ConverterFailure {}
+
+/**
+ * KTInternalFailure - An unexpected, internal error has occured,
+ * and the score could not be processed.
+ */
+export class InternalFailure extends ConverterFailure {}

server/src/lib/score-import/framework/common/get-input-parser.ts

@@ -0,0 +1,27 @@
+import { Parsers } from "lib/score-import/import-types/parsers";
+import type { KtLogger } from "lib/logger/logger";
+import type {
+	ImportTypeContextMap,
+	ImportTypeDataMap,
+	ParserFunctionReturns,
+} from "lib/score-import/import-types/common/types";
+import type { ScoreImportJobData } from "lib/score-import/worker/types";
+import type { ImportTypes } from "tachi-common";
+
+export function GetInputParser<I extends ImportTypes>(jobData: ScoreImportJobData<I>) {
+	// Retrieve the set parser function for this import type.
+	const ParserFunction = Parsers[jobData.importType];
+
+	const InputParser = (logger: KtLogger) =>
+		// eslint-disable-next-line lines-around-comment
+		// @ts-expect-error TypeScript doesn't like the fact that we
+		// pass this as a rest parameter, since none of the parsers
+		// actually take rest args. However, this is the only way to
+		// achieve the dynamic passing we need to, so lets just override
+		// it here.
+		ParserFunction(...jobData.parserArguments, logger) as Promise<
+			ParserFunctionReturns<ImportTypeDataMap[I], ImportTypeContextMap[I]>
+		>;
+
+	return InputParser;
+}

server/src/lib/score-import/framework/common/import-logger.ts

@@ -0,0 +1,22 @@
+import { rootLogger } from "lib/logger/logger";
+import { FormatUserDoc } from "utils/user";
+import type { KtLogger } from "lib/logger/logger";
+import type { ImportTypes, PublicUserDocument } from "tachi-common";
+
+export function CreateScoreLogger(
+	user: PublicUserDocument,
+	importID: string,
+	importType: ImportTypes
+): KtLogger {
+	const meta = {
+		context: ["Score Import", importType, FormatUserDoc(user)],
+		importID,
+	};
+
+	// used so appendLogCtx works
+	const childLogger = rootLogger.child(meta);
+
+	childLogger.defaultMeta = meta;
+
+	return childLogger as KtLogger;
+}

server/src/lib/score-import/framework/common/score-utils.test.ts

@@ -0,0 +1,141 @@
+import { GenericCalculatePercent, GetGradeFromPercent, ValidatePercent } from "./score-utils";
+import t from "tap";
+import { isApproximately } from "test-utils/asserts";
+import { Testing511SPA } from "test-utils/test-data";
+import type { ChartDocument, Game } from "tachi-common";
+
+t.test("#GetGradeForPercent", (t) => {
+	t.equal(
+		GetGradeFromPercent("chunithm", "Single", 101),
+		"SSS",
+		"Should return SSS for the edge case of chunithms 101% == SS."
+	);
+	t.equal(
+		GetGradeFromPercent("iidx", "SP", 100),
+		"MAX",
+		"Should return SSS for the edge case of 100% == MAX (iidx SP)."
+	);
+	t.equal(
+		GetGradeFromPercent("iidx", "DP", 100),
+		"MAX",
+		"Should return SSS for the edge case of 100% == MAX (iidx DP)."
+	);
+	t.equal(
+		GetGradeFromPercent("iidx", "SP", 100 * (1932 / 2484)),
+		"AA",
+		"Should return AA for the edge case of 77.77...% (iidx SP)."
+	);
+	t.equal(
+		GetGradeFromPercent("iidx", "SP", 100 * (7 / 9)),
+		"AA",
+		"Should return AA for the edge case of 77.77...% (iidx SP)."
+	);
+	t.equal(
+		GetGradeFromPercent("iidx", "SP", 77.77777777777777),
+		"AA",
+		"Should return AA for the edge case of 77.77...% (iidx SP)."
+	);
+
+	t.equal(
+		GetGradeFromPercent("iidx", "SP", 0),
+		"F",
+		"Should return SSS for the edge case of 0% == F (iidx SP)."
+	);
+	t.equal(
+		GetGradeFromPercent("iidx", "DP", 0),
+		"F",
+		"Should return SSS for the edge case of 0% == F (iidx DP)."
+	);
+
+	t.end();
+});
+
+t.test("#GenericCalculatePercent", (t) => {
+	const f = (game: Game, score: number, equal: number) =>
+		isApproximately(t, GenericCalculatePercent(game, score), equal);
+
+	for (const game of ["iidx", "bms", "pms"] as const) {
+		isApproximately(t, GenericCalculatePercent(game, 1240, Testing511SPA), 78.88);
+		isApproximately(t, GenericCalculatePercent(game, 0, Testing511SPA), 0);
+	}
+
+	f("popn", 100_000, 100);
+	f("popn", 50_000, 50);
+	f("popn", 0, 0);
+
+	f("maimai", 10, 10);
+	f("maimai", 101.12, 101.12);
+
+	for (const game of ["usc", "sdvx"] as const) {
+		f(game, 10_000_000, 100);
+		f(game, 5_000_000, 50);
+		f(game, 9_950_000, 99.5);
+		f(game, 1_370_000, 13.7);
+		f(game, 0, 0);
+	}
+
+	for (const game of ["ddr", "museca", "chunithm", "wacca"] as const) {
+		f(game, 1_000_000, 100);
+		f(game, 500_000, 50);
+		f(game, 993_121, 99.3121);
+		f(game, 1_000, 0.1);
+	}
+
+	t.throws(() => GenericCalculatePercent("unknown_game" as Game, 100), {
+		message: /Invalid game/u,
+	});
+	t.throws(() => GenericCalculatePercent("iidx", 100), {
+		message: /No Chart passed/u,
+	});
+
+	t.end();
+});
+
+t.test("#ValidatePercent", (t) => {
+	for (const playtype of ["SP", "DP"] as const) {
+		t.doesNotThrow(() => {
+			ValidatePercent("iidx", playtype, 90, Testing511SPA);
+		});
+		t.doesNotThrow(() => {
+			ValidatePercent("iidx", playtype, 0, Testing511SPA);
+		});
+		t.doesNotThrow(() => {
+			ValidatePercent("iidx", playtype, 100, Testing511SPA);
+		});
+
+		t.throws(() => {
+			ValidatePercent("iidx", playtype, -50, Testing511SPA);
+		});
+		t.throws(() => {
+			ValidatePercent("iidx", playtype, 101, Testing511SPA);
+		});
+	}
+
+	const m = (maxPercent: number) => ({ data: { maxPercent } } as ChartDocument);
+
+	t.doesNotThrow(() => {
+		ValidatePercent("maimai", "Single", 90, m(100));
+	});
+	t.doesNotThrow(() => {
+		ValidatePercent("maimai", "Single", 100, m(100));
+	});
+	t.doesNotThrow(() => {
+		ValidatePercent("maimai", "Single", 0, m(100));
+	});
+	t.doesNotThrow(() => {
+		ValidatePercent("maimai", "Single", 110, m(120));
+	});
+	t.doesNotThrow(() => {
+		ValidatePercent("maimai", "Single", 120, m(120));
+	});
+	t.throws(
+		() => {
+			ValidatePercent("maimai", "Single", 130, m(120));
+		},
+		{
+			message: /expected a number less than 120/u,
+		}
+	);
+
+	t.end();
+});

server/src/lib/score-import/framework/common/score-utils.ts

@@ -0,0 +1,217 @@
+import { InternalFailure, InvalidScoreFailure } from "./converter-failures";
+import CreateLogCtx from "lib/logger/logger";
+import { ESDCore, GetGamePTConfig } from "tachi-common";
+import { NotNullish } from "utils/misc";
+import type {
+	ChartDocument,
+	Game,
+	GameToIDStrings,
+	Grades,
+	IDStrings,
+	Playtype,
+} from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * Util for getting a games' grade for a given percent.
+ */
+export function GetGradeFromPercent<I extends IDStrings = IDStrings>(
+	game: Game,
+	playtype: Playtype,
+	percent: number
+): Grades[I] {
+	const gptConfig = GetGamePTConfig(game, playtype);
+	const boundaries = gptConfig.gradeBoundaries;
+	const grades = gptConfig.grades;
+
+	if (!boundaries) {
+		throw new Error(
+			`Invalid call to GetGradeFromPercent! GPT ${game}:${playtype} does not use grade boundaries.`
+		);
+	}
+
+	// (hey, this for loop is backwards!)
+	for (let i = boundaries.length - 1; i >= 0; i--) {
+		if (percent + Number.EPSILON >= NotNullish(boundaries[i])) {
+			return grades[i] as Grades[I];
+		}
+	}
+
+	logger.error(`Could not resolve grade for percent ${percent} on game ${game}`);
+	throw new InternalFailure(`Could not resolve grade for percent ${percent} on game ${game}.`);
+}
+
+/**
+ * A Generic function for calculating a percent from a given score on
+ * a given game.
+ */
+export function GenericCalculatePercent(game: Game, score: number, chart?: ChartDocument): number {
+	switch (game) {
+		case "ddr":
+		case "museca":
+		case "chunithm":
+		case "wacca":
+			return (score / 1_000_000) * 100;
+		case "sdvx":
+		case "usc":
+			return (score / 10_000_000) * 100;
+		case "popn":
+			return (score / 100_000) * 100;
+		case "maimai":
+			// score in maimai is actually just your percent, since nobody cares about
+			// money score.
+			return score;
+		case "bms":
+		case "pms":
+		case "iidx": {
+			if (!chart) {
+				logger.severe("No Chart passed to GenericCalcPercent but game was iidx/bms/pms.");
+				throw new InternalFailure(
+					"No Chart passed to GenericCalcPercent but game was iidx/bms/pms."
+				);
+			}
+
+			// Yeah, we declare it like this so the below return is actually clear.
+			// eslint-disable-next-line no-case-declarations
+			const MAX =
+				(
+					chart as ChartDocument<
+						| "bms:7K"
+						| "bms:14K"
+						| "iidx:DP"
+						| "iidx:SP"
+						| "pms:Controller"
+						| "pms:Keyboard"
+					>
+				).data.notecount * 2;
+
+			return (100 * score) / MAX;
+		}
+
+		case "jubeat":
+		case "gitadora":
+		case "itg":
+			return score;
+
+		default: {
+			logger.severe(`Invalid game passed of ${game} to GenericCalcPercent.`);
+			throw new InternalFailure(`Invalid game passed of ${game} to GenericCalcPercent.`);
+		}
+	}
+}
+
+/**
+ * Helper utility for validating percents on a game. This throws an InvalidScoreFailure if the percent is
+ * invalid, and returns void on success.
+ *
+ * This exists to support maimai, as it has a dynamic "max percent".
+ */
+export function ValidatePercent(
+	game: Game,
+	playtype: Playtype,
+	percent: number,
+	chart: ChartDocument
+) {
+	// i love needing a helper function for *ONE* game.
+	if (game === "maimai") {
+		const mmChart = chart as ChartDocument<"maimai:Single">;
+
+		if (percent > mmChart.data.maxPercent) {
+			throw new InvalidScoreFailure(
+				`Invalid percent - expected a number less than ${mmChart.data.maxPercent}.`
+			);
+		}
+	} else {
+		const gptConfig = GetGamePTConfig(game, playtype);
+
+		if (percent > gptConfig.percentMax) {
+			throw new InvalidScoreFailure(
+				`Invalid percent of ${percent} - expected a value less than ${gptConfig.percentMax}% (${chart.songID} ${chart.playtype} ${chart.difficulty}).`
+			);
+		}
+	}
+
+	if (percent < 0) {
+		throw new InvalidScoreFailure(
+			`Invalid percent of ${percent} - Expected a positive number? (${chart.songID} ${chart.playtype} ${chart.difficulty})`
+		);
+	}
+}
+
+/**
+ * Generically gets the grade and percent for a given score on a given game. This only works for games where
+ * grades are just percent boundaries. This will throw an InvalidScoreFailure if the percent is invalid,
+ * or if the grade is invalid.
+ */
+export function GenericGetGradeAndPercent<G extends Game>(
+	game: G,
+	score: number,
+	chart: ChartDocument
+): { percent: number; grade: Grades[GameToIDStrings[G]] } {
+	const percent = GenericCalculatePercent(game, score, chart);
+
+	ValidatePercent(game, chart.playtype, percent, chart);
+
+	const grade: Grades[GameToIDStrings[G]] = GetGradeFromPercent(game, chart.playtype, percent);
+
+	return { percent, grade };
+}
+
+export function JubeatGetGrade(score: number): Grades["jubeat:Single"] {
+	if (score === 1_000_000) {
+		return "EXC";
+	} else if (score >= 980_000) {
+		return "SSS";
+	} else if (score >= 950_000) {
+		return "SS";
+	} else if (score >= 900_000) {
+		return "S";
+	} else if (score >= 850_000) {
+		return "A";
+	} else if (score >= 800_000) {
+		return "B";
+	} else if (score >= 700_000) {
+		return "C";
+	} else if (score >= 500_000) {
+		return "D";
+	}
+
+	return "E";
+}
+
+/**
+ * Calculates the ESD for a given game + percent combo. This function returns
+ * null if the game does not support support ESD.
+ */
+export function CalculateESDForGame(
+	game: Game,
+	playtype: Playtype,
+	percent: number
+): number | null {
+	const gptConfig = GetGamePTConfig(game, playtype);
+
+	if (!gptConfig.supportsESD) {
+		return null;
+	}
+
+	return ESDCore.CalculateESD(gptConfig.judgementWindows, percent);
+}
+
+/**
+ * Parses and validates a date from a string.
+ * @returns Milliseconds from the unix epoch, or null if the initial argument was null or undefined.
+ */
+export function ParseDateFromString(str: string | null | undefined): number | null {
+	if (!str) {
+		return null;
+	}
+
+	const date = Date.parse(str);
+
+	if (Number.isNaN(date)) {
+		throw new InvalidScoreFailure(`Invalid/Unparsable score timestamp of ${str}.`);
+	}
+
+	return date;
+}

server/src/lib/score-import/framework/common/string-asserts.test.ts

@@ -0,0 +1,186 @@
+import { InvalidScoreFailure } from "./converter-failures";
+import { AssertStrAsPositiveInt, AssertStrAsPositiveNonZeroInt } from "./string-asserts";
+import t from "tap";
+
+function astr(v: string) {
+	try {
+		const r = AssertStrAsPositiveInt(v, "err");
+
+		return r;
+	} catch (e) {
+		return e;
+	}
+}
+
+function astrp(v: string) {
+	try {
+		const r = AssertStrAsPositiveNonZeroInt(v, "err");
+
+		return r;
+	} catch (e) {
+		return e;
+	}
+}
+
+t.test("#AssertStrAsPositiveInt", (t) => {
+	t.strictSame(
+		astr("---"),
+		new InvalidScoreFailure(`err (Not an integer -- ---.)`),
+		"Should reject non-int input"
+	);
+
+	t.strictSame(
+		astr("1.4"),
+		new InvalidScoreFailure(`err (Not an integer -- 1.4.)`),
+		"Should reject float input"
+	);
+
+	t.strictSame(
+		astr("NaN"),
+		new InvalidScoreFailure(`err (Not an integer -- NaN.)`),
+		"Should reject NaN"
+	);
+	t.strictSame(
+		astr("-0.3"),
+		new InvalidScoreFailure(`err (Not an integer -- -0.3.)`),
+		"Should reject negative float input"
+	);
+
+	t.strictSame(
+		astr("0xFF"),
+		new InvalidScoreFailure(`err (Not an integer -- 0xFF.)`),
+		"Should reject hex input"
+	);
+
+	t.strictSame(
+		astr("0b11"),
+		new InvalidScoreFailure(`err (Not an integer -- 0b11.)`),
+		"Should reject binary input"
+	);
+
+	t.strictSame(
+		astr("0o77"),
+		new InvalidScoreFailure(`err (Not an integer -- 0o77.)`),
+		"Should reject oct input"
+	);
+
+	t.strictSame(
+		astr("--1"),
+		new InvalidScoreFailure(`err (Not an integer -- --1.)`),
+		"Should reject double neg input"
+	);
+
+	t.strictSame(
+		astr("12f"),
+		new InvalidScoreFailure(`err (Not an integer -- 12f.)`),
+		"Should reject valid parseInt but invalid Number() input"
+	);
+
+	t.strictSame(
+		astr(`${Number.MAX_SAFE_INTEGER.toString()}000`),
+
+		// @warn Counterintuitive errmsg here. It is an int!
+		new InvalidScoreFailure(
+			`err (Not an integer -- ${Number.MAX_SAFE_INTEGER.toString()}000.)`
+		),
+		"Should reject numbers > max_safe_integer."
+	);
+
+	t.strictSame(
+		astr("-1"),
+		new InvalidScoreFailure(`err (Was negative -- -1.)`),
+		"Should reject negative integer input"
+	);
+
+	t.strictSame(astr("-0"), 0, "Should allow negative 0 input");
+	t.strictSame(astr("0"), 0, "Should allow 0 input");
+	t.strictSame(astr("13"), 13, "Should allow valid int input");
+
+	t.end();
+});
+
+t.test("#AssertStrAsPositiveNonZeroInt", (t) => {
+	t.strictSame(
+		astrp("---"),
+		new InvalidScoreFailure(`err (Not an integer -- ---.)`),
+		"Should reject non-int input"
+	);
+
+	t.strictSame(
+		astrp("1.4"),
+		new InvalidScoreFailure(`err (Not an integer -- 1.4.)`),
+		"Should reject float input"
+	);
+
+	t.strictSame(
+		astrp("NaN"),
+		new InvalidScoreFailure(`err (Not an integer -- NaN.)`),
+		"Should reject NaN"
+	);
+	t.strictSame(
+		astrp("-0.3"),
+		new InvalidScoreFailure(`err (Not an integer -- -0.3.)`),
+		"Should reject negative float input"
+	);
+
+	t.strictSame(
+		astrp("0xFF"),
+		new InvalidScoreFailure(`err (Not an integer -- 0xFF.)`),
+		"Should reject hex input"
+	);
+
+	t.strictSame(
+		astrp("0b11"),
+		new InvalidScoreFailure(`err (Not an integer -- 0b11.)`),
+		"Should reject binary input"
+	);
+
+	t.strictSame(
+		astrp("0o77"),
+		new InvalidScoreFailure(`err (Not an integer -- 0o77.)`),
+		"Should reject oct input"
+	);
+
+	t.strictSame(
+		astrp("--1"),
+		new InvalidScoreFailure(`err (Not an integer -- --1.)`),
+		"Should reject double neg input"
+	);
+
+	t.strictSame(
+		astrp("12f"),
+		new InvalidScoreFailure(`err (Not an integer -- 12f.)`),
+		"Should reject valid parseInt but invalid Number() input"
+	);
+
+	t.strictSame(
+		astrp(`${Number.MAX_SAFE_INTEGER.toString()}000`),
+
+		// @warn Counterintuitive errmsg here. It is an int!
+		new InvalidScoreFailure(
+			`err (Not an integer -- ${Number.MAX_SAFE_INTEGER.toString()}000.)`
+		),
+		"Should reject numbers > max_safe_integer."
+	);
+
+	t.strictSame(
+		astrp("-1"),
+		new InvalidScoreFailure(`err (Was negative or zero -- -1.)`),
+		"Should reject negative integer input"
+	);
+	t.strictSame(
+		astrp("-0"),
+		new InvalidScoreFailure(`err (Was negative or zero -- 0.)`),
+		"Should reject negative 0 input"
+	);
+
+	t.strictSame(
+		astrp("0"),
+		new InvalidScoreFailure(`err (Was negative or zero -- 0.)`),
+		"Should reject 0 input"
+	);
+
+	t.strictSame(astrp("13"), 13, "Should allow valid int input");
+
+	t.end();
+});

server/src/lib/score-import/framework/common/string-asserts.ts

@@ -0,0 +1,59 @@
+import { InvalidScoreFailure } from "./converter-failures";
+import { GetGamePTConfig } from "tachi-common";
+import type { Difficulties, Game, IDStrings, Playtype } from "tachi-common";
+
+export function AssertStrAsDifficulty(
+	strVal: string,
+	game: Game,
+	playtype: Playtype
+): Difficulties[IDStrings] {
+	const validDifficulties = GetGamePTConfig(game, playtype).difficulties;
+
+	if (!validDifficulties.includes(strVal)) {
+		throw new InvalidScoreFailure(
+			`Invalid Difficulty for ${game} ${playtype} - Expected any of ${validDifficulties.join(
+				", "
+			)} (Got ${strVal})`
+		);
+	}
+
+	return strVal;
+}
+
+const isIntegerRegex = /^-?\d+$/u;
+
+export function AssertStrAsPositiveInt(strVal: string, errorMessage: string) {
+	const isInt = isIntegerRegex.test(strVal);
+
+	if (!isInt) {
+		throw new InvalidScoreFailure(`${errorMessage} (Not an integer -- ${strVal}.)`);
+	}
+
+	const val = Number(strVal);
+
+	if (!Number.isSafeInteger(val)) {
+		throw new InvalidScoreFailure(`${errorMessage} (Not an integer -- ${strVal}.)`);
+	} else if (val < 0) {
+		throw new InvalidScoreFailure(`${errorMessage} (Was negative -- ${strVal}.)`);
+	}
+
+	return val;
+}
+
+export function AssertStrAsPositiveNonZeroInt(strVal: string, errorMessage: string) {
+	const isInt = isIntegerRegex.test(strVal);
+
+	if (!isInt) {
+		throw new InvalidScoreFailure(`${errorMessage} (Not an integer -- ${strVal}.)`);
+	}
+
+	const val = Number(strVal);
+
+	if (!Number.isSafeInteger(val)) {
+		throw new InvalidScoreFailure(`${errorMessage} (Not an integer -- ${val}.)`);
+	} else if (val <= 0) {
+		throw new InvalidScoreFailure(`${errorMessage} (Was negative or zero -- ${val}.)`);
+	}
+
+	return val;
+}

server/src/lib/score-import/framework/common/types.ts

@@ -0,0 +1,14 @@
+import type { IDStrings, Playtype, ScoreDocument } from "tachi-common";
+
+/**
+ * An intermediate score format that will be filled out by
+ * HydrateScore.
+ */
+export type DryScore<I extends IDStrings = IDStrings> = Pick<
+	ScoreDocument<I>,
+	"comment" | "game" | "importType" | "scoreMeta" | "service" | "timeAchieved"
+> & {
+	scoreData: Omit<ScoreDocument<I>["scoreData"], "esd" | "gradeIndex" | "lampIndex">;
+};
+
+export type ScorePlaytypeMap = Partial<Record<Playtype, Array<ScoreDocument>>>;

server/src/lib/score-import/framework/express-wrapper.ts

@@ -0,0 +1,74 @@
+import { MakeScoreImport } from "./score-import";
+import ScoreImportFatalError from "./score-importing/score-import-error";
+import CreateLogCtx from "lib/logger/logger";
+import { Random20Hex } from "utils/misc";
+import type { ParserArguments } from "../worker/types";
+import type {
+	ImportDocument,
+	ImportTypes,
+	integer,
+	SuccessfulAPIResponse,
+	UnsuccessfulAPIResponse,
+} from "tachi-common";
+
+export interface WrappedAPIResponse {
+	statusCode: number;
+	body: SuccessfulAPIResponse<ImportDocument> | UnsuccessfulAPIResponse;
+}
+
+const logger = CreateLogCtx(__filename);
+
+/**
+ * A thin(ish) wrapper for ScoreImportMain which converts thrown
+ * errors and import documents into a WrappedAPIResponse, which can
+ * be immediately sent with res.json().
+ */
+export async function ExpressWrappedScoreImportMain<I extends ImportTypes>(
+	userID: integer,
+	userIntent: boolean,
+	importType: I,
+	parserArguments: ParserArguments<I>
+): Promise<WrappedAPIResponse> {
+	const importID = Random20Hex();
+
+	logger.debug("Received import request.");
+
+	try {
+		const res = await MakeScoreImport({
+			importID,
+			importType,
+			userIntent,
+			userID,
+			parserArguments,
+		});
+
+		return {
+			statusCode: 200,
+			body: {
+				success: true,
+				description: "Import successful.",
+				body: res,
+			},
+		};
+	} catch (err) {
+		if (err instanceof ScoreImportFatalError) {
+			logger.info(err.message);
+			return {
+				statusCode: err.statusCode,
+				body: {
+					success: false,
+					description: err.message,
+				},
+			};
+		}
+
+		logger.error(err);
+		return {
+			statusCode: 500,
+			body: {
+				success: false,
+				description: "An internal service error has occured. This has been reported!",
+			},
+		};
+	}
+}

server/src/lib/score-import/framework/goals/goals.test.ts

@@ -0,0 +1,487 @@
+import { GetRelevantFolderGoals, GetRelevantGoals, UpdateGoalsForUser, ProcessGoal } from "./goals";
+import deepmerge from "deepmerge";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+import {
+	GetKTDataJSON,
+	Testing511SPA,
+	TestingIIDXSPScorePB,
+	HC511Goal,
+	HC511UserGoal,
+	TestingIIDXFolderSP10,
+} from "test-utils/test-data";
+import { CreateFolderChartLookup } from "utils/folder";
+import crypto from "crypto";
+import type {
+	ChartDocument,
+	GoalDocument,
+	GoalSubscriptionDocument,
+	SongDocument,
+} from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+t.test("#GetRelevantFolderGoals", (t) => {
+	t.beforeEach(ResetDBState);
+
+	const fakeFolderGoalDocument: GoalDocument = {
+		charts: {
+			type: "folder",
+			data: "ed9d8c734447ce67d7135c0067441a98cc81aeaf",
+		},
+		game: "iidx",
+		goalID: "fake_goal_id",
+		playtype: "SP",
+		timeAdded: 0,
+		name: "get > 1 ex score on any level 10.",
+		criteria: {
+			mode: "single",
+			value: 1,
+			key: "scoreData.score",
+		},
+	};
+
+	const notFolderGoalDocument: GoalDocument = {
+		charts: {
+			type: "folder",
+			data: "some_fake_folder_id",
+		},
+		game: "iidx",
+		goalID: "fake_bad_goal_id",
+		playtype: "SP",
+		timeAdded: 0,
+		name: "get > 1 ex score on some other folder.",
+		criteria: {
+			mode: "single",
+			value: 1,
+			key: "scoreData.score",
+		},
+	};
+
+	t.beforeEach(async () => {
+		// @ts-expect-error garbage types
+		const sp11folder = deepmerge(TestingIIDXFolderSP10, {
+			data: { level: "11" },
+			folderID: "foo",
+		});
+
+		await db.folders.insert(TestingIIDXFolderSP10);
+		await db.folders.insert(sp11folder);
+		await db.goals.insert(fakeFolderGoalDocument);
+		await db.goals.insert(notFolderGoalDocument);
+		await CreateFolderChartLookup(TestingIIDXFolderSP10, true);
+		await CreateFolderChartLookup(sp11folder, true);
+	});
+
+	t.test("Should correctly find the goals on this folder.", async (t) => {
+		const res = await GetRelevantFolderGoals(
+			["fake_goal_id", "fake_bad_goal_id"],
+			[Testing511SPA.chartID]
+		);
+
+		t.strictSame(
+			res,
+			[fakeFolderGoalDocument],
+			"Should correctly return only the 511 goal document."
+		);
+
+		t.end();
+	});
+
+	t.end();
+});
+
+t.test("#GetRelevantGoals", (t) => {
+	t.beforeEach(ResetDBState);
+	t.beforeEach(async () => {
+		// use the real data so we have enough charts loaded for this to test properly.
+		await db.songs.iidx.remove({});
+		await db.charts.iidx.remove({});
+		await db.charts.iidx.insert(
+			GetKTDataJSON("./tachi/tachi-charts-iidx.json") as Array<
+				ChartDocument<"iidx:DP" | "iidx:SP">
+			>
+		);
+		await db.songs.iidx.insert(
+			GetKTDataJSON("./tachi/tachi-songs-iidx.json") as Array<SongDocument<"iidx">>
+		);
+
+		const lotsOfCharts = await db.charts.iidx.find({}, { limit: 20 });
+		const goals: Array<GoalDocument> = lotsOfCharts.map((e) => ({
+			charts: {
+				type: "single",
+				data: e.chartID,
+			},
+			game: "iidx",
+			goalID: crypto.randomBytes(20).toString("hex"),
+			playtype: "SP",
+			timeAdded: 0,
+			name: "get > 1 ex score on some other folder.",
+			criteria: {
+				mode: "single",
+				value: 1,
+				key: "scoreData.score",
+			},
+		}));
+
+		await db.goals.insert(goals);
+
+		await db["goal-subs"].insert(
+			goals.map((e) => ({
+				achieved: false,
+				wasInstantlyAchieved: false,
+				timeAchieved: null,
+				game: "iidx",
+				playtype: "SP",
+				goalID: e.goalID,
+				lastInteraction: null,
+				outOf: 5,
+				outOfHuman: "HARD CLEAR",
+				progress: null,
+				progressHuman: "NO DATA",
+				timeSet: Date.now(),
+				userID: 1,
+			}))
+		);
+	});
+
+	t.test(
+		"Should successfully filter goals to only those that are relevant to the session.",
+		async (t) => {
+			// lets pretend our session had scores on charts 0->4 and 20->25. We also only have goals on
+			// charts 1->20, so only 5 of these should resolve.
+			const ourCharts = [
+				...(await db.charts.iidx.find({}, { limit: 5 })),
+				...(await db.charts.iidx.find({}, { skip: 20, limit: 5 })),
+			];
+
+			const chartIDs = new Set(ourCharts.map((e) => e.chartID));
+
+			const res = await GetRelevantGoals("iidx", 1, chartIDs, logger);
+
+			t.equal(res.goals.length, 5, "Should correctly resolve to 5 goals.");
+			t.equal(res.goalSubsMap.size, 5, "Should also return 5 goalSubs.");
+
+			t.end();
+		}
+	);
+
+	t.end();
+});
+
+t.test("#UpdateGoalsForUser", (t) => {
+	t.beforeEach(ResetDBState);
+
+	const baseGoalDocument: GoalDocument = {
+		charts: {
+			type: "single" as const,
+			data: Testing511SPA.chartID,
+		},
+		game: "iidx",
+		goalID: "FAKE_GOAL_ID",
+		playtype: "SP",
+		timeAdded: 0,
+		name: "get > 1 ex score on some other folder.",
+		criteria: {
+			mode: "single",
+			value: 1,
+			key: "scoreData.score",
+		},
+	};
+
+	const baseGoalSubscriptionDocument: GoalSubscriptionDocument = {
+		achieved: false,
+		wasInstantlyAchieved: false,
+		game: "iidx",
+		playtype: "SP",
+		goalID: "FAKE_GOAL_ID",
+		lastInteraction: null,
+		outOf: 1,
+		outOfHuman: "1",
+		progress: 0,
+		progressHuman: "0",
+		timeAchieved: null,
+		timeSet: 0,
+		userID: 1,
+	};
+
+	t.test("Should correctly update goals when user achieves goal.", async (t) => {
+		await db.goals.insert(baseGoalDocument);
+		delete baseGoalDocument._id;
+
+		await db["goal-subs"].insert(baseGoalSubscriptionDocument);
+
+		// we dont delete _id here because updategoalsforuser
+		// depends on usergoal _id
+
+		await db["personal-bests"].insert(TestingIIDXSPScorePB);
+		delete TestingIIDXSPScorePB._id;
+
+		const ugMap = new Map([["FAKE_GOAL_ID", baseGoalSubscriptionDocument]]);
+
+		const res = await UpdateGoalsForUser([baseGoalDocument], ugMap, 1, logger);
+
+		t.strictSame(res, [
+			{
+				goalID: "FAKE_GOAL_ID",
+				old: {
+					progress: 0,
+					progressHuman: "0",
+					outOf: 1,
+					outOfHuman: "1",
+					achieved: false,
+				},
+				new: {
+					progress: 1479,
+					progressHuman: "1479",
+					outOf: 1,
+					outOfHuman: "1",
+					achieved: true,
+				},
+			},
+		]);
+
+		const r = await db["goal-subs"].findOne({ goalID: "FAKE_GOAL_ID", userID: 1 });
+
+		t.hasStrict(
+			r,
+			{
+				progress: 1479,
+				progressHuman: "1479",
+				outOf: 1,
+				outOfHuman: "1",
+				achieved: true,
+			},
+			"Should update goals in the database."
+		);
+
+		delete baseGoalSubscriptionDocument._id;
+
+		t.end();
+	});
+
+	t.test("Should correctly update goals when user does not achieve goal.", async (t) => {
+		const goal = deepmerge(baseGoalDocument, { criteria: { value: 2 } });
+
+		await db.goals.insert(goal);
+
+		const goalSub = deepmerge(baseGoalSubscriptionDocument, {
+			outOf: 2,
+			outOfHuman: "2",
+		}) as unknown as GoalSubscriptionDocument;
+
+		await db["goal-subs"].insert(goalSub);
+
+		// we dont delete _id here because updategoalsforuser
+		// depends on usergoal _id
+
+		await db["personal-bests"].insert(
+			deepmerge(TestingIIDXSPScorePB, { scoreData: { score: 1 } })
+		);
+
+		const ugMap = new Map([["FAKE_GOAL_ID", goalSub]]);
+
+		const res = await UpdateGoalsForUser([goal], ugMap, 1, logger);
+
+		t.strictSame(res, [
+			{
+				goalID: "FAKE_GOAL_ID",
+				old: {
+					progress: 0,
+					progressHuman: "0",
+					outOf: 2,
+					outOfHuman: "2",
+					achieved: false,
+				},
+				new: {
+					progress: 1,
+					progressHuman: "1",
+					outOf: 2,
+					outOfHuman: "2",
+					achieved: false,
+				},
+			},
+		]);
+
+		const r = await db["goal-subs"].findOne({ goalID: "FAKE_GOAL_ID", userID: 1 });
+
+		t.hasStrict(
+			r,
+			{
+				progress: 1,
+				progressHuman: "1",
+				outOf: 2,
+				outOfHuman: "2",
+				achieved: false,
+			},
+			"Should update goals in the database."
+		);
+
+		delete baseGoalSubscriptionDocument._id;
+
+		t.end();
+	});
+
+	t.test("Should return [] if no data is to be changed.", async (t) => {
+		const res = await UpdateGoalsForUser([], new Map(), 1, logger);
+
+		t.strictSame(res, []);
+
+		t.end();
+	});
+
+	t.test("Should handle (skip) goals if no usergoal is set.", async (t) => {
+		const res = await UpdateGoalsForUser([baseGoalDocument], new Map(), 1, logger);
+
+		t.strictSame(res, []);
+
+		t.end();
+	});
+
+	t.test("Should handle (skip) invalid goals.", async (t) => {
+		const res = await UpdateGoalsForUser(
+			[deepmerge(baseGoalDocument, { charts: { type: "INVALID" } })],
+			new Map([["FAKE_GOAL_ID", baseGoalSubscriptionDocument]]),
+			1,
+			logger
+		);
+
+		t.strictSame(res, []);
+
+		t.end();
+	});
+
+	t.end();
+});
+
+t.todo("#GetAndUpdateUsersGoals", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.end();
+});
+
+t.test("#ProcessGoal", (t) => {
+	t.beforeEach(ResetDBState);
+	t.beforeEach(async () => {
+		await db.goals.insert(HC511Goal);
+	});
+
+	t.test("Should process the users goal if a score has changed.", async (t) => {
+		await db["goal-subs"].insert(HC511UserGoal);
+
+		// score is EX HARD CLEAR by default.
+		await db["personal-bests"].insert(TestingIIDXSPScorePB);
+
+		const res = await ProcessGoal(HC511Goal, HC511UserGoal, 1, logger);
+
+		t.not(res, undefined, "Should NOT return undefined.");
+
+		t.strictSame(
+			res?.import,
+			{
+				goalID: "mock_goalID",
+				old: {
+					progress: null,
+					progressHuman: "NO DATA",
+					outOf: 5,
+					outOfHuman: "HARD CLEAR",
+					achieved: false,
+				},
+				new: {
+					progress: 6,
+					progressHuman: "EX HARD CLEAR (BP: 2)",
+					outOf: 5,
+					outOfHuman: "HARD CLEAR",
+					achieved: true,
+				},
+			},
+			"Should return a valid import goal fragment."
+		);
+
+		t.end();
+	});
+
+	t.test("Should unset wasInstantlyAchieved if the goal became unachieved.", async (t) => {
+		const achievedGoalSub: GoalSubscriptionDocument = {
+			achieved: true,
+			game: "iidx",
+			goalID: "mock_goalID",
+			lastInteraction: null,
+			outOf: 5,
+			outOfHuman: "HARD CLEAR",
+			playtype: "SP",
+			progress: 6,
+			progressHuman: "EX HARD CLEAR",
+			timeAchieved: 1000,
+			timeSet: 1000,
+			wasInstantlyAchieved: true,
+			userID: 1,
+		};
+
+		await db["goal-subs"].insert(achievedGoalSub);
+
+		const res = await ProcessGoal(HC511Goal, achievedGoalSub, 1, logger);
+
+		t.not(res, undefined, "Should NOT return undefined.");
+
+		t.hasStrict(
+			res?.import,
+			{
+				goalID: "mock_goalID",
+				old: {
+					progress: 6,
+					outOf: 5,
+					achieved: true,
+				},
+				new: {
+					progress: null,
+					outOf: 5,
+					achieved: false,
+				},
+			},
+			"Should unachieve the goal."
+		);
+
+		t.equal(
+			res?.bwrite.updateOne.update.$set.wasInstantlyAchieved,
+			false,
+			"Goal is to be set as not instantly achieved."
+		);
+
+		t.end();
+	});
+
+	t.test("Should return undefined if there's no score.", async (t) => {
+		await db["goal-subs"].insert(HC511UserGoal);
+
+		const res = await ProcessGoal(HC511Goal, HC511UserGoal, 1, logger);
+
+		t.equal(res, undefined, "Should return undefined.");
+
+		t.end();
+	});
+
+	t.test("Should return undefined if the progress has not changed.", async (t) => {
+		await db["goal-subs"].insert(HC511UserGoal);
+		await db["personal-bests"].insert(TestingIIDXSPScorePB);
+
+		const firstUpdate = await ProcessGoal(HC511Goal, HC511UserGoal, 1, logger);
+
+		// ignore this one
+		t.not(firstUpdate, undefined, "Should NOT return undefined.");
+
+		await db["goal-subs"].bulkWrite([firstUpdate!.bwrite]);
+
+		const goalSub = await db["goal-subs"].findOne({ userID: 1, goalID: HC511Goal.goalID });
+
+		const secondUpdate = await ProcessGoal(HC511Goal, goalSub!, 1, logger);
+
+		t.equal(secondUpdate, undefined, "Should return undefined.");
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/score-import/framework/goals/goals.ts

@@ -0,0 +1,289 @@
+import db from "external/mongo/db";
+import { EvaluateGoalForUser } from "lib/targets/goals";
+import { EmitWebhookEvent } from "lib/webhooks/webhooks";
+import type { KtLogger } from "lib/logger/logger";
+import type { Game, GoalDocument, GoalSubscriptionDocument, integer } from "tachi-common";
+
+/**
+ * Update a user's progress on all of their set goals.
+ */
+export async function GetAndUpdateUsersGoals(
+	game: Game,
+	userID: integer,
+	chartIDs: Set<string>,
+	logger: KtLogger
+) {
+	const { goals, goalSubsMap } = await GetRelevantGoals(game, userID, chartIDs, logger);
+
+	if (!goals.length) {
+		// if we hit the below code with an empty array mongodb will flip out on the bulkwrite op
+		return [];
+	}
+
+	logger.verbose(`Found ${goals.length} relevant goals.`);
+
+	return UpdateGoalsForUser(goals, goalSubsMap, userID, logger);
+}
+
+export async function UpdateGoalsForUser(
+	goals: Array<GoalDocument>,
+	goalSubsMap: Map<string, GoalSubscriptionDocument>,
+	userID: integer,
+	logger: KtLogger
+) {
+	const returns = await Promise.all(
+		goals.map((goal: GoalDocument) => {
+			const goalSub = goalSubsMap.get(goal.goalID);
+
+			if (!goalSub) {
+				logger.error(
+					`UserGoal:GoalID mismatch ${goal.goalID} - this user has no goalSub for this, yet it is set.`
+				);
+
+				return null;
+			}
+
+			return ProcessGoal(goal, goalSub, userID, logger).catch((err: Error) => {
+				logger.warn(
+					`Failed to process goal '${goal.name}' for ${userID}, ${err.message}. Skipping.`,
+					{ goal, err, userID, goalSub }
+				);
+
+				return undefined;
+			});
+		})
+	);
+
+	const importInfo = [];
+	const bulkWrite = [];
+	const webhookEventContent = [];
+
+	for (const ret of returns) {
+		if (!ret) {
+			continue;
+		}
+
+		importInfo.push(ret.import);
+		bulkWrite.push(ret.bwrite);
+
+		if (ret.webhookEvent) {
+			webhookEventContent.push(ret.webhookEvent);
+		}
+	}
+
+	if (bulkWrite.length === 0) {
+		// bulkwrite cannot be an empty array -- this means there's nothing to update or return, then.
+		// i.e. goals was non empty but returns was entirely [undefined, undefined...].
+		return [];
+	}
+
+	if (webhookEventContent.length !== 0 && goals[0]) {
+		await EmitWebhookEvent({
+			type: "goals-achieved/v1",
+			content: { goals: webhookEventContent, userID, game: goals[0].game },
+		});
+	}
+
+	await db["goal-subs"].bulkWrite(bulkWrite, { ordered: false });
+
+	return importInfo;
+}
+
+/**
+ * Calls EvaluateGoalForUser, then processes the returns into a bulkWrite
+ * operation and an import statistic.
+ * @returns undefined on error (i.e. EvaluateGoalForUser) OR if there's nothing
+ * to say (i.e. user didnt raise the goal).
+ */
+export async function ProcessGoal(
+	goal: GoalDocument,
+	goalSub: GoalSubscriptionDocument,
+	userID: integer,
+	logger: KtLogger
+) {
+	const res = await EvaluateGoalForUser(goal, userID, logger);
+
+	if (!res) {
+		// some sort of error occured - its logged by the previous function.
+		return;
+	}
+
+	// nothing has changed
+	if (goalSub.progress === res.progress && goalSub.outOf === res.outOf) {
+		return;
+	}
+
+	const newData = {
+		progress: res.progress,
+		progressHuman: res.progressHuman,
+		outOf: res.outOf,
+		outOfHuman: res.outOfHuman,
+		achieved: res.achieved,
+	};
+
+	const oldData = {
+		progress: goalSub.progress,
+		progressHuman: goalSub.progressHuman,
+		outOf: goalSub.outOf,
+		outOfHuman: goalSub.outOfHuman,
+		achieved: goalSub.achieved,
+	};
+
+	let webhookEvent = null;
+
+	// if this is a newly-achieved goal
+
+	if (res.achieved && !goalSub.achieved) {
+		webhookEvent = {
+			goalID: goal.goalID,
+			old: oldData,
+			new: newData,
+			playtype: goal.playtype,
+		};
+	}
+
+	const setData = {
+		...newData,
+		timeAchieved: newData.achieved ? Date.now() : null,
+
+		// we're guaranteed that this works, because things
+		// that haven't changed return nothing instead of
+		// getting to this point.
+		lastInteraction: Date.now(),
+	} as unknown as Partial<GoalSubscriptionDocument>;
+
+	// If this goal was achieved, and is now *not* achieved, we need to unset
+	// some things.
+	if (goalSub.achieved && !res.achieved) {
+		logger.info(`User ${userID} lost their achieved status on ${goal.name}.`, {
+			goal,
+			res,
+			goalSub,
+		});
+
+		// This goal can't be marked as instantly achieved, since it was lost.
+		setData.wasInstantlyAchieved = false;
+	}
+
+	const bulkWrite = {
+		updateOne: {
+			filter: { goalID: goalSub.goalID, userID: goalSub.userID },
+			update: {
+				$set: setData,
+			},
+		},
+	};
+
+	return {
+		bwrite: bulkWrite,
+		import: {
+			goalID: goal.goalID,
+			old: oldData,
+			new: newData,
+		},
+		webhookEvent,
+	};
+}
+
+/**
+ * Gets the goals the user has set for this game and playtype.
+ * Then, filters it based on the chartIDs involved in this import.
+ *
+ * This optimisation allows users to have *lots* of goals, but only ever
+ * evaluate the ones we need to.
+ * @returns An array of Goals, and an array of goalSubs.
+ */
+export async function GetRelevantGoals(
+	game: Game,
+	userID: integer,
+	chartIDs: Set<string>,
+	logger: KtLogger
+): Promise<{ goals: Array<GoalDocument>; goalSubsMap: Map<string, GoalSubscriptionDocument> }> {
+	const goalSubs = await db["goal-subs"].find({ game, userID });
+
+	logger.verbose(`Found user has ${goalSubs.length} goals.`);
+
+	if (!goalSubs.length) {
+		return { goals: [], goalSubsMap: new Map() };
+	}
+
+	const goalIDs = goalSubs.map((e) => e.goalID);
+
+	const chartIDsArr: Array<string> = [];
+
+	for (const c of chartIDs) {
+		chartIDsArr.push(c);
+	}
+
+	const goals = await Promise.all([
+		// this gets the relevantGoals for direct and multi
+		db.goals.find({
+			"charts.type": { $in: ["single", "multi"] },
+			"charts.data": { $in: chartIDsArr },
+			goalID: { $in: goalIDs },
+		}),
+		db.goals.find({
+			"charts.type": "any",
+			goalID: { $in: goalIDs },
+		}),
+		GetRelevantFolderGoals(goalIDs, chartIDsArr),
+	]).then((r) => r.flat(1));
+
+	const goalSet = new Set(goals.map((e) => e.goalID));
+
+	const goalSubsMap: Map<string, GoalSubscriptionDocument> = new Map();
+
+	for (const goalSub of goalSubs) {
+		if (!goalSet.has(goalSub.goalID)) {
+			continue;
+		}
+
+		// since these are guaranteed to be unique, lets make a hot map of goalID -> goalSubDocument, so we can
+		// pull them in for post-processing and filter out the goalSubDocuments that aren't relevant.
+		goalSubsMap.set(goalSub.goalID, goalSub);
+	}
+
+	return {
+		goals,
+		goalSubsMap,
+	};
+}
+
+/**
+ * Returns the set of goals where its folder contains any member
+ * of chartIDsArr.
+ */
+export function GetRelevantFolderGoals(goalIDs: Array<string>, chartIDsArr: Array<string>) {
+	// Slightly black magic - this is kind of like doing an SQL join.
+	// it's weird to do this in mongodb, but this seems like the right
+	// way to actually handle this.
+
+	const result: Promise<Array<GoalDocument>> = db.goals.aggregate([
+		{
+			$match: {
+				"charts.type": "folder",
+				goalID: { $in: goalIDs },
+			},
+		},
+		{
+			$lookup: {
+				from: "folder-chart-lookup",
+				localField: "charts.data",
+				foreignField: "folderID",
+				as: "folderCharts",
+			},
+		},
+		{
+			$match: {
+				"folderCharts.chartID": { $in: chartIDsArr },
+			},
+		},
+		{
+			$project: {
+				folderCharts: 0,
+			},
+		},
+	]);
+
+	return result;
+}

server/src/lib/score-import/framework/import-locks/lock.ts

@@ -0,0 +1,50 @@
+import db from "external/mongo/db";
+import type { integer } from "tachi-common";
+
+/**
+ * If a user has no ongoing import, enable the import lock and return true.
+ * If a user has an ongoing import, return false.
+ *
+ * @param userID - The user this import lock is for.
+ * @returns True if the lock was set successfully, false if the user already
+ * has a lock.
+ */
+export async function CheckAndSetOngoingImportLock(userID: integer) {
+	const lockExists = await db["import-locks"].findOne({
+		userID,
+	});
+
+	if (!lockExists) {
+		await db["import-locks"].insert({
+			userID,
+			locked: false,
+		});
+	}
+
+	const lockWasSet = await db["import-locks"].findOneAndUpdate(
+		{
+			userID,
+			locked: false,
+		},
+		{
+			$set: { locked: true },
+		}
+	);
+
+	return !lockWasSet;
+}
+
+/**
+ * Disable a users import lock.
+ */
+export function UnsetOngoingImportLock(userID: integer) {
+	return db["import-locks"].findOneAndUpdate(
+		{
+			userID,
+			locked: true,
+		},
+		{
+			$set: { locked: false },
+		}
+	);
+}

server/src/lib/score-import/framework/milestones/milestones.test.ts

@@ -0,0 +1,369 @@
+import { UpdateUsersMilestones } from "./milestones";
+import deepmerge from "deepmerge";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+import { TestingIIDXSPMilestone } from "test-utils/test-data";
+import type { GoalImportInfo } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+function CreateMockGII(...garr: Array<[string, boolean]>) {
+	return garr.map((e) => ({
+		goalID: e[0],
+		old: {},
+		new: { achieved: e[1] },
+	})) as unknown as Array<GoalImportInfo>;
+}
+
+t.test("#UpdateUsersMilestones", (t) => {
+	t.beforeEach(ResetDBState);
+	t.beforeEach(() => db.milestones.insert(TestingIIDXSPMilestone));
+	t.beforeEach(() =>
+		db["milestone-subs"].insert({
+			achieved: false,
+			wasInstantlyAchieved: false,
+			game: "iidx",
+			milestoneID: TestingIIDXSPMilestone.milestoneID,
+			playtype: "SP",
+			progress: 0,
+			timeAchieved: null,
+			lastInteraction: null,
+			timeSet: 0,
+			userID: 1,
+		})
+	);
+
+	t.test("Test with clean achieved milestone.", async (t) => {
+		const res = await UpdateUsersMilestones(
+			CreateMockGII(
+				["eg_goal_1", true],
+				["eg_goal_2", true],
+				["eg_goal_3", true],
+				["eg_goal_4", true]
+			),
+			"iidx",
+			["SP"],
+			1,
+			logger
+		);
+
+		t.strictSame(
+			res,
+			[
+				{
+					milestoneID: "example_milestone_id",
+					old: {
+						progress: 0,
+						achieved: false,
+					},
+					new: {
+						progress: 4,
+						achieved: true,
+					},
+				},
+			],
+			"Should correctly assert the milestone is achieved."
+		);
+
+		t.end();
+	});
+
+	t.test("Test with unclean achieved milestone.", async (t) => {
+		const res = await UpdateUsersMilestones(
+			CreateMockGII(
+				["eg_goal_1", true],
+				["eg_goal_2", true],
+				["eg_goal_3", true],
+				["eg_goal_4", true],
+				["x_goal_1", true],
+				["x_goal_2", true],
+				["x_goal_3", true]
+			),
+			"iidx",
+			["SP"],
+			1,
+			logger
+		);
+
+		t.strictSame(
+			res,
+			[
+				{
+					milestoneID: "example_milestone_id",
+					old: {
+						progress: 0,
+						achieved: false,
+					},
+					new: {
+						progress: 4,
+						achieved: true,
+					},
+				},
+			],
+			"Should correctly assert the milestone is achieved."
+		);
+
+		t.end();
+	});
+
+	t.test("Test with increased progress on milestone.", async (t) => {
+		const res = await UpdateUsersMilestones(
+			CreateMockGII(["eg_goal_1", true], ["eg_goal_2", true]),
+			"iidx",
+			["SP"],
+			1,
+			logger
+		);
+
+		t.strictSame(
+			res,
+			[
+				{
+					milestoneID: "example_milestone_id",
+					old: {
+						progress: 0,
+						achieved: false,
+					},
+					new: {
+						progress: 2,
+						achieved: false,
+					},
+				},
+			],
+			"Should correctly assert the milestone progress has increased."
+		);
+
+		t.end();
+	});
+
+	t.test("Test with no new goals on milestone.", async (t) => {
+		const res = await UpdateUsersMilestones([], "iidx", ["SP"], 1, logger);
+
+		t.strictSame(res, [], "Should correctly return no changes.");
+
+		t.end();
+	});
+
+	t.test("Test with new goals on multiple milestones.", async (t) => {
+		delete TestingIIDXSPMilestone._id;
+		await db.milestones.insert(
+			// eslint-disable-next-line lines-around-comment
+			// @ts-expect-error lol
+			deepmerge(TestingIIDXSPMilestone, {
+				milestoneID: "some_other_milestone_with_mutual_goals",
+			})
+		);
+		await db["milestone-subs"].insert({
+			achieved: false,
+			wasInstantlyAchieved: false,
+			game: "iidx",
+			milestoneID: "some_other_milestone_with_mutual_goals",
+			playtype: "SP",
+			progress: 0,
+			timeAchieved: null,
+			lastInteraction: null,
+			timeSet: 0,
+			userID: 1,
+		});
+
+		const res = await UpdateUsersMilestones(
+			CreateMockGII(
+				["eg_goal_1", true],
+				["eg_goal_2", true],
+				["eg_goal_3", true],
+				["eg_goal_4", true],
+				["x_goal_1", true],
+				["x_goal_2", true],
+				["x_goal_3", true]
+			),
+			"iidx",
+			["SP", "DP"],
+			1,
+			logger
+		);
+
+		t.strictSame(
+			res,
+			[
+				{
+					milestoneID: "example_milestone_id",
+					old: {
+						progress: 0,
+						achieved: false,
+					},
+					new: {
+						progress: 4,
+						achieved: true,
+					},
+				},
+				{
+					milestoneID: "some_other_milestone_with_mutual_goals",
+					old: {
+						progress: 0,
+						achieved: false,
+					},
+					new: {
+						progress: 4,
+						achieved: true,
+					},
+				},
+			],
+			"Should correctly assert the milestones are achieved."
+		);
+		t.end();
+	});
+
+	t.test("Test with multiple milestones that only some match", async (t) => {
+		delete TestingIIDXSPMilestone._id;
+		await db.milestones.insert([
+			// @ts-expect-error lol
+			deepmerge(TestingIIDXSPMilestone, {
+				milestoneID: "some_other_milestone_with_mutual_goals",
+			}),
+			deepmerge(
+				TestingIIDXSPMilestone,
+				{
+					milestoneID: "dp_milestone",
+					playtype: "DP",
+					milestoneData: [
+						{
+							goals: [{ goalID: "foobar" }],
+						},
+					],
+				},
+				{ arrayMerge: (d, s) => s }
+			),
+			deepmerge(
+				TestingIIDXSPMilestone,
+				{
+					milestoneID: "other_game_milestone",
+					game: "museca",
+					playtype: "Single",
+					milestoneData: [
+						{
+							goals: [{ goalID: "foo" }],
+						},
+					],
+				},
+				{ arrayMerge: (d, s) => s }
+			),
+			deepmerge(
+				TestingIIDXSPMilestone,
+				{
+					milestoneID: "iidx_with_not_goal",
+					game: "iidx",
+					playtype: "SP",
+					milestoneData: [
+						{
+							goals: [{ goalID: "not_real_goal_id" }],
+						},
+					],
+				},
+				{ arrayMerge: (d, s) => s }
+			),
+		]);
+
+		await db["milestone-subs"].insert([
+			{
+				achieved: false,
+				wasInstantlyAchieved: false,
+				game: "iidx",
+				milestoneID: "some_other_milestone_with_mutual_goals",
+				playtype: "SP",
+				progress: 0,
+				timeAchieved: null,
+				lastInteraction: null,
+				timeSet: 0,
+				userID: 1,
+			},
+			{
+				achieved: false,
+				wasInstantlyAchieved: false,
+
+				game: "iidx",
+				milestoneID: "dp_milestone",
+				playtype: "DP",
+				progress: 0,
+				timeAchieved: null,
+				lastInteraction: null,
+				timeSet: 0,
+				userID: 1,
+			},
+			{
+				achieved: false,
+				wasInstantlyAchieved: false,
+
+				game: "museca",
+				milestoneID: "other_game_milestone",
+				playtype: "Single",
+				progress: 0,
+				timeAchieved: null,
+				lastInteraction: null,
+				timeSet: 0,
+				userID: 1,
+			},
+			{
+				achieved: false,
+				wasInstantlyAchieved: false,
+				game: "iidx",
+				milestoneID: "iidx_with_not_goal",
+				playtype: "SP",
+				progress: 0,
+				timeAchieved: null,
+				lastInteraction: null,
+				timeSet: 0,
+				userID: 1,
+			},
+		]);
+
+		const res = await UpdateUsersMilestones(
+			CreateMockGII(
+				["eg_goal_1", true],
+				["eg_goal_2", true],
+				["eg_goal_3", true],
+				["eg_goal_4", true],
+				["x_goal_1", true],
+				["x_goal_2", true],
+				["x_goal_3", true]
+			),
+			"iidx",
+			["SP", "DP"],
+			1,
+			logger
+		);
+
+		t.strictSame(
+			res,
+			[
+				{
+					milestoneID: "example_milestone_id",
+					old: {
+						progress: 0,
+						achieved: false,
+					},
+					new: {
+						progress: 4,
+						achieved: true,
+					},
+				},
+				{
+					milestoneID: "some_other_milestone_with_mutual_goals",
+					old: {
+						progress: 0,
+						achieved: false,
+					},
+					new: {
+						progress: 4,
+						achieved: true,
+					},
+				},
+			],
+			"Should correctly assert the milestones are achieved."
+		);
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/score-import/framework/milestones/milestones.ts

@@ -0,0 +1,184 @@
+import db from "external/mongo/db";
+import { CalculateMilestoneOutOf, GetGoalIDsFromMilestone } from "lib/targets/milestones";
+import { EmitWebhookEvent } from "lib/webhooks/webhooks";
+import type { KtLogger } from "lib/logger/logger";
+import type { BulkWriteUpdateOneOperation } from "mongodb";
+import type {
+	Game,
+	GoalImportInfo,
+	integer,
+	MilestoneDocument,
+	MilestoneImportInfo,
+	Playtype,
+	MilestoneSubscriptionDocument,
+	GoalImportStat,
+} from "tachi-common";
+
+/**
+ * Processes and updates a user's milestones from their Goal Import Info (i.e. what is returned
+ * about goals from imports)
+ */
+export function ProcessMilestoneFromGII(
+	milestone: MilestoneDocument,
+	gii: Map<string, GoalImportInfo["new"]>
+) {
+	const goalIDs = GetGoalIDsFromMilestone(milestone);
+
+	let progress = 0;
+
+	for (const goalID of goalIDs) {
+		const userInfo = gii.get(goalID);
+
+		if (!userInfo) {
+			continue;
+		}
+
+		if (!userInfo.achieved) {
+			continue;
+		}
+
+		progress++;
+	}
+
+	const outOf = CalculateMilestoneOutOf(milestone);
+
+	// milestone achieved!
+	if (progress >= outOf) {
+		return { achieved: true, progress };
+	}
+
+	return { achieved: false, progress };
+}
+
+export async function UpdateUsersMilestones(
+	importGoalInfo: Array<GoalImportInfo>,
+	game: Game,
+	playtypes: Array<Playtype>,
+	userID: integer,
+	logger: KtLogger
+) {
+	const goalSubInfoMap: Map<string, GoalImportInfo["new"]> = new Map();
+
+	const goalIDs = [];
+
+	for (const e of importGoalInfo) {
+		goalSubInfoMap.set(e.goalID, e.new);
+		goalIDs.push(e.goalID);
+	}
+
+	const { milestones, milestoneSubs } = await GetRelevantMilestones(
+		goalIDs,
+		game,
+		playtypes,
+		userID,
+		logger
+	);
+
+	// create a map here to avoid linear searching when
+	// co-iterating
+	const milestoneSubMap = new Map<string, MilestoneSubscriptionDocument>();
+
+	for (const um of milestoneSubs) {
+		milestoneSubMap.set(um.milestoneID, um);
+	}
+
+	const importGoalMap = new Map<string, GoalImportStat>();
+
+	for (const ig of importGoalInfo) {
+		importGoalMap.set(ig.goalID, ig.new);
+	}
+
+	const bwrite: Array<BulkWriteUpdateOneOperation<MilestoneSubscriptionDocument>> = [];
+
+	const importMilestoneInfo: Array<MilestoneImportInfo> = [];
+
+	for (const milestone of milestones) {
+		const { achieved, progress } = ProcessMilestoneFromGII(milestone, importGoalMap);
+
+		const milestoneSub = milestoneSubMap.get(milestone.milestoneID);
+
+		if (!milestoneSub) {
+			logger.severe(
+				`Invalid state achieved in milestone processing - processed milestone that user did not have? ${milestone.milestoneID}`
+			);
+
+			throw new Error("Invalid state achieved in milestone processing.");
+		}
+
+		const bwriteOp: BulkWriteUpdateOneOperation<MilestoneSubscriptionDocument> = {
+			updateOne: {
+				filter: { milestoneID: milestone.milestoneID },
+				update: {
+					$set: {
+						achieved,
+						progress,
+					},
+				},
+			},
+		};
+
+		const milestoneInfo = {
+			milestoneID: milestoneSub.milestoneID,
+			old: {
+				progress: milestoneSub.progress,
+				achieved: milestoneSub.achieved,
+			},
+			new: {
+				progress,
+				achieved,
+			},
+		};
+
+		if (progress !== milestoneSub.progress) {
+			importMilestoneInfo.push(milestoneInfo);
+
+			// @ts-expect-error This property isn't read only, because I said so.
+			bwriteOp.updateOne.update.$set!.lastInteraction = Date.now();
+		}
+
+		bwrite.push(bwriteOp);
+
+		if (achieved && !milestoneSub.achieved) {
+			void EmitWebhookEvent({
+				type: "milestone-achieved/v1",
+				content: {
+					userID,
+					...milestoneInfo,
+					game,
+					playtype: milestone.playtype,
+				},
+			});
+		}
+	}
+
+	if (bwrite.length !== 0) {
+		await db["milestone-subs"].bulkWrite(bwrite, { ordered: false });
+	}
+
+	return importMilestoneInfo;
+}
+
+async function GetRelevantMilestones(
+	goalIDs: Array<string>,
+	game: Game,
+	playtypes: Array<Playtype>,
+	userID: integer,
+	logger: KtLogger
+) {
+	const milestoneSubs = await db["milestone-subs"].find({
+		game,
+		playtype: { $in: playtypes },
+		userID,
+	});
+
+	logger.debug(`Found ${milestoneSubs.length} milestone-subs.`);
+
+	const milestones = await db.milestones.find({
+		milestoneID: { $in: milestoneSubs.map((e) => e.milestoneID) },
+		"milestoneData.goals.goalID": { $in: goalIDs },
+	});
+
+	logger.debug(`Found ${milestones.length} relevant milestones.`);
+
+	return { milestoneSubs, milestones };
+}

server/src/lib/score-import/framework/orphans/orphans.test.ts

@@ -0,0 +1,251 @@
+import { OrphanScore, ReprocessOrphan } from "./orphans";
+import deepmerge from "deepmerge";
+import db from "external/mongo/db";
+import fjsh from "fast-json-stable-hash";
+import CreateLogCtx from "lib/logger/logger";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+import type { BatchManualContext } from "../../import-types/common/batch-manual/types";
+import type { OrphanScoreDocument } from "../../import-types/common/types";
+import type { BatchManualScore } from "tachi-common";
+
+const logger = CreateLogCtx(__filename);
+
+const batchManualScore: BatchManualScore = {
+	score: 500,
+	lamp: "HARD CLEAR",
+	matchType: "songTitle",
+	identifier: "5.1.1.",
+	difficulty: "ANOTHER",
+};
+
+const batchManualContext: BatchManualContext = {
+	game: "iidx",
+	playtype: "SP",
+	service: "foo",
+	version: "27",
+};
+
+t.test("#OrphanScore", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.test("Should orphan data and context information.", async (t) => {
+		const res = await OrphanScore(
+			"ir/direct-manual",
+			1,
+			batchManualScore,
+			batchManualContext,
+			"Example Error Message",
+			"iidx",
+			logger
+		);
+
+		t.equal(res.success, true);
+		t.equal(
+			res.orphanID,
+			`O${fjsh.hash(
+				{
+					importType: "ir/direct-manual",
+					data: batchManualScore,
+					context: batchManualContext,
+					userID: 1,
+				},
+				"sha256"
+			)}`,
+			"Should be a checksum of the orphan's contents."
+		);
+
+		const dbCheck = await db["orphan-scores"].findOne({
+			orphanID: res.orphanID,
+		});
+
+		t.hasStrict(dbCheck, {
+			orphanID: res.orphanID,
+			userID: 1,
+			data: batchManualScore,
+			context: batchManualContext,
+			importType: "ir/direct-manual",
+			errMsg: "Example Error Message",
+		});
+
+		t.ok(
+			Math.abs(dbCheck!.timeInserted - Date.now()) < 10_000,
+			"timeInserted should be decently close to now."
+		);
+
+		t.end();
+	});
+
+	t.test("Should skip orphans that already exist.", async (t) => {
+		const res1 = await OrphanScore(
+			"ir/direct-manual",
+			1,
+			batchManualScore,
+			batchManualContext,
+			"Example Error Message",
+			"iidx",
+			logger
+		);
+
+		const res2 = await OrphanScore(
+			"ir/direct-manual",
+			1,
+			batchManualScore,
+			batchManualContext,
+			"Example Error Message",
+			"iidx",
+			logger
+		);
+
+		t.equal(res2.success, false);
+		t.equal(res1.orphanID, res2.orphanID);
+
+		const dbCheck = await db["orphan-scores"].count();
+
+		t.equal(dbCheck, 1);
+	});
+
+	t.end();
+});
+
+t.test("#ReprocessOrphan", (t) => {
+	t.beforeEach(ResetDBState);
+
+	const orphanDoc: OrphanScoreDocument = {
+		context: batchManualContext,
+		data: batchManualScore,
+		errMsg: "foo",
+		importType: "ir/direct-manual",
+		orphanID: "foo",
+		timeInserted: 0,
+		game: "iidx",
+		userID: 1,
+	};
+
+	t.test("Should convert and insert an orphan if parents are found", async (t) => {
+		// this orphan doc is for 5.1.1. SPA, which definitely exists in
+		// the test DB.
+		const res = await ReprocessOrphan(orphanDoc, [], logger);
+
+		t.hasStrict(
+			res,
+			{
+				success: true,
+				type: "ScoreImported",
+				message:
+					"Imported score R7a3a2b04bd4882ec06c198d78297fe3d56561502c7b134c067214bbfdf4f1602.",
+				content: {
+					score: {
+						game: "iidx",
+						service: "foo (DIRECT-MANUAL)",
+						comment: null,
+						importType: "ir/direct-manual",
+						timeAchieved: null,
+						scoreData: {
+							lampIndex: 5,
+							lamp: "HARD CLEAR",
+							score: 500,
+						},
+						highlight: false,
+						userID: 1,
+						songID: 1,
+						chartID: "c2311194e3897ddb5745b1760d2c0141f933e683",
+						scoreID:
+							"R7a3a2b04bd4882ec06c198d78297fe3d56561502c7b134c067214bbfdf4f1602",
+						playtype: "SP",
+						isPrimary: true,
+					},
+				},
+			},
+			"Should successfully import the score."
+		);
+
+		const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
+
+		t.equal(orphan, null, "Should remove the orphan document.");
+
+		t.end();
+	});
+
+	t.test("Should delete the orphan doc and return null if the score is invalid.", async (t) => {
+		await db["orphan-scores"].insert(orphanDoc);
+		const res = await ReprocessOrphan(
+			deepmerge(orphanDoc, {
+				data: {
+					score: 99999,
+				},
+			}),
+			[],
+			logger
+		);
+
+		t.equal(res, null);
+
+		const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
+
+		t.equal(orphan, null, "Should remove the orphan document.");
+
+		t.end();
+	});
+
+	t.test(
+		"Should keep the orphan doc and return false if no parents could be found.",
+		async (t) => {
+			await db["orphan-scores"].insert(orphanDoc);
+			const res = await ReprocessOrphan(
+				deepmerge(orphanDoc, {
+					data: {
+						identifier: "NONSENSE CHART TITLE",
+					},
+				}),
+				[],
+				logger
+			);
+
+			t.equal(res, false);
+
+			const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
+
+			t.not(orphan, null, "Should not remove the orphan document.");
+
+			t.end();
+		}
+	);
+
+	t.test(
+		"Should keep the orphan doc and return false if an internal failure was triggered.",
+		async (t) => {
+			await db["orphan-scores"].insert(orphanDoc);
+			const GAZER_SHA256 = "195fe1be5c3e74fccd04dc426e05f8a9cfa8a1059c339d0a23e99f63661f0b7d";
+
+			// force an internal failure by removing all bms songs and then
+			// leaving a valid bms chart to match with.
+			await db.songs.bms.remove({});
+
+			const res = await ReprocessOrphan(
+				deepmerge(orphanDoc, {
+					context: {
+						game: "bms",
+						playtype: "7K",
+					},
+					data: {
+						matchType: "bmsChartHash",
+						identifier: GAZER_SHA256,
+					},
+				}),
+				[],
+				logger
+			);
+
+			t.equal(res, false);
+
+			const orphan = await db["orphan-scores"].findOne({ orphanID: orphanDoc.orphanID });
+
+			t.not(orphan, null, "Should not remove the orphan document.");
+
+			t.end();
+		}
+	);
+
+	t.end();
+});

server/src/lib/score-import/framework/orphans/orphans.ts

@@ -0,0 +1,169 @@
+import { Converters } from "../../import-types/converters";
+import {
+	ConverterFailure,
+	InternalFailure,
+	KTDataNotFoundFailure,
+} from "../common/converter-failures";
+import { HandlePostImportSteps } from "../score-importing/score-import-main";
+import { ProcessSuccessfulConverterReturn } from "../score-importing/score-importing";
+import db from "external/mongo/db";
+import fjsh from "fast-json-stable-hash";
+import { GetUserWithID } from "utils/user";
+import type {
+	ConverterFunction,
+	ConverterFnReturnOrFailure,
+	ImportTypeContextMap,
+	ImportTypeDataMap,
+	OrphanScoreDocument,
+} from "../../import-types/common/types";
+import type { KtLogger } from "lib/logger/logger";
+import type { Game, ImportTypes, integer } from "tachi-common";
+
+/**
+ * Creates an OrphanedScore document from the data and context,
+ * and inserts it into the DB if it is not already in there.
+ *
+ * @returns Returns { success: true | false, orphanID }
+ */
+export async function OrphanScore<T extends ImportTypes = ImportTypes>(
+	importType: T,
+	userID: integer,
+	data: ImportTypeDataMap[T],
+	context: ImportTypeContextMap[T],
+	errMsg: string | null,
+	game: Game,
+	logger: KtLogger
+) {
+	const orphan: Pick<OrphanScoreDocument, "context" | "data" | "importType" | "userID"> = {
+		importType,
+		data,
+		context,
+		userID,
+	};
+
+	logger.debug("Orphaning document", orphan);
+
+	let orphanID;
+
+	try {
+		orphanID = `O${fjsh.hash(orphan, "sha256")}`;
+	} catch (err) {
+		logger.error(`Failed to orphan chart -- `, { err, orphan });
+		throw new Error(`Failed to orphan chart. ${(err as Error).message}`);
+	}
+
+	const exists = await db["orphan-scores"].findOne({ orphanID });
+
+	if (exists) {
+		logger.debug(`Skipped orphaning score ${orphanID} because it already exists.`);
+		return { success: false, orphanID };
+	}
+
+	const orphanScoreDoc: OrphanScoreDocument = {
+		...orphan,
+		orphanID,
+		game,
+		errMsg,
+		timeInserted: Date.now(),
+	};
+
+	logger.debug(`Inserting orphanScoreDoc...`, orphanScoreDoc);
+
+	await db["orphan-scores"].insert(orphanScoreDoc);
+
+	return { success: true, orphanID };
+}
+
+/**
+ * Takes an orphan document and re-runs the converter->scoreimport pipeline on its data.
+ *
+ * @returns False if no parent documents could be found for the score again,
+ * Null if the orphan document was removed, but no score was inserted (i.e. score was orphaned AND invalid, so nothing
+ * could be imported when parents were found).
+ * ImportProcessingInfo on success.
+ */
+export async function ReprocessOrphan(
+	orphan: OrphanScoreDocument,
+	blacklist: Array<string>,
+	logger: KtLogger
+) {
+	const ConverterFunction = Converters[orphan.importType] as ConverterFunction<
+		ImportTypeDataMap[ImportTypes],
+		ImportTypeContextMap[ImportTypes]
+	>;
+
+	let res: ConverterFnReturnOrFailure;
+
+	try {
+		res = await ConverterFunction(orphan.data, orphan.context, orphan.importType, logger);
+	} catch (err) {
+		// this is impossible to test, so we're going to ignore it
+		/* istanbul ignore next */
+		if (!(err instanceof ConverterFailure)) {
+			logger.error(`Converter function ${orphan.importType} returned unexpected error.`, {
+				err,
+			});
+
+			// throw this higher up, i guess.
+			throw err;
+		}
+
+		res = err;
+	}
+
+	// If the data still can't be found, we do nothing about it.
+	if (res instanceof KTDataNotFoundFailure) {
+		logger.debug(`Unorphaning ${orphan.orphanID} failed. (${res.message})`);
+		return false;
+	} else if (res instanceof InternalFailure) {
+		logger.error(`Orphan Internal Failure - ${res.message}, OrphanID ${orphan.orphanID}`);
+
+		return false;
+	} else if (res instanceof ConverterFailure) {
+		logger.warn(
+			`received ConverterFailure ${res.message} on orphan ${orphan.orphanID}. Removing orphan.`
+		);
+
+		// @danger - This could go terribly, if there's a mistake in the converterFN we might accidentally
+		// remove a users score.
+		await db["orphan-scores"].remove({ orphanID: orphan.orphanID });
+
+		return null;
+	}
+
+	await db["orphan-scores"].remove({ orphanID: orphan.orphanID });
+
+	// else, import the orphan.
+	const converterReturns = await ProcessSuccessfulConverterReturn(
+		orphan.userID,
+		res,
+		blacklist,
+		logger,
+		true
+	);
+
+	if (converterReturns === null || !converterReturns.success) {
+		return null;
+	}
+
+	const user = await GetUserWithID(orphan.userID);
+
+	if (!user) {
+		logger.severe(
+			`Orphan ${orphan.orphanID} belongs to ${orphan.userID}, but that user no longer exists in the database. Going to skip this and remove the orphan.`
+		);
+		return null;
+	}
+
+	await HandlePostImportSteps(
+		[converterReturns],
+		user,
+		orphan.importType,
+		orphan.game,
+		null,
+		logger,
+		undefined
+	);
+
+	return converterReturns;
+}

server/src/lib/score-import/framework/pb/create-pb-doc.test.ts

@@ -0,0 +1,200 @@
+import { CreatePBDoc } from "./create-pb-doc";
+import deepmerge from "deepmerge";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import { GetGamePTConfig } from "tachi-common";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+import { Testing511SPA, TestingBMS7KScore, TestingIIDXSPScore } from "test-utils/test-data";
+import type { KtLogger } from "lib/logger/logger";
+
+const IIDXScore = TestingIIDXSPScore;
+
+const logger = CreateLogCtx(__filename);
+
+const lamps = GetGamePTConfig("iidx", "SP").lamps;
+
+t.test("#CreatePBDoc", (t) => {
+	t.beforeEach(ResetDBState);
+
+	const chartID = Testing511SPA.chartID;
+
+	const ExamplePBDoc = {
+		chartID,
+		userID: 1,
+		songID: 1,
+
+		// rankingData -- is not present because it is not added until post-processing.
+		highlight: false,
+		isPrimary: true,
+		timeAchieved: 1619454485988,
+		game: "iidx",
+		playtype: "SP",
+		composedFrom: {
+			scorePB: IIDXScore.scoreID,
+			lampPB: "LAMP_PB_ID",
+		},
+		scoreData: {
+			score: IIDXScore.scoreData.score,
+			percent: IIDXScore.scoreData.percent,
+			esd: IIDXScore.scoreData.esd,
+			grade: IIDXScore.scoreData.grade,
+			gradeIndex: IIDXScore.scoreData.gradeIndex,
+			judgements: IIDXScore.scoreData.judgements,
+			lamp: "FULL COMBO",
+			lampIndex: lamps.indexOf("FULL COMBO"),
+			hitMeta: { bp: 1 },
+		},
+		calculatedData: {
+			ktLampRating: 12,
+		},
+	};
+
+	t.test(
+		"(IIDX) Should use the GameSpecificMergeFN to also join the BP PB if necessary.",
+		async (t) => {
+			await db.scores.remove({});
+			await db.scores.insert([
+				IIDXScore,
+				deepmerge(IIDXScore, {
+					scoreData: {
+						lamp: "FULL COMBO",
+						lampIndex: lamps.indexOf("FULL COMBO"),
+						score: 0,
+						percent: 0,
+						hitMeta: {
+							bp: 15,
+						},
+					},
+					calculatedData: {
+						ktLampRating: 12,
+					},
+					scoreID: "LAMP_PB_ID",
+				}),
+				deepmerge(IIDXScore, {
+					scoreData: {
+						lamp: "CLEAR",
+						lampIndex: lamps.indexOf("CLEAR"),
+						score: 1,
+						percent: 1,
+						hitMeta: {
+							bp: 5,
+						},
+					},
+					calculatedData: {
+						ktLampRating: 10,
+					},
+					scoreID: "BP_PB_ID",
+				}),
+			]);
+
+			const res = await CreatePBDoc(1, chartID, logger);
+
+			t.not(res, undefined, "Should actually return something.");
+
+			t.strictSame(
+				res,
+				deepmerge(ExamplePBDoc, {
+					composedFrom: {
+						other: [{ name: "Best BP", scoreID: "BP_PB_ID" }],
+					},
+					scoreData: {
+						hitMeta: {
+							bp: 5,
+							gsm: null,
+							gauge: null,
+							gaugeHistory: null,
+						},
+					},
+				}),
+				"Should correctly return a merged PBDocument with BP"
+			);
+
+			t.end();
+		}
+	);
+
+	t.test("Should merge a score and lamp PB into one document.", async (t) => {
+		const d = deepmerge(IIDXScore, {
+			scoreData: {
+				lamp: "FULL COMBO",
+				lampIndex: lamps.indexOf("FULL COMBO"),
+				score: 0,
+				percent: 0,
+				hitMeta: {
+					bp: 1,
+				},
+			},
+			calculatedData: {
+				ktLampRating: 12,
+			},
+			scoreID: "LAMP_PB_ID",
+		});
+
+		await db.scores.remove({});
+		await db.scores.insert([IIDXScore, d]);
+
+		const res = await CreatePBDoc(1, chartID, logger);
+
+		t.not(res, undefined, "Should actually return something.");
+
+		t.strictSame(res, ExamplePBDoc, "Should correctly return a merged PBDocument");
+
+		t.end();
+	});
+
+	t.test("Should bail safely if no score exists when one should", async (t) => {
+		// a work of genius
+		const fakeLogger = {
+			severe: () => (severeCalled = true),
+		} as unknown as KtLogger;
+
+		let severeCalled = false;
+
+		await db.scores.remove({});
+
+		const res = await CreatePBDoc(1, chartID, fakeLogger);
+
+		t.equal(res, undefined, "Should return nothing (and emit a warning)");
+
+		t.equal(severeCalled, true, "Severe logging should have been called.");
+
+		t.end();
+	});
+
+	t.test("(BMS) Should inherit sieglinde from the higher rated score.", async (t) => {
+		await db.scores.remove({});
+		await db.scores.insert([
+			TestingBMS7KScore,
+			deepmerge(TestingBMS7KScore, {
+				scoreData: {
+					lamp: "FULL COMBO",
+					lampIndex: lamps.indexOf("FULL COMBO"),
+					score: 0,
+					percent: 0,
+					hitMeta: {
+						bp: 15,
+					},
+				},
+				calculatedData: {
+					sieglinde: 500,
+				},
+				scoreID: "LAMP_PB_ID",
+			}),
+		]);
+
+		const res = await CreatePBDoc(1, TestingBMS7KScore.chartID, logger);
+
+		t.not(res, undefined, "Should actually return something.");
+
+		t.equal(
+			res?.calculatedData.sieglinde,
+			500,
+			"Should select the lampPBs sieglinde and not the score PBs."
+		);
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/score-import/framework/pb/create-pb-doc.ts

@@ -0,0 +1,192 @@
+import {
+	BMSMergeFn,
+	IIDXMergeFn,
+	PopnMergeFn,
+	SDVXMergeFn,
+	USCMergeFn,
+} from "./game-specific-merge";
+import db from "external/mongo/db";
+import type { KtLogger } from "lib/logger/logger";
+import type { BulkWriteUpdateOneOperation } from "mongodb";
+import type { Game, integer, PBScoreDocument, ScoreDocument } from "tachi-common";
+
+export type PBScoreDocumentNoRank = Omit<PBScoreDocument, "rankingData">;
+
+export async function CreatePBDoc(userID: integer, chartID: string, logger: KtLogger) {
+	const scorePB = await db.scores.findOne(
+		{
+			userID,
+			chartID,
+		},
+		{
+			sort: {
+				"scoreData.percent": -1,
+			},
+		}
+	);
+
+	if (!scorePB) {
+		logger.severe(
+			`User ${userID} has no scores on chart, but a PB was attempted to be created?`,
+			{
+				chartID,
+				userID,
+			}
+		);
+		return;
+	}
+
+	const lampPB = (await db.scores.findOne(
+		{
+			userID,
+			chartID,
+		},
+		{
+			sort: {
+				"scoreData.lampIndex": -1,
+			},
+		}
+	)) as ScoreDocument;
+
+	// ^ guaranteed to not be null, as this always resolves
+	// to atleast one score (and we got ScorePB above, so we know there's
+	// atleast one).
+
+	const pbDoc = await MergeScoreLampIntoPB(userID, scorePB, lampPB, logger);
+
+	if (!pbDoc) {
+		return;
+	}
+
+	// finally, return our full pbDoc, that does NOT have the ranking props.
+	// (We will add those later)
+	return pbDoc;
+}
+
+/**
+ * Updates user's rankings on a given chart.
+ */
+export async function UpdateChartRanking(chartID: string) {
+	const scores = await db["personal-bests"].find(
+		{ chartID },
+		{
+			sort: {
+				"scoreData.percent": -1,
+				timeAchieved: -1,
+			},
+		}
+	);
+
+	const bwrite: Array<BulkWriteUpdateOneOperation<PBScoreDocument>> = [];
+
+	let rank = 0;
+
+	for (const score of scores) {
+		rank++;
+
+		bwrite.push({
+			updateOne: {
+				filter: { chartID: score.chartID, userID: score.userID },
+				update: {
+					$set: {
+						rankingData: {
+							rank,
+							outOf: scores.length,
+						},
+					},
+				},
+			},
+		});
+	}
+
+	// If a score is deleted such that the chart is now empty of
+	// scores, the below statement will crash with no op specified.
+	if (bwrite.length === 0) {
+		return;
+	}
+
+	await db["personal-bests"].bulkWrite(bwrite, { ordered: false });
+}
+
+async function MergeScoreLampIntoPB(
+	userID: integer,
+	scorePB: ScoreDocument,
+	lampPB: ScoreDocument,
+	logger: KtLogger
+): Promise<PBScoreDocumentNoRank | undefined> {
+	// @hack
+	// since time cannot be negative, this is a rough hack
+	// to resolve nullable timeAchieveds without hitting NaN.
+	let timeAchieved: number | null = Math.max(
+		scorePB.timeAchieved ?? -1,
+		lampPB.timeAchieved ?? -1
+	);
+
+	if (timeAchieved === -1) {
+		timeAchieved = null;
+	}
+
+	const pbDoc: PBScoreDocumentNoRank = {
+		composedFrom: {
+			scorePB: scorePB.scoreID,
+			lampPB: lampPB.scoreID,
+		},
+		chartID: scorePB.chartID,
+		userID,
+		songID: scorePB.songID,
+		highlight: scorePB.highlight || lampPB.highlight,
+		timeAchieved,
+		game: scorePB.game,
+		playtype: scorePB.playtype,
+		isPrimary: scorePB.isPrimary,
+		scoreData: {
+			score: scorePB.scoreData.score,
+			percent: scorePB.scoreData.percent,
+			esd: scorePB.scoreData.esd,
+			grade: scorePB.scoreData.grade,
+			gradeIndex: scorePB.scoreData.gradeIndex,
+			lamp: lampPB.scoreData.lamp,
+			lampIndex: lampPB.scoreData.lampIndex,
+			judgements: scorePB.scoreData.judgements,
+
+			// this will probably be overrode by game-specific fns
+			hitMeta: scorePB.scoreData.hitMeta,
+		},
+		calculatedData: scorePB.calculatedData,
+	};
+
+	const GameSpecificMergeFn = GetGameSpecificMergeFn(scorePB.game);
+
+	if (GameSpecificMergeFn) {
+		// @ts-expect-error Yeah, this call sucks. It correctly warns us that scorePB and lampPB
+		// might've diverged, but we know they haven't.
+		const success = await GameSpecificMergeFn(pbDoc, scorePB, lampPB, logger);
+
+		// If the mergeFn returns false, this means something has gone
+		// rather wrong. We just return undefined here, which in turn
+		// tells our calling code to skip this PB entirely.
+		if (!success) {
+			return;
+		}
+	}
+
+	return pbDoc;
+}
+
+function GetGameSpecificMergeFn(game: Game) {
+	switch (game) {
+		case "iidx":
+			return IIDXMergeFn;
+		case "usc":
+			return USCMergeFn;
+		case "sdvx":
+			return SDVXMergeFn;
+		case "popn":
+			return PopnMergeFn;
+		case "bms":
+		case "pms":
+			return BMSMergeFn;
+		default:
+			return null;
+	}
+}

server/src/lib/score-import/framework/pb/game-specific-merge.ts

@@ -0,0 +1,143 @@
+import { InternalFailure } from "../common/converter-failures";
+import db from "external/mongo/db";
+import { Volforce } from "rg-stats";
+import { FindChartWithChartID } from "utils/queries/charts";
+import type { KtLogger } from "lib/logger/logger";
+import type { PBScoreDocument, ScoreDocument } from "tachi-common";
+
+export async function IIDXMergeFn(
+	pbDoc: PBScoreDocument<"iidx:DP" | "iidx:SP">,
+	scorePB: ScoreDocument<"iidx:DP" | "iidx:SP">,
+	lampPB: ScoreDocument<"iidx:DP" | "iidx:SP">,
+	logger: KtLogger
+): Promise<boolean> {
+	// lampRating needs to be updated.
+	pbDoc.calculatedData.ktLampRating = lampPB.calculatedData.ktLampRating;
+
+	// bad+poor PB document. This is a weird, third indepdenent metric that IIDX players sometimes care about.
+	const bpPB = (await db.scores.findOne(
+		{
+			userID: scorePB.userID,
+			chartID: scorePB.chartID,
+			"scoreData.hitMeta.bp": { $exists: true },
+		},
+		{
+			sort: {
+				// bp 0 is the best BP, bp 1 is worse, so on
+				"scoreData.hitMeta.bp": 1,
+			},
+		}
+	)) as ScoreDocument<"iidx:DP" | "iidx:SP"> | null;
+
+	if (!bpPB) {
+		logger.verbose(
+			`Could not find BP PB for ${scorePB.userID} ${scorePB.chartID} in PB joining. User likely has no scores with BP defined.`,
+			{ pbDoc }
+		);
+
+		// this isn't actually an error! we just don't have to do anything.
+		return true;
+	}
+
+	// by default scorePB is chosen for hitMeta fields, so, we can skip any assignments here by returning here.
+	if (bpPB.scoreID === scorePB.scoreID) {
+		logger.debug(`Skipped merging BP PB as scorePB was also BP PB.`);
+		return true;
+	} else if (bpPB.scoreID === lampPB.scoreID) {
+		pbDoc.scoreData.hitMeta.bp = lampPB.scoreData.hitMeta.bp;
+		logger.debug(`Skipped adding BP PB as composedFrom because lampPB was also BP PB.`);
+		return true;
+	}
+
+	pbDoc.scoreData.hitMeta.bp = bpPB.scoreData.hitMeta.bp!;
+
+	pbDoc.composedFrom.other = [{ name: "Best BP", scoreID: bpPB.scoreID }];
+
+	// Update lamp related iidx-specific info from the lampPB.
+	pbDoc.scoreData.hitMeta.gsm = lampPB.scoreData.hitMeta.gsm ?? null;
+	pbDoc.scoreData.hitMeta.gauge = lampPB.scoreData.hitMeta.gauge ?? null;
+	pbDoc.scoreData.hitMeta.gaugeHistory = lampPB.scoreData.hitMeta.gaugeHistory ?? null;
+
+	return true;
+}
+
+export function PopnMergeFn(
+	pbDoc: PBScoreDocument<"popn:9B">,
+	scorePB: ScoreDocument<"popn:9B">,
+	lampPB: ScoreDocument<"popn:9B">,
+	_logger: KtLogger
+) {
+	pbDoc.scoreData.hitMeta.specificClearType = lampPB.scoreData.hitMeta.specificClearType;
+
+	return true;
+}
+
+export function BMSMergeFn(
+	pbDoc: PBScoreDocument<"bms:7K" | "bms:14K">,
+	scorePB: ScoreDocument<"bms:7K" | "bms:14K">,
+	lampPB: ScoreDocument<"bms:7K" | "bms:14K">,
+	_logger: KtLogger
+) {
+	pbDoc.calculatedData.sieglinde = lampPB.calculatedData.sieglinde;
+
+	return true;
+}
+
+/**
+ * This function recalculates and applies VF6 to the PB document.
+ *
+ * This is near-identical to the SDVXMergeFn. See that.
+ */
+export async function USCMergeFn(
+	pbDoc: PBScoreDocument<"usc:Controller" | "usc:Keyboard">,
+	scorePB: ScoreDocument<"usc:Controller" | "usc:Keyboard">,
+	lampPB: ScoreDocument<"usc:Controller" | "usc:Keyboard">,
+	logger: KtLogger
+) {
+	// @optimisable - see SDVXMergeFn
+	const chart = await FindChartWithChartID("usc", pbDoc.chartID);
+
+	if (!chart) {
+		logger.severe(`Chart ${pbDoc.chartID} disappeared underfoot?`);
+		throw new InternalFailure(`Chart ${pbDoc.chartID} disappeared underfoot?`);
+	}
+
+	pbDoc.calculatedData.VF6 = Volforce.calculateVF6(
+		pbDoc.scoreData.score,
+		pbDoc.scoreData.lamp,
+		chart.levelNum
+	);
+
+	return true;
+}
+
+/**
+ * This function recalculates and applies VF6 to the PB document.
+ *
+ * SDVX cannot just select the larger volforce - instead, volforce has to be
+ * re-calculated for any different permutation of scorePB + lampPB.
+ */
+export async function SDVXMergeFn(
+	pbDoc: PBScoreDocument<"sdvx:Single">,
+	scorePB: ScoreDocument<"sdvx:Single">,
+	lampPB: ScoreDocument<"sdvx:Single">,
+	logger: KtLogger
+): Promise<boolean> {
+	// @optimisable
+	// This is a re-fetch, but it's difficult to pass the chart all
+	// the way down here due to how chartIDs (set) works. :(
+	const chart = await FindChartWithChartID("sdvx", pbDoc.chartID);
+
+	if (!chart) {
+		logger.severe(`Chart ${pbDoc.chartID} disappeared underfoot?`);
+		throw new InternalFailure(`Chart ${pbDoc.chartID} disappeared underfoot?`);
+	}
+
+	pbDoc.calculatedData.VF6 = Volforce.calculateVF6(
+		pbDoc.scoreData.score,
+		pbDoc.scoreData.lamp,
+		chart.levelNum
+	);
+
+	return true;
+}

server/src/lib/score-import/framework/pb/process-pbs.test.ts

@@ -0,0 +1,61 @@
+import { ProcessPBs } from "./process-pbs";
+import deepmerge from "deepmerge";
+import db from "external/mongo/db";
+import CreateLogCtx from "lib/logger/logger";
+import t from "tap";
+import ResetDBState from "test-utils/resets";
+import { Testing511SPA, TestingIIDXSPScore } from "test-utils/test-data";
+import crypto from "crypto";
+
+const logger = CreateLogCtx(__filename);
+
+t.test("#ProcessPBs", (t) => {
+	t.beforeEach(ResetDBState);
+
+	t.test("Should successfully insert a pb into the score-pb database", async (t) => {
+		await db["personal-bests"].remove({});
+
+		// scores on 511 SPA are pre-loaded into the database
+		await ProcessPBs(1, new Set([Testing511SPA.chartID]), logger);
+
+		const pbs = await db["personal-bests"].find({});
+
+		t.equal(pbs.length, 1, "Should match the amount of PBs inserted into the DB.");
+
+		t.end();
+	});
+
+	t.test("Should successfully insert multiple pbs into the score-pb database", async (t) => {
+		await db["personal-bests"].remove({});
+
+		await db.scores.insert([
+			// @ts-expect-error lol
+			deepmerge(TestingIIDXSPScore, {
+				chartID: "test1",
+				scoreID: crypto.randomBytes(20).toString("hex"),
+			}),
+
+			// @ts-expect-error lol
+			deepmerge(TestingIIDXSPScore, {
+				chartID: "test2",
+				scoreID: crypto.randomBytes(20).toString("hex"),
+			}),
+
+			// @ts-expect-error lol
+			deepmerge(TestingIIDXSPScore, {
+				chartID: "test3",
+				scoreID: crypto.randomBytes(20).toString("hex"),
+			}),
+		]);
+
+		await ProcessPBs(1, new Set([Testing511SPA.chartID, "test1", "test2", "test3"]), logger);
+
+		const pbs = await db["personal-bests"].find({});
+
+		t.equal(pbs.length, 4, "Should match the amount of PBs inserted into the DB.");
+
+		t.end();
+	});
+
+	t.end();
+});

server/src/lib/score-import/framework/pb/process-pbs.ts

@@ -0,0 +1,64 @@
+import { CreatePBDoc, UpdateChartRanking } from "./create-pb-doc";
+import db from "external/mongo/db";
+import type { PBScoreDocumentNoRank } from "./create-pb-doc";
+import type { KtLogger } from "lib/logger/logger";
+import type { integer } from "tachi-common";
+
+/**
+ * Process, recalculate and update a users PBs for this set of chartIDs.
+ */
+export async function ProcessPBs(
+	userID: integer,
+	chartIDs: Set<string>,
+	logger: KtLogger
+): Promise<void> {
+	if (chartIDs.size === 0) {
+		return;
+	}
+
+	const promises = [];
+
+	for (const chartID of chartIDs) {
+		promises.push(CreatePBDoc(userID, chartID, logger));
+	}
+
+	const pbDocsReturn = await Promise.all(promises);
+
+	const pbDocs: Array<PBScoreDocumentNoRank> = [];
+
+	for (const doc of pbDocsReturn) {
+		if (!doc) {
+			continue;
+		}
+
+		pbDocs.push(doc);
+	}
+
+	if (pbDocsReturn.length === 0) {
+		return;
+	}
+
+	// so here's the kinda awkward part - for the time between this operation
+	// and the next one - THE SCORE PBS ARE IN THE DATABASE WITHOUT RANKINGDATA.
+	// this *is* bad behaviour, but I don't have a nice way to fix it.
+	// This should be fixed in the future to avoid crashes between these two
+	// calls - but that is unlikely.
+	await db["personal-bests"].bulkWrite(
+		pbDocs.map((e) => ({
+			updateOne: {
+				filter: { chartID: e.chartID, userID: e.userID },
+				update: { $set: e },
+				upsert: true,
+			},
+		})),
+		{
+			ordered: false,
+		}
+	);
+
+	// now that everything has been updated or inserted, we can refresh
+	// the chart rankings.
+	await Promise.all(pbDocs.map((e) => UpdateChartRanking(e.chartID)));
+
+	// and we're done!
+}