Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-06 13:58:05 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #486 from TNG-dev/develop

Browse Source
This commit is contained in:
zkldi
2021-11-30 22:40:30 +00:00
committed by GitHub
parent b577ca514f 7a7dd99bd9
commit 534bcdb176
638 changed files with 403906 additions and 63431 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/.dockerignore
+1
View File
@@ -0,0 +1 @@
Dockerfile
server/.editorconfig
+8
View File
@@ -0,0 +1,8 @@
root = true
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
indent_style = tab
trim_trailing_whitespace = true
server/.eslintrc
+112 -80
View File
@@ -1,81 +1,113 @@
{
"root": true,
"env": {
"node": true,
"es6": true
},
"plugins": [
"@typescript-eslint",
"import"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"sourceType": "module",
"ecmaVersion": 2020
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended",
"plugin:prettier/recommended",
"plugin:import/typescript",
"plugin:import/warnings",
"plugin:import/errors"
],
"rules": {
"prettier/prettier": ["error", {
"printWidth": 100,
"tabWidth": 4,
"semi": true,
"singleQuote": false,
"trailingComma": "es5",
"bracketSpacing": true,
"endOfLine": "auto"
}],
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/no-non-null-assertion": "off",
"require-await": "warn",
"no-await-in-loop": "warn",
"no-unused-labels": "error",
"prefer-const": "error",
"no-console": "error",
"eqeqeq": "error",
"no-eval": "error",
"no-var": "error",
"no-new-object": "error",
"quote-props": ["error", "as-needed"],
"no-array-constructor": "error",
"prefer-template": "error",
"prefer-promise-reject-errors": "error",
"no-useless-escape": "error",
"no-loop-func": "error",
"prefer-rest-params": "error",
"default-param-last": "error",
"no-new-func": "error",
"arrow-body-style": "error",
// "arrow-parens": ["error", "as-needed"] prettier hates this
"no-useless-constructor": "error",
"no-duplicate-imports": "error",
"dot-notation": "error",
"space-before-blocks": "error",
"no-prototype-builtins": "off",
"prefer-arrow-callback": "error",
"curly": "error",
"no-template-curly-in-string": "error",
"no-implied-eval": "error",
"yoda": "error",
"no-labels": "error",
"no-useless-concat": "error",
"radix": "error",
"no-octal-escape": "error",
"no-param-reassign": "error",
"no-proto": "error",
"no-return-await": "error",
"no-throw-literal": "error",
"no-unmodified-loop-condition": "error",
"no-unused-expressions": "error",
"no-useless-call": "error",
"no-useless-return": "error",
"require-unicode-regexp": "error",
}
{
"root": true,
"env": {
"node": true,
"es6": true
},
"plugins": [
"@typescript-eslint",
"import"
],
"overrides": [
{
"files": [
"*.ts",
"*.tsx"
],
"parserOptions": {
"project": [
"./tsconfig.json"
]
}
}
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"sourceType": "module",
"ecmaVersion": 2020
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended",
"plugin:prettier/recommended",
"plugin:import/typescript",
"plugin:import/warnings",
"plugin:import/errors"
],
"rules": {
"prettier/prettier": [
"error",
{
"printWidth": 100,
"tabWidth": 4,
"semi": true,
"singleQuote": false,
"trailingComma": "es5",
"bracketSpacing": true,
"endOfLine": "lf",
"useTabs": true
}
],
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/no-non-null-assertion": "off",
"@typescript-eslint/no-misused-promises": [
"error",
{
"checksVoidReturn": false
}
],
"@typescript-eslint/no-unused-vars": [
"warn",
{
"argsIgnorePattern": "logger"
}
],
// breaks with baseUrl
"import/no-unresolved": "off",
"require-await": "warn",
"no-await-in-loop": "warn",
"no-unused-labels": "error",
"prefer-const": "error",
"no-console": "error",
"eqeqeq": "error",
"no-eval": "error",
"no-var": "error",
"no-new-object": "error",
"quote-props": [
"error",
"as-needed"
],
"no-array-constructor": "error",
"prefer-template": "error",
"prefer-promise-reject-errors": "error",
"no-useless-escape": "error",
"no-loop-func": "error",
"prefer-rest-params": "error",
"default-param-last": "error",
"no-new-func": "error",
"arrow-body-style": "error",
"no-useless-constructor": "error",
"no-duplicate-imports": "error",
"dot-notation": "error",
"space-before-blocks": "error",
"no-prototype-builtins": "off",
"prefer-arrow-callback": "error",
"curly": "error",
"no-template-curly-in-string": "error",
"no-implied-eval": "error",
"yoda": "error",
"no-labels": "error",
"no-useless-concat": "error",
"radix": "error",
"no-octal-escape": "error",
"no-param-reassign": "error",
"no-proto": "error",
"no-return-await": "error",
"no-throw-literal": "error",
"no-unmodified-loop-condition": "error",
"no-unused-expressions": "error",
"no-useless-call": "error",
"no-useless-return": "error",
"require-unicode-regexp": "error"
}
}
server/.git-blame-ignore-revs
+8
View File
@@ -0,0 +1,8 @@
#188 - Spaces to tabs
47c981f4b3189e9cf995ed03256609f3bf56b930
#219 - Refactor entire repo to use baseUrl "/src"
897e832b01fafade554f99c4fba629c42a50468f
#435 - Standardise line breaks to LF.
7d477970c610b223d42fbd0a3fccffd53498271e
server/.github/FUNDING.yml
Vendored
+12
View File
@@ -0,0 +1,12 @@
# These are supported funding model platforms
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: zkldi
open_collective: # Replace with a single Open Collective username
ko_fi: zkldi
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
server/.github/codecov.yml
Vendored
+26
View File
@@ -0,0 +1,26 @@
codecov:
require_ci_to_pass: yes
branch: develop
coverage:
precision: 2
round: down
range: "50...100"
status:
patch:
default:
informational: true
project:
default:
target: 80
threshold: 2
base: auto
informational: true
parsers:
gcov:
branch_detection:
conditional: yes
loop: yes
method: no
macro: no
server/.github/test.conf.json5
Vendored
+77
View File
@@ -0,0 +1,77 @@
// Config file that the CI deployment uses. This is moved to $pwd/conf.json5 before CI tests are ran.
{
MONGO_DATABASE_NAME: "testingdb",
CAPTCHA_SECRET_KEY: "unused",
SESSION_SECRET: "unused",
FLO_API_URL: "https://flo.example.com",
EAG_API_URL: "https://eag.example.com",
MIN_API_URL: "https://min.example.com",
ARC_API_URL: "https://arc.example.com",
FLO_OAUTH2_INFO: {
CLIENT_ID: "DUMMY_CLIENT_ID",
CLIENT_SECRET: "DUMMY_CLIENT_SECRET",
REDIRECT_URI: "https://example.com",
},
EAG_OAUTH2_INFO: {
CLIENT_ID: "DUMMY_CLIENT_ID",
CLIENT_SECRET: "DUMMY_CLIENT_SECRET",
REDIRECT_URI: "https://example.com",
},
ARC_AUTH_TOKEN: "unused",
ENABLE_SERVER_HTTPS: false,
OUR_URL: "https://example.com",
INVITE_CODE_CONFIG: {
BATCH_SIZE: 2,
INVITE_CAP: 100,
BETA_USER_BONUS: 5,
},
CDN_CONFIG: {
WEB_LOCATION: "/cdn",
SAVE_LOCATION: {
TYPE: "LOCAL_FILESYSTEM",
LOCATION: "./test-cdn",
SERVE_OWN_CDN: true,
},
},
TACHI_CONFIG: {
TYPE: "omni",
NAME: "Tachi Testing",
GAMES: [
"iidx",
"museca",
"maimai",
"sdvx",
"ddr",
"bms",
"chunithm",
"usc",
],
IMPORT_TYPES: [
"file/eamusement-iidx-csv",
"file/batch-manual",
"file/solid-state-squad",
"file/mer-iidx",
"file/pli-iidx-csv",
"ir/direct-manual",
"ir/barbatos",
"ir/fervidex",
"ir/fervidex-static",
"ir/beatoraja",
"ir/usc",
"ir/kshook-sv3c",
"api/arc-iidx",
"api/arc-sdvx",
"api/eag-iidx",
"api/eag-sdvx",
"api/flo-iidx",
"api/flo-sdvx",
"api/min-sdvx",
],
},
LOGGER_CONFIG: {
FILE: false,
CONSOLE: true,
LOG_LEVEL: "info",
},
}
server/.github/workflows/test-build-deploy.yml
Vendored
+116
View File
@@ -0,0 +1,116 @@
name: Test, Build, Deploy
on:
push:
branches: [master, develop]
pull_request:
branches: [master, develop]
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
env:
NODE_ENV: "test"
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install Node.js 16
uses: actions/setup-node@v2
with:
node-version: 16.x
- name: Install PNPM
uses: pnpm/action-setup@v2.0.1
with:
version: 6.0.2
- name: Install Dependencies
run: pnpm i
# configure external dbs
- uses: supercharge/mongodb-github-action@1.4.1
- uses: supercharge/redis-github-action@1.1.0
- name: Move Config File
run: mv ./.github/test.conf.json5 ./test.conf.json5
- name: Run Tests
run: pnpm test
env:
NODE_ENV: "test"
PORT: 8080
MONGO_URL: "127.0.0.1"
REDIS_URL: "127.0.0.1"
- name: Lint Code
run: pnpm lint
- name: Upload Coverage
if: always()
run: |
pnpm install -g codecov
cat coverage/lcov.info | codecov
docker-push:
runs-on: ubuntu-latest
needs: [test]
if: github.event_name == 'push'
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Docker Hub login
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Develop tag
if: github.ref == 'refs/heads/develop'
run: echo "BRANCH_TAG=develop" >> $GITHUB_ENV
- name: Stable tag
if: github.ref == 'refs/heads/master'
run: echo "BRANCH_TAG=stable" >> $GITHUB_ENV
- name: Version tag
run: 'echo "VERSION_TAG=$(cat package.json | grep version | head -1 | awk -F: ''{ print $2 }'' | sed ''s/[", ]//g'')" >> $GITHUB_ENV'
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
with:
context: ./
file: ./Dockerfile
push: true
tags: ${{ secrets.DOCKER_USERNAME }}/tachi-server:${{ env.BRANCH_TAG }},${{ secrets.DOCKER_USERNAME }}/tachi-server:${{ env.VERSION_TAG }},
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
deploy:
runs-on: ubuntu-latest
needs: [docker-push]
if: github.event_name == 'push'
steps:
- name: Deploy to staging
if: github.ref == 'refs/heads/develop'
uses: appleboy/ssh-action@1a8b3784eaa665f677fa114edd5683bb6a6bfaa8
with:
host: ${{ secrets.TACHI_HOST }}
username: tachi
key: ${{ secrets.TACHI_KEY }}
script: tachi-server-deploy -s
- name: Deploy to live
if: github.ref == 'refs/heads/master'
uses: appleboy/ssh-action@1a8b3784eaa665f677fa114edd5683bb6a6bfaa8
with:
host: ${{ secrets.TACHI_HOST }}
username: tachi
key: ${{ secrets.TACHI_KEY }}
script: tachi-server-deploy
server/.github/workflows/tests.yml
Vendored
-89
View File
@@ -1,89 +0,0 @@
name: KTBlack-Server CI
env:
KTBKS_MONGO_BASE_URL: "127.0.0.1"
NODE_ENV: "test"
KTBKS_LOG_LEVEL: "info"
KTBKS_CAPTCHA_SECRET_KEY: "unused"
KTBKS_SESSION_SECRET: "unused"
KTBKS_FLO_API_URL: "unused"
KTBKS_EAG_API_URL: "unused"
KTBKS_ARC_API_URL: "unused"
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
test:
name: Build and Test
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v2
- name: Install Node.js 15
uses: actions/setup-node@v2
with:
node-version: 15.x
- name: Install PNPM
uses: pnpm/action-setup@v2.0.1
with:
version: 6.0.2
# configure external dbs
- uses: supercharge/mongodb-github-action@1.4.1
- uses: supercharge/redis-github-action@1.1.0
- name: Install Dependencies
run: pnpm i
# for the time being, disable lint checking.
# - name: Lint Code
# run: pnpm lint
# enable indexing for our testingdb, this results in faster action time.
- name: Set Mongo Indexes
run: pnpx ts-node scripts/set-indexes.ts
- name: Run Tests
run: pnpm test
- name: Upload Coverage
if: always()
run: |
pnpm install -g codecov
cat coverage/lcov.info | codecov
# deploy:
# if: ${{ github.ref == 'master' }}
# needs: test
# name: Deploy
# runs-on: ubuntu-latest
# steps:
# - name: Checkout Code
# uses: actions/checkout@v2
# - name: Install Node.js 15
# uses: actions/setup-node@v2
# with:
# node-version: 15.x
# - name: Install PNPM
# uses: pnpm/action-setup@v2.0.1
# with:
# version: 6.0.2
# # configure external dbs
# - uses: supercharge/mongodb-github-action@1.4.1
# - uses: supercharge/redis-github-action@1.1.0
# - name: Install Dependencies
# run: pnpm i
# # for the time being, disable lint checking.
# # - name: Lint Code
# # run: pnpm lint
server/.gitignore
+39 -17
View File
@@ -1,17 +1,39 @@
node_modules
js
# .vscode
logs
.nyc_output
coverage
.env
.setup_env
scripts/arc-db-sync/charts.json
scripts/arc-db-sync/songs.json
scripts/validate-database-errs/*
scripts/__JSON-TO-KTDB/**/*.json
/src/lib/env/config.json
# General node stuffs
node_modules
js
logs
.pnpm-debug.log
# Test stuffs
.nyc_output
coverage
.env
.setup_env
failed-tests.txt
# Config files
conf.json5
*.conf.json5
# HTTPS local certs
cert
# Error Files
scripts/validate-database-errs/*
# IDE Specific files
.vscode
.idea
# Inline CDN
local-cdn
test-cdn
# Weird Redis Dumps
dump.rdb
scripts/quick-insert/insert.ts
scripts/bpi-poyashi-data/output.json
scripts/sync-database/tachi-database-seeds
scripts/arc-sync/cache.json
server/.taprc
+11 -7
View File
@@ -1,12 +1,16 @@
files: "src/**/*.test.ts"
test-env: NODE_ENV=test
jobs: 1 # disable parallel to avoid db clashes
timeout: 500 # might need to be increased
files: "src/test-utils/single-process-tap.ts"
test-env: [NODE_ENV=test, TCHIS_CONF_LOCATION=test.conf.json5]
# On a local machine, these tests run in under 2-3 minutes.
# On github actions, these tests take an inordinate amount of time.
# I do not know why, and it suddenly started happening one day.
# The timeout is set to 16 minutes, which is ridiculous.
# Doesn't matter for real machines.
timeout: 960
ts: true
before: "./src/test-utils/setup.ts"
before: "./src/test-utils/hack-setup.js"
check-coverage: true
branches: 80
branches: 60
functions: 80
lines: 80
statements: 80
reporter: terse
reporter: terse
server/.vscode/execTsNode.sh
Vendored
-1
View File
@@ -1 +0,0 @@
export $(xargs < .env) && ts-node $1
server/.vscode/launch.json
Vendored
-24
View File
@@ -1,24 +0,0 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "Run Server",
"preLaunchTask": "npm: build",
"envFile": "${workspaceFolder}/.env",
"program": "${workspaceFolder}/src/main.ts",
"outFiles": [
"${workspaceFolder}/js/**/*.js"
],
"skipFiles": [
"<node_internals>/**",
"${workspaceFolder}/node_modules/**/*.js",
],
"outputCapture": "std",
},
]
}
server/.vscode/settings.json
Vendored
-3
View File
@@ -1,3 +0,0 @@
{
"typescript.tsdk": "node_modules\\typescript\\lib"
}
server/.vscode/tasks.json
Vendored
-42
View File
@@ -1,42 +0,0 @@
{
"version": "2.0.0",
"tasks": [
{
"type": "npm",
"script": "devrun",
"problemMatcher": [],
"label": "npm: devrun",
"detail": "tsc && export NODE_ENV=\"dev\" && node js/main.js",
"group": {
"kind": "build",
"isDefault": true
}
},
{
"type": "npm",
"script": "fulltest",
"problemMatcher": [],
"label": "npm: fulltest",
"detail": "tap",
"group": {
"kind": "test",
"isDefault": true
}
},
{
"label": "runWithTsnode",
"type": "shell",
"command": "./.vscode/execTsNode.sh",
"args": [
"${relativeFile}"
],
"group": "test",
"presentation": {
"echo": true,
"reveal": "always",
"focus": false,
"panel": "dedicated"
}
}
]
}
server/@types/express-request-mock/index.d.ts
Vendored
+12 -12
View File
@@ -2,17 +2,17 @@
// this is a hack fix for express-request-mock's lack of typing.
declare module "express-request-mock" {
import { Request, RequestHandler, Response } from "express";
import { RequestOptions, MockResponse } from "node-mocks-http";
import { Request, RequestHandler, Response } from "express";
import { RequestOptions, MockResponse } from "node-mocks-http";
export default function (
callback: RequestHandler,
options?: RequestOptions,
decorators?: Record<string, unknown>
): Promise<{
req: Request;
res: MockResponse<Response<any, Record<string, any>>>;
request: Request;
response: MockResponse<Response<any, Record<string, any>>>;
}>;
export default function (
callback: RequestHandler,
options?: RequestOptions,
decorators?: Record<string, unknown>
): Promise<{
req: Request;
res: MockResponse<Response<any, Record<string, any>>>;
request: Request;
response: MockResponse<Response<any, Record<string, any>>>;
}>;
}
server/@types/express/index.d.ts
Vendored
+27 -23
View File
@@ -1,23 +1,27 @@
import { Session, SessionData } from "express-session";
import { SYMBOL_KtchiData } from "../../src/lib/constants/ktchi";
import { KtchiRequestData, KtchiSessionData } from "../../src/utils/types";
export {};
// this only exists for testing context - it doesn't seem to work properly otherwise.
declare module "express-session" {
// Inject additional properties on express-session
interface SessionData {
ktchi: KtchiSessionData;
}
}
declare global {
namespace Express {
interface Request {
session: Session & Partial<SessionData>;
[SYMBOL_KtchiData]?: Partial<KtchiRequestData>;
}
}
}
import { Session, SessionData } from "express-session";
import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "../../src/lib/constants/tachi";
import { TachiRequestData, TachiSessionData } from "../../src/utils/types";
import { APITokenDocument } from "tachi-common";
export {};
// this only exists for testing context - it doesn't seem to work properly otherwise.
declare module "express-session" {
// Inject additional properties on express-session
interface SessionData {
tachi: TachiSessionData;
}
}
declare global {
namespace Express {
interface Request {
session: Session & Partial<SessionData>;
[SYMBOL_TachiData]?: Partial<TachiRequestData>;
// even though this is technically *not* present on every request
// it's always assigned in the main router, so its functionally equivalent.
[SYMBOL_TachiAPIAuth]: APITokenDocument;
}
}
}
server/@types/mongodb/index.d.ts
Vendored
+8
View File
@@ -0,0 +1,8 @@
declare module "mongodb" {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
interface FindOneOptions<T> {
projectID?: boolean;
}
}
export {};
server/@types/why-is-node-running/index.d.ts
Vendored
+1 -1
View File
@@ -1,3 +1,3 @@
declare module "why-is-node-running" {
export default function log(): void;
export default function log(): void;
}
server/Dockerfile
+9
View File
@@ -0,0 +1,9 @@
FROM node:16
WORKDIR /app
COPY --chown=node:node . /app
RUN npm install --silent -g pnpm && groupmod -g 1003 node && chown node:node /app
USER node
RUN pnpm i --silent && pnpx tsc --project tsconfig.build.json
HEALTHCHECK --interval=15s --timeout=5s CMD curl -f http://localhost:8080/api/v1/status || exit 1
ENV NODE_PATH=js/
CMD ["node", "js/main.js"]
server/README.md
+35 -5
View File
@@ -1,8 +1,38 @@
![](https://raw.githubusercontent.com/zkldi/kamaitachi-api/master/Kamaitachi-TX.png)
# Tachi-Server
(Black)
[![codecov](https://codecov.io/gh/zkldi/tachi-server/branch/develop/graph/badge.svg?token=RAZSDSH1Y9)](https://codecov.io/gh/zkldi/tachi-server)
Any colour you like...
Tachi-Server is the backend for Tachi.
[![codecov](https://codecov.io/gh/zkldi/ktblack-server/branch/master/graph/badge.svg?token=RAZSDSH1Y9)](https://codecov.io/gh/zkldi/ktblack-server)
[![KTBlack-Server Master](https://github.com/zkldi/ktblack-server/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/zkldi/ktblack-server/actions/workflows/tests.yml)
Tachi itself is a supercharged Rhythm Game Score Tracker, powering multiple games!
## What's it do?
- Hosts an express backend server.
- Opens a public API on `/api`.
- Supports various IR frameworks under `/ir`
- Powers a ridiculously overengineered score import framework.
## Other Repos
`tachi-server` is one part of the larger Tachi framework. Other repositories include:
[tachi-common](https://github.com/tng-dev/tachi-common): Common Types and Utilities for Tachi. (Stuff that's shared between the client and the server, basically.)
[tachi-docs](https://github.com/tng-dev/tachi-docs): Documentation for Tachi.
[tachi-import-scripts](https://github.com/tng-dev/tachi-import-scripts): A UI for converting local files into Batch-Manual format, then uploading to Tachi.
[tachi-beatoraja-ir](https://github.com/tng-dev/tachi-beatoraja-ir): A beatoraja IR client for integrating with Tachi.
[tachi-database-seeds](https://github.com/tng-dev/tachi-database-seeds): Backbone data for Tachi, such as song and chart information. Free to use.
[tachi-bot](https://github.com/tng-dev/tachi-bot): A discord bot that integrates with the Tachi API.
## Where's the client?
The client code is private. However, making a client is *definitely* not the hard part here, and you're completely free to make your own.
## Documentation
[Over here!](https://tachi.rtfd.io)
server/hosts.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
hostname=$(hostname)
ip=$(hostname -i)
cp /etc/hosts ~/hosts
sed -i 's/127\.0\.0\.1\s+/127\.0\.0\.1 localhost.localdomain /' ~/hosts
sed -i "s/${ip}\s+/${ip} ${hostname}.localdomain /" ~/hosts
cp ~/hosts /etc/hosts
rm ~/hosts
server/make-selfsigned-https.sh
Executable
+10
View File
@@ -0,0 +1,10 @@
#! /bin/bash
# This command is for dev servers only! You should use this to
# create a self-signed HTTPS certificate for local dev.
# That is it. This is not secure.
mkdir -p cert
yes "" | openssl req -x509 -newkey rsa:4096 -keyout ./cert/key.pem -out ./cert/cert.pem -sha256 -days 365 --nodes &> /dev/null
echo "Created HTTPS Certificates!"
server/package.json
+105 -78
View File
@@ -1,79 +1,106 @@
{
"name": "ktblack-server",
"version": "0.1.0",
"description": "The server for Kamaitachi Black",
"main": "js/index.js",
"scripts": {
"fulltest": "tap",
"test": "tap",
"watchtest": "tap --watch",
"build": "tsc --project tsconfig.build.json",
"lint": "eslint ./src --ext .ts --fix",
"devrun": "tsc --project tsconfig.build.json && node -r dotenv/config js/main.js"
},
"author": "zkldi",
"license": "ISC",
"devDependencies": {
"@types/bcrypt": "^3.0.1",
"@types/connect-redis": "^0.0.16",
"@types/express": "^4.17.11",
"@types/express-session": "^1.17.3",
"@types/multer": "^1.4.5",
"@types/node": "^15.0.1",
"@types/node-fetch": "^2.5.10",
"@types/range-parser": "^1.2.3",
"@types/supertest": "^2.0.11",
"@types/tap": "^15.0.0",
"@types/uuid": "^8.3.0",
"@typescript-eslint/eslint-plugin": "^4.22.0",
"@typescript-eslint/parser": "^4.22.0",
"eslint": "^7.25.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-prettier": "^3.4.0",
"express-request-mock": "^3.1.0",
"node-mocks-http": "^1.10.1",
"prettier": "^2.2.1",
"supertest": "^6.1.3",
"tap": "^15.0.5",
"ts-node": "^9.1.1",
"why-is-node-running": "^2.2.0"
},
"dependencies": {
"bcrypt": "^5.0.1",
"bee-queue": "^1.3.1",
"commander": "^7.2.0",
"connect-redis": "^5.2.0",
"csv-parse": "^4.15.4",
"deepmerge": "^4.2.2",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"express-async-errors": "^3.1.1",
"express-session": "^1.17.1",
"fast-json-stable-hash": "^1.0.2",
"fast-xml-parser": "^3.19.0",
"kamaitachi-common": "github:zkldi/Kamaitachi-common",
"mongodb": "^3.6.6",
"monk": "^7.3.4",
"multer": "^1.4.2",
"node-fetch": "^2.6.1",
"prudence": "^0.9.6",
"redis": "^3.1.2",
"typescript": "^4.2.4",
"uuid": "^8.3.2",
"winston": "^3.3.3"
},
"nyc": {
"reporter": [
"html",
"lcov"
],
"all": true,
"include": [
"src/*/!(test-utils)/**/*.ts"
],
"exclude": [
"src/**/*.test.ts"
]
}
}
"name": "tachi-server",
"version": "2.0.0-prerelease",
"description": "A score tracking server.",
"main": "js/index.js",
"private": true,
"scripts": {
"test": "tap",
"watchtest": "tap --watch",
"build": "tsc --project tsconfig.build.json",
"lint": "eslint ./src --ext .ts --fix",
"start": "tsc --project tsconfig.build.json && export NODE_PATH=js/ && node js/main.js",
"run": "export NODE_PATH=js/ && node js/main.js",
"runscoreworker": "export NODE_PATH=js/ && node js/lib/score-import/worker/worker.js"
},
"author": "zkldi",
"license": "AGPL3",
"devDependencies": {
"@types/bcrypt": "5.0.0",
"@types/bcryptjs": "^2.4.2",
"@types/bunyan": "^1.8.7",
"@types/connect-redis": "0.0.16",
"@types/express": "4.17.12",
"@types/express-rate-limit": "5.1.2",
"@types/express-session": "1.17.3",
"@types/mkdirp": "1.0.1",
"@types/multer": "1.4.6",
"@types/node": "15.12.4",
"@types/node-fetch": "2.5.10",
"@types/nodemailer": "^6.4.4",
"@types/range-parser": "1.2.3",
"@types/rate-limit-redis": "1.7.1",
"@types/rimraf": "3.0.0",
"@types/safe-json-stringify": "1.1.2",
"@types/supertest": "2.0.11",
"@types/tap": "15.0.4",
"@types/uuid": "8.3.0",
"@typescript-eslint/eslint-plugin": "4.28.0",
"@typescript-eslint/parser": "4.28.0",
"eslint": "7.29.0",
"eslint-config-prettier": "8.3.0",
"eslint-plugin-import": "2.23.4",
"eslint-plugin-prettier": "3.4.0",
"express-request-mock": "3.1.0",
"node-mocks-http": "1.10.1",
"prettier": "2.3.1",
"supertest": "6.1.3",
"tap": "15.0.9",
"tsconfig-paths": "3.10.1",
"why-is-node-running": "2.2.0"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.41.0",
"@types/mocha": "^9.0.0",
"@valuabletouch/winston-seq": "^1.2.0",
"bcryptjs": "^2.4.3",
"bullmq": "^1.54.4",
"bunyan": "^1.8.15",
"commander": "7.2.0",
"connect-redis": "6.0.0",
"csv-parse": "4.16.0",
"deepmerge": "4.2.2",
"dotenv": "10.0.0",
"express": "4.17.1",
"express-async-errors": "3.1.1",
"express-rate-limit": "5.2.6",
"express-session": "1.17.2",
"fast-json-stable-hash": "1.0.3",
"fast-xml-parser": "3.19.0",
"helmet": "4.6.0",
"json5": "2.2.0",
"mkdirp": "1.0.4",
"mongodb": "3.6.3",
"monk": "7.3.4",
"multer": "1.4.2",
"node-cache": "^5.1.2",
"node-fetch": "2.6.1",
"nodemailer": "^6.6.3",
"prudence": "0.9.7",
"rate-limit-redis": "2.1.0",
"redis": "3.1.2",
"rimraf": "3.0.2",
"safe-json-stringify": "1.2.0",
"seq-logging": "^1.1.1",
"tachi-common": "0.3.10",
"ts-node": "10.0.0",
"typescript": "4.3.4",
"winston": "3.3.3",
"winston-daily-rotate-file": "^4.5.5",
"winston-transport": "^4.4.0"
},
"nyc": {
"reporter": [
"html",
"lcov"
],
"all": true,
"include": [
"src/*/!(test-utils)/**/*.ts"
],
"exclude": [
"src/**/*.test.ts",
"src/external/mongo/schemas.ts"
]
}
}
server/pnpm-lock.yaml
Generated
+1917 -882
View File
File diff suppressed because it is too large Load Diff
server/scripts/.pnpm-debug.log
-14
View File
@@ -1,14 +0,0 @@
{
"0 debug pnpm:scope": {
"selected": 1
},
"1 error pnpm": {
"code": "ERR_PNPM_CANNOT_REMOVE_MISSING_DEPS",
"err": {
"name": "Error",
"message": "Cannot remove 'better-sqlite3': project has no dependencies of any kind",
"code": "ERR_PNPM_CANNOT_REMOVE_MISSING_DEPS",
"stack": "Error: Cannot remove 'better-sqlite3': project has no dependencies of any kind\n at Object.handler (/home/zkldi/pnpm-global/5/node_modules/.pnpm/pnpm@6.4.0/node_modules/pnpm/dist/pnpm.cjs:116381:15)\n at async Timeout._onTimeout (/home/zkldi/pnpm-global/5/node_modules/.pnpm/pnpm@6.4.0/node_modules/pnpm/dist/pnpm.cjs:124690:24)"
}
}
}
server/scripts/_README
+1 -3
View File
@@ -3,6 +3,4 @@ https://github.com/TypeStrong/ts-node
They have only been tested for ts-node.
npm install -g ts-node
it's probably installed on the machine you're seeing this on, winter(?). love ya.
pnpm install -g ts-node
server/scripts/__JSON-TO-KTDB/beatoraja/import.ts
-20
View File
@@ -1,20 +0,0 @@
import fs from "fs";
import path from "path";
import CreateLogCtx from "../../../src/common/logger";
import db from "../../../src/db/db";
const logger = CreateLogCtx(__filename);
const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8"));
const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8"));
(async () => {
logger.info("starting chart import.");
await db.charts.bms.remove({});
await db.charts.bms.insert(charts);
logger.info("imported charts.");
await db.songs.bms.remove({});
await db.songs.bms.insert(songs);
logger.info("done.");
})();
server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts
-100
View File
@@ -1,100 +0,0 @@
/* eslint-disable no-await-in-loop */
// diffs an out JSON with the kamaitachi db, optionally syncs with -s.
import fs from "fs";
import path from "path";
import db from "../../../src/db/db";
import { MDBJson } from "./types";
import { Difficulties, Playtypes, ChartDocument } from "kamaitachi-common";
import CreateLogCtx from "../../../src/common/logger";
import { Command } from "commander";
const program = new Command();
program.option("-s --sync", "Sync KTDB with the file.");
program.parse(process.argv);
const options = program.opts();
const dataset = JSON.parse(
fs.readFileSync(path.join(__dirname, "./out.json"), "utf-8")
) as MDBJson[];
const logger = CreateLogCtx(__filename);
(async () => {
for (const data of dataset) {
for (const d in data.difficulties) {
const diff = d as keyof typeof data.difficulties;
if (data.difficulties[diff] === 0) {
logger.verbose(`Skipping chart ${data.title} ${diff}`);
continue;
}
const spl = diff.split("-");
const chart = (await db.charts.iidx.findOne({
"data.inGameID": data.songID,
playtype: spl[0] as Playtypes["iidx"],
difficulty: spl[1] as Difficulties["iidx:SP" | "iidx:DP"],
isPrimary: true,
})) as ChartDocument<"iidx:SP" | "iidx:DP">;
if (!chart) {
logger.warn(`No chart exists for ${data.title} (${diff}).`);
continue;
}
if (chart.levelNum !== data.difficulties[diff]) {
logger.warn(
`Level Mismatch with ${data.title} (${diff}) [${chart.levelNum} -> ${data.difficulties[diff]}]`
);
if (options.sync) {
await db.charts.iidx.update(
{
_id: chart._id,
},
{
$set: {
levelNum: data.difficulties[diff],
level: data.difficulties[diff].toString(),
},
}
);
}
}
if (chart.data.notecount !== data.notecounts[diff]) {
if (
data.notecounts[diff] === undefined ||
Math.abs(chart.data.notecount - data.notecounts[diff]!) > 2
) {
logger.error(
`Significant Notecount Mismatch with ${data.title} (${diff}) [${chart.data.notecount} -> ${data.notecounts[diff]}] - skipping.`
);
// continue;
}
logger.warn(
`Notecount Mismatch with ${data.title} (${diff}) [${chart.data.notecount} -> ${data.notecounts[diff]}]`
);
if (options.sync) {
await db.charts.iidx.update(
{
_id: chart._id,
},
{
$set: {
"data.notecount": data.notecounts[diff],
},
}
);
}
}
}
}
logger.info("done");
})();
server/scripts/__JSON-TO-KTDB/iidx/types.ts
-12
View File
@@ -1,12 +0,0 @@
export interface MDBJson {
title: string;
artist: string;
marquee: string;
folder: number;
genre: string;
difficulties: Record<DiffNames, number>;
notecounts: Partial<Record<DiffNames, number>>;
songID: number;
}
type DiffNames = `${"SP" | "DP"}-${"BEGINNER" | "NORMAL" | "HYPER" | "ANOTHER" | "LEGGENDARIA"}`;
server/scripts/__KT_DATABASE_MIGRATION/charts-chunithm.ts
-56
View File
@@ -1,56 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { ChartDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
import { gameOrders } from "kamaitachi-common/js/config";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ChartDocument<"chunithm:Single">> {
const song = await db.songs.chunithm.findOne({
id: c.id,
});
if (!song) {
logger.severe(`Cannot find song with ID ${c.id}?`);
throw new Error(`Cannot find song with ID ${c.id}?`);
}
const newChartDoc: ChartDocument<"chunithm:Single"> = {
rgcID: null,
chartID: c.chartID,
difficulty: c.difficulty,
songID: c.id,
playtype: c.playtype,
levelNum: c.levelNum,
level: c.level.toString(),
flags: {
"IN BASE GAME": !!c.flags["IN BASE GAME"],
OMNIMIX: !!c.flags.OMNIMIX,
},
data: {
inGameID: c.internals.inGameID,
},
isPrimary: true,
versions: [], // sentinel
};
const idx = gameOrders.chunithm.indexOf(song.firstVersion!);
if (idx === -1) {
logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
newChartDoc.versions = [song.firstVersion!];
} else {
newChartDoc.versions = gameOrders.chunithm.slice(idx);
}
return newChartDoc;
}
(async () => {
await MigrateRecords(db.charts.chunithm, "charts-chunithm", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/charts-ddr.ts
-62
View File
@@ -1,62 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { ChartDocument, SongDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
import { gameOrders } from "kamaitachi-common/js/config";
import { oldKTDB } from "./old-db";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ChartDocument<"ddr:SP" | "ddr:DP">> {
const song = (await db.songs.ddr.findOne({
id: c.id,
})) as SongDocument<"ddr">;
const oldSong = await oldKTDB.get("songs-ddr").findOne({
id: c.id,
});
if (!song) {
logger.severe(`Cannot find song with ID ${c.id}?`);
throw new Error(`Cannot find song with ID ${c.id}?`);
}
const newChartDoc: ChartDocument<"ddr:SP" | "ddr:DP"> = {
rgcID: null,
chartID: c.chartID,
difficulty: c.difficulty,
songID: c.id,
playtype: c.playtype,
levelNum: c.levelNum,
level: c.level.toString(),
flags: {
"IN BASE GAME": true,
"N-1": true,
},
data: {
inGameID: c.internals.inGameID,
songHash: oldSong.internals.songHash,
},
isPrimary: true,
versions: [], // sentinel
};
const idx = gameOrders.ddr.indexOf(song.firstVersion!);
if (idx === -1) {
logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
newChartDoc.versions = [song.firstVersion!];
} else {
newChartDoc.versions = gameOrders.ddr.slice(idx);
}
return newChartDoc;
}
(async () => {
await MigrateRecords(db.charts.ddr, "charts-ddr", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/charts-gitadora.ts
-60
View File
@@ -1,60 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { ChartDocument, SongDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
import { gameOrders } from "kamaitachi-common/js/config";
import { oldKTDB } from "./old-db";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ChartDocument<"gitadora:Gita" | "gitadora:Dora">> {
const song = (await db.songs.gitadora.findOne({
id: c.id,
})) as SongDocument<"gitadora">;
const oldSong = await oldKTDB.get("songs-gitadora").findOne({
id: c.id,
});
if (!song) {
logger.severe(`Cannot find song with ID ${c.id}?`);
throw new Error(`Cannot find song with ID ${c.id}?`);
}
const newChartDoc: ChartDocument<"gitadora:Gita" | "gitadora:Dora"> = {
rgcID: null,
chartID: c.chartID,
difficulty: c.difficulty,
songID: c.id,
playtype: c.playtype,
levelNum: c.levelNum,
level: c.level.toString(),
flags: {
"IN BASE GAME": true,
OMNIMIX: true,
},
data: {
inGameID: c.internals.inGameINTID,
},
isPrimary: true,
versions: [], // sentinel
};
const idx = gameOrders.gitadora.indexOf(song.firstVersion!);
if (idx === -1) {
logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
newChartDoc.versions = [song.firstVersion!];
} else {
newChartDoc.versions = gameOrders.gitadora.slice(idx);
}
return newChartDoc;
}
(async () => {
await MigrateRecords(db.charts.gitadora, "charts-gitadora", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/charts-iidx.ts
-59
View File
@@ -1,59 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { ChartDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
import { gameOrders } from "kamaitachi-common/js/config";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ChartDocument<"iidx:SP" | "iidx:DP">> {
const song = await db.songs.iidx.findOne({
id: c.id,
});
if (!song) {
logger.severe(`Cannot find song with ID ${c.id}?`);
throw new Error(`Cannot find song with ID ${c.id}?`);
}
const newChartDoc: ChartDocument<"iidx:SP" | "iidx:DP"> = {
rgcID: null,
chartID: c.chartID,
difficulty: c.difficulty,
songID: c.id,
playtype: c.playtype,
levelNum: c.levelNum,
level: c.level.toString(),
flags: {
"IN BASE GAME": !!c.flags["IN BASE GAME"],
OMNIMIX: !!c.flags.OMNIMIX,
"N-1": !!c.flags["N-1"],
},
data: {
inGameID: c.internals.inGameINTID,
notecount: c.notedata.notecount,
arcChartID: null,
},
isPrimary: true,
versions: [], // sentinel
};
const idx = gameOrders.iidx.indexOf(song.firstVersion!);
if (idx === -1) {
logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
newChartDoc.versions = [song.firstVersion!];
} else {
newChartDoc.versions = gameOrders.iidx.slice(idx);
}
return newChartDoc;
}
(async () => {
await MigrateRecords(db.charts.iidx, "charts-iidx", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/charts-museca.ts
-53
View File
@@ -1,53 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { ChartDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
import { gameOrders } from "kamaitachi-common/js/config";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ChartDocument<"museca:Single">> {
const song = await db.songs.museca.findOne({
id: c.id,
});
if (!song) {
logger.severe(`Cannot find song with ID ${c.id}?`);
throw new Error(`Cannot find song with ID ${c.id}?`);
}
const newChartDoc: ChartDocument<"museca:Single"> = {
rgcID: null,
chartID: c.chartID,
difficulty: c.difficulty,
songID: c.id,
playtype: c.playtype,
levelNum: c.levelNum,
level: c.level.toString(),
flags: {
"IN BASE GAME": true,
OMNIMIX: true, // false, needs to be overrode for 3 or so charts
},
data: {},
isPrimary: true,
versions: [], // sentinel
};
const idx = gameOrders.museca.indexOf(song.firstVersion!);
if (idx === -1) {
logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
newChartDoc.versions = [song.firstVersion!];
} else {
newChartDoc.versions = gameOrders.museca.slice(idx);
}
return newChartDoc;
}
(async () => {
await MigrateRecords(db.charts.museca, "charts-museca", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/charts-sdvx.ts
-56
View File
@@ -1,56 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { ChartDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
import { gameOrders } from "kamaitachi-common/js/config";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ChartDocument<"sdvx:Single">> {
const song = await db.songs.sdvx.findOne({
id: c.id,
});
if (!song) {
logger.severe(`Cannot find song with ID ${c.id}?`);
throw new Error(`Cannot find song with ID ${c.id}?`);
}
const newChartDoc: ChartDocument<"sdvx:Single"> = {
rgcID: null,
chartID: c.chartID,
difficulty: c.difficulty,
songID: c.id,
playtype: c.playtype,
levelNum: c.levelNum,
level: c.level.toString(),
flags: {
"IN BASE GAME": !!c.flags["IN BASE GAME"],
OMNIMIX: !!c.flags.OMNIMIX,
"N-1": !!c.flags["N-1"],
},
data: {
inGameID: c.internals.inGameINTID,
},
isPrimary: true,
versions: [], // sentinel
};
const idx = gameOrders.sdvx.indexOf(song.firstVersion!);
if (idx === -1) {
logger.warn(`Invalid firstAppearance of ${song.firstVersion!}, running anyway.`);
newChartDoc.versions = [song.firstVersion!];
} else {
newChartDoc.versions = gameOrders.sdvx.slice(idx);
}
return newChartDoc;
}
(async () => {
await MigrateRecords(db.charts.sdvx, "charts-sdvx", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/folders.ts
-48
View File
@@ -1,48 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { FolderDocument, Game } from "kamaitachi-common";
import { validPlaytypes, gameHuman } from "kamaitachi-common/js/config";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
import crypto from "crypto";
const logger = CreateLogCtx(__filename);
function ConvertFn(c: any): FolderDocument[] {
throw new Error("This doesn't work, It was migrated elsewhere.");
// let docs = [];
// for (const playtype of validPlaytypes[c.game as Game]) {
// const fd: FolderDocument = {
// title: c.title,
// game: c.game,
// playtype,
// type: "query" as const,
// query: {
// collection: c.query.collection,
// body: c.query.query,
// },
// folderID: crypto.randomBytes(20).toString("hex"),
// table: c.table,
// tableIndex: 0, // ???
// };
// let gh = gameHuman[c.game as Game];
// fd.title = fd.title.replace(new RegExp(`${gh} +`, "giu"), "");
// logger.info(`Porting folder ${fd.title} (${playtype})`);
// docs.push(fd);
// }
// return docs;
}
(async () => {
await MigrateRecords(db.folders, "folders", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/goals.ts
+56 -56
View File
@@ -1,79 +1,79 @@
// oh boy
/* eslint-disable @typescript-eslint/no-explicit-any */
import { GoalDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import db from "external/mongo/db";
import { GoalDocument } from "tachi-common";
import CreateLogCtx from "../../src/common/logger";
import MigrateRecords from "./migrate";
const logger = CreateLogCtx(__filename);
function ConvertFn(c: any): GoalDocument | null {
let charts: GoalDocument["charts"];
let charts: GoalDocument["charts"];
if (c.directChartID) {
charts = {
type: "single",
data: c.directChartID,
};
} else if (c.directChartIDs) {
charts = {
type: "multi",
data: c.directChartIDs,
};
} else if (c.chartQuery) {
logger.warn(`Skipped goal ${c.title}, as it used cQuery.`);
return null;
} else {
logger.warn(`Any Goal ${c.goalID}, ${c.title}. Skipping.`);
return null;
}
if (c.directChartID) {
charts = {
type: "single",
data: c.directChartID,
};
} else if (c.directChartIDs) {
charts = {
type: "multi",
data: c.directChartIDs,
};
} else if (c.chartQuery) {
logger.warn(`Skipped goal ${c.title}, as it used cQuery.`);
return null;
} else {
logger.warn(`Any Goal ${c.goalID}, ${c.title}. Skipping.`);
return null;
}
const partialCriteria: Partial<GoalDocument["criteria"]> = {
mode: "single",
};
const partialCriteria: Partial<GoalDocument["criteria"]> = {
mode: "single",
};
if (c.scoreQuery["scoreData¬lampIndex"]) {
partialCriteria.key = "scoreData.lampIndex";
} else if (c.scoreQuery["scoreData¬gradeIndex"]) {
partialCriteria.key = "scoreData.gradeIndex";
} else if (c.scoreQuery["scoreData¬score"]) {
partialCriteria.key = "scoreData.score";
} else if (c.scoreQuery["scoreData¬percent"]) {
partialCriteria.key = "scoreData.percent";
} else {
throw new Error(`Invalid goal ${c.goalID} ${c.title}.`);
}
if (c.scoreQuery["scoreData¬lampIndex"]) {
partialCriteria.key = "scoreData.lampIndex";
} else if (c.scoreQuery["scoreData¬gradeIndex"]) {
partialCriteria.key = "scoreData.gradeIndex";
} else if (c.scoreQuery["scoreData¬score"]) {
partialCriteria.key = "scoreData.score";
} else if (c.scoreQuery["scoreData¬percent"]) {
partialCriteria.key = "scoreData.percent";
} else {
throw new Error(`Invalid goal ${c.goalID} ${c.title}.`);
}
const v = c.scoreQuery[partialCriteria.key.replace(/\./gu, "¬")];
const v = c.scoreQuery[partialCriteria.key.replace(/\./gu, "¬")];
const value = v ? v["~gte"] : undefined;
const value = v ? v["~gte"] : undefined;
// would break for value == 0, but a goal would never be
// zero.
if (!value) {
throw new Error(`Invalid goal ${c.goalID} ${c.title}.`);
}
// would break for value == 0, but a goal would never be
// zero.
if (!value) {
throw new Error(`Invalid goal ${c.goalID} ${c.title}.`);
}
partialCriteria.value = value;
partialCriteria.value = value;
// @ts-expect-error ts cant resolve this union properly
const newGoalDoc: GoalDocument = {
game: c.game,
playtype: c.playtype,
title: c.title,
timeAdded: c.timeAdded,
goalID: c.goalID, // maybe
createdBy: c.createdBy,
charts,
criteria: partialCriteria as GoalDocument["criteria"],
};
// @ts-expect-error ts cant resolve this union properly
const newGoalDoc: GoalDocument = {
game: c.game,
playtype: c.playtype,
title: c.title,
timeAdded: c.timeAdded,
goalID: c.goalID, // maybe
createdBy: c.createdBy,
charts,
criteria: partialCriteria as GoalDocument["criteria"],
};
return newGoalDoc;
return newGoalDoc;
}
(async () => {
await MigrateRecords(db.goals, "goals", ConvertFn);
await MigrateRecords(db.goals, "goals", ConvertFn);
process.exit(0);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/migrate.ts
+46 -33
View File
@@ -1,49 +1,62 @@
import CreateLogCtx from "lib/logger/logger";
import { ICollection } from "monk";
import CreateLogCtx from "../../src/common/logger";
import { oldKTDB } from "./old-db";
const logger = CreateLogCtx(__filename);
const newDocuments: unknown[] = [];
export default async function MigrateRecords(
existingCollection: ICollection,
collectionName: string,
HandlerFN: (c: unknown) => unknown
existingCollection: ICollection,
collectionName: string,
HandlerFN: (c: unknown) => unknown,
filter: Record<string, any> = {},
force = false
) {
const existingRecords = await existingCollection.count({});
logger.info(`Starting migration for ${collectionName}...`);
const existingRecords = await existingCollection.count({});
logger.info(`Starting migration for ${collectionName}...`);
if (existingRecords > 0) {
logger.error(`${existingRecords} documents already exist in db, terminating.`);
process.exit(1);
}
if (existingRecords > 0) {
if (force) {
logger.warn(
`Forcing migration of documents - appending to ${existingRecords} existing records.`
);
} else {
logger.error(`${existingRecords} documents already exist in db, terminating.`);
process.exit(1);
}
}
let i = 0;
let i = 0;
const bucketSize = 10_000;
await oldKTDB
.get(collectionName)
.find({})
// @ts-expect-error it exists.
.each(async (c, { pause, resume }) => {
i++;
pause();
// eslint-disable-next-line no-constant-condition
while (true) {
// eslint-disable-next-line no-await-in-loop
const docs = await oldKTDB.get(collectionName).find(filter, { limit: bucketSize, skip: i });
if (i % 10000 === 0) {
logger.info(`Processed ${i} documents.`);
}
const newDoc = await HandlerFN(c);
logger.info(`Migrating ${i} - ${i + docs.length} documents.`);
if (newDoc !== null) {
newDocuments.push(newDoc);
}
if (docs.length === 0) {
break;
}
resume();
});
// eslint-disable-next-line no-await-in-loop
const newDocs = await Promise.all(docs.map(HandlerFN));
logger.info(`Inserting ${newDocuments.length} documents.`);
await existingCollection.insert(newDocuments.flat());
logger.info(`Done!`);
i += bucketSize;
process.exit(0);
const notNulls = newDocs.filter((e) => e !== null);
logger.info(`Success: ${notNulls.length}, Failures: ${newDocs.length - notNulls.length}.`);
// eslint-disable-next-line no-await-in-loop
await existingCollection.insert(notNulls);
logger.info(`Migrated ${docs.length} documents.`);
if (docs.length < bucketSize) {
break;
}
}
logger.info(`Done!`);
}
server/scripts/__KT_DATABASE_MIGRATION/old-db.ts
+16 -1
View File
@@ -1,3 +1,18 @@
import CreateLogCtx from "lib/logger/logger";
import { Environment } from "lib/setup/config";
import monk from "monk";
export const oldKTDB = monk(`${process.env.MONGO_BASE_URL}/kamaitachidb`);
const logger = CreateLogCtx(__filename);
logger.info(`Connecting to Old KTDB at ${Environment.mongoUrl}/kamaitachidb.`);
export const oldKTDB = monk(`${Environment.mongoUrl}/kamaitachidb`);
oldKTDB
.then(() => {
logger.info(`Done.`);
})
.catch((err) => {
logger.crit(`Fatal in connecting to old KT DB.`, { err });
process.exit(1);
});
server/scripts/__KT_DATABASE_MIGRATION/other/museca-tierlist.ts
-46
View File
@@ -1,46 +0,0 @@
import db from "../../../src/db/db";
import { TierlistDataDocument } from "kamaitachi-common";
import { CalculateTierlistDataID } from "../../../src/common/tierlist";
import { oldKTDB } from "../old-db";
import CreateLogCtx from "../../../src/common/logger";
const logger = CreateLogCtx(__filename);
const TIERLIST_ID = "ed45403c2cd93922d8a45b3c5bc352a962a8f115";
function ConvertFn(c: any) {
const tdd: TierlistDataDocument<"Individual Difference"> = {
chartID: c.chartID,
type: "score",
key: null,
tierlistDataID: CalculateTierlistDataID(c.chartID, "score", null, TIERLIST_ID),
tierlistID: TIERLIST_ID,
data: {
flags: {
"Individual Difference": c.idvDifference === "FALSE",
},
value: c.tiers["score-tier"],
humanised: `☆${c.tiers["score-tier"]}`,
},
};
return tdd;
}
(async () => {
const exists = await db["tierlist-data"].count({
tierlistID: TIERLIST_ID,
});
if (exists > 0) {
logger.error(`Cancelling migration request as ${exists} documents already exist.`);
return process.exit(1);
}
const docs = await oldKTDB.get("tierlistdata").find({
tierlistID: TIERLIST_ID,
});
const newDocs = await Promise.all(docs.map(ConvertFn));
await db["tierlist-data"].insert(newDocs);
})();
server/scripts/__KT_DATABASE_MIGRATION/scores.ts
-144
View File
@@ -1,144 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { PrivateUserDocument, ScoreDocument } from "kamaitachi-common";
import { grades, lamps, supportedGames, validPlaytypes } from "kamaitachi-common/js/config";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
import CreateLogCtx from "../../src/common/logger";
import { CreateScoreID } from "../../src/score-import/framework/common/score-utils";
const logger = CreateLogCtx(__filename);
// HERE. WE. GO!
function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
base[baseProp] = other[otherProp];
}
}
function ConvertFn(c: any): ScoreDocument | null {
if (!supportedGames.includes(c.game)) {
logger.warn(`Ignored game ${c.game}`);
return null;
}
if (["popn", "jubeat", "gitadora", "usc"].includes(c.game)) {
return null;
}
if (validPlaytypes[c.game].includes(c.playtype)) {
logger.warn(`Ignored game pt ${c.game}, ${c.playtype}`);
}
const base: Omit<ScoreDocument, "scoreID"> = {
userID: c.userID,
songID: c.songID,
playtype: c.scoreData.playtype,
chartID: c.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
calculatedData: c.calculatedData,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: grades[c.game].indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: lamps[c.game].indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
hitData: c.scoreData.hitData,
hitMeta: c.scoreData.hitMeta,
},
scoreMeta: {},
};
if (c.scoreMeta) {
if (base.game === "iidx") {
ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom");
ConditionalAssign(base.scoreMeta, "gauge", c.scoreMeta, "optionsGauge");
ConditionalAssign(base.scoreMeta, "assist", c.scoreMeta, "optionsAssist");
ConditionalAssign(base.scoreMeta, "range", c.scoreMeta, "optionsRange");
ConditionalAssign(base.scoreMeta, "pacemaker", c.scoreMeta, "pacemaker");
ConditionalAssign(base.scoreMeta, "pacemakerName", c.scoreMeta, "pacemakerName");
ConditionalAssign(base.scoreMeta, "pacemakerTarget", c.scoreMeta, "pacemakerTarget");
if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
base.scoreMeta.random = null;
}
// @ts-expect-error asdf
if (base.scoreMeta.range === "") {
// @ts-expect-error asdf
base.scoreMeta.range = "NONE";
}
} else if (base.game === "bms") {
ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom");
ConditionalAssign(base.scoreMeta, "inputDevice", c.scoreData.hitMeta, "inputDevice");
}
}
// @ts-expect-error shut
if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
// @ts-expect-error shut
base.scoreData.hitMeta.bp = null;
}
if (base.game === "iidx") {
// @ts-expect-error nope
if (base.scoreData.hitMeta.gauge > 200) {
// @ts-expect-error nope
base.scoreData.hitMeta.gauge = null;
}
// @ts-expect-error nope
if (base.scoreData.hitMeta.gaugeHistory) {
// @ts-expect-error nope
base.scoreData.hitMeta.gaugeHistory = base.scoreData.hitMeta.gaugeHistory.map((e) =>
e > 200 ? null : e
);
}
} else if (base.game === "bms") {
// @ts-expect-error shut
if (base.scoreData.hitMeta.gauge === -1) {
// @ts-expect-error shut
base.scoreData.hitMeta.gauge = null;
}
// @ts-expect-error shut
delete base.scoreData.hitMeta.inputDevice;
} else if (base.game === "sdvx" || base.game === "usc") {
ConditionalAssign(base.scoreData.hitData, "miss", base.scoreData.hitData, "error");
//@ts-expect-error yea
delete base.scoreData.hitData.error;
} else if (base.game === "chunithm") {
if (base.scoreData.percent === 101) {
base.scoreData.grade === "SSS";
}
}
if (base.service === "mm") {
base.service = "maimagic";
}
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const score: ScoreDocument = {
...base,
scoreID,
};
return score;
}
(async () => {
await MigrateRecords(db.scores, "scores", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts
+153
View File
@@ -0,0 +1,153 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
import { GetGamePTConfig, ScoreDocument, StaticConfig } from "tachi-common";
import MigrateRecords from "../migrate";
import { oldKTDB } from "../old-db";
const logger = CreateLogCtx(__filename);
function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
base[baseProp] = other[otherProp];
}
}
async function ConvertFn(c: any): Promise<ScoreDocument | null> {
if (!StaticConfig.allSupportedGames.includes(c.game)) {
logger.warn(`Ignored game ${c.game}`);
return null;
}
const game = c.game;
const playtype = c.scoreData.playtype;
const oldChartDoc = await oldKTDB.get("charts-bms").findOne({
chartID: c.chartID,
});
if (!oldChartDoc) {
logger.warn(`Cannot find oldChartDoc for ${c.chartID}`);
}
// @ts-expect-error implicit any is expected
const chartDoc = await db.charts[game].findOne({
"data.hashMD5": oldChartDoc.internals.hash,
});
if (!chartDoc) {
const oldSong = await oldKTDB.get("songs-bms").findOne({ id: c.songID });
logger.warn(`Cannot find ChartDoc for ${oldSong.title} ${oldChartDoc.level} ${playtype}`);
return null;
}
const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
if (!gptConfig) {
logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
throw new Error(`Screwed`);
}
const base: Omit<ScoreDocument, "scoreID"> = {
userID: c.userID,
songID: chartDoc.songID,
playtype: chartDoc.playtype,
chartID: chartDoc.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
// we'll just recalc calculated data instead of trying to update
calculatedData: {},
isPrimary: chartDoc.isPrimary,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
judgements: c.scoreData.hitData,
hitMeta: {
fast: c.scoreData.hitMeta.fast,
slow: c.scoreData.hitMeta.slow,
maxCombo: c.scoreData.hitMeta.maxCombo,
bp: c.scoreData.hitMeta.bp,
gauge: c.scoreData.hitMeta.gauge,
epg: c.scoreData.hitMeta.epg,
egr: c.scoreData.hitMeta.egr,
egd: c.scoreData.hitMeta.egd,
ebd: c.scoreData.hitMeta.ebd,
epr: c.scoreData.hitMeta.epr,
lpg: c.scoreData.hitMeta.lpg,
lgr: c.scoreData.hitMeta.lgr,
lgd: c.scoreData.hitMeta.lgd,
lbd: c.scoreData.hitMeta.lbd,
lpr: c.scoreData.hitMeta.lpr,
},
},
scoreMeta: {
random: c.scoreMeta?.optionsRandom ?? null,
inputDevice: c.scoreMeta?.inputDevice ?? null,
},
};
// @ts-expect-error shut
if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
// @ts-expect-error shut
base.scoreData.hitMeta.bp = null;
}
// @ts-expect-error shut
if (Number(base.scoreData.hitMeta.bp) < 0) {
// @ts-expect-error shut
base.scoreData.hitMeta.bp = null;
}
// @ts-expect-error shut
if (base.scoreData.hitMeta.gauge === -1) {
// @ts-expect-error shut
base.scoreData.hitMeta.gauge = null;
}
// @ts-expect-error shut
delete base.scoreData.hitMeta.inputDevice;
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const exists = await db.scores.findOne({ scoreID });
if (exists) {
logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
return null;
}
const score: ScoreDocument = {
...base,
scoreID,
};
await oldKTDB.get("score-id-lookup").insert({
old: c.scoreID,
new: scoreID,
});
return score;
}
export async function MigrateBMSScores() {
await MigrateRecords(db.scores, "scores", ConvertFn, { game: "bms" }, true);
}
if (require.main === module) {
(async () => {
await MigrateBMSScores();
process.exit(0);
})();
}
server/scripts/__KT_DATABASE_MIGRATION/scores/chunithm.ts
+95
View File
@@ -0,0 +1,95 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
import { GetGameConfig, GetGamePTConfig, ScoreDocument } from "tachi-common";
import MigrateRecords from "../migrate";
import { oldKTDB } from "../old-db";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ScoreDocument | null> {
const game = c.game;
const playtype = c.scoreData.playtype;
const gameConfig = GetGameConfig(game);
if (!gameConfig.validPlaytypes.includes(playtype)) {
logger.warn(`Ignored game pt ${game}, ${playtype}`);
throw new Error(`Ignored game pt ${game}, ${playtype}`);
}
const gptConfig = GetGamePTConfig(game, playtype);
if (!gptConfig) {
logger.error(`No gptconfig for ${game} ${playtype}?`);
throw new Error(`No gptconfig for ${game} ${playtype}?`);
}
// @ts-expect-error implicit any is expected
const chartDoc = await db.charts[game].findOne({
songID: c.songID,
difficulty: c.scoreData.difficulty,
playtype,
});
if (!chartDoc) {
logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.scoreData.difficulty} ${playtype} `);
return null;
}
const base: Omit<ScoreDocument<"chunithm:Single">, "scoreID"> = {
userID: c.userID,
songID: chartDoc.songID,
playtype,
chartID: chartDoc.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
// we'll just recalc calculated data instead of trying to update
calculatedData: {},
isPrimary: chartDoc.isPrimary,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
judgements: c.scoreData.hitData,
hitMeta: c.scoreData.hitMeta,
},
scoreMeta: {},
};
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const exists = await db.scores.findOne({ scoreID });
if (exists) {
logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
return null;
}
const score: ScoreDocument = {
...base,
scoreID,
};
await oldKTDB.get("score-id-lookup").insert({
old: c.scoreID,
new: scoreID,
});
return score;
}
export async function MigrateCHUNITHMScores() {
await MigrateRecords(db.scores, "scores", ConvertFn, { game: "chunithm" }, true);
}
server/scripts/__KT_DATABASE_MIGRATION/scores/ddr.ts
+101
View File
@@ -0,0 +1,101 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
import { GetGamePTConfig, ScoreDocument, StaticConfig } from "tachi-common";
import MigrateRecords from "../migrate";
import { oldKTDB } from "../old-db";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ScoreDocument | null> {
if (!StaticConfig.allSupportedGames.includes(c.game)) {
logger.warn(`Ignored game ${c.game}`);
return null;
}
const game = c.game;
// @ts-expect-error implicit any is expected
const chartDoc = await db.charts[game].findOne({
chartID: c.chartID,
});
if (!chartDoc) {
logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
return null;
}
const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
if (!gptConfig) {
logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
throw new Error(`Screwed`);
}
const base: Omit<ScoreDocument<"ddr:SP" | "ddr:DP">, "scoreID"> = {
userID: c.userID,
songID: chartDoc.songID,
playtype: chartDoc.playtype,
chartID: chartDoc.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
// we'll just recalc calculated data instead of trying to update
calculatedData: {},
isPrimary: chartDoc.isPrimary,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
judgements: c.scoreData.hitData,
hitMeta: c.scoreData.hitMeta,
},
scoreMeta: {},
};
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const exists = await db.scores.findOne({ scoreID });
if (exists) {
logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
return null;
}
const score: ScoreDocument = {
...base,
scoreID,
};
try {
await oldKTDB.get("score-id-lookup").insert({
old: c.scoreID,
new: scoreID,
});
} catch (err) {
// whoops
}
return score;
}
export async function MigrateDDRScores() {
await MigrateRecords(db.scores, "scores", ConvertFn, { game: "ddr" }, true);
}
if (require.main === module) {
(async () => {
await MigrateDDRScores();
process.exit(0);
})();
}
server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts
+149
View File
@@ -0,0 +1,149 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
import { GetGameConfig, GetGamePTConfig, ScoreDocument } from "tachi-common";
import MigrateRecords from "../migrate";
import { oldKTDB } from "../old-db";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ScoreDocument | null> {
const game = c.game;
const playtype = c.scoreData.playtype;
const gameConfig = GetGameConfig(game);
if (!gameConfig.validPlaytypes.includes(playtype)) {
logger.warn(`Ignored game pt ${game}, ${playtype}`);
throw new Error(`Ignored game pt ${game}, ${playtype}`);
}
const gptConfig = GetGamePTConfig(game, playtype);
if (!gptConfig) {
logger.error(`No gptconfig for ${game} ${playtype}?`);
throw new Error(`No gptconfig for ${game} ${playtype}?`);
}
if (c.songID === 1819) {
c.songID = 1121; // praludium has two songs in kt1 for some reason (tricoro 6)
// this overrides that
}
// @ts-expect-error implicit any is expected
const chartDoc = await db.charts[game].findOne({
songID: c.songID,
difficulty: c.scoreData.difficulty,
playtype,
});
if (!chartDoc) {
logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.scoreData.difficulty} ${playtype} `);
return null;
}
const base: Omit<ScoreDocument<"iidx:SP" | "iidx:DP">, "scoreID"> = {
userID: c.userID,
songID: chartDoc.songID,
playtype,
chartID: chartDoc.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
// we'll just recalc calculated data instead of trying to update
calculatedData: {},
isPrimary: chartDoc.isPrimary,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
judgements: c.scoreData.hitData,
hitMeta: c.scoreData.hitMeta,
},
scoreMeta: {
random: c.scoreMeta?.optionsRandom ?? null,
gauge: c.scoreMeta?.optionsGauge ?? null,
assist: c.scoreMeta?.optionsAssist ?? null,
range: c.scoreMeta?.optionsRange ?? null,
},
};
if (c.scoreMeta) {
if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
base.scoreMeta.random = null;
}
if (c.scoreData.hitMeta.gauge < 0) {
c.scoreData.hitMeta.gauge = null;
}
if (!Array.isArray(c.scoreData.hitMeta.gaugeHistory)) {
c.scoreData.hitMeta.gaugeHistory = null;
}
// @ts-expect-error asdf
if (base.scoreMeta.range === "") {
base.scoreMeta.range = "NONE";
}
if (c.scoreMeta.deadMeasure || c.scoreMeta.deadNote) {
base.scoreData.hitMeta.bp = null;
}
}
if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
base.scoreData.hitMeta.bp = null;
}
if ((base.scoreData.hitMeta.gauge ?? 0) > 200) {
base.scoreData.hitMeta.gauge = null;
}
if (base.scoreData.hitMeta.gaugeHistory) {
base.scoreData.hitMeta.gaugeHistory = base.scoreData.hitMeta.gaugeHistory.map((e) =>
(e ?? 0) > 200 ? null : e
);
}
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const exists = await db.scores.findOne({ scoreID });
if (exists) {
logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
return null;
}
const score: ScoreDocument = {
...base,
scoreID,
};
await oldKTDB.get("score-id-lookup").insert({
old: c.scoreID,
new: scoreID,
});
return score;
}
export async function MigrateIIDXScores() {
await MigrateRecords(db.scores, "scores", ConvertFn, { game: "iidx" }, true);
}
if (require.main === module) {
(async () => {
await MigrateIIDXScores();
process.exit(0);
})();
}
server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts
+33
View File
@@ -0,0 +1,33 @@
import CreateLogCtx from "lib/logger/logger";
import { MigrateBMSScores } from "./bms";
import { MigrateCHUNITHMScores } from "./chunithm";
import { MigrateIIDXScores } from "./iidx";
import { MigrateMusecaScores } from "./museca";
import { MigrateSDVXScores } from "./sdvx";
const logger = CreateLogCtx(__filename);
(async () => {
logger.info(`Migrating IIDX Scores.`);
await MigrateIIDXScores();
logger.info(`Migrating BMS Scores.`);
await MigrateBMSScores();
logger.info(`Migrating MUSECA Scores.`);
await MigrateMusecaScores();
logger.info(`Migrating SDVX Scores.`);
await MigrateSDVXScores();
logger.info(`Migrating CHUNITHM Scores.`);
await MigrateCHUNITHMScores();
logger.info("Done.");
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts
+101
View File
@@ -0,0 +1,101 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
import { GetGamePTConfig, ScoreDocument, StaticConfig } from "tachi-common";
import MigrateRecords from "../migrate";
import { oldKTDB } from "../old-db";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ScoreDocument | null> {
if (!StaticConfig.allSupportedGames.includes(c.game)) {
logger.warn(`Ignored game ${c.game}`);
return null;
}
const game = c.game;
// @ts-expect-error implicit any is expected
const chartDoc = await db.charts[game].findOne({
chartID: c.chartID,
});
if (!chartDoc) {
logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
return null;
}
const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
if (!gptConfig) {
logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
throw new Error(`Screwed`);
}
const base: Omit<ScoreDocument<"museca:Single">, "scoreID"> = {
userID: c.userID,
songID: chartDoc.songID,
playtype: chartDoc.playtype,
chartID: chartDoc.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
// we'll just recalc calculated data instead of trying to update
calculatedData: {},
isPrimary: chartDoc.isPrimary,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
judgements: c.scoreData.hitData,
hitMeta: c.scoreData.hitMeta,
},
scoreMeta: {},
};
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const exists = await db.scores.findOne({ scoreID });
if (exists) {
logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
return null;
}
const score: ScoreDocument = {
...base,
scoreID,
};
try {
await oldKTDB.get("score-id-lookup").insert({
old: c.scoreID,
new: scoreID,
});
} catch (err) {
// whoops
}
return score;
}
export async function MigrateMusecaScores() {
await MigrateRecords(db.scores, "scores", ConvertFn, { game: "museca" }, true);
}
if (require.main === module) {
(async () => {
await MigrateMusecaScores();
process.exit(0);
})();
}
server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts
+174
View File
@@ -0,0 +1,174 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
import { GetGameConfig, GetGamePTConfig, ScoreDocument, StaticConfig } from "tachi-common";
import MigrateRecords from "../migrate";
const logger = CreateLogCtx(__filename);
function ConditionalAssign(base: any, baseProp: string, other: any, otherProp: string) {
if (Object.prototype.hasOwnProperty.call(other, otherProp)) {
base[baseProp] = other[otherProp];
}
}
async function ConvertFn(c: any): Promise<ScoreDocument | null> {
if (!StaticConfig.allSupportedGames.includes(c.game)) {
logger.warn(`Ignored game ${c.game}`);
return null;
}
const game = c.game;
const playtype = c.scoreData.playtype;
if (["popn", "jubeat", "gitadora", "usc"].includes(c.game)) {
return null;
}
const gameConfig = GetGameConfig(game);
if (!gameConfig.validPlaytypes.includes(playtype)) {
logger.warn(`Ignored game pt ${game}, ${playtype}`);
throw new Error(`Ignored game pt ${game}, ${playtype}`);
}
const gptConfig = GetGamePTConfig(game, playtype);
if (!gptConfig) {
logger.error(`No gptconfig for ${game} ${playtype}?`);
throw new Error(`No gptconfig for ${game} ${playtype}?`);
}
if (game === "iidx" && c.songID === 1819) {
c.songID = 1121; // praludium has two songs in kt1 for some reason (tricoro 6)
// this overrides that
}
// @ts-expect-error implicit any is expected
const chartDoc = await db.charts[game].findOne({
songID: c.songID,
difficulty: c.scoreData.difficulty,
playtype,
});
if (!chartDoc) {
logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.scoreData.difficulty} ${playtype} `);
return null;
}
const base: Omit<ScoreDocument, "scoreID"> = {
userID: c.userID,
songID: chartDoc.songID,
playtype,
chartID: chartDoc.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
// we'll just recalc calculated data instead of trying to update
calculatedData: {},
isPrimary: chartDoc.isPrimary,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
judgements: c.scoreData.hitData,
hitMeta: c.scoreData.hitMeta,
},
scoreMeta: {},
};
if (c.scoreMeta) {
if (base.game === "iidx") {
ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom");
ConditionalAssign(base.scoreMeta, "gauge", c.scoreMeta, "optionsGauge");
ConditionalAssign(base.scoreMeta, "assist", c.scoreMeta, "optionsAssist");
ConditionalAssign(base.scoreMeta, "range", c.scoreMeta, "optionsRange");
ConditionalAssign(base.scoreMeta, "pacemaker", c.scoreMeta, "pacemaker");
ConditionalAssign(base.scoreMeta, "pacemakerName", c.scoreMeta, "pacemakerName");
ConditionalAssign(base.scoreMeta, "pacemakerTarget", c.scoreMeta, "pacemakerTarget");
// @ts-expect-error this game is iidx
if (base.playtype === "DP" && !Array.isArray(base.scoreMeta.random)) {
// @ts-expect-error this game is iidx
base.scoreMeta.random = null;
}
// @ts-expect-error asdf
if (base.scoreMeta.range === "") {
// @ts-expect-error asdf
base.scoreMeta.range = "NONE";
}
} else if (base.game === "bms") {
ConditionalAssign(base.scoreMeta, "random", c.scoreMeta, "optionsRandom");
ConditionalAssign(base.scoreMeta, "inputDevice", c.scoreData.hitMeta, "inputDevice");
}
}
// @ts-expect-error shut
if (base.scoreData.hitMeta.bp === -1 || Number.isNaN(base.scoreData.hitMeta.bp)) {
// @ts-expect-error shut
base.scoreData.hitMeta.bp = null;
}
if (base.game === "iidx") {
// @ts-expect-error nope
if (base.scoreData.hitMeta.gauge > 200) {
// @ts-expect-error nope
base.scoreData.hitMeta.gauge = null;
}
// @ts-expect-error nope
if (base.scoreData.hitMeta.gaugeHistory) {
// @ts-expect-error nope
base.scoreData.hitMeta.gaugeHistory = base.scoreData.hitMeta.gaugeHistory.map((e) =>
e > 200 ? null : e
);
}
} else if (base.game === "bms") {
// @ts-expect-error shut
if (base.scoreData.hitMeta.gauge === -1) {
// @ts-expect-error shut
base.scoreData.hitMeta.gauge = null;
}
// @ts-expect-error shut
delete base.scoreData.hitMeta.inputDevice;
} else if (base.game === "sdvx" || base.game === "usc") {
ConditionalAssign(base.scoreData.judgements, "miss", base.scoreData.judgements, "error");
//@ts-expect-error yea
delete base.scoreData.judgements.error;
} else if (base.game === "chunithm") {
if (base.scoreData.percent === 101) {
base.scoreData.grade = "SSS";
}
}
if (base.service === "mm") {
base.service = "maimagic";
}
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const score: ScoreDocument = {
...base,
scoreID,
};
return score;
}
(async () => {
await MigrateRecords(db.scores, "scores", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts
+114
View File
@@ -0,0 +1,114 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id";
import { GetGamePTConfig, ScoreDocument, StaticConfig } from "tachi-common";
import MigrateRecords from "../migrate";
import { oldKTDB } from "../old-db";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<ScoreDocument | null> {
if (!StaticConfig.allSupportedGames.includes(c.game)) {
logger.warn(`Ignored game ${c.game}`);
return null;
}
const game = c.game;
// @ts-expect-error implicit any is expected
const chartDoc = await db.charts[game].findOne({
chartID: c.chartID,
});
if (!chartDoc) {
logger.warn(`Cannot find ChartDoc for ${c.songID} ${c.chartID} ${c.scoreData.difficulty}`);
return null;
}
const gptConfig = GetGamePTConfig(game, chartDoc.playtype);
if (!gptConfig) {
logger.error(`Cannot find GPTConfig for ${game} ${chartDoc.playtype}?`);
throw new Error(`Screwed`);
}
const base: Omit<ScoreDocument<"sdvx:Single">, "scoreID"> = {
userID: c.userID,
songID: chartDoc.songID,
playtype: chartDoc.playtype,
chartID: chartDoc.chartID,
game: c.game,
timeAdded: c.timeAdded,
timeAchieved: Number.isNaN(c.timeAchieved) ? null : c.timeAchieved,
comment: c.comment ?? null,
highlight: c.highlight ?? false,
service: c.service,
importType: null,
// we'll just recalc calculated data instead of trying to update
calculatedData: {},
isPrimary: chartDoc.isPrimary,
scoreData: {
esd: c.scoreData.esd ?? null,
grade: c.scoreData.grade,
gradeIndex: gptConfig.grades.indexOf(c.scoreData.grade),
lamp: c.scoreData.lamp,
lampIndex: gptConfig.lamps.indexOf(c.scoreData.lamp),
percent: c.scoreData.percent,
score: c.scoreData.score,
judgements: c.scoreData.hitData,
hitMeta: {
fast: c.scoreData.hitMeta.fast,
slow: c.scoreData.hitMeta.slow,
maxCombo: c.scoreData.hitMeta.maxCombo,
gauge: c.scoreData.hitMeta.gauge,
},
},
scoreMeta: {
inSkillAnalyser: null,
},
};
if (c.scoreData.hitData.error !== undefined) {
base.scoreData.judgements.miss = c.scoreData.hitData.error;
// @ts-expect-error yeah we know
delete base.scoreData.judgements.error;
}
const scoreID = CreateScoreID(base.userID, base, base.chartID);
const exists = await db.scores.findOne({ scoreID });
if (exists) {
logger.warn(`Skipping duplicate score ${base.chartID} ${base.userID}.`);
return null;
}
const score: ScoreDocument = {
...base,
scoreID,
};
try {
await oldKTDB.get("score-id-lookup").insert({
old: c.scoreID,
new: scoreID,
});
} catch (err) {
// lol
}
return score;
}
export async function MigrateSDVXScores() {
await MigrateRecords(db.scores, "scores", ConvertFn, { game: "sdvx" }, true);
}
if (require.main === module) {
(async () => {
await MigrateSDVXScores();
process.exit(0);
})();
}
server/scripts/__KT_DATABASE_MIGRATION/sessions.ts
+109
View File
@@ -0,0 +1,109 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { CreateSessionID } from "lib/score-import/framework/sessions/sessions";
import { GetGameConfig, integer, SessionDocument, SessionScoreInfo } from "tachi-common";
import { allSupportedGames } from "tachi-common/js/config/static-config";
import MigrateRecords from "./migrate";
import { oldKTDB } from "./old-db";
const logger = CreateLogCtx(__filename);
interface LegacySessionScoreFormat {
pbInfo: {
isGeneralPB: boolean;
isGradePB: boolean;
isLampPB: boolean;
isNewScore: boolean;
lampDelta: integer;
percentDelta: number;
gradeDelta: integer;
scoreDelta: number;
};
scoreID: string;
}
async function ConvertScoreToScoreInfo(
score: LegacySessionScoreFormat
): Promise<SessionScoreInfo | null> {
const scoreIDs = await oldKTDB.get("score-id-lookup").findOne({
old: score.scoreID,
});
if (!scoreIDs) {
logger.verbose(`No score for ${score.scoreID}? Skipping.`);
return null;
}
logger.verbose(`Found score for ${score.scoreID}.`);
if (score.pbInfo.isNewScore) {
return {
isNewScore: true,
scoreID: scoreIDs.new,
};
}
if (score.pbInfo.scoreDelta === null) {
return null;
}
return {
scoreID: scoreIDs.new,
isNewScore: false,
gradeDelta: score.pbInfo.gradeDelta,
lampDelta: score.pbInfo.lampDelta,
percentDelta: score.pbInfo.percentDelta,
scoreDelta: score.pbInfo.scoreDelta,
};
}
async function ConvertFn(c: any): Promise<SessionDocument | null> {
if (!allSupportedGames.includes(c.game)) {
return null;
}
const gameConfig = GetGameConfig(c.game);
if (!gameConfig.validPlaytypes.includes(c.playtype)) {
return null;
}
const scores = [];
for (const sc of c.scores) {
// eslint-disable-next-line no-await-in-loop
const conv = await ConvertScoreToScoreInfo(sc);
if (conv === null) {
continue;
}
scores.push(conv);
}
const sessionDocument: SessionDocument = {
name: c.name,
desc: c.desc,
timeStarted: c.timeStarted,
timeEnded: c.timeEnded,
timeInserted: c.timestamp,
game: c.game,
playtype: c.playtype,
highlight: !!c.highlight,
sessionID: CreateSessionID(),
views: 0,
userID: c.userID,
calculatedData: {},
scoreInfo: scores,
importType: null,
};
return sessionDocument;
}
(async () => {
await MigrateRecords(db.sessions, "sessions", ConvertFn, {}, true);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/songs-chunithm.ts
-29
View File
@@ -1,29 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { SongDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): SongDocument<"chunithm"> {
const newSongDoc: SongDocument<"chunithm"> = {
title: c.title,
artist: c.artist,
id: c.id,
firstVersion: c.firstAppearance,
"alt-titles": c["alt-titles"].filter((e: string) => e !== c.title),
"search-titles": c["search-titles"]
.map((e: string) => e.toString())
.filter((e: string) => e !== c.title),
data: {
genre: c.genre,
},
};
return newSongDoc;
}
(async () => {
await MigrateRecords(db.songs.chunithm, "songs-chunithm", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/songs-ddr.ts
-28
View File
@@ -1,28 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { SongDocument } from "kamaitachi-common";
import { gameOrders } from "kamaitachi-common/js/config";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): SongDocument<"ddr"> {
const newSongDoc: SongDocument<"ddr"> = {
title: c.title,
artist: c.artist,
id: c.id,
firstVersion: c.firstAppearance,
"alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [],
"search-titles": c["search-title"]
.filter((e: string) => !!e)
.map((e: string) => e.toString())
.filter((e: string) => e !== c.title),
data: {},
};
return newSongDoc;
}
(async () => {
await MigrateRecords(db.songs.ddr, "songs-ddr", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/songs-gitadora.ts
-27
View File
@@ -1,27 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { SongDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): SongDocument<"gitadora"> {
const newSongDoc: SongDocument<"gitadora"> = {
title: c.title,
artist: c.artist,
id: c.id,
firstVersion: c.firstAppearance,
"alt-titles": c["alt-titles"] ? c["alt-titles"].filter((e: string) => e !== c.title) : [],
"search-titles": c["search-titles"]
.filter((e: string) => !!e)
.map((e: string) => e.toString())
.filter((e: string) => e !== c.title),
data: {},
};
return newSongDoc;
}
(async () => {
await MigrateRecords(db.songs.gitadora, "songs-gitadora", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/songs-iidx.ts
-29
View File
@@ -1,29 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { SongDocument } from "kamaitachi-common";
import { gameOrders } from "kamaitachi-common/js/config";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): SongDocument<"iidx"> {
const newSongDoc: SongDocument<"iidx"> = {
title: c.title,
artist: c.artist,
id: c.id,
firstVersion: c.firstAppearance,
"alt-titles": c["alt-titles"].filter((e: string) => e !== c.title),
"search-titles": c["search-titles"]
.map((e: string) => e.toString())
.filter((e: string) => e !== c.title),
data: {
genre: c.genre,
},
};
return newSongDoc;
}
(async () => {
await MigrateRecords(db.songs.iidx, "songs-iidx", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/songs-museca.ts
-27
View File
@@ -1,27 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { SongDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): SongDocument<"museca"> {
const newSongDoc: SongDocument<"museca"> = {
title: c.title,
artist: c.artist,
id: c.id,
firstVersion: c.firstAppearance,
"alt-titles": [],
"search-titles": [],
data: {
titleJP: c.title_jp,
artistJP: c.artist_jp,
},
};
return newSongDoc;
}
(async () => {
await MigrateRecords(db.songs.museca, "songs-museca", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/songs-sdvx.ts
-28
View File
@@ -1,28 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { SongDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): SongDocument<"sdvx"> {
const newSongDoc: SongDocument<"sdvx"> = {
title: c.title,
artist: c.artist,
id: c.id,
firstVersion: c.firstAppearance,
"alt-titles": c["alt-titles"].filter((e: string) => e !== c.title),
"search-titles": c["search-titles"]
.map((e: string) => e.toString())
.filter((e: string) => e !== c.title),
data: {
uscEquiv: null,
},
};
return newSongDoc;
}
(async () => {
await MigrateRecords(db.songs.sdvx, "songs-sdvx", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/state-sync/create-game-profiles.ts
+43
View File
@@ -0,0 +1,43 @@
/* eslint-disable no-await-in-loop */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { UpdateUsersGamePlaytypeStats } from "lib/score-import/framework/user-game-stats/update-ugs";
import { FormatUserDoc } from "utils/user";
const logger = CreateLogCtx(__filename);
export async function CreateGameProfiles() {
const users = await db.users.find({});
for (const user of users) {
const gpts = await db.scores.aggregate([
{
$match: {
userID: user.id,
},
},
{
$group: {
_id: {
game: "$game",
playtype: "$playtype",
},
},
},
]);
logger.info(`Found ${gpts.length} GPTs for ${FormatUserDoc(user)}`);
for (const gpt of gpts) {
const { game, playtype } = gpt._id;
logger.info(`Updating ${FormatUserDoc(user)}'s ${game} ${playtype} stats.`);
await UpdateUsersGamePlaytypeStats(game, playtype, user.id, null, logger);
}
}
logger.info(`Done.`);
}
if (require.main === module) {
CreateGameProfiles().then(() => process.exit(0));
}
server/scripts/__KT_DATABASE_MIGRATION/state-sync/dedupe-score-ids.ts
+36
View File
@@ -0,0 +1,36 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
async function DedupeScoreIDs() {
const dups = await db.scores.aggregate(
[
{
$group: {
_id: "$scoreID",
dups: { $addToSet: "$_id" },
count: { $sum: 1 },
},
},
{
$match: {
count: { $gt: 1 },
},
},
],
{ allowDiskUse: true }
);
logger.info(`Found ${dups.length} dups.`);
for (const dup of dups) {
dup.dups.shift();
await db.scores.remove({ _id: { $in: dup.dups } });
}
logger.info(`Done.`);
process.exit(0);
}
DedupeScoreIDs();
server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores.ts
+5
View File
@@ -0,0 +1,5 @@
import { RecalcAllScores } from "utils/calculations/recalc-scores";
if (require.main === module) {
RecalcAllScores();
}
server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-sessions.ts
+5
View File
@@ -0,0 +1,5 @@
import { RecalcSessions } from "utils/calculations/recalc-sessions";
if (require.main === module) {
RecalcSessions().then(() => process.exit(0));
}
server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts
+16
View File
@@ -0,0 +1,16 @@
import CreateLogCtx from "lib/logger/logger";
import { RecalcAllScores, UpdateAllPBs } from "utils/calculations/recalc-scores";
import { RecalcSessions } from "utils/calculations/recalc-sessions";
import { CreateGameProfiles } from "./create-game-profiles";
const logger = CreateLogCtx(__filename);
(async () => {
await RecalcAllScores();
await UpdateAllPBs();
await CreateGameProfiles();
await RecalcSessions();
logger.info(`Completely done!`);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/state-sync/update-all-pbs.ts
+6
View File
@@ -0,0 +1,6 @@
/* eslint-disable no-await-in-loop */
import { UpdateAllPBs } from "utils/calculations/recalc-scores";
if (require.main === module) {
UpdateAllPBs();
}
server/scripts/__KT_DATABASE_MIGRATION/tierlists.ts
-41
View File
@@ -1,41 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { TierlistParent, Game } from "kamaitachi-common";
import { gameHuman, validPlaytypes } from "kamaitachi-common/js/config";
import db from "../../src/db/db";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): TierlistParent {
const tierlistParent: TierlistParent = {
createdAt: Date.now(),
tierlistID: c.tierlistID,
createdBy: 1,
description: `The official Kamaitachi Tierlist for ${gameHuman[c.game as Game]}${
validPlaytypes[c.game as Game].length > 1 ? ` (${c.playtype})` : ""
}.`,
game: c.game,
playtype: c.playtype,
isDefault: true,
name: c.title,
lastUpdated: Date.now(),
permissions: {
anyPlayer: {
edit: 0,
submit: 1,
vote: 1,
},
},
config: {
autoHumanise: false,
flags: ["Individual Difference"],
requireState: "clear",
},
};
return tierlistParent;
}
(async () => {
await MigrateRecords(db.tierlists, "tierlist", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/user-goals.ts
+11 -11
View File
@@ -1,26 +1,26 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { UserGoalDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { UserGoalDocument } from "tachi-common";
import MigrateRecords from "./migrate";
const logger = CreateLogCtx(__filename);
async function ConvertFn(c: any): Promise<UserGoalDocument | null> {
// const newUG: UserGoalDocument = {};
// const newUG: UserGoalDocument = {};
let relatedGoal = await db.goals.findOne({ goalID: c.goalID });
const relatedGoal = await db.goals.findOne({ goalID: c.goalID });
if (!relatedGoal) {
return null;
}
if (!relatedGoal) {
return null;
}
return c;
return c;
}
(async () => {
await MigrateRecords(db["user-goals"], "user-goals", ConvertFn);
await MigrateRecords(db["user-goals"], "user-goals", ConvertFn);
process.exit(0);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/user-private-info.ts
+19
View File
@@ -0,0 +1,19 @@
import db from "external/mongo/db";
import { PrivateUserInfoDocument } from "tachi-common";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): PrivateUserInfoDocument {
const privateUserInfoDoc: PrivateUserInfoDocument = {
userID: c.id,
password: c.password,
email: c.email,
};
return privateUserInfoDoc;
}
(async () => {
await MigrateRecords(db["user-private-information"], "users", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/user-settings.ts
+15
View File
@@ -0,0 +1,15 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import db from "external/mongo/db";
import { DEFAULT_USER_SETTINGS } from "server/router/api/v1/auth/auth";
import MigrateRecords from "./migrate";
function ConvertFn(c: any) {
return { userID: c.id, preferences: DEFAULT_USER_SETTINGS };
}
(async () => {
await MigrateRecords(db["user-settings"], "users", ConvertFn);
process.exit(0);
})();
server/scripts/__KT_DATABASE_MIGRATION/users.ts
+28 -32
View File
@@ -1,40 +1,36 @@
import { PrivateUserDocument } from "kamaitachi-common";
import db from "../../src/db/db";
import db from "external/mongo/db";
import { PublicUserDocument, UserAuthLevels } from "tachi-common";
import MigrateRecords from "./migrate";
function ConvertFn(c: any): PrivateUserDocument {
const newUserDoc: PrivateUserDocument = {
username: c.username,
usernameLowercase: c.username.toLowerCase(),
about: c.about,
clan: c.clan,
customBanner: c.custombanner,
customPfp: c.custompfp,
email: c.email,
friends: c.friends,
id: c.id,
lastSeen: c.lastSeen,
password: c.password,
permissions: {},
settings: {
invisible: !!c.settings.invisible,
nsfwSplashes: !!c.settings.nsfwsplashes,
},
socialMedia: {
discord: c.socialmedia.discord || null,
twitter: c.socialmedia.twitter || null,
youtube: c.socialmedia.youtube || null,
twitch: c.socialmedia.twitch || null,
github: c.socialmedia.github || null,
steam: c.socialmedia.steam || null,
},
};
function ConvertFn(c: any): PublicUserDocument {
const newUserDoc: PublicUserDocument = {
username: c.username,
usernameLowercase: c.username.toLowerCase(),
about: c.about,
clan: c.clan,
customBannerLocation: null,
customPfpLocation: null,
id: c.id,
lastSeen: c.lastSeen,
socialMedia: {
discord: c.socialmedia.discord || null,
twitter: c.socialmedia.twitter || null,
youtube: c.socialmedia.youtube || null,
twitch: c.socialmedia.twitch || null,
github: c.socialmedia.github || null,
steam: c.socialmedia.steam || null,
},
joinDate: 0,
authLevel: UserAuthLevels.USER,
status: null,
badges: ["beta"],
};
return newUserDoc;
return newUserDoc;
}
(async () => {
await MigrateRecords(db.users, "users", ConvertFn);
await MigrateRecords(db.users, "users", ConvertFn);
process.exit(0);
process.exit(0);
})();
server/scripts/arc-db-sync/arc-db-fetch.ts
-91
View File
@@ -1,91 +0,0 @@
import { Command } from "commander";
import CreateLogCtx from "../../src/common/logger";
import fetch from "../../src/common/fetch";
import fs from "fs";
import path from "path";
// note - this program was never run due to WSL2 errors
// causing network requests to screech to a halt.
const logger = CreateLogCtx(__filename);
const program = new Command();
program.option("-u, --url <URL>", "The base URL to sync from.");
program.option("-b, --bearer <token>", "The authorization bearer token to use.");
program.parse(process.argv);
const options = program.opts();
async function GetIIDXSongs() {
const allSongs = [];
let url = `https://${options.url}/api/v1/iidx/27/music/`;
let moreData = true;
while (moreData) {
// eslint-disable-next-line no-await-in-loop
const songs = await fetch(url, {
headers: {
Authorization: `Bearer ${options.bearer}`,
},
}).then((r) => r.json());
if (songs._links._next) {
url = songs._links._next;
} else {
moreData = false;
}
allSongs.push(...songs._items);
}
fs.writeFileSync(path.join(__dirname, "./songs.json"), JSON.stringify(allSongs));
}
async function GetCharts() {
const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8"));
logger.info("Parsed songs.json");
const allCharts: any[] = [];
const failed: any[] = [];
async function DoStuff(song: any) {
try {
logger.info(
`Starting Request https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true.`
);
// eslint-disable-next-line no-await-in-loop
const charts = await fetch(
`https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true`,
{
headers: {
Authorization: `Bearer ${options.bearer}`,
},
}
).then((r) => r.json());
logger.info(`Parsed: ${charts._related.music[0].title}`);
allCharts.push(charts);
} catch (err) {
logger.error(song);
logger.error(err);
failed.push(err);
}
}
const promises = songs.map((s: any) => DoStuff(s));
await Promise.all(promises);
fs.writeFileSync(path.join(__dirname, "./charts.json"), JSON.stringify(allCharts));
logger.info(failed);
logger.info("done?");
}
GetCharts();
server/scripts/arc-db-sync/arc-db-merge.ts
-86
View File
@@ -1,86 +0,0 @@
import fs from "fs";
import { ICollection } from "monk";
import path from "path";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
import { ChartDocument } from "kamaitachi-common";
import { FindSongOnTitle } from "../../src/common/database-lookup/song";
import { FindChartWithPTDFVersion } from "../../src/common/database-lookup/chart";
const logger = CreateLogCtx(__filename);
async function MergeIDs() {
const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8"));
for (const chartData of charts) {
const songTitle = chartData._related.music[0].title;
// eslint-disable-next-line no-await-in-loop
const song = await FindSongOnTitle("iidx", songTitle);
if (!song) {
logger.error(`Could not resolve ${songTitle}?`);
continue;
}
for (const chart of chartData._items) {
const playtype: "SP" | "DP" = chart.play_style === "DOUBLE" ? "DP" : "SP";
let difficulty = chart.difficulty === "BLACK" ? "LEGGENDARIA" : chart.difficulty;
const version = "27-omni";
// yeah this fails for wacky playtype mismatches -- i don't care.
if (
["ミッドナイト堕天使", "Y&Co. is dead or alive", "State Of The Art"].includes(
songTitle
) &&
difficulty === "LEGGENDARIA"
) {
difficulty = "ANOTHER";
}
// eslint-disable-next-line no-await-in-loop
const ktchiChart = await FindChartWithPTDFVersion(
"iidx",
song.id,
playtype,
difficulty,
version
);
if (!ktchiChart) {
logger.error(
`Could not find chart ${songTitle} ${playtype} ${difficulty} ${version}.`
);
continue;
}
// eslint-disable-next-line no-await-in-loop
await ((db.charts.iidx as unknown) as ICollection<
ChartDocument<"iidx:SP" | "iidx:DP">
>).update(
{
_id: ktchiChart._id,
},
{
$set: {
"data.arcChartID": chart._id,
},
}
);
}
}
const r = await db.charts.iidx.update(
{
"data.arcChartID": { $exists: false },
},
{ $set: { "data.arcChartID": null } },
{
multi: true,
}
);
logger.info("done.", { r });
}
MergeIDs();
server/scripts/arc-sync/arc-db-sync.ts
+213
View File
@@ -0,0 +1,213 @@
/* eslint-disable no-await-in-loop */
import { Command } from "commander";
import db from "external/mongo/db";
import fs from "fs";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import path from "path";
import fetch from "utils/fetch";
import { FindChartWithPTDFVersion, FindSDVXChartOnDFVersion } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
const logger = CreateLogCtx(__filename);
const program = new Command();
program.option("-u, --url <URL>", "The base URL to sync from.");
program.option("-b, --bearer <token>", "The authorization bearer token to use.");
program.option("-g, --game <game>", "Should be exactly iidx or sdvx.");
program.option("-c, --cache", "Use a local cached JSON file, instead of requesting from the api.");
program.parse(process.argv);
const options = program.opts();
if (!options.game) {
throw new Error(`Expected -g or --game parameter.`);
}
if (!["iidx", "sdvx"].includes(options.game)) {
throw new Error(`Expected provided game to be either iidx or sdvx.`);
}
async function GetSongs() {
const allSongs = [];
let url =
options.game === "iidx"
? `${ServerConfig.ARC_API_URL}/api/v1/iidx/28/music/`
: `${ServerConfig.ARC_API_URL}/api/v1/sdvx/5/music/`;
let moreData = true;
while (moreData) {
// eslint-disable-next-line no-await-in-loop
const songs = await fetch(url, {
headers: {
Authorization: `Bearer ${options.bearer}`,
},
}).then((r) => r.json());
if (songs._links._next) {
url = songs._links._next;
} else {
moreData = false;
}
allSongs.push(...songs._items);
}
return allSongs;
}
async function GetCharts() {
const songs = await GetSongs();
const allCharts: any[] = [];
const failed: any[] = [];
async function DoStuff(song: any) {
const url =
options.game === "iidx"
? `${ServerConfig.ARC_API_URL}/api/v1/iidx/28/charts/?music_id=${song._id}&omnimix=true`
: `${ServerConfig.ARC_API_URL}/api/v1/sdvx/5/charts/?music_id=${song._id}`;
try {
logger.info(`Starting Request ${url}.`);
// eslint-disable-next-line no-await-in-loop
const charts = await fetch(url, {
headers: {
Authorization: `Bearer ${options.bearer}`,
},
}).then((r) => r.json());
logger.info(`Parsed: ${charts._related.music[0].title}`);
allCharts.push(charts);
} catch (err) {
logger.error(err.message, { song, err });
failed.push(err);
}
}
for (const song of songs) {
await DoStuff(song);
}
// fast, but ARCs server cant keep up.
// const promises = songs.map((s: any) => DoStuff(s));
// await Promise.all(promises);
logger.info(failed);
logger.info("Retrieved charts.");
return allCharts;
}
const cacheLoc = path.join(__dirname, "cache.json");
async function MergeIDs() {
let charts;
if (options.cache) {
logger.info(`--cache provided, reading cache.`);
try {
charts = JSON.parse(fs.readFileSync(cacheLoc, "utf-8"));
} catch (err) {
throw new Error(`No cache, cannot use --cache!`);
}
} else {
logger.info(`Reading from arc.`);
charts = await GetCharts();
}
fs.writeFileSync(cacheLoc, JSON.stringify(charts));
for (const chartData of charts) {
const songTitle = chartData._related.music[0].title;
// eslint-disable-next-line no-await-in-loop
const song = await FindSongOnTitle(options.game, songTitle);
if (!song) {
logger.error(`Could not resolve song ${songTitle}?`);
continue;
}
for (const chart of chartData._items) {
if (chart.rating === 0) {
continue;
}
const playtype: "SP" | "DP" | "Single" =
options.game === "iidx" ? (chart.play_style === "DOUBLE" ? "DP" : "SP") : "Single";
let difficulty = chart.difficulty === "BLACK" ? "LEGGENDARIA" : chart.difficulty;
const version = options.game === "iidx" ? "28-omni" : "vivid";
// yeah this fails for wacky playtype mismatches -- i don't care.
if (
["ミッドナイト堕天使", "Y&Co. is dead or alive", "State Of The Art"].includes(
songTitle
) &&
difficulty === "LEGGENDARIA"
) {
difficulty = "ANOTHER";
}
let tachiChart;
if (options.game === "sdvx") {
tachiChart = await FindSDVXChartOnDFVersion(
song.id,
difficulty === "INF" ? "ANY_INF" : difficulty,
version as "vivid"
);
} else {
tachiChart = await FindChartWithPTDFVersion(
options.game,
song.id,
playtype,
difficulty,
version
);
}
if (!tachiChart) {
logger.error(
`Could not find chart ${songTitle} ${playtype} ${difficulty} ${version} (SongID ${song.id}).`
);
continue;
}
// eslint-disable-next-line no-await-in-loop
await db.charts[options.game as "sdvx" | "iidx"].update(
{
chartID: tachiChart.chartID,
},
{
$set: {
"data.arcChartID": chart._id,
},
}
);
}
}
await db.charts[options.game as "sdvx" | "iidx"].update(
{
"data.arcChartID": { $exists: false },
},
{ $set: { "data.arcChartID": null } },
{
multi: true,
}
);
logger.info("done.");
process.exit(0);
}
if (require.main === module) {
MergeIDs();
}
server/scripts/bpi-poyashi-data/output.json
-1
View File
File diff suppressed because one or more lines are too long
server/scripts/bpi-poyashi-data/update-bpi-poyashi.ts
+118
View File
@@ -0,0 +1,118 @@
/* eslint-disable no-await-in-loop */
import { Command } from "commander";
import db from "external/mongo/db";
import fs from "fs";
import CreateLogCtx from "lib/logger/logger";
import fetch from "node-fetch";
import path from "path";
import { ChartDocument, Difficulties, IIDXBPIData } from "tachi-common";
import { FindChartWithPTDFVersion } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
import { RecalcAllScores } from "../__KT_DATABASE_MIGRATION/state-sync/recalc-all-scores";
const program = new Command();
const logger = CreateLogCtx(__filename);
program.option("-f, --fetch", "Fetch the latest data from the poyashi repo.");
program.parse(process.argv);
const options = program.opts();
const dataLoc = path.join(__dirname, "./output.json");
const difficultyResolve: Record<string, [string, string]> = {
3: ["SP", "HYPER"],
4: ["SP", "ANOTHER"],
8: ["DP", "HYPER"],
9: ["DP", "ANOTHER"],
10: ["SP", "LEGGENDARIA"],
11: ["DP", "LEGGENDARIA"],
};
async function UpdatePoyashiData() {
let data;
if (options.fetch) {
// lol!
logger.info("Fetching data from github...");
const rj = await fetch("https://proxy.poyashi.me/?type=bpi").then((r) => r.json());
logger.info("Fetched data.");
fs.writeFileSync(dataLoc, JSON.stringify(rj));
logger.info("Saved data to output.json.");
data = rj;
} else {
logger.info("Reading data from output.json...");
data = JSON.parse(fs.readFileSync(dataLoc, "utf-8"));
}
const realData: IIDXBPIData[] = [];
for (const d of data.body) {
const res = difficultyResolve[d.difficulty];
if (!res) {
throw new Error(`Unknown difficulty ${d.difficulty}`);
}
const [playtype, diff] = res;
const tachiSong = await FindSongOnTitle("iidx", d.title);
if (!tachiSong) {
logger.warn(`Cannot find song ${d.title}?`);
continue;
}
const tachiChart = (await FindChartWithPTDFVersion(
"iidx",
tachiSong.id,
playtype as "SP" | "DP",
diff as Difficulties["iidx:DP" | "iidx:SP"],
"27"
)) as ChartDocument<"iidx:SP" | "iidx:DP">;
if (!tachiChart) {
logger.warn(
`Cannot find chart ${tachiSong.title} (${tachiSong.id}) ${playtype}, ${diff}?`
);
continue;
}
const kavg = Number(d.avg);
if (kavg < 0) {
logger.warn(
`${tachiSong.title} (${playtype} ${diff}). Invalid kavg ${d.avg}, Skipping.`
);
continue;
}
if (d.removed) {
logger.info(`Skipping removed chart ${tachiSong.title}.`);
continue;
}
realData.push({
coef: d.coef === -1 ? null : d.coef,
kavg: Number(d.avg),
wr: Number(d.wr),
chartID: tachiChart.chartID,
});
}
await db["iidx-bpi-data"].remove({});
await db["iidx-bpi-data"].insert(realData);
logger.info(`Removed all, and inserted ${realData.length} documents.`);
logger.info(`Triggering IIDX Recalc.`);
await RecalcAllScores({ game: "iidx" });
logger.info(`Done.`);
}
if (require.main === module) {
UpdatePoyashiData();
}
server/scripts/clone-and-anonymise-db.ts
+105
View File
@@ -0,0 +1,105 @@
import { execSync } from "child_process";
import { Command } from "commander";
import CreateLogCtx from "lib/logger/logger";
import monk from "monk";
const program = new Command();
program
.option("--nsFrom <Database to clone from.>")
.option("--nsTo <Database to clone and anonymise to.>");
program.parse(process.argv);
const options = program.opts();
if (!options.nsTo || !options.nsFrom) {
throw new Error(`Expected --nsFrom and --nsTo options.`);
}
const logger = CreateLogCtx(__filename);
/**
* Strips all name and private information from the database.
* For use for things like future tachi-db-dumps?
*
* @param nsTo The database to anonymise. DO NOT PASS THE PRODUCTION DATABASE TO THIS!
*/
async function AnonymiseDB(nsTo: string) {
logger.info(`Connecting to ${nsTo}.`);
const clonedDB = monk(`127.0.0.1/${nsTo}`);
logger.info(`Connected to ${nsTo}.`);
const r1 = await clonedDB.get("user-private-information").update(
{},
[
{
$set: {
// This is "password" encrypted in bcrypt12.
password: "$2b$12$QRFCAxvFoNI2spszFPgt/e.qLy55GvYWlSHioa0AujRbFpChLwHmu",
// emails are uniquely indexed. We need to anonymise these in
// a way that they dont duplicate.
email: { $concat: [{ $toString: "$userID" }, "@example.com"] },
},
},
],
{
multi: true,
}
);
logger.info(`Stripped private info.`, { r1 });
const r2 = await clonedDB.get("users").update(
{},
[
{
$set: {
socialMedia: { $const: {} },
},
},
{
$set: {
username: { $concat: ["u", { $toString: "$id" }] },
usernameLowercase: { $concat: ["u", { $toString: "$id" }] },
},
},
],
{
multi: true,
}
);
logger.info(`Stripped username info.`, { r2 });
for (const collection of [
"api-clients",
"oauth2-auth-codes",
"password-reset-codes",
"api-tokens",
"kai-auth-tokens",
"arc-saved-profiles",
]) {
// eslint-disable-next-line no-await-in-loop
const r3 = await clonedDB.get("collection").remove({});
logger.info(`Removed all ${collection} documents.`, { r3 });
}
logger.info(`Done! Closing.`);
process.exit(0);
}
function CloneDB(nsFrom: string, nsTo: string) {
logger.info(`Cloning Database. This will take a while.`);
execSync(`mongodump --gzip --archive=temp-dump --db=${nsFrom}`);
execSync(`mongorestore --archive=temp-dump --gzip --nsFrom='${nsFrom}.*' --nsTo='${nsTo}.*'`);
execSync(`rm temp-dump`);
logger.info(`Cloned!`);
}
if (require.main === module) {
CloneDB(options.nsFrom, options.nsTo);
AnonymiseDB(options.nsTo);
}
server/scripts/email/send-test-email.ts
+30
View File
@@ -0,0 +1,30 @@
import { Command } from "commander";
import { SendEmail } from "lib/email/client";
import { MainHTMLWrapper } from "lib/email/formats";
import CreateLogCtx from "lib/logger/logger";
const program = new Command();
program.option("-e, --email <Email to send to>");
const logger = CreateLogCtx(__filename);
program.parse(process.argv);
const options = program.opts();
if (!options.email) {
throw new Error(`Need an --email to send to.`);
}
if (require.main === module) {
(async () => {
logger.info(`Sending email to ${options.email}.`);
await SendEmail(
options.email,
"Hello World",
MainHTMLWrapper("Hello world! This is a test email for doing things."),
"Hello world! This is a test email for doing things."
);
logger.info(`Done.`);
process.exit(0);
})();
}
server/scripts/fix-iidx-versions/update-versions.ts
+117
View File
@@ -0,0 +1,117 @@
/* eslint-disable no-await-in-loop */
import db from "external/mongo/db";
import fs from "fs";
import CreateLogCtx from "lib/logger/logger";
import path from "path";
import { FindSongOnTitleInsensitive } from "utils/queries/songs";
const logger = CreateLogCtx(__filename);
const comicallyLargeJSONData = JSON.parse(
fs.readFileSync(path.join(__dirname, "./versions.json"), "utf-8")
);
const DIFF_MAP = {
A: "ANOTHER",
H: "HYPER",
N: "NORMAL",
B: "BEGINNER",
X: "LEGGENDARIA",
} as any;
function ConvertVersion(c: string): any {
if (c.startsWith("AC")) {
return c.split("AC")[1];
} else if (c.startsWith("OM")) {
return `${c.split("OM")[1]}-omni`;
} else if (c.startsWith("CS")) {
return `${c.split("CS")[1]}-cs`;
} else if (c === "INF") {
return "inf";
} else if (c === "BMUS") {
return "bmus";
} else {
logger.error(`Invalid Chart Version ${c}.`);
throw new Error("Invalid Chart Version");
}
}
(async () => {
for (const data of comicallyLargeJSONData) {
const song = await FindSongOnTitleInsensitive("iidx", data["MUSIC NAME"]);
if (!song) {
logger.error(`Can't find song with title ${data["MUSIC NAME"]}.`);
continue;
}
const playtype = data.STYLE === "S" ? "SP" : "DP";
const difficulty = DIFF_MAP[data.DIFF];
const relevantVersions = [
"OM27",
"OM26",
"AC27",
"AC26",
"AC25",
"AC24",
"AC23",
"AC22",
"AC21",
"AC20",
"INF",
"CS16",
"CS15",
"CS14",
"CS13",
"CS12",
"CS11",
"CS10",
"CS9",
"CS8",
"CS7",
"BMUS",
"CS6",
"CS5",
"CS4",
"CS3",
]
.filter((e) => data[e] === "1")
.map(ConvertVersion);
if (!relevantVersions.includes("27-omni") && !relevantVersions.includes("inf")) {
if (relevantVersions.includes("26-omni")) {
logger.info(
`Song ${song.title} (${playtype} ${difficulty}) was a very recent removal. Consider deprimarifying it. (had ${relevantVersions})`,
{ keys: Object.keys(data).filter((e) => data[e] === "1") }
);
} else {
logger.verbose(
`Song ${song.title} (${playtype} ${difficulty}) has no relevant versions. (had ${relevantVersions})`,
{ keys: Object.keys(data).filter((e) => data[e] === "1") }
);
}
continue;
}
const chart = await db.charts.iidx.findOne(
{ songID: song.id, difficulty, playtype },
{ projectID: true }
);
if (!chart) {
logger.error(`Can't find chart ${song.title} (${playtype} ${difficulty})`);
continue;
}
relevantVersions.push(...chart.versions.filter((e) => !relevantVersions.includes(e)));
await db.charts.iidx.update({ _id: chart._id }, { $set: { versions: relevantVersions } });
}
logger.info("done");
process.exit(0);
})();
server/scripts/single-use/fix-iidx-versions/versions.json → server/scripts/fix-iidx-versions/versions.json
View File
server/scripts/folders/create-tables-and-folders.ts
+172
View File
@@ -0,0 +1,172 @@
/* eslint-disable no-await-in-loop */
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import {
FormatGame,
GetGameConfig,
GetGamePTConfig,
IDStrings,
integer,
StaticConfig,
} from "tachi-common";
import { CreateFolderID } from "utils/folder";
const logger = CreateLogCtx(__filename);
function StrRange(min: integer, max: integer) {
const arr = [];
for (let i = min; i <= max; i++) {
arr.push(i.toString());
}
return arr;
}
// These are the games that have simple, discrete folders.
const STATIC_LEVELS: Partial<Record<IDStrings, string[]>> = {
"iidx:SP": StrRange(1, 12),
"iidx:DP": StrRange(1, 12),
"bms:7K": [],
"bms:14K": [],
"chunithm:Single": [
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"7+",
"8",
"8+",
"9",
"9+",
"10",
"10+",
"11",
"11+",
"12",
"12+",
"13",
"13+",
"14",
"14+",
],
"maimai:Single": [
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"7+",
"8",
"8+",
"9",
"9+",
"10",
"10+",
"11",
"11+",
"12",
"12+",
"13",
"13+",
"14",
],
"museca:Single": StrRange(1, 15),
"sdvx:Single": StrRange(1, 20),
"ddr:SP": StrRange(1, 20),
"ddr:DP": StrRange(1, 20),
};
async function InsertStaticLevelVersionFolders() {
// Little known fact: I invented the cyclomatic complexity warning
for (const { game, playtype } of GPTYielder()) {
const gameConfig = GetGameConfig(game);
const gptConfig = GetGamePTConfig(game, playtype);
for (const version of gptConfig.supportedVersions) {
// @ts-expect-error dont worry
const staticLevels = STATIC_LEVELS[`${game}:${playtype}`];
if (!staticLevels) {
continue;
}
let inactive = false;
if (
!(
gptConfig.currentGameOmniVersion === version ||
gptConfig.currentLatestVersion === version ||
gptConfig.currentLocalVersion === version
)
) {
inactive = true;
}
await db.tables.insert({
tableID: `${game}-${playtype}-${version}-levels`,
description: `Levels for ${FormatGame(game, playtype)} in version ${version}.`,
title: `${FormatGame(game, playtype)} (${version})`,
folders: [],
game,
playtype,
inactive,
});
for (const level of staticLevels) {
const query = {
level,
versions: version,
};
const folderID = CreateFolderID(query, game, playtype);
await db.folders.insert({
folderID,
game,
playtype,
title: `Level ${level} (${version})`,
type: "charts",
data: query,
searchTerms: [],
inactive,
});
await db.tables.update(
{
tableID: `${game}-${playtype}-${version}-levels`,
},
{
$push: {
folders: folderID,
},
}
);
logger.info(`Inserted ${gameConfig.name} Level ${level} (${version})`);
}
}
}
logger.info(`Done.`);
}
function* GPTYielder() {
for (const game of StaticConfig.allSupportedGames) {
const gameConfig = GetGameConfig(game);
for (const playtype of gameConfig.validPlaytypes) {
yield { game, playtype };
}
}
}
// Some games don't use "fixed" levels per se, and as such breaking
// the folders up are quite hard. This is for *those* games.
async function InsertCustomStuff() {}
(async () => {
await InsertStaticLevelVersionFolders();
})();
server/scripts/folders/folder-cache.ts
+3
View File
@@ -0,0 +1,3 @@
import { InitaliseFolderChartLookup } from "../../src/utils/folder";
InitaliseFolderChartLookup().then(() => process.exit(1));
server/scripts/iidx-tierlist/get-sp12-data.ts
+159 -153
View File
@@ -1,174 +1,180 @@
/* eslint-disable no-await-in-loop */
/* eslint-disable no-case-declarations */
/* eslint-disable no-param-reassign */
import db from "../../src/db/db";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import fetch from "node-fetch";
import { FindSongOnTitle } from "../../src/common/database-lookup/song";
import { FindChartWithPTDF } from "../../src/common/database-lookup/chart";
import { Difficulties, TierlistDataDocument } from "kamaitachi-common";
import { CalculateTierlistDataID } from "../../src/common/tierlist";
import CreateLogCtx from "../../src/common/logger";
import { Difficulties, integer } from "tachi-common";
import { FindChartWithPTDF } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
const logger = CreateLogCtx(__filename);
const TIERLIST_ID = "ee9b756e50cff8282091102257b01f423ef855f2";
async function FetchSP12Data() {
const rj = await fetch("https://sp12.iidx.app/api/v1/sheets").then((r) => r.json()); // will throw if somethings wrong, anyway
if (!rj.sheets) {
throw new Error(`No sheets in return from sp12?`);
}
const tdd: TierlistDataDocument<"Individual Difference">[] = [];
for (const sh of rj.sheets) {
let chart;
try {
chart = await HumanisedTitleLookup(sh.title);
} catch (err) {
logger.error((err as Error).message);
continue;
}
for (const key of ["NORMAL CLEAR", "HARD CLEAR", "EX HARD CLEAR"]) {
let fakeKey;
let val;
switch (key) {
case "NORMAL CLEAR":
fakeKey = "n_clear";
const v = Math.floor(sh[fakeKey] / 2);
if (v === 9) {
val = 11.8;
} else if (v === 8) {
val = 12.0;
} else if (v === 7) {
val = 12.2;
} else if (v === 6) {
val = 12.4;
} else if (v === 5) {
val = 12.6;
} else if (v < 0) {
continue;
} else {
val = 12.6 + (5 - v) * 0.1;
}
break;
case "HARD CLEAR":
fakeKey = "hard";
const v2 = Math.floor(sh[fakeKey] / 2);
if (v2 === 9) {
val = 11.9;
} else if (v2 === 8) {
val = 12.1;
} else if (v2 === 7) {
val = 12.3;
} else if (v2 === 6) {
val = 12.5;
} else if (v2 === 5) {
val = 12.7;
} else if (v2 < 0) {
continue;
} else {
val = 12.7 + (5 - v2) * 0.1;
}
break;
case "EX HARD CLEAR":
fakeKey = "exh";
const v3 = sh[fakeKey];
if (v3 < 0) {
continue;
}
val = 12.4 + (12 - v3) * 0.1;
break;
default:
throw new Error("??");
}
const humanised = sh[`${fakeKey}_string`];
if (humanised === "難易度未定") {
continue;
}
if (val < 11.8) {
logger.info(`Skipping ${sh.title} ${key}, ${humanised}.`);
continue;
}
val = parseFloat(val.toFixed(2));
tdd.push({
chartID: chart.chartID,
type: "lamp",
tierlistID: TIERLIST_ID,
tierlistDataID: CalculateTierlistDataID(chart.chartID, "lamp", key, TIERLIST_ID),
key,
data: {
humanised,
value: val,
flags: {
"Individual Difference": !!humanised.match(/個人差/u),
},
},
});
}
}
logger.info(`Inserting ${tdd.length} documents...`);
await db["tierlist-data"].insert(tdd);
logger.info("Done!");
process.exit(0);
interface SP12Data {
id: integer;
title: string;
n_clear: integer;
hard: integer;
exh: integer;
n_clear_string: string;
hard_string: string;
exh_string: string;
version: integer;
}
function isOdd(num: number) {
return num % 2 !== 0;
async function FetchSP12Data() {
const rj: { sheets: SP12Data[] } = await fetch("https://sp12.iidx.app/api/v1/sheets").then(
(r) => r.json()
); // will throw if somethings wrong, anyway
if (!rj.sheets) {
throw new Error(`No sheets in return from sp12?`);
}
for (const sh of rj.sheets) {
let chart;
try {
chart = await HumanisedTitleLookup(sh.title);
} catch (err) {
logger.error((err as Error).message);
continue;
}
if (!chart) {
logger.warn(`Couldn't find chart with title ${sh.title}`);
}
for (const key of ["n_clear", "hard", "exh"] as const) {
let val;
switch (key) {
case "n_clear":
const v = Math.floor(sh[key] / 2);
if (v === 9) {
val = 11.8;
} else if (v === 8) {
val = 12.0;
} else if (v === 7) {
val = 12.2;
} else if (v === 6) {
val = 12.4;
} else if (v === 5) {
val = 12.6;
} else if (v < 0) {
continue;
} else {
val = 12.6 + (5 - v) * 0.1;
}
break;
case "hard":
const v2 = Math.floor(sh[key] / 2);
if (v2 === 9) {
val = 11.9;
} else if (v2 === 8) {
val = 12.1;
} else if (v2 === 7) {
val = 12.3;
} else if (v2 === 6) {
val = 12.5;
} else if (v2 === 5) {
val = 12.7;
} else if (v2 < 0) {
continue;
} else {
val = 12.7 + (5 - v2) * 0.1;
}
break;
case "exh":
const v3 = sh[key];
if (v3 >= 12 || v3 <= 0) {
continue;
}
val = 12.4 + (12 - v3) * 0.1;
break;
default:
throw new Error("??");
}
const stringVal = sh[`${key}_string` as const];
if (stringVal === "難易度未定") {
continue;
}
val = parseFloat(val.toFixed(2));
const ktKey = key === "exh" ? "kt-EXHC" : key === "hard" ? "kt-HC" : "kt-NC";
// If EXH, just save the string version as 12.xx
// Else, use the A/S+ thing that people are used to.
const text =
ktKey === "kt-EXHC"
? val.toFixed(2)
: `12${stringVal.replace(/(個人差|地力)/u, "")}`;
await db.charts.iidx.update(
{
chartID: chart.chartID,
},
{
$set: {
[`tierlistInfo.${ktKey}`]: {
text,
value: val,
individualDifference: stringVal.includes("個人差"),
},
},
}
);
logger.info(`Saved ${sh.title} value ${key} = ${val} (${text}).`);
}
}
logger.info("Done!");
process.exit(0);
}
async function HumanisedTitleLookup(originalTitle: string) {
let difficulty: Difficulties["iidx:SP"] = "ANOTHER";
let difficulty: Difficulties["iidx:SP"] = "ANOTHER";
let title = originalTitle;
let title = originalTitle;
if (title.match(/(†|†LEGGENDARIA)$/u)) {
difficulty = "LEGGENDARIA";
title = title.split("†")[0]; // this will break if there is ever a ptp leggendaria
} else if (title.match(/\[H\]$/u)) {
difficulty = "HYPER";
title = title.split("[")[0];
} else if (title.match(/\[A\]$/u)) {
difficulty = "ANOTHER"; // lmao
title = title.split("[")[0];
}
if (title.match(/(†|†LEGGENDARIA)$/u)) {
difficulty = "LEGGENDARIA";
title = title.split(/(†|†LEGGENDARIA)$/u)[0];
} else if (title.match(/\[H\]$/u)) {
difficulty = "HYPER";
title = title.split("[")[0];
} else if (title.match(/\[A\]$/u)) {
difficulty = "ANOTHER"; // lmao
title = title.split("[")[0];
}
const song = await FindSongOnTitle("iidx", title);
const song = await FindSongOnTitle("iidx", title);
if (!song) {
throw new Error(
`Could not resolve song ${title} (${difficulty}) (Original ${originalTitle}).`
);
}
if (!song) {
throw new Error(
`Could not resolve song ${title} (${difficulty}) (Original ${originalTitle}).`
);
}
const chart = await FindChartWithPTDF("iidx", song.id, "SP", difficulty);
const chart = await FindChartWithPTDF("iidx", song.id, "SP", difficulty);
if (!chart) {
throw new Error(
`Could not resolve chart ${title} ${difficulty} (Original ${originalTitle}).`
);
}
if (!chart) {
throw new Error(
`Could not resolve chart ${title} ${difficulty} (Original ${originalTitle}).`
);
}
return chart;
return chart;
}
FetchSP12Data();
server/scripts/iidx-tierlist/iidx-pe-kr/_README
+1 -1
View File
@@ -2,4 +2,4 @@ get the data by going to https://iidx.insane.pe.kr/!/sp10/ and typing copy(table
parsers here handle the rest.
the sp8h tierlist *sucks*, so, we're ignoring it.
the sp8h, 9 and 10 tierlists *suck*, so, we're ignoring them.
server/scripts/iidx-tierlist/iidx-pe-kr/sp10H.json
-15704
View File
File diff suppressed because it is too large Load Diff
server/scripts/iidx-tierlist/iidx-pe-kr/sp9H.json
-11423
View File
File diff suppressed because it is too large Load Diff
server/scripts/iidx-tierlist/parse-iidxkr-data.ts
+90 -137
View File
@@ -1,164 +1,117 @@
import db from "../../src/db/db";
/* eslint-disable no-await-in-loop */
import db from "external/mongo/db";
import fs from "fs";
import CreateLogCtx from "lib/logger/logger";
import path from "path";
import { CalculateTierlistDataID } from "../../src/common/tierlist";
import { FindSongOnTitle } from "../../src/common/database-lookup/song";
import CreateLogCtx from "../../src/common/logger";
import { FindChartWithPTDF } from "../../src/common/database-lookup/chart";
import { Difficulties } from "kamaitachi-common";
import { Difficulties } from "tachi-common";
import { FindChartWithPTDF } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
const logger = CreateLogCtx(__filename);
const TIERLIST_ID = "ee9b756e50cff8282091102257b01f423ef855f2";
async function parseKr(
dir: string,
key: "NORMAL CLEAR" | "HARD CLEAR",
catVals: { h: string; v: number }[]
file: string,
mode: "NC" | "HC",
catVals: { text: string; value: number; idv: boolean }[]
) {
const krdata = JSON.parse(fs.readFileSync(path.join(__dirname, "./iidx-pe-kr", dir), "utf-8"));
const krdata = JSON.parse(fs.readFileSync(path.join(__dirname, "./iidx-pe-kr", file), "utf-8"));
const tdd = [];
for (let i = 0; i < krdata.categories.length; i++) {
const data = krdata.categories[i];
if (data.sortindex < 0) {
continue;
}
for (let i = 0; i < krdata.categories.length; i++) {
const data = krdata.categories[i];
if (data.sortindex < 0) {
continue;
}
const cv = catVals[i];
const cv = catVals[i];
if (!cv) {
throw new Error(`cv krdata mismatch ${krdata.categories.length} ${catVals.length}?`);
}
if (!cv) {
throw new Error(`cv krdata mismatch ${krdata.categories.length} ${catVals.length}?`);
}
for (const item of data.items) {
const song = await FindSongOnTitle("iidx", item.data.title.replace(/†$/u, "").trim());
for (const item of data.items) {
const song = await FindSongOnTitle("iidx", item.data.title.replace(/†$/u, "").trim());
if (!song) {
logger.warn(`Could not find song with title ${item.data.title}`);
continue;
}
if (!song) {
logger.warn(`Could not find song with title ${item.data.title}`);
continue;
}
const t = item.data.type;
const t = item.data.type;
let diff: Difficulties["iidx:SP"];
if (t === "A") {
diff = "ANOTHER";
} else if (t === "L") {
diff = "LEGGENDARIA";
} else if (t === "H") {
diff = "HYPER";
} else if (t === "N") {
diff = "NORMAL";
} else {
logger.warn(`${song.title} Unknown difficulty ${t}.`);
continue;
}
let diff: Difficulties["iidx:SP"];
if (t === "A") {
diff = "ANOTHER";
} else if (t === "L") {
diff = "LEGGENDARIA";
} else if (t === "H") {
diff = "HYPER";
} else if (t === "N") {
diff = "NORMAL";
} else {
logger.warn(`${song.title} Unknown difficulty ${t}.`);
continue;
}
const chart = await FindChartWithPTDF("iidx", song.id, "SP", diff);
const chart = await FindChartWithPTDF("iidx", song.id, "SP", diff);
if (!chart) {
logger.warn(`${song.title} ${diff} - Couldn't find chart?`);
continue;
}
if (!chart) {
logger.warn(`${song.title} ${diff} - Couldn't find chart?`);
continue;
}
// Update this part of the tierlist, we've got everything.
tdd.push({
chartID: chart.chartID,
type: "lamp" as const,
tierlistID: TIERLIST_ID,
tierlistDataID: CalculateTierlistDataID(chart.chartID, "lamp", key, TIERLIST_ID),
key,
data: {
humanised: cv.h,
value: cv.v,
flags: {
"Individual Difference": !!data.category.match(/個人差/u),
},
},
});
}
}
await db.charts.iidx.update(
{
chartID: chart.chartID,
},
{
$set: {
[`tierlistInfo.kt-${mode}`]: {
text: cv.text,
value: cv.value,
individualDifference: cv.idv,
},
},
}
);
logger.info(`Inserting ${tdd.length} documents...`);
logger.verbose(`Updated ${chart.chartID} to value ${cv.value}.`);
}
}
const x = await db["tierlist-data"].bulkWrite(
tdd.map((e) => ({
updateOne: {
filter: {
tierlistDataID: e.tierlistDataID,
},
update: {
$set: e,
},
upsert: true,
},
}))
);
logger.info(x);
logger.info("done...");
logger.info(`Finished parsing ${file}`);
}
function h(h: string, v: number) {
return { h, v };
function h(text: string, value: number, idv = false) {
return { text, value, idv };
}
parseKr("sp9H.json", "HARD CLEAR", [
h("S+", 10.2),
h("S", 10),
h("S 個人差", 10),
h("A+", 9.9),
h("A+ 個人差", 9.9),
h("A", 9.8),
h("B+", 9.7),
h("B", 9.6),
h("C+", 9.5),
h("C", 9.4),
h("D", 9.2),
h("E", 9.0),
h("F", 8.8),
parseKr("sp11N.json", "NC", [
h("11S+", 12.2),
h("11S", 12),
h("11A+", 11.9),
h("11A", 11.8),
h("11B", 11.6),
h("11C", 11.4),
h("11D", 11.2),
h("11E", 11.0),
h("11F", 10.8),
]);
parseKr("sp10H.json", "HARD CLEAR", [
h("S", 11),
h("A+", 10.9),
h("A", 10.8),
h("B+", 10.7),
h("B", 10.6),
h("C+", 10.5),
h("C", 10.4),
h("D", 10.2),
h("E", 10.0),
h("F", 9.8),
]);
parseKr("sp11N.json", "NORMAL CLEAR", [
h("S+", 12.2),
h("S", 12),
h("A+", 11.9),
h("A", 11.8),
h("B", 11.6),
h("C", 11.4),
h("D", 11.2),
h("E", 11.0),
h("F", 10.8),
]);
parseKr("sp11H.json", "HARD CLEAR", [
h("S+", 12.2),
h("S+ 個人差", 12.2),
h("S", 12),
h("S 個人差", 12),
h("A+", 11.9),
h("A+ 個人差", 11.9),
h("A", 11.8),
h("A 個人差", 11.0),
h("B+", 11.7),
h("B", 11.6),
h("C", 11.4),
h("D", 11.2),
h("E", 11.0),
h("F", 10.8),
parseKr("sp11H.json", "HC", [
h("11S+", 12.2),
h("11S+", 12.2, true), // 個人差
h("11S", 12),
h("11S", 12, true), // 個人差
h("11A+", 11.9),
h("11A+", 11.9, true), // 個人差
h("11A", 11.8),
h("11A", 11.0, true), // 個人差
h("11B+", 11.7),
h("11B", 11.6),
h("11C", 11.4),
h("11D", 11.2),
h("11E", 11.0),
h("11F", 10.8),
]);
server/scripts/init-folder-cache/folder-cache.ts
-3
View File
@@ -1,3 +0,0 @@
import { InitaliseFolderChartLookup } from "../../src/common/folder";
InitaliseFolderChartLookup();
server/scripts/init-sequence-docs.ts
+5
View File
@@ -0,0 +1,5 @@
import { InitSequenceDocs } from "external/mongo/sequence-docs";
if (require.main === module) {
InitSequenceDocs().then(() => process.exit(0));
}
server/scripts/initialise-counters.ts
-22
View File
@@ -1,22 +0,0 @@
import db from "../db/db";
import CreateLogCtx from "../common/logger";
const logger = CreateLogCtx(__filename);
(async () => {
let users = await db.counters.findOne({
counterName: "users",
});
if (users) {
throw new Error(`"users" document already exists, exiting.`);
}
await db.counters.insert({
counterName: "users",
value: 1,
});
logger.info("Successfully initialised counter documents. Exiting.");
process.exit(0);
})();
server/scripts/invites/create-invite-code.ts
+31
View File
@@ -0,0 +1,31 @@
import { Command } from "commander";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { Random20Hex } from "utils/misc";
const logger = CreateLogCtx(__filename);
const program = new Command();
program.option("-c, --code <code>", "The code for this invite.");
program.parse(process.argv);
const options = program.opts();
if (!options.code) {
options.code = Random20Hex();
}
db.invites
.insert({
code: options.code,
createdBy: 1,
consumed: false,
createdAt: Date.now(),
consumedBy: null,
consumedAt: null,
})
.then(() => {
console.log(`Created invite ${options.code}.`);
process.exit(0);
});
server/scripts/package.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "ktblack-server-scripts",
"version": "0.0.0",
"description": "Scripts and their dependencies for KTBlack. This lets us separate dependencies to avoid unecessary overhead in the CI/CD pipeline.",
"main": "",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC"
}
server/scripts/quick-insert/quick-insert.ts
+21
View File
@@ -0,0 +1,21 @@
import deepmerge from "deepmerge";
import { ICollection } from "monk";
import { ChartDocument, IDStrings } from "tachi-common";
import { Random20Hex } from "utils/misc";
export async function QuickInsert<I extends IDStrings>(
base: Omit<ChartDocument<I>, "chartID">,
content: Partial<ChartDocument<I>>[],
coll: ICollection
) {
const docs = [];
for (const c of content) {
const doc = deepmerge(base, c);
doc.chartID = Random20Hex();
docs.push(doc);
}
await coll.insert(docs);
}
server/scripts/save-testdata-from-ktblack.ts
-33
View File
@@ -1,33 +0,0 @@
import monk from "monk";
import fs from "fs";
import path from "path";
import { Command } from "commander";
import CreateLogCtx from "../src/common/logger";
const program = new Command();
const logger = CreateLogCtx(__filename);
program.option("-c, --collection <collection>", "The collection to fetch ktblack data from.");
program.parse(process.argv);
const options = program.opts();
const collection = options.collection;
logger.info(`Connecting to ${process.env.MONGO_BASE_URL}/ktblackdb...`);
const ktBlackDB = monk(`${process.env.MONGO_BASE_URL}/ktblackdb`);
(async () => {
logger.info(`Fetching data for ${collection}...`);
const data = await ktBlackDB.get(collection).find({}, { projection: { _id: 0 } });
logger.info(`Fetched ${data.length} documents, Writing...`);
fs.writeFileSync(
path.join(__dirname, `../src/test-utils/test-data/kamaitachi/ktblack-${collection}.json`),
JSON.stringify(data)
);
logger.info("Done!");
process.exit(0);
})();
server/scripts/set-indexes.ts
+5 -124
View File
@@ -1,135 +1,16 @@
import monk from "monk";
/* eslint-disable no-await-in-loop */
import { Command } from "commander";
import CreateLogCtx from "../src/lib/logger/logger";
import { IndexOptions } from "mongodb";
import { ValidDatabases } from "kamaitachi-common";
import { supportedGames } from "kamaitachi-common/js/config";
const logger = CreateLogCtx(__filename);
import { SetIndexes } from "../src/external/mongo/indexes";
const program = new Command();
program.option("-d, --db <database>", "The database to index.");
program.option(
"-r, --reset",
"Whether to reset all indexes on this database before indexing or not."
"-r, --reset",
"Whether to reset all indexes on this database before indexing or not."
);
program.parse(process.argv);
const options = program.opts();
const db = monk(`${process.env.MONGO_BASE_URL ?? "127.0.0.1"}/${options.db ?? "ktblackdb"}`);
interface Index {
fields: Record<string, unknown>;
options?: IndexOptions;
}
function index(fields: Record<string, unknown>, options?: IndexOptions) {
return { fields, options };
}
const UNIQUE = { unique: true };
const staticIndexes: Partial<Record<ValidDatabases, Index[]>> = {
scores: [index({ scoreID: 1 }, UNIQUE)],
"score-pbs": [
index({ chartID: 1, userID: 1 }, UNIQUE),
index({ chartID: 1, "scoreData.percent": 1 }),
],
sessions: [
// lol
index({ timeStarted: 1, timeEnded: 1, userID: 1, game: 1, playtype: 1 }),
],
"game-stats": [index({ userID: 1, game: 1, playtype: 1 }, UNIQUE)],
"folder-chart-lookup": [index({ chartID: 1, folderID: 1 }, UNIQUE)],
"tierlist-data": [
index({ tierlistDataID: 1 }, UNIQUE),
index({ chartID: 1, tierlistID: 1 }),
index({ chartID: 1, tierlistID: 1, type: 1 }),
index({ chartID: 1, tierlistID: 1, type: 1, key: 1 }, UNIQUE),
],
goals: [index({ goalID: 1 }, UNIQUE)],
"user-goals": [index({ goalID: 1, userID: 1 }, UNIQUE), index({ goalID: 1 })],
milestones: [index({ milestoneID: 1 }, UNIQUE), index({ group: 1, game: 1, playtype: 1 })],
"user-milestones": [
index({ milestoneID: 1, userID: 1 }, UNIQUE),
index({ userID: 1, game: 1, playtype: 1 }),
],
imports: [index({ importID: 1 }, UNIQUE)],
"import-timings": [
index({ importID: 1 }, UNIQUE),
index({ timestamp: 1 }),
index({ total: 1 }),
],
// @todo #96 Add more indexes to the users collection.
users: [index({ id: 1 }, UNIQUE)],
tierlists: [
index({ tierlistID: 1 }, UNIQUE),
index({ game: 1, playtype: 1, isDefault: 1 }, UNIQUE),
],
folders: [
index({ folderID: 1 }, UNIQUE),
index({ game: 1, playtype: 1 }),
index({ game: 1, playtype: 1, table: 1 }),
index({ game: 1, playtype: 1, table: 1, tableIndex: 1 }),
],
"kai-auth-tokens": [index({ userID: 1, service: 1 }, UNIQUE)],
"charts-iidx": [
index(
{ "data.arcChartID": 1 },
{ unique: true, partialFilterExpression: { "data.arcChartID": { $type: "string" } } }
),
index({ "data.hashSHA256": 1 }),
],
};
const indexes: Partial<Record<ValidDatabases, Index[]>> = staticIndexes;
for (const game of supportedGames) {
if (indexes[`charts-${game}` as "charts-iidx"]) {
indexes[`charts-${game}` as "charts-iidx"]!.push(
index({ chartID: 1 }, UNIQUE),
index(
{ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 },
{ unique: true, partialFilterExpression: { isPrimary: { $eq: true } } }
)
);
} else {
indexes[`charts-${game}` as "charts-iidx"] = [
index({ chartID: 1 }, UNIQUE),
index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE),
];
}
if (indexes[`songs-${game}` as "songs-iidx"]) {
indexes[`songs-${game}` as "songs-iidx"]!.push(
index({ id: 1 }, UNIQUE),
index({ title: 1 })
);
} else {
indexes[`songs-${game}` as "songs-iidx"] = [index({ id: 1 }, UNIQUE), index({ title: 1 })];
}
}
(async () => {
logger.info(`Starting indexing for ${options.db ?? "ktblackdb"}...`);
for (const collection in indexes) {
if (options.reset) {
// eslint-disable-next-line no-await-in-loop
await db.get(collection).dropIndexes();
}
// @ts-expect-error dru(n)kts
for (const index of indexes[collection]) {
// eslint-disable-next-line no-await-in-loop
const r = await db.get(collection).createIndex(index.fields, index.options);
logger.info(r);
}
}
logger.info("Done.");
process.exit(0);
})();
SetIndexes(options.db, options.reset).then(() => process.exit(0));
server/scripts/single-use/add-2dxtra-flag.ts
-21
View File
@@ -1,21 +0,0 @@
// @ts-nocheck
import db from "../../src/db/db";
db.charts.iidx
.find({})
.each(async (c, { pause, resume }) => {
pause();
await db.charts.iidx.update(
{ _id: c._id },
{
$set: {
"flags.2dxtra": false,
},
}
);
resume();
})
.then(() => console.log("done"));
server/scripts/single-use/add-folders-array.ts
+16
View File
@@ -0,0 +1,16 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
const logger = CreateLogCtx(__filename);
(async () => {
logger.info(`Setting tableFolders.`);
const r = await db.charts.bms.update(
{},
{ $set: { "data.tableFolders": [] } },
{ multi: true }
);
logger.info(`Done!`, { r });
process.exit(0);
})();
server/scripts/single-use/add-hcnver.ts
-24
View File
@@ -1,24 +0,0 @@
import fs from "fs";
import path from "path";
import { PRUDENCE_CHART_SCHEMAS } from "../../src/db/schemas";
import p from "prudence";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/common/logger";
const logger = CreateLogCtx(__filename);
let data = JSON.parse(fs.readFileSync(path.join(__dirname, "./hcnverdata.json"), "utf-8"));
for (const d of data) {
let r = p(d, PRUDENCE_CHART_SCHEMAS.iidx);
if (r) {
logger.error(r.userVal);
logger.error(r);
throw r;
}
}
db.charts.iidx.insert(data).then(() => {
logger.info("Done.");
process.exit(0);
});
server/scripts/single-use/add-tierlist-info.ts
+20
View File
@@ -0,0 +1,20 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
const logger = CreateLogCtx(__filename);
if (require.main === module) {
(async () => {
for (const game of TachiConfig.GAMES) {
// eslint-disable-next-line no-await-in-loop
const res = await db.charts[game].update(
{ tierlistInfo: { $exists: false } },
{ $set: { tierlistInfo: {} } },
{ multi: true }
);
logger.info(`Added tierlistInfo to ${game}'s charts.`, { res });
}
})();
}
server/scripts/single-use/convert-2dxtra-charts.ts
+71
View File
@@ -0,0 +1,71 @@
/* eslint-disable no-await-in-loop */
import db from "external/mongo/db";
import { rootLogger } from "lib/logger/logger";
import { ChartDocument, Difficulties } from "tachi-common";
import { FindIIDXChartOnInGameID } from "utils/queries/charts";
async function ConvertChart(chart: ChartDocument<"iidx:SP" | "iidx:DP">) {
// We need to get the related official song. The best way to do this is
// probably with song IDs.
const originalChart = await FindIIDXChartOnInGameID(
chart.data.inGameID,
chart.playtype,
chart.difficulty
);
if (!originalChart) {
rootLogger.error(`Can't find original chart.`, { chart });
throw new Error(`Can't find original chart.`);
}
return {
difficulty: `${chart.data["2dxtraSet"]!} ${chart.difficulty}` as Difficulties[
| "iidx:SP"
| "iidx:DP"],
songID: originalChart!.songID,
};
}
if (require.main === module) {
(async () => {
const modCharts = (await db.charts.iidx.find({
"data.2dxtraSet": { $ne: null },
difficulty: { $in: ["BEGINNER", "NORMAL", "HYPER", "ANOTHER", "LEGGENDARIA"] },
})) as ChartDocument<"iidx:SP" | "iidx:DP">[];
rootLogger.info(`Modifying ${modCharts.length} charts.`);
for (const chart of modCharts) {
const setDoc = await ConvertChart(chart);
await db.charts.iidx.update(
{
chartID: chart.chartID,
},
{ $set: setDoc }
);
// We're also going to have to port over old scores.
await db.scores.update(
{
chartID: chart.chartID,
},
{
$set: {
songID: setDoc.songID,
},
}
);
}
rootLogger.info(`Removing ${modCharts.length} songs.`);
await db.songs.iidx.remove({
id: { $in: modCharts.map((e) => e.songID) },
});
rootLogger.info(`Done.`);
})();
}
server/scripts/single-use/convert-kt1-userimages.ts
+29
View File
@@ -0,0 +1,29 @@
import { Command } from "commander";
import fs from "fs";
import path from "path";
const program = new Command();
program.option("-d, --dir <dir>");
program.parse(process.argv);
const options = program.opts();
if (!options.dir) {
throw new Error(`No dir provided, yet one is necessary.`);
}
// @warn THIS SCRIPT IS OUTDATED!
// Tachi now uses /users/${userID}/banner instead of
// /users/banner/${userID}
fs.mkdirSync("users/banner", { recursive: true });
fs.mkdirSync("users/pfp", { recursive: true });
for (const file of fs.readdirSync(options.dir)) {
const dfile = path.join(options.dir, file);
if (file.endsWith("-banner.png")) {
fs.copyFileSync(dfile, `users/banner/${file.split("-banner.png")[0]}`);
} else {
fs.copyFileSync(dfile, `users/pfp/${file.split("-pfp.png")[0]}`);
}
}
server/scripts/single-use/create-fake-password.ts
-4
View File
@@ -1,4 +0,0 @@
/* eslint-disable no-console */
import bcrypt from "bcrypt";
console.log(bcrypt.hashSync("password", 12));
server/scripts/single-use/create-tables.ts
+44
View File
@@ -0,0 +1,44 @@
import db from "external/mongo/db";
(async () => {
const folders = await db.folders.find({}, { sort: { tableIndex: 1 } });
for (const folder of folders) {
const { game, playtype } = folder;
const tableID = ((folder as any).table as string)
.toLowerCase()
.replace(/ /gu, "_")
.replace(/\(/gu, "")
.replace(/\)/gu, "");
console.log(game, playtype, tableID, folder.title);
const existingTable = await db.tables.findOne({ game, playtype, tableID });
if (!existingTable) {
await db.tables.insert({
tableID,
game,
playtype,
description: "todo",
folders: [folder.folderID],
// @ts-expect-error yea
title: folder.table,
});
} else {
await db.tables.update(
{
game,
playtype,
tableID,
},
{
$push: {
folders: folder.folderID,
},
}
);
}
}
})();

Some files were not shown because too many files have changed in this diff Show More