From 377df6495bb3dcd9daee2536fde651a46c102132 Mon Sep 17 00:00:00 2001 From: zkldi <20380519+zkldi@users.noreply.github.com> Date: Mon, 23 Jan 2023 14:43:35 +0000 Subject: [PATCH] feat: first overhaul --- README.md | 14 +- docs/docs/api/routes/users.md | 4 +- docs/docs/api/webhooks/class-update-v1.md | 6 +- .../batch-manual/direct-manual.md | 0 .../batch-manual/index.md | 0 .../implementation-details/details.md | 0 .../game-configuration.md | 0 .../implementation-details/goal-id.md | 0 .../implementation-details/goals-quests.md | 11 +- .../implementation-details/score-id.md | 43 ++ .../implementation-details/search.md | 0 .../implementation-details/songs-charts.md | 0 .../import/conv-failures.md | 0 .../import/goals.md | 2 - .../import/import-doc-time.md | 2 - .../import/import-types.md | 0 .../import/importing.md | 0 .../{tachi-server => codebase}/import/main.md | 0 .../import/orphans.md | 0 .../import/overview.md | 0 .../import/parse-conv.md | 0 .../import/parse-ipi.md | 0 .../{tachi-server => codebase}/import/pbs.md | 0 .../import/quests.md | 6 +- .../import/sessions.md | 19 +- .../{tachi-server => codebase}/import/ugs.md | 0 docs/docs/codebase/index.md | 43 ++ .../infrastructure/api-clients.md | 0 docs/docs/codebase/infrastructure/branches.md | 25 + .../infrastructure/database-seeds.md | 0 .../infrastructure/file-flow.md | 0 .../infrastructure/logging.md | 0 .../infrastructure/oauth2.md | 0 .../setup/config.md | 17 +- .../structure/filesystem.md | 0 docs/docs/codebase/structure/style.md | 10 + .../structure/testing.md | 0 docs/docs/tachi-bot/index.md | 5 - .../implementation-details/esd.md | 102 --- .../implementation-details/score-id.md | 52 -- .../implementation-details/statistics.md | 178 ------ docs/docs/tachi-server/index.md | 23 - .../tachi-server/infrastructure/branches.md | 80 --- .../tachi-server/infrastructure/toolchain.md | 106 --- .../tachi-server/infrastructure/versions.md | 51 -- docs/docs/tachi-server/structure/style.md | 148 ----- docs/docs/wiki/pbs-scores.md | 1 + docs/docs/wiki/rules.md | 7 +- docs/docs/wiki/score-oddities.md | 2 +- docs/docs/wiki/stats/esd.md | 56 -- docs/docs/wiki/stats/tachi.md | 605 ------------------ docs/mkdocs.yml | 83 ++- server/package.json | 2 +- 53 files changed, 193 insertions(+), 1510 deletions(-) rename docs/docs/{tachi-server => codebase}/batch-manual/direct-manual.md (100%) rename docs/docs/{tachi-server => codebase}/batch-manual/index.md (100%) rename docs/docs/{tachi-server => codebase}/implementation-details/details.md (100%) rename docs/docs/{tachi-server => codebase}/implementation-details/game-configuration.md (100%) rename docs/docs/{tachi-server => codebase}/implementation-details/goal-id.md (100%) rename docs/docs/{tachi-server => codebase}/implementation-details/goals-quests.md (92%) create mode 100644 docs/docs/codebase/implementation-details/score-id.md rename docs/docs/{tachi-server => codebase}/implementation-details/search.md (100%) rename docs/docs/{tachi-server => codebase}/implementation-details/songs-charts.md (100%) rename docs/docs/{tachi-server => codebase}/import/conv-failures.md (100%) rename docs/docs/{tachi-server => codebase}/import/goals.md (97%) rename docs/docs/{tachi-server => codebase}/import/import-doc-time.md (92%) rename docs/docs/{tachi-server => codebase}/import/import-types.md (100%) rename docs/docs/{tachi-server => codebase}/import/importing.md (100%) rename docs/docs/{tachi-server => codebase}/import/main.md (100%) rename docs/docs/{tachi-server => codebase}/import/orphans.md (100%) rename docs/docs/{tachi-server => codebase}/import/overview.md (100%) rename docs/docs/{tachi-server => codebase}/import/parse-conv.md (100%) rename docs/docs/{tachi-server => codebase}/import/parse-ipi.md (100%) rename docs/docs/{tachi-server => codebase}/import/pbs.md (100%) rename docs/docs/{tachi-server => codebase}/import/quests.md (63%) rename docs/docs/{tachi-server => codebase}/import/sessions.md (70%) rename docs/docs/{tachi-server => codebase}/import/ugs.md (100%) create mode 100644 docs/docs/codebase/index.md rename docs/docs/{tachi-server => codebase}/infrastructure/api-clients.md (100%) create mode 100644 docs/docs/codebase/infrastructure/branches.md rename docs/docs/{tachi-server => codebase}/infrastructure/database-seeds.md (100%) rename docs/docs/{tachi-server => codebase}/infrastructure/file-flow.md (100%) rename docs/docs/{tachi-server => codebase}/infrastructure/logging.md (100%) rename docs/docs/{tachi-server => codebase}/infrastructure/oauth2.md (100%) rename docs/docs/{tachi-server => codebase}/setup/config.md (95%) rename docs/docs/{tachi-server => codebase}/structure/filesystem.md (100%) create mode 100644 docs/docs/codebase/structure/style.md rename docs/docs/{tachi-server => codebase}/structure/testing.md (100%) delete mode 100644 docs/docs/tachi-bot/index.md delete mode 100644 docs/docs/tachi-server/implementation-details/esd.md delete mode 100644 docs/docs/tachi-server/implementation-details/score-id.md delete mode 100644 docs/docs/tachi-server/implementation-details/statistics.md delete mode 100644 docs/docs/tachi-server/index.md delete mode 100644 docs/docs/tachi-server/infrastructure/branches.md delete mode 100644 docs/docs/tachi-server/infrastructure/toolchain.md delete mode 100644 docs/docs/tachi-server/infrastructure/versions.md delete mode 100644 docs/docs/tachi-server/structure/style.md delete mode 100644 docs/docs/wiki/stats/esd.md delete mode 100644 docs/docs/wiki/stats/tachi.md diff --git a/README.md b/README.md index 3cf80e881..d26feb650 100644 --- a/README.md +++ b/README.md @@ -27,25 +27,25 @@ You can then check the component-specific guides to see how to run those compone This monorepo contains the following codebases: -- `client/`, Which is a React frontend for Tachi. +- `client/`, Which is a React frontend for Tachi. (AGPL3) The client and the server are fairly decoupled. Someone could trivially create their own frontend client for Tachi. -- `server/`, Which is an Express-Typescript backend for Tachi. +- `server/`, Which is an Express-Typescript backend for Tachi. (AGPL3) This contains all of our API calls, and interfaces with our database, and powers the actual score import engine. -- `database-seeds/`, Which is a git-tracked set of data to be synced with Tachi. +- `database-seeds/`, Which is a git-tracked set of data to be synced with Tachi. (unlicense) **This is the source of truth for the songs, charts, and more on the site!** By submitting PRs to this, you can fix bugs on the website, add new charts, and more. -- `bot/`, Which is a discord bot frontend for Tachi. +- `bot/`, Which is a discord bot frontend for Tachi. (MIT) -- `common/`, Which contains common types, utils and functions shared between all other packages. +- `common/`, Which contains common types, utils and functions shared between all other packages. (MIT) This is also published to NPM when it hits production. -- `docs/`, Which contains Tachi documentation. +- `docs/`, Which contains Tachi documentation. (MIT) -- `sieglinde/`, Which contains our BMS/PMS analysis functions. +- `sieglinde/`, Which contains our BMS/PMS analysis functions. (MIT) diff --git a/docs/docs/api/routes/users.md b/docs/docs/api/routes/users.md index b7dfcf5ff..e4c4162e6 100644 --- a/docs/docs/api/routes/users.md +++ b/docs/docs/api/routes/users.md @@ -407,7 +407,7 @@ None. ## Retrieve your notifications -`GET /api/v1/users/:userID/games/:game/notifications` +`GET /api/v1/users/:userID/notifications` ### Permissions @@ -462,7 +462,7 @@ None. (Empty Object) ### Permissions -- Must be a session-level request from the user who owns these notifications.d +- Must be a session-level request from the user who owns these notifications. ### Parameters diff --git a/docs/docs/api/webhooks/class-update-v1.md b/docs/docs/api/webhooks/class-update-v1.md index bb0abd23c..a36b6945a 100644 --- a/docs/docs/api/webhooks/class-update-v1.md +++ b/docs/docs/api/webhooks/class-update-v1.md @@ -8,8 +8,8 @@ The content is as follows: | `set` | Game Class Set | The name of the class set that was updated, such as `genocideDan` or `vfClass`. | | `game` | Game | The game this class update was for. | | `playtype` | Playtype | The playtype this class update was for. | -| `old` | Null \| Integer | The old value for this class. If null, the user had no class here before. | -| `new` | Integer | The new value for this class. | +| `old` | Null \| String | The old value for this class. If null, the user had no class here before. | +| `new` | String | The new value for this class. | ## Example @@ -20,6 +20,6 @@ The content is as follows: "game": "iidx", "playtype": "SP", "old": null, - "new": 12 + "new": "CHUUDEN" } ``` \ No newline at end of file diff --git a/docs/docs/tachi-server/batch-manual/direct-manual.md b/docs/docs/codebase/batch-manual/direct-manual.md similarity index 100% rename from docs/docs/tachi-server/batch-manual/direct-manual.md rename to docs/docs/codebase/batch-manual/direct-manual.md diff --git a/docs/docs/tachi-server/batch-manual/index.md b/docs/docs/codebase/batch-manual/index.md similarity index 100% rename from docs/docs/tachi-server/batch-manual/index.md rename to docs/docs/codebase/batch-manual/index.md diff --git a/docs/docs/tachi-server/implementation-details/details.md b/docs/docs/codebase/implementation-details/details.md similarity index 100% rename from docs/docs/tachi-server/implementation-details/details.md rename to docs/docs/codebase/implementation-details/details.md diff --git a/docs/docs/tachi-server/implementation-details/game-configuration.md b/docs/docs/codebase/implementation-details/game-configuration.md similarity index 100% rename from docs/docs/tachi-server/implementation-details/game-configuration.md rename to docs/docs/codebase/implementation-details/game-configuration.md diff --git a/docs/docs/tachi-server/implementation-details/goal-id.md b/docs/docs/codebase/implementation-details/goal-id.md similarity index 100% rename from docs/docs/tachi-server/implementation-details/goal-id.md rename to docs/docs/codebase/implementation-details/goal-id.md diff --git a/docs/docs/tachi-server/implementation-details/goals-quests.md b/docs/docs/codebase/implementation-details/goals-quests.md similarity index 92% rename from docs/docs/tachi-server/implementation-details/goals-quests.md rename to docs/docs/codebase/implementation-details/goals-quests.md index 038d7eaca..ef0c3b378 100644 --- a/docs/docs/tachi-server/implementation-details/goals-quests.md +++ b/docs/docs/codebase/implementation-details/goals-quests.md @@ -103,18 +103,13 @@ webhook event is emitted. ### Unsubscribing -Unsubscribing from a quest is a two-pass process. Firstly, for all `goalID`s in the quest, the subscription document has the relevant `questID` pulled from the array. +For all `goalID`s in the quest, we check if this goal subscription has any other parent quests. -The second part of the process involves unsubscribing the user from any goals that now have -an empty array as a result of this process. - -!!! warning - If this process sounds like it has race conditions, it's because it probably does. - Ah well. +If this quest was the only reason the goal was assigned (i.e. it wasn't assigned directly, or isn't part of another quest), this quest will be unsubscribed from. ## Questlines Questline are ordered lists of quests. Their purpose is to group quests together -visually. +visually in an ordered manner. Users *can not* "subscribe" to questlines, but they can use questlines as a utility for subscribing to all the related quests. diff --git a/docs/docs/codebase/implementation-details/score-id.md b/docs/docs/codebase/implementation-details/score-id.md new file mode 100644 index 000000000..d741d864b --- /dev/null +++ b/docs/docs/codebase/implementation-details/score-id.md @@ -0,0 +1,43 @@ +# Score ID implementation + +Score IDs exist to dedupe scores when a user re-submits +the same scores. This happens frequently with `file/` +and `api/` [Import Types](../import/import-types.md), +as they typically resubmit the same scores. + +If a user only got a score once, we don't want to store +it twice. + +Sadly, we can't depend on things like timestamps to assert +whether or whether not we've saw a score before. Many services +alter/tamper their timestamps such that they're unreliable. + +!!! example + E-Amusement IIDX CSVs will change the timestamp of every single + score when a new version comes out to the second the user created + their new account. + +***** + +## Hashing + +The score ID is created by joining the following properties: + +- The `userID` that got this score +- The `chartID` that this score was on +- All [Provided Metrics](todo) for this GPT +- Any [Optional Metrics](todo) that have been marked as `partOfScoreID` + +and then hashing them with SHA256. + +This is then prefixed with `T`, and returned. + +## Clobbering + +Since a scoreID isn't necessarily all of the possible statistics for a score, it's +possible for users to "clobber" their scores, by importing a score without as many +pieces of info (i.e. no judgements), then trying to import that same score with +judgements later will not work, as the scoreID sees it as a duplicate. + +See [Clobbering](todo). + diff --git a/docs/docs/tachi-server/implementation-details/search.md b/docs/docs/codebase/implementation-details/search.md similarity index 100% rename from docs/docs/tachi-server/implementation-details/search.md rename to docs/docs/codebase/implementation-details/search.md diff --git a/docs/docs/tachi-server/implementation-details/songs-charts.md b/docs/docs/codebase/implementation-details/songs-charts.md similarity index 100% rename from docs/docs/tachi-server/implementation-details/songs-charts.md rename to docs/docs/codebase/implementation-details/songs-charts.md diff --git a/docs/docs/tachi-server/import/conv-failures.md b/docs/docs/codebase/import/conv-failures.md similarity index 100% rename from docs/docs/tachi-server/import/conv-failures.md rename to docs/docs/codebase/import/conv-failures.md diff --git a/docs/docs/tachi-server/import/goals.md b/docs/docs/codebase/import/goals.md similarity index 97% rename from docs/docs/tachi-server/import/goals.md rename to docs/docs/codebase/import/goals.md index bf1b0eaf7..ee5db8163 100644 --- a/docs/docs/tachi-server/import/goals.md +++ b/docs/docs/codebase/import/goals.md @@ -20,8 +20,6 @@ as a result of the import. This is calculated as follows: - Multi Goals are matched if their data contains any chartIDs that were affected. -- Any Goals are always matched. - - Folder goals are matched if the folder contains any charts inside the set of chartIDs affected. ## Processing Goals diff --git a/docs/docs/tachi-server/import/import-doc-time.md b/docs/docs/codebase/import/import-doc-time.md similarity index 92% rename from docs/docs/tachi-server/import/import-doc-time.md rename to docs/docs/codebase/import/import-doc-time.md index 8f157f676..ddce2f7ed 100644 --- a/docs/docs/tachi-server/import/import-doc-time.md +++ b/docs/docs/codebase/import/import-doc-time.md @@ -4,8 +4,6 @@ The final step of the process is to coalesce all the returns of the various steps above into one analysable document and return it. -Details on what this document looks like can be found [(todo) here](todo). - ***** ## Logging diff --git a/docs/docs/tachi-server/import/import-types.md b/docs/docs/codebase/import/import-types.md similarity index 100% rename from docs/docs/tachi-server/import/import-types.md rename to docs/docs/codebase/import/import-types.md diff --git a/docs/docs/tachi-server/import/importing.md b/docs/docs/codebase/import/importing.md similarity index 100% rename from docs/docs/tachi-server/import/importing.md rename to docs/docs/codebase/import/importing.md diff --git a/docs/docs/tachi-server/import/main.md b/docs/docs/codebase/import/main.md similarity index 100% rename from docs/docs/tachi-server/import/main.md rename to docs/docs/codebase/import/main.md diff --git a/docs/docs/tachi-server/import/orphans.md b/docs/docs/codebase/import/orphans.md similarity index 100% rename from docs/docs/tachi-server/import/orphans.md rename to docs/docs/codebase/import/orphans.md diff --git a/docs/docs/tachi-server/import/overview.md b/docs/docs/codebase/import/overview.md similarity index 100% rename from docs/docs/tachi-server/import/overview.md rename to docs/docs/codebase/import/overview.md diff --git a/docs/docs/tachi-server/import/parse-conv.md b/docs/docs/codebase/import/parse-conv.md similarity index 100% rename from docs/docs/tachi-server/import/parse-conv.md rename to docs/docs/codebase/import/parse-conv.md diff --git a/docs/docs/tachi-server/import/parse-ipi.md b/docs/docs/codebase/import/parse-ipi.md similarity index 100% rename from docs/docs/tachi-server/import/parse-ipi.md rename to docs/docs/codebase/import/parse-ipi.md diff --git a/docs/docs/tachi-server/import/pbs.md b/docs/docs/codebase/import/pbs.md similarity index 100% rename from docs/docs/tachi-server/import/pbs.md rename to docs/docs/codebase/import/pbs.md diff --git a/docs/docs/tachi-server/import/quests.md b/docs/docs/codebase/import/quests.md similarity index 63% rename from docs/docs/tachi-server/import/quests.md rename to docs/docs/codebase/import/quests.md index 1290bcada..914673423 100644 --- a/docs/docs/tachi-server/import/quests.md +++ b/docs/docs/codebase/import/quests.md @@ -8,11 +8,11 @@ get those that are affected. ## Returns -We evaluate every quest, and for each one, -create a bulkwrite operation to update the user's quest +We evaluate every quest that contain a goal that has just had it's status change. +For each one we create a bulkwrite operation to update the user's quest progress. -If the user has newly achieved a quest, a Redis Event +If the user has newly achieved a quest, a Webhook Event is emitted, which could be hooked into by our Discord Bot. If the progress has changed at all, the data is diff --git a/docs/docs/tachi-server/import/sessions.md b/docs/docs/codebase/import/sessions.md similarity index 70% rename from docs/docs/tachi-server/import/sessions.md rename to docs/docs/codebase/import/sessions.md index 18df19670..27f6b8b0c 100644 --- a/docs/docs/tachi-server/import/sessions.md +++ b/docs/docs/codebase/import/sessions.md @@ -33,20 +33,8 @@ This means that if there was more than two hours between any two successive scores, the session is marked as terminated and a new bucket of scores is created. -For every bucket of scores we have, we iterate over the -scores inside the bucket, and compare them against the -users' current PB. Since PBs are updated *after* sessions -are processed, this means we are processing them against -their past PBs. - -!!! bug - If a user ends up making sessions in the past, - this step will compare against their current PBs - - instead of the PBs they would've had at the time - of that session. - -Once we have compared all the scores against the PBs, we -can create a session from it. We need to generate a random +Once we have gotten all the scores we +can create a session from them. We need to generate a random name (we have some stuff in `src/datasets` for this). We also need to calculate statistics for this session, @@ -62,3 +50,6 @@ then that score is appended to the session. get a score between two sessions such that there was now less than two hours between the first sessions last score and the second sessions first score. + + This is so difficult to properly resolve that Tachi simply + makes no attempt to stop it. diff --git a/docs/docs/tachi-server/import/ugs.md b/docs/docs/codebase/import/ugs.md similarity index 100% rename from docs/docs/tachi-server/import/ugs.md rename to docs/docs/codebase/import/ugs.md diff --git a/docs/docs/codebase/index.md b/docs/docs/codebase/index.md new file mode 100644 index 000000000..36d231b31 --- /dev/null +++ b/docs/docs/codebase/index.md @@ -0,0 +1,43 @@ +# Codebase Overview + +This part of the documentation is for the [Tachi-Server](https://github.com/TNG-dev/Tachi/tree/staging/server) codebase. + +## Codebase Documentation vs. Code Documentation + +This is documentation for the **Codebase**. **NOT** documentation for the code. + +The distinction is because we aren't writing a library here - there's no need to document function +signatures or what function calls are meant to do. That can all be done inline because no other +projects depend on our function calls! + +This documentation is more meta-level. Why things are in certain folders, what certain enums +correspond to, how `thing` works, etc. + +## Repos and Licenses + +Tachi is a monorepo, and is made up of many projects. These are: + +- `client/`, Which is a React frontend for Tachi. + +The client and the server are fairly decoupled. Someone could trivially create their own frontend client for Tachi. + +- `server/`, Which is an Express-Typescript backend for Tachi. + +This contains all of our API calls, and interfaces with our database, and powers the actual score import engine. + +- `database-seeds/`, Which is a git-tracked set of data to be synced with Tachi. + +**This is the source of truth for the songs, charts, and more on the site!** +By submitting PRs to this, you can fix bugs on the website, add new charts, and more. + +- `bot/`, Which is a discord bot frontend for Tachi. + +- `common/`, Which contains common types, utils and functions shared between all other packages. + +This is also published to NPM when it hits production. + +- `docs/`, Which contains Tachi documentation. + +- `sieglinde/`, Which contains our BMS/PMS analysis functions. + +Of these, `server/` and `client/` are licensed under the AGPL3. The `database-seeds/` are licensed under the unlicense, and everything else is MIT. \ No newline at end of file diff --git a/docs/docs/tachi-server/infrastructure/api-clients.md b/docs/docs/codebase/infrastructure/api-clients.md similarity index 100% rename from docs/docs/tachi-server/infrastructure/api-clients.md rename to docs/docs/codebase/infrastructure/api-clients.md diff --git a/docs/docs/codebase/infrastructure/branches.md b/docs/docs/codebase/infrastructure/branches.md new file mode 100644 index 000000000..b22e22f0b --- /dev/null +++ b/docs/docs/codebase/infrastructure/branches.md @@ -0,0 +1,25 @@ +# Branching Model + +`Tachi` maintains two long-running branches, and uses a remarkably simple model for merging. + +***** + +## Branches + +### `release/v2.x` + +This is the current release version of `Tachi`, and is automatically deployed into production. + +!!! note + Our CI automatically selects the largest value of `x` to use as the production + branch. + +### `staging` + +This is the development branch, and is where pull requests +are merged to. This will be automatically deployed to the Tachi staging servers +for further testing. + +## How should I PR? + +You should submit your PRs for `staging`. If this change should be backported into production, note that in your PR. diff --git a/docs/docs/tachi-server/infrastructure/database-seeds.md b/docs/docs/codebase/infrastructure/database-seeds.md similarity index 100% rename from docs/docs/tachi-server/infrastructure/database-seeds.md rename to docs/docs/codebase/infrastructure/database-seeds.md diff --git a/docs/docs/tachi-server/infrastructure/file-flow.md b/docs/docs/codebase/infrastructure/file-flow.md similarity index 100% rename from docs/docs/tachi-server/infrastructure/file-flow.md rename to docs/docs/codebase/infrastructure/file-flow.md diff --git a/docs/docs/tachi-server/infrastructure/logging.md b/docs/docs/codebase/infrastructure/logging.md similarity index 100% rename from docs/docs/tachi-server/infrastructure/logging.md rename to docs/docs/codebase/infrastructure/logging.md diff --git a/docs/docs/tachi-server/infrastructure/oauth2.md b/docs/docs/codebase/infrastructure/oauth2.md similarity index 100% rename from docs/docs/tachi-server/infrastructure/oauth2.md rename to docs/docs/codebase/infrastructure/oauth2.md diff --git a/docs/docs/tachi-server/setup/config.md b/docs/docs/codebase/setup/config.md similarity index 95% rename from docs/docs/tachi-server/setup/config.md rename to docs/docs/codebase/setup/config.md index a7a6d1c79..1f52f49aa 100644 --- a/docs/docs/tachi-server/setup/config.md +++ b/docs/docs/codebase/setup/config.md @@ -176,7 +176,7 @@ used as the privateKey and the certificate, respectively. - Type: String - Default: Null -If present, and a string, this points to the webpack dev server for a react app. Having this +If present, and a string, this points to the local dev server for a react app. Having this option set results in CORS being enabled for *that* specific URL. This is useful for local development, but should not be used in production. @@ -212,7 +212,8 @@ If true, all `OPTIONS` requests to the server will return `200`, no matter what. If true, an external worker process will be used to handle score imports. !!! warning - You have to run this worker yourself. The entry point is `src/lib/score-import/worker/worker.ts`. + You have to run this worker yourself. `pnpm build && pnpm start-score-worker` will + run the external worker process. ### EXTERNAL_SCORE_IMPORT_WORKER_CONCURRENCY @@ -221,16 +222,6 @@ If true, an external worker process will be used to handle score imports. How many score imports one worker should be allowed to work on at a time. This improves parallelisation of score imports. -### USC_QUEUE_SIZE - -- Type: Integer -- Default: 3 - -How many unique players have to have a score on a chart for it to be de-orphaned. - -!!! warning - The lowest legal value for this field is 2. - ### BEATORAJA_QUEUE_SIZE - Type: Integer @@ -422,7 +413,7 @@ The process environment also contains necessary things for functional Tachi Serv !!! info These variables are put into the process environment instead of the conf.json5 file because - they're easier to change between docker instances. Helps us scale and deploy. + they're easier to change between docker instances. This Helps us scale and deploy. ### PORT diff --git a/docs/docs/tachi-server/structure/filesystem.md b/docs/docs/codebase/structure/filesystem.md similarity index 100% rename from docs/docs/tachi-server/structure/filesystem.md rename to docs/docs/codebase/structure/filesystem.md diff --git a/docs/docs/codebase/structure/style.md b/docs/docs/codebase/structure/style.md new file mode 100644 index 000000000..18d0e35ee --- /dev/null +++ b/docs/docs/codebase/structure/style.md @@ -0,0 +1,10 @@ +# Style + +For style, we use a custom tool called [Cadence](https://github.com/CadenceJS/Cadence). + +This is effectively a huge ESLint config with everything I care about enabled. + +It's worth noting that this is an *extremely strict* linter. You are **expected** to have "Format on Save" turned on, in order to use this linter properly. + +!!! info + You can run ESLint in the repo any time with `pnpm lint`. diff --git a/docs/docs/tachi-server/structure/testing.md b/docs/docs/codebase/structure/testing.md similarity index 100% rename from docs/docs/tachi-server/structure/testing.md rename to docs/docs/codebase/structure/testing.md diff --git a/docs/docs/tachi-bot/index.md b/docs/docs/tachi-bot/index.md deleted file mode 100644 index 110d7e8d5..000000000 --- a/docs/docs/tachi-bot/index.md +++ /dev/null @@ -1,5 +0,0 @@ -# Bot Overview - -This part of the documentation is for the [Tachi-Bot](https://github.com/TNG-dev/Tachi/tree/staging/bot). - -At the moment, it's empty. I'll get around to filling it out soon. diff --git a/docs/docs/tachi-server/implementation-details/esd.md b/docs/docs/tachi-server/implementation-details/esd.md deleted file mode 100644 index 81ac70bba..000000000 --- a/docs/docs/tachi-server/implementation-details/esd.md +++ /dev/null @@ -1,102 +0,0 @@ -# ESD Implementation - -!!! info - ESD is not currently displayed or used anywhere in the site. In the future, it will - likely be used for rival comparisons, as it was in Kamaitachi v1. - -ESD uses the fact that [A binomial distribution can approximate a normal one](https://en.wikipedia.org/wiki/Central_limit_theorem) in order to derive an estimate for standard deviations. - -***** - -## Method Outline - -Our overview is as follows. We are given a percent and -judgement windows for a game. - -From that, we want to return the standard deviation that -would result in that percent - given that game's judgement -windows. - -### Deriving Percent From Standard Deviation - -We assume that the mean of the players hits is always 0. - -Then, we work backwards. With the knowledge of -the judgement windows for a game, we can estimate the -percent a standard deviation would typically give. - -We construct a distribution with a mean of 0 and a standard -deviation of S, and then see roughly where hits would -end up on that distribution. - -We multiply how many hits we'd *expect* to be within a -certain judgement window by the value of that judgement. - -So in our scenario of S standard deviation, we would -expect X% of hits to be between, say, -16.67 and +16.67 (IIDX's PGREAT window). - -!!! note - To calculate that percent we need to use the - cumulative distribution function. That is not covered - here, but guides are all over the internet. - -We can multiply that percent by the value of a PGREAT (100%). - -Then, we repeat for the great window at 50%, and so on. - -When we've summed all that up, we get an estimate of -the percent this standard deviation is worth. - -### Reversing That - -This is good, but this is backwards! - -Turns out, there's no algebraic way to reverse this function! - -So, let's do a little approximating. - -We can start with an ESD of 100, which is halfway between -the lowest ESD (0), and the highest (200). - -```ts -for (let i = 0; i < MAX_ITERATIONS; i++) { - const estimatedPercent = StdDeviationToPercent(judgements, estSD, largestValue); - - if (Math.abs(estimatedPercent - percent) < ACCEPTABLE_ERROR) { - return estSD; - } - - if (estimatedPercent < percent) { - maxSD = estSD; - } else { - minSD = estSD; - } - - if (estSD === (minSD + maxSD) / 2) { - // if it isn't moving, just terminate - break; - } - - estSD = (minSD + maxSD) / 2; -} -``` - -This code is then ran to approximate the standard deviation -needed to get a percent *like* the one we were given. - -`ACCEPTABLE_ERROR` is set to `0.001` by default. -`MAX_ITERATIONS` is set to `50`. - -With this, we can "intelligently brute force" standard deviations, -getting closer to our provided percent until its -within 0.001%. Then, we can return the standard deviation -we used to get that percent! - -!!! note - Performance of this is incredibly fast, while - "intelligently brute forcing" isn't ideal, 50 - iterations are almost never hit, and most ESDs - are calculated in about 10 iterations. - - All of this happens in significantly under 1 milisecond, - so it is not exactly a significant performance hit. diff --git a/docs/docs/tachi-server/implementation-details/score-id.md b/docs/docs/tachi-server/implementation-details/score-id.md deleted file mode 100644 index 848870a69..000000000 --- a/docs/docs/tachi-server/implementation-details/score-id.md +++ /dev/null @@ -1,52 +0,0 @@ -# Score ID implementation - -Score IDs exist to dedupe scores when a user re-submits -the same scores. This happens frequently with `file/` -and `api/` [Import Types](../import/import-types.md), -as they typically resubmit the same scores. - -If a user only got a score once, we don't want to store -it twice. - -***** - -## Hashing - -The score ID is created by hashing the following template string: - -```ts -`${userID}|${chartID}|${dryScore.scoreData.lamp}|${dryScore.scoreData.grade}|${dryScore.scoreData.score}|${dryScore.scoreData.percent}` -``` - -with SHA256. - -This is then prefixed with `R`, and returned. - -## Why those fields? - -Games do a lot of wacky things. We'd rather not discard -non-duplicates, so we're a little more strict than we -maybe should be. - -UserID and chartID are so we don't accidentally collide -with other user's scores or charts. - -Score and Lamp make sense - since if those change we no -longer really have the same score. - -Grade and Percent are the slightly more strict ones. Some -games have very strange external grade requirements, such -as osu!standard enforcing a Full Combo for an S rank. - -Percent may not correlate with score. This was in anticipation -to support BMS's `#RANDOM` instruction - where charts may -have a dynamic amount of notes - but I've decided it -wasn't worth it. Still, this is left in for futureproofing. - -!!! warning - An open issue exists to refactor how score IDs work. - - It would make significantly more sense for every game to get its own scoreID algorithm, - since not all games have the same significant properties. - - This would also let us support things like SDVX6's EX Score. \ No newline at end of file diff --git a/docs/docs/tachi-server/implementation-details/statistics.md b/docs/docs/tachi-server/implementation-details/statistics.md deleted file mode 100644 index cbb4f4b6e..000000000 --- a/docs/docs/tachi-server/implementation-details/statistics.md +++ /dev/null @@ -1,178 +0,0 @@ -# Statistic Implementation - -This page documents the maths behind various algorithms -in Tachi. - -***** - -## BPI - -Our implementation of Poyashi BPI is leveraged from [here](https://github.com/potakusan/iidx_score_manager/blob/f21ba6b85fcc0bf8b7ca888fa2239a3951a9c9c2/src/components/bpi/index.tsx#L120). - -To be honest, I do not really understand *why* BPI looks -like this. I couldn't justify basically any line of this -function, nor any of the magic numbers it references. - -## MFCP - -MFCP is implemented as follows. - -If the score is not an MFC, it is worth `null`. - -If the score on a BEGINNER or BASIC chart, it is worth `null`. - -If the level of the chart is worth less than 8, it is worth `null`. - -Else, it follows this table: - -| Levels | MFCP | -| :: | :: | -| 8, 9, 10 | 1 | -| 11, 12 | 2 | -| 13 | 4 | -| 14 | 8 | -| 15 | 15 | -| 16, 17, 18, 19, 20 | 25 | - -## VF6 - -VF6 is calculated as follows. - -The grade of the score is converted into a coefficent -according to this table. - -```ts -const VF5GradeCoefficients = { - S: 1.05, - "AAA+": 1.02, - AAA: 1.0, - "AA+": 0.97, - AA: 0.94, - "A+": 0.91, // everything below this point (incl. this) is marked with a (?) in bemaniwiki. - A: 0.88, - B: 0.85, - C: 0.82, - D: 0.8, -}; -``` - -Lamps are converted into a coefficent similarly. - -```ts -const VF5LampCoefficients = { - "PERFECT ULTIMATE CHAIN": 1.1, - "ULTIMATE CHAIN": 1.05, - "EXCESSIVE CLEAR": 1.02, - CLEAR: 1.0, - FAILED: 0.5, -}; -``` - -Then, we perform the following calculation: - -$$ -f(l, p, c1, c2) = 2l * p * c1 * c2 * 0.01 -$$ - -Where L is the chart's level, P is the percent of the score, -C1 is the grade coefficent, and C2 is the lamp coefficient. - -For VF6, this result is returned floored to 3 decimal places. - -For VF5, this result is returned floored to **2** decimal places. - -## KtRating - -KtRating is a generic exponential algorithm that takes -three tunable parameters. These are changed depending -on the game that uses this algorithm. - -!!! note - This is not meant to be a perfect algorithm for all - scenarios. It's meant to cover for games that don't - have a sensible default rating algorithm. - -The three parameters are as follows: - -| Parameter | Description | -| :: | :: | -| `pivotPercent` | The percent below which a score is considered a 'fail', and should be negatively punished. | -| `failHarshnessMultiplier` | By how much fails should be punished. A higher value implies fails are worth less. | -| `clearExpMultiplier` | How much to exponentially reward clears. A higher value implies that timing at the highest level is more difficult. | - -If the score's percent is below the `pivotPercent`, the -fail calculator is invoked, which uses the following -function: - -```ts - percentDiv100 ** (parameters.failHarshnessMultiplier * levelNum) * - (levelNum / parameters.pivotPercent ** (parameters.failHarshnessMultiplier * levelNum)) -``` - -In mathematical notation, this is represented as: - -$$ -f(x, f, l, c) = \left(x^{f}\cdot\left(\frac{l}{c^{f}}\right)\right) -$$ - -Where X is the percent divided by 100, -F is the `failHarshnessMultiplier` multiplied by L, -L is the level of the chart and -C is the `pivotPercent`. - -If the score is above the `pivotPercent`, then the -clear calculator is invoked, which uses the following -function: - -```ts - Math.cosh( - parameters.clearExpMultiplier * levelNum * (percentDiv100 - parameters.pivotPercent) - ) + - (levelNum - 1); -``` - -In mathematical notation, this is expressed as: - -$$ -f(x, c, l) = \cosh\left(n\left(x-c\right)\right)+l-1 -$$ - -Where X is the percent divided by 100, -L is the level of the chart and -C is the `pivotPercent`. - -Cosh is used because it's essentially e^x, and easier -to work with in this (contrived) scenario. - -!!! note - L will be replaced with the timing difficulty - declared for that chart in the tierlists. - - If one does not exist, it will fall back to - the provided level as a number. - -## KtLampRating - -Unlike KTRating, KTLampRating is a fairly sensible -function. - -It has no tunable parameters. - -It starts by getting the tierlist information for the -given chart. - -If there is tierlist information, then we iterate -over the lamp ratings they declare. - -We select the largest lamp rating that the lamp meets -the requirements of. -This is to fix things like Hard Clears sometimes being worth -less than Normal Clears (especially in IIDX). This also -means that we safely fall back to lower values if the -users lamp was unsupported - i.e. a FULL COMBO will fall -down to an EX HARD CLEAR. - -If there is no tierlist data available for this chart, -we fall to a simple question of whether the score was -considered a clear or not. If it is, give the level -of the chart as a number as points. \ No newline at end of file diff --git a/docs/docs/tachi-server/index.md b/docs/docs/tachi-server/index.md deleted file mode 100644 index 8d7fae7f9..000000000 --- a/docs/docs/tachi-server/index.md +++ /dev/null @@ -1,23 +0,0 @@ -# Codebase Overview - -This part of the documentation is for the [Tachi-Server](https://github.com/TNG-dev/Tachi/tree/staging/server) codebase. - -## Codebase Documentation vs. Code Documentation - -This is documentation for the **Codebase**. **NOT** documentation for the code. - -The distinction is because we aren't writing a library here - there's no need to document function -signatures or what function calls are meant to do. That can all be done inline because no other -projects depend on our function calls! - -This documentation is more meta-level. Why things are in certain folders, what certain enums -correspond to, how `thing` works, etc. - -## Repos and Licenses - -Tachi is made up of four components: - -- `tachi-common`: Common types and values for Tachi. [GitHub](https://github.com/TNG-dev/Tachi/tree/staging/common). This is licensed under MIT. -- `tachi-server`: The API, IR implementations and 'business logic' behind Tachi. [GitHub](https://github.com/TNG-dev/Tachi/tree/staging/server). This is licensed under AGPLv3. -- `tachi-client`: The front-end code for Tachi. [GitHub](https://github.com/TNG-dev/Tachi/tree/staging/client). This is licensed under AGPLv3.. -- `tachi-docs`: The documentation you're reading right now! [GitHub](https://github.com/TNG-dev/Tachi/tree/staging/docs). This is licensed under MIT. diff --git a/docs/docs/tachi-server/infrastructure/branches.md b/docs/docs/tachi-server/infrastructure/branches.md deleted file mode 100644 index 596beec23..000000000 --- a/docs/docs/tachi-server/infrastructure/branches.md +++ /dev/null @@ -1,80 +0,0 @@ -# Branching Model - -`tachi-server` has a very specific branching model. - -Breaking the rules in this branching model will result -in your Pull Request being rejected. - -***** - -## Acknowledgements - -This branching model is identical to [A Successful Git Branching Model](https://nvie.com/posts/a-successful-git-branching-model/). - -You can read that post instead of this one, if you prefer, -but I will provide a simplified explaination below. - -## Static Branches - -These branches are always present, and never to be deleted. - -### `master` - -This is the current release version of `tachi-server`, and -is deployed onto production. - -This should never be committed to directly from anything other than `release-` branches. - -Every commit to master is a new release. - -### `develop` - -This is the development branch, and is where pull requests -are merged to. You should merge `issue-` branches with `develop`. - -## Ephemeral Branches - -These branches are intended to be created, merged, and then -deleted. - -### Feature Branches (`issue-`) - -These branches **may** start with the text `issue-`, and are -followed by the issue number they reference. - -You may call these branches whatever you want, -except things like `master` or `develop` or `hotfix-` etc. - -Our convention is to use `issue-` - -These are forked from `develop`, and should be merged -with `develop`. - -!!! note - Despite the name `issue-`, these branches aren't - bug fixes. They are anything that fix an Issue - on the repository. - - This means these are also feature branches, and other - things. - -### `hotfix-` - -These branches start with the text `hotfix-` and are followed by the issue number they fix. - -These are forked from `master`, and should be merged with -both `master` and `develop`. - -### `release-` - -These branches start with the text `release-` and are -followed by the Major.Minor version of the release. - -These are typically the final draft for a new release. - -These are created when all the features we want in the -new release are present in develop, and we want to -get it ready for production. - -These are forked from `develop`, and should be merged with -`master`. diff --git a/docs/docs/tachi-server/infrastructure/toolchain.md b/docs/docs/tachi-server/infrastructure/toolchain.md deleted file mode 100644 index 336f5dc14..000000000 --- a/docs/docs/tachi-server/infrastructure/toolchain.md +++ /dev/null @@ -1,106 +0,0 @@ -# Toolchain - -This page documents all the various tools involved with -`tachi-server` and why they were chosen specifically. - -A lot of this is personal preference, but hopefully the -justifications provide enough insight into why. - -***** - -## Programming Language: TypeScript - -Everything is wrote in [TypeScript](https://www.typescriptlang.org). TypeScript provides -substantial usability improvements for JS at scale, -and integrates very nicely with IDEs. - -## Package Manager: pnpm - -`pnpm` is the preferred package manager for all `tachi-` projects. - -It can be acquired with `npm install -g pnpm`, and documentation can be found [here](https://pnpm.io). - -### Why not `npm`? - -`npm` duplicates packages across my whole system, and it takes up a lot of package space. Furthermore, each new install requires new network calls, and my internet is pretty poor. - -`pnpm` dedupes packages everywhere, and pulls packages from the local cache if possible. - -It's also way faster. The developers behind `pnpm` have seriously -put a lot of work into it, and it's a great tool. - -## Main Database: MongoDB - -We use [MongoDB](https://mongodb.com) because it works nicely with JSON, and has -good support for Javascript and TypeScript. - -!!! tip - If you're debugging MongoDB state, I cannot recommend - [MongoDB Compass](https://www.mongodb.com/products/compass) enough. It's one of the best ways to interact - with Mongo and analyse queries. - -### Why not SQL? - -We take advantage of the more dynamic nature of MongoDB documents in some places to get away with writing less code, -which means a port to SQL is non-trivial. - -The document model also meshes nicer with the rest of the codebase. - -!!! note - We also use Redis for sessions and inter-process communication. - -## Database Driver: Monk - -[Monk](https://github.com/automattic/monk) is a simple wrapper around the MongoDB native NodeJS Driver. Although it's a simple wrapper, it is a massive -UX improvement over the native wrapper. - -### Why not Mongoose? - -Mongoose mandates schemas, and works with a much more OOP-style -approach to handling documents. I personally think it's an unecessary -abstraction for this project specifically, but is a great library nonetheless. - -## Logger: Winston - -We use the [Winston](https://github.com/winstonjs/winston) framework to provide logging to the application. You can read more about this in [Logging](./logging). - -!!! note - Winston seems to be an abandoned project. Furthermore, - the level of documentation is questionable at best. - - Despite this, Winston is an incredibly mature and - stable logging framework, and is a good fit for - our project as a result. - -## HTTP Client: Express - -[Express](https://github.com/expressjs/express) is a battle tested HTTP framework for NodeJS, and -the middleware chain is incredibly useful. - -### Why Not Koa or TinyHTTP? - -It lacks a lot of the ecosystem Express has - mainly -with regards to testing. - -## Test Runner: TAP - -[Node TAP](https://node-tap.org) is an opinionated testing framework by the guy -who made NPM. - -The opinionated parts of it line up exactly with what I -want from a testing library, and it has batteries included for everything I want. - -This also handles our coverage reports by calling `nyc` -and automatically uploads to CodeCov with our CI setup. - -## CI-CD: Github Actions - -Handles our deployment and test running. We also integrate -with CodeCov for the coverage reports. - -## Documentation: MKDocs + MKDocs Material - -Our documentation is wrote with MKDocs with Material -as a theme. MKDocs Material is a beautiful theme, and -writing documentation in markdown is incredibly intuitive -(Unlike sphinx's RST, which is almost impossible to follow.) \ No newline at end of file diff --git a/docs/docs/tachi-server/infrastructure/versions.md b/docs/docs/tachi-server/infrastructure/versions.md deleted file mode 100644 index e92633506..000000000 --- a/docs/docs/tachi-server/infrastructure/versions.md +++ /dev/null @@ -1,51 +0,0 @@ -# Versioning - -***** - -## Semver - -`tachi-server` and `tachi-client` follow [Semantic Versioning](https://semver.org). - -The two repositories have completely separate versioning, but generally will move in -lockstep with one-another. - -!!! example - For quick reference, MAJOR, MINOR and PATCH correspond as follows: - - ```2.3.1 -> MAJOR.MINOR.PATCH``` - -## `master` Branch - -Every push to `master` must involve a change to the versioning of that repository. If -it's a hotfix, it should bump the PATCH version. If it's a feature, it should update the -MINOR version. - -The MAJOR version will only be bumped if there are significant changes to almost everything, -which I don't anticipate. - -## Version Names - -`tachi-server` and `tachi-client` have version names that change in correspondence with -their `MINOR` versions. - -The version names follow the song titles of an album. In `tachi-server`'s case, it follows -[Portishead - Dummy](https://en.wikipedia.org/wiki/Dummy_(album)) - -For `tachi-client`, we follow [The Cure - Disintegration](https://en.wikipedia.org/wiki/Disintegration_(The_Cure_album)) - -### Why? - -Versions following album songs saves the hassle of having to come up with nice sounding version -names. It's also an excuse to show off albums I really like. - -As for why these specific albums: - -Portishead's Dummy was chosen because Tachi V1 followed -[Massive Attack - Mezzanine](https://en.wikipedia.org/wiki/Mezzanine_(album)). The two albums -are both standout albums in the same genre, so it was fitting to follow it up. - -The Cure's Disintegration was chosen because I wanted another album with -the same amount of tracks as Dummy, and it's also a great album. - -!!! tip - These albums are great. Do yourself a favour and check them out. diff --git a/docs/docs/tachi-server/structure/style.md b/docs/docs/tachi-server/structure/style.md deleted file mode 100644 index 799b80886..000000000 --- a/docs/docs/tachi-server/structure/style.md +++ /dev/null @@ -1,148 +0,0 @@ -# Style - -This document explains the code style rules for the repo. - -Please make sure any changes you commit adhere to the defined linting rules. - -For style, we use ESLint + Prettier. We have some options set for prettier, and a lot of ESLint rules set. - -Since it's redundant (You should just read the eslint docs), and mostly just personal preference, each individual rule isn't going to be explained here. - -ESLint is set up to automatically perform all of these changes when ran. - -!!! info - You can run ESLint in the repo any time with `pnpm lint` or `eslint ./src --ext .ts --fix`. - -***** - -## Prettier Rules - -- Tab Indenting. - -I'd prefer to use tabs, honestly, but Prettier and JSDoc like to align things with spaces and it messes with them. - -It turns out there's only one rare scenario where prettier mixes tabs and spaces (Rare as in, it happens -once in an obscure place in the entire codebase), so we've switched to tabs. - -- Semicolons. - -No-Semicolons causes issues with IIFEs. - -- Try to keep things under 100 characters. - -Absolutely **DO NOT** insert random line breaks to keep stuff under 100 characters. It's fine for things to go a bit over. -Seriously, your editor is definitely capable of wrapping text if it goes too far. - -Prettier has its own opinions on where these line breaks should happen, just trust them. - -- Double Quotes instead of Single Quotes - -JSON does it and that's pretty much the only reason why. - -- Line Break is LF, **not CRLF** - -Your editor will handle this properly. If it does not -automatically set, check the bottom right of your editor. -For Atom, VSCode and most others it will let you switch between -CRLF and LF. - -## Commenting Style - -The Tachi-Server codebase handles code comments against these rules: - -1. Do not write redundant comments. -2. Return signatures are obvious from TypeScript most of the time. If they aren't, declare them with TypeScript rather than commenting them. -3. Most of the time, comments should describe *why* code does something. -4. The exception is when a line or function **needs** to do something complicated - in which case, comments can be *how*. -5. That said, code should be self-explanatory. -6. If it isn't, it should be refactored until it is. -7. If that isn't possible (The logic is complex for good reason), the documentation should be next to the code. - -### Example - -For this example, we're going to write a function for calculating -[Standard Deviation](https://en.wikipedia.org/wiki/Standard_deviation). - -```ts -function sd(arr: number[]) { - const m = arr.reduce((a, r) => a + r) / arr.length; - - return Math.sqrt(arr.reduce((a, r) => (r + m) ** 2) / arr.length); -} -``` - -This is bad code. Very bad code. It is not at all clear what -this code does from any of the variable names, and the function signature barely helps. - -First, let's try and make this code more self documenting. - -We'll give everything proper variable names, and then -expand the second `reduce` call into a simpler for loop.[^1] - -```ts -function CalculateStandardDeviation(dataset: number[]) { - const mean = arr.reduce((a, r) => a + r) / dataset.length; - - let variance = 0; - - for (const value of dataset) { - variance += (value - mean) ** 2; - } - - return Math.sqrt(variance / dataset.length); -} -``` - -Now that is much cleaner, but it's still not perfect. - -For example, what if someone didn't know what the standard deviation was? -Standard Deviation is *external knowledge*, and as such, we should document what this is. - -The lazy way, is just this: -```ts -/** - * @see {@link https://en.wikipedia.org/wiki/Standard_deviation} - */ -function CalculateStandardDeviation(dataset: number[]) { - // ... -} -``` - -This is a perfectly acceptable way to document code. We -don't need to describe any returns or even the function here. As always, use your head to judge whether something needs external knowledge or not. - -## Comment Directives - -Tachi uses comment directives to allow for quick searching -of certain comments. Directives are used like this: - -```ts -// @todo A bug with the foo is here. -1 + 1; -``` - -The list of directives and their meaning is here: - -| Directive | Description | -| :: | :: | -| `@todo` | This line has a bug or something unfinished that needs to be done. A GitHub issue should be created about this. | -| `@danger` | The below code is dangerous, and if called wrong could cause issues. This directive should make the programmer act cautiously around the code before making changes. | -| `@hack` | A hack is used here. | -| `@optimisable` | This code is optimisable, and how to optimise it is known, but has not been implemented yet. | -| `@inefficient` | This code is slow, and how to optimise it is not figured out yet. | - -!!! info - TypeScript also uses comment directives for things - like `@ts-expect-error`. What these do is documented - in [TypeScript's documentation](https://www.typescriptlang.org). - -## Strictness - -Don't worry about this too much, At the end of the day, as -long as the code is understandable and the linter is happy, -it's good. - -[^1]: JS's ES6 array methods are the devil if used improperly. For some reason, lots of people in -react and react-adjacent scenes seem to love (ab)using these array methods for everything. Complex -`reduce` operations should always be turned into a `for loop`, and that's to say nothing of my opinions -on `forEach`. diff --git a/docs/docs/wiki/pbs-scores.md b/docs/docs/wiki/pbs-scores.md index 2f6db0e08..db5e63fb4 100644 --- a/docs/docs/wiki/pbs-scores.md +++ b/docs/docs/wiki/pbs-scores.md @@ -1,4 +1,5 @@ # What's the difference between a PB and a Score? + A PB is all of your best scores on that chart joined together. In most games, this means joining your best score with your best lamp. diff --git a/docs/docs/wiki/rules.md b/docs/docs/wiki/rules.md index 213dbd25d..7eb8356dc 100644 --- a/docs/docs/wiki/rules.md +++ b/docs/docs/wiki/rules.md @@ -75,8 +75,11 @@ The valid input devices are listed below. | :: | :: | :: | | unnamed_sdvx_clone (Controller) | Any Arcade Size Controller | The controller leaderboards are **exclusively** for Arcade **SIZE** controllers. **POCKET VOLTEXES DO NOT COUNT AS ARCADE SIZE CONTROLLERS.** | | unnamed_sdvx_clone (Keyboard) | Keyboard | The keyboard leaderboards are **exclusively** for keyboard players. You must not play on these leaderboards with a controller! | -| BMS (7K) | Keyboard, any IIDX Controller | Keyboard play is allowed for BMS 7K. | -| BMS (14K) | any two IIDX Controllers | Keyboard play is **NOT** allowed for BMS 14K, due to some ridiculous advantages. | +| PMS (Controller) | Any Arcade Size Controller | The controller leaderboards are **exclusively** for Arcade **SIZE** controllers. **MINI POP'N CONTROLLERS DO NOT COUNT AS ARCADE SIZE CONTROLLERS.** | +| PMS (Keyboard) | Keyboard | The keyboard leaderboards are **exclusively** for keyboard players. You must not play on these leaderboards with a controller! | +| ITG (Stamina) | Standard Pad | Any pad that you step on with your feet is probably fine. **DO NOT SUBMIT KEYBOARD SCORES.** | +| BMS (7K) | Anything | Do whatever. | +| BMS (14K) | Anything | Do Whatever. | !!! info As always though, exercise some common sense. I'm not going to whitelist controllers on here, because that's asking for trouble. Use your head as for whether something is fair or not, and if you're not certain still, ask in the discord. diff --git a/docs/docs/wiki/score-oddities.md b/docs/docs/wiki/score-oddities.md index 6f28a0057..424007a84 100644 --- a/docs/docs/wiki/score-oddities.md +++ b/docs/docs/wiki/score-oddities.md @@ -62,7 +62,7 @@ into their original form, changing the difficulty from ANOTHER to LEGGENDARIA. ## Deduplication False Positives (All Games) -Tachi identifies scores using a `scoreID`. This is calculated from some properties on the score, such as who got it (`userID`), their `score`, `percent` and `lamp`. +Tachi identifies scores using a `scoreID`. This is calculated from some properties on the score, such as who got it (`userID`) and the various metrics for this game (`score`, `percent`, `lamp`, etc.) `scoreID`s are *unique* across all of Tachi. The motivation for this is to avoid score duplication. diff --git a/docs/docs/wiki/stats/esd.md b/docs/docs/wiki/stats/esd.md deleted file mode 100644 index 9a33d9d89..000000000 --- a/docs/docs/wiki/stats/esd.md +++ /dev/null @@ -1,56 +0,0 @@ -# What is ESD? - -ESD is short for "Estimated Standard Deviation". - -Standard deviation is a way of measuring how *dispersed* -data is. [Wikipedia](https://en.wikipedia.org/wiki/Standard_deviation) explains what this is and why it works. - -!!! warning - ESD is not publically displayed on Tachi at the moment for any game. - This page is technically redundant. - -***** - -## Motivation - -Standard Deviation is an interesting statistic for rhythm -games, as the dispersion of a users hits is quite interesting. - -That is, if a user is hitting all over the place timing wise, -they probably aren't doing very good. Vice versa, -if a user has incredibly close together hits timing wise, -they're probably good. - -ESD let's us derive standard deviation from just the *percent* -of a score, and the judgement windows for a game! - -It's surprisingly accurate, and is a useful statistic -to derive other statistics from. - -!!! note - Only some games support ESD. These are only games that - have strict hit windows that correlate perfectly with - percent. - - An example would be IIDX, where percent is only - derived from EX Score, and hit windows are constant. - - BMS cannot support ESD as it has dynamic hit windows, - depending on the chart. - - GITADORA cannot support ESD as it's percent is influenced - by combo-based scoring. - - SDVX cannot support ESD because things like holds - count as multiple repeated hits, but you do not have - to time those repeated hits! - - In short, one hit needs to correspond to one judgement, - and every hit has to involve a timing window. - -For details on the implementation of ESD, you can see -[here](../../tachi-server/implementation-details/esd.md). - -!!! warning - Implementation details about ESD require some - external knowledge about statistics and distributions. \ No newline at end of file diff --git a/docs/docs/wiki/stats/tachi.md b/docs/docs/wiki/stats/tachi.md deleted file mode 100644 index fea25f52c..000000000 --- a/docs/docs/wiki/stats/tachi.md +++ /dev/null @@ -1,605 +0,0 @@ -# Tachi Statistics - -Tachi has a lot of statistics involved. This page documents all of the -statistics used for each game, and what they mean. - -!!! note - These explainations brush over the technical details a bit. If you're interested - in that, you might want to see the [Implementation Details](../../tachi-server/implementation-details/statistics.md). - -***** - -## Statistics Overview - -Tachi needs statistics in three main places. The first -place is on each *individual* score. - -As an example, if I get a score, it should have a rating -attached onto it. - -The second place it needs a rating algorithm is for -a user's profile. This is likely to be combined from -individual score statistics, either by averaging or -totalling. - -The third place it needs a rating algorithm is for -a user's sessions. This is also likely to be combined -from individual score statistics. - -## Score Statistics - -The below statistics apply to individual scores. - -!!! note - Every game needs to have a default rating algorithm. A default rating - algorithm needs to work well on all scores from all skill levels. - - This means that some more 'top player' oriented statistics cannot be the default. - - Some games might not have *any* rating algorithms. If we dont have - a good built-in contender for a default rating algorithm, we will have to invent our own. - -***** - -### BPI (IIDX) - -Pros: - -- Estimates timing difficulty for a chart reasonably. -- Is an understood standard by Kaiden and Post-Kaiden players. - -Cons: - -- Depends on Kaiden Average and World Record, which can be highly fluctuative. -- Not able to accurately cross-compare values (i.e. 20BPI on one song is often not equivalent in 'skill'[^1] to 20BPI on another.) -- Only practically works on 12s. It can be extended to 11s, but it doesn't work as well. It does not work at all below 11. - -There are two implementations of BPI - We'll call them Nori BPI and Poyashi BPI, Tachi uses the -more recent Poyashi BPI. - -!!! info - As mentioned above in the cons of BPI, Kaiden Average and WR can be rather unreliable estimates - of difficulty. When HV came out, 120hz caused almost all WRs to jump up significantly, which - pretty much completely broke Nori's BPI. To fix this, some parameters were adjusted for - poyashi's BPI, which is implemented [here](https://bpi.poyashi.me). - Namely, reducing the impact of high WRs on average BPI. - - As a consequence, Poyashi BPI is significantly easier than Nori BPI in every circumstance. - Whether this is an issue or not is up to you, but I think most players enjoy seeing the - larger number. - -BPI looks at the kaiden average and world record for a chart, and constructs an exponential -graph between the two points. A BPI of 0 is equivalent to Kaiden Average, a BPI of 100 is -equivalent to the world record. - -!!! info - For scores less than the Kaiden Average, Poyashi BPI uses a negative extension that caps - at -15. This capping decision appears to be arbitrary. - - For Nori BPI, scores less than the Kaiden Average become [Complex Numbers](https://en.wikipedia.org/wiki/Complex_number). - -BPI is intended for use for kaidens and players significantly beyond kaiden. For that, it works -decently. As mentioned above in the cons, BPI is not very cross-comparable. 20BPI on one song -is not necessarily as good as 20BPI on another. - -!!! example - At the time of writing, 20BPI on Verflucht Leggendaria is AAA+66. 20BPI on FAKE TIME is also AAA+66. - Experienced players will notice a problem here. - -***** - -### ktLampRating (IIDX) - -- **Default for IIDX SP and IIDX DP** - -Pros: - -- Corresponds 100% with well-agreed-upon tierlists -- Works on all charts, and uses tierlists down to SP10. - -Cons: - -- Does not support additional points for Full Combos (will give EXHard points) or Easy Clears (will give 0). - -KtLampRating (Kamaitachi Lamp Rating) is a generic algorithm that gives points based -on the quality of the lamp. - -This is entirely done with tierlists that are converted into decimal form. So, if a chart -is marked as 11.3 for Hard Clear, HCing it will give 11.3 points. - -BP is not taken into account, and the only lamps that give rating are Normal, Hard and EXHard. - -!!! note - In the scenario where, say, a NC is worth more than a HC, HCing the chart will give the NC rating. - - This also applies to EXHCs. - -***** - -### VF6 (SDVX, USC) - -- **Default for SDVX and USC** - -Pros: - -- Built-in to the game, and understood by all players. -- Unlike VF4, doesn't massively reward fails on high level charts -- Unlike VF5, has more than 33 unique values. - -Cons: - -- Values on an individual score are small decimals, which can be difficult to parse. - -VF6 (Volforce 6) is the Volforce algorithm used in SDVX 6. -This algorithm is identical to VF5, but with the addition -of another decimal place. This fixes a long standing issue -with VF5 where there were only 33 possible values for a -given score, which made the function painfully discrete. - -!!! note - VF5 and VF4 are deprecated in Tachi, and not displayed - anywhere. - - VF5 is deprecated because VF6 is strictly better. - - VF4 is deprecated because it's 4 years old at this - point, and its flaws make it incredibly abusable. - -***** - - - -### KtRating (MÚSECA) - -- **Default for MÚSECA** - -Pros: - -- Works on all scores. -- Doesn't reward weak passes on high rated charts. -- Takes tierlists into account. - -Cons: - -- Not well understood by players. - -We need a generic rating algorithm for MÚSECA's scores. - -MÚSECA's built-in CURATOR RANK is built in a similar -vein to Volforce, but is broken by some -questionable chart rating decisions. This makes it -undesirable for score comparison. - -This is the KTRating algorithm, with parameters tuned for MÚSECA. - -Scores below 900k are heavily nerfed. Timing is rewarded -significantly. Clear type is ignored. - -***** - -### Sieglinde (BMS, PMS) - -- **Default for BMS 7K, BMS 14K, and PMS** - -Pros: - -- Derives how difficult a lamp is to get by scores on LR2IR. -- An update to walkure without certain vulnerabilities. -- Only gives rating for popular tables. - -Cons: - -- Does not support Groove Clears, EX Hard Clears or FCs. -- Likely contentious for individual difference stuff. (What algorithm isn't!) - -Sieglinde is a modern implementation of Walkure which -aims to fix a couple of issues with Walkure as an -individual score rating algorithm. - -It uses data from IRs to derive an EC and HC value for -a chart, which is then given if you get EC or HC -respectively. - -!!! note - Due to poor data on LR2IR, and complete lack of support - in LR2, Groove Clears and EX Hard Clears will be - treated as Easy Clears and Hard Clears, respectively. - - Full Combos are similarly removed due to incredibly - poor data. Grinding Full Combos on low level insane - charts was the best way to raise your walkure! - -!!! info - For 7K, the currently supported tables are: - - - Insane1 - - Insane2 - - Normal1 - - Normal2 - - Satellite - - Stella - - Overjoy - - For 14K, no tables are currently supported. - -***** - -### Rating (CHUNITHM) - -- **Default for CHUNITHM** - -Pros: - -- Built-in to the game. -- Universally understood by players. - -Cons: - -- ??? - -Rating refers to CHUNITHM's built-in rating algorithm. -This takes into account an internal tierlist, and looks -at the accuracy of the provided score. - -!!! note - I don't play CHUNITHM, so I don't really know the - implementation flaws of this algorithm for individual - scores. - -***** - -### Skill (Gitadora) - -- **Default for GITADORA** - -Pros: - -- Built-in to the game. -- Universally understood by players. - -Cons: - -- Algorithm is very naive, and doesn't increase exponentially with respect to accuracy. - -Gitadora's Skill algorithm is the in-game algorithm -for determining a players 'skill' on a given chart. - -The algorithm itself is incredibly simple, and likely -breaks horribly for a lot of scenarios, but, it works -decently for what it is. - -!!! note - Unlike almost every other algorithm on this page, - the GITADORA Skill algorithm can be expressed in a single line. - - $$ - f(p,l) = 0.01p * 20l - $$ - - Where P is the users percent, and L is the level - of the chart. - -***** - -### Rate (WACCA) - -- **Default for WACCA** - -Pros: - -- Built-in to the game. -- Generally well-understood by players. -- Uses the game's internal precise chart rating. - -Cons: - -- Uses wide cutoffs that don't reward AMs over SSS+. - -Although a better score statistic could be implemented, Rate -is preferred since it is well-understood and can get the job -done. - -***** - -## Profile Statistics - -This section explains the statistics on a user's profile. -These are almost always derived from individual score -statistics as listed above. - -***** - -### IIDX - -The below statistics apply to SP and DP. - -- BPI - -The average of your highest 20 BPIs. - -- KtLampRating **(Default)** - -The average of your highest 20 ktLampRatings. - -***** - -### SDVX, USC - -The below statistics apply to both SDVX and USC. - -- VF6 **(Default)** - -Your highest 50 VF6s added together. This has the benefit -of making VF6 not a small decimal, and is generally how -people talk about their volforce. - -***** - - -### MUSECA - -The below statistics apply to maimai and museca. - -- KtRating **(Default)** - -The average of your highest 20 ktRatings. - -***** - -### BMS, PMS - -- Sieglinde **(Default)** - -The average of your best 50 Sieglinde scores. - -***** -### CHUNITHM - -- Naive Rating **(Default)** - -The average of your highest 20 CHUNITHM Ratings. - -!!! warning - This is *different* to what you'd expect! CHUNITHM - has a built-in profile rating mechanism, but it has - some awful flaws with respect to losing rating after - playing poorly, and is generally *very* hard to - implement. - -***** - -### GITADORA - -- Skill **(Default)** - -Your profile skill as it appears in game. This is the -sum of all your skills on 50 HOT songs and 50 'NOT HOT' -songs. - -!!! info - This might be slightly different to your in-game - skill. This may be due to rerates or things like - certain songs no longer being hot. - -***** - -### WACCA - -- Rate - -The sum of your best 15 Rate scores from the most recent -version of the game, and your best 35 Rate scores from -previous versions of the game. - -- Naive Rate **(Default)** - -The sum of your best 50 Rate scores. - -!!! info - Your Rate should pretty much match what you see - in-game. However, the in-game Rate advantages players on - the newest version of the game by weighting those charts - much more heavily, which is kind of dumb. NaiveRate - should eliminate this bias. In practice, for active - players on the newest version of the game, these values - are usually quite similar. - -***** - -## Session Ratings - -This section explains the statistics on a session. -These are almost always derived from individual score -statistics as listed above. - -If a session has less than 10 scores, all of the below -statistics are marked as N/A except for MFCP. - -***** - -### IIDX - -The below information applies to SP and DP. - -- BPI, KtLampRating - -The average of your highest 10 values for that statistic. - -***** - -### SDVX, USC - -- VF6 - -The average of your highest 10 VF6's that session. - -- Profile VF6 **(Default)** - -The above statistic, but multiplied by 50. This is -to scale it up to what you'd normally see on a profile. - -The reason for this multiplication is that, generally, -people don't like dealing with decimals. Furthermore, -SDVX players generally talk about their profile volforce, -rather than their individual volforce. - - -***** - -### maimai, MÚSECA - -- KtRating **(Default)** - -The average of the highest 10 KtRatings that session. - -***** - -### BMS, PMS - -The below information applies to both 7K and 14K (and all of PMS). - -- Sieglinde **(Default)** - -The average of the highest 10 Sieglinde ratings that session. - -***** - -### CHUNITHM - -- Naive Rating **(Default)** - -The average of the highest 10 ratings that session. - -***** - -### GITADORA - -- Skill **(Default)** - -The average of the highest 10 skills achieved that session. - -!!! note - Unlike SDVX, we do not have a ProfileSkill here. - There's no good reason for this, though. - - If people want it, it can be added! Feel - free to report it as an issue if you think it - should be added. - -***** - -### WACCA - -- Rate **(Default)** - -The average of the highest 10 Rates achieved that session. - -[^1]: Skill is obviously very loosely used here. If there was a consensus on what "skill" something was, we would already have a perfect rating algorithm. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 41fe17737..a5d9449d1 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -48,10 +48,6 @@ nav: - "wiki/lamps.md" - "wiki/score-oddities.md" - - Statistics: - - "wiki/stats/tachi.md" - # - "wiki/stats/esd.md" - - Documents: - "schemas/index.md" - "schemas/user.md" @@ -94,60 +90,55 @@ nav: - "api/webhooks/main.md" - "api/webhooks/class-update-v1.md" - - Tachi Server Reference: - - "tachi-server/index.md" + - Codebase Reference: + - "codebase/index.md" - Setup: - - "tachi-server/setup/config.md" + - "codebase/setup/config.md" - Infrastructure: - - "tachi-server/infrastructure/toolchain.md" - - "tachi-server/infrastructure/logging.md" - - "tachi-server/infrastructure/branches.md" - - "tachi-server/infrastructure/versions.md" - - "tachi-server/infrastructure/api-clients.md" - - "tachi-server/infrastructure/oauth2.md" - - "tachi-server/infrastructure/file-flow.md" - - "tachi-server/infrastructure/database-seeds.md" + - "codebase/infrastructure/logging.md" + - "codebase/infrastructure/branches.md" + - "codebase/infrastructure/versions.md" + - "codebase/infrastructure/database-seeds.md" + + - OAuth2: + - "codebase/infrastructure/api-clients.md" + - "codebase/infrastructure/oauth2.md" + - "codebase/infrastructure/file-flow.md" - Structure: - - "tachi-server/structure/style.md" - - "tachi-server/structure/filesystem.md" - - "tachi-server/structure/testing.md" + - "codebase/structure/style.md" + - "codebase/structure/filesystem.md" + - "codebase/structure/testing.md" - BATCH-MANUAL: - - "tachi-server/batch-manual/index.md" - - "tachi-server/batch-manual/direct-manual.md" + - "codebase/batch-manual/index.md" + - "codebase/batch-manual/direct-manual.md" - Score Importing: - - "tachi-server/import/index.md" - - "tachi-server/import/main.md" - - "tachi-server/import/import-types.md" - - "tachi-server/import/parse-conv.md" - - "tachi-server/import/conv-failures.md" - - "tachi-server/import/importing.md" - - "tachi-server/import/orphans.md" - - "tachi-server/import/parse-ipi.md" - - "tachi-server/import/sessions.md" - - "tachi-server/import/pbs.md" - - "tachi-server/import/ugs.md" - - "tachi-server/import/goals.md" - - "tachi-server/import/quests.md" - - "tachi-server/import/import-doc-time.md" + - "codebase/import/main.md" + - "codebase/import/import-types.md" + - "codebase/import/parse-conv.md" + - "codebase/import/conv-failures.md" + - "codebase/import/importing.md" + - "codebase/import/orphans.md" + - "codebase/import/parse-ipi.md" + - "codebase/import/sessions.md" + - "codebase/import/pbs.md" + - "codebase/import/ugs.md" + - "codebase/import/goals.md" + - "codebase/import/quests.md" + - "codebase/import/import-doc-time.md" - Implementation Details: - - "tachi-server/implementation-details/details.md" - - "tachi-server/implementation-details/search.md" - - "tachi-server/implementation-details/statistics.md" - - "tachi-server/implementation-details/songs-charts.md" - - "tachi-server/implementation-details/game-configuration.md" - - "tachi-server/implementation-details/esd.md" - - "tachi-server/implementation-details/score-id.md" - - "tachi-server/implementation-details/goal-id.md" - - "tachi-server/implementation-details/goals-quests.md" - - - Tachi Bot Reference: - - "tachi-bot" + - "codebase/implementation-details/details.md" + - "codebase/implementation-details/search.md" + - "codebase/implementation-details/songs-charts.md" + - "codebase/implementation-details/game-configuration.md" + - "codebase/implementation-details/score-id.md" + - "codebase/implementation-details/goal-id.md" + - "codebase/implementation-details/goals-quests.md" markdown_extensions: - admonition diff --git a/server/package.json b/server/package.json index 713f70404..2e2787196 100644 --- a/server/package.json +++ b/server/package.json @@ -16,7 +16,7 @@ "lint": "eslint ./src --ext .ts --fix", "start": "pnpm build && pnpm start-no-build", "start-no-build": "NODE_PATH=js/ node js/main.js", - "runscoreworker": "NODE_PATH=js/ node js/lib/score-import/worker/worker.js", + "start-score-worker": "NODE_PATH=js/ node js/lib/score-import/worker/worker.js", "sync-database": "ts-node src/scripts/sync-database", "sync-database-local": "ts-node src/scripts/sync-database --localPath '../database-seeds/collections'", "recalc-everything": "ts-node src/scripts/state-sync/sync-state.ts",