From 7593143492ad843ecb8b95058e035c9c06eb2d1c Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 15 Jun 2021 18:42:37 +0100 Subject: [PATCH] Initial Commit --- docs/docs/api/auth.md | 50 +++ docs/docs/api/overview.md | 98 ++++++ docs/docs/api/routes/auth.md | 108 ++++++ docs/docs/api/routes/example.md | 113 ++++++ docs/docs/api/routes/import.md | 85 +++++ docs/docs/api/routes/status.md | 70 ++++ docs/docs/codebase/contributing.md | 118 +++++++ docs/docs/codebase/import/conv-failures.md | 60 ++++ docs/docs/codebase/import/goals.md | 42 +++ docs/docs/codebase/import/import-doc-time.md | 37 ++ docs/docs/codebase/import/import-types.md | 66 ++++ docs/docs/codebase/import/importing.md | 108 ++++++ docs/docs/codebase/import/main.md | 156 +++++++++ docs/docs/codebase/import/milestones.md | 20 ++ docs/docs/codebase/import/orphans.md | 33 ++ docs/docs/codebase/import/overview.md | 32 ++ docs/docs/codebase/import/parse-conv.md | 323 ++++++++++++++++++ docs/docs/codebase/import/parse-ipi.md | 32 ++ docs/docs/codebase/import/pbs.md | 59 ++++ docs/docs/codebase/import/sessions.md | 64 ++++ docs/docs/codebase/import/ugs.md | 116 +++++++ docs/docs/codebase/infrastructure/branches.md | 80 +++++ docs/docs/codebase/infrastructure/logging.md | 181 ++++++++++ docs/docs/codebase/infrastructure/testing.md | 57 ++++ .../docs/codebase/infrastructure/toolchain.md | 99 ++++++ docs/docs/codebase/overview.md | 55 +++ docs/docs/codebase/setup/config.md | 147 ++++++++ docs/docs/codebase/setup/setup.md | 82 +++++ docs/docs/codebase/structure/filesystem.md | 134 ++++++++ docs/docs/codebase/structure/style.md | 139 ++++++++ docs/docs/index.md | 46 ++- docs/includes/abbreviations.md | 1 + docs/mkdocs.yml | 73 +++- 33 files changed, 2878 insertions(+), 6 deletions(-) create mode 100644 docs/docs/api/auth.md create mode 100644 docs/docs/api/overview.md create mode 100644 docs/docs/api/routes/auth.md create mode 100644 docs/docs/api/routes/example.md create mode 100644 docs/docs/api/routes/import.md create mode 100644 docs/docs/api/routes/status.md create mode 100644 docs/docs/codebase/contributing.md create mode 100644 docs/docs/codebase/import/conv-failures.md create mode 100644 docs/docs/codebase/import/goals.md create mode 100644 docs/docs/codebase/import/import-doc-time.md create mode 100644 docs/docs/codebase/import/import-types.md create mode 100644 docs/docs/codebase/import/importing.md create mode 100644 docs/docs/codebase/import/main.md create mode 100644 docs/docs/codebase/import/milestones.md create mode 100644 docs/docs/codebase/import/orphans.md create mode 100644 docs/docs/codebase/import/overview.md create mode 100644 docs/docs/codebase/import/parse-conv.md create mode 100644 docs/docs/codebase/import/parse-ipi.md create mode 100644 docs/docs/codebase/import/pbs.md create mode 100644 docs/docs/codebase/import/sessions.md create mode 100644 docs/docs/codebase/import/ugs.md create mode 100644 docs/docs/codebase/infrastructure/branches.md create mode 100644 docs/docs/codebase/infrastructure/logging.md create mode 100644 docs/docs/codebase/infrastructure/testing.md create mode 100644 docs/docs/codebase/infrastructure/toolchain.md create mode 100644 docs/docs/codebase/overview.md create mode 100644 docs/docs/codebase/setup/config.md create mode 100644 docs/docs/codebase/setup/setup.md create mode 100644 docs/docs/codebase/structure/filesystem.md create mode 100644 docs/docs/codebase/structure/style.md create mode 100644 docs/includes/abbreviations.md diff --git a/docs/docs/api/auth.md b/docs/docs/api/auth.md new file mode 100644 index 000000000..5d4fad422 --- /dev/null +++ b/docs/docs/api/auth.md @@ -0,0 +1,50 @@ +# API Authorisation + +Certain endpoints on the Tachi API require permissions. +This page covers how to present your permissions to the server, and what those permissions are. + +***** + +## Authorising Requests + +There are two ways to authorise a request. The first one involves API keys. + +To use authentication with a request, you should set a HTTP header of: + +``` +Authorization: Bearer API_KEY +``` + +Where API_KEY is the api key you wish to use. + +The other way to authorise a request is with your session cookie. This is **NOT** recommended +for normal use, and is instead a way for logged-in users to interact with the API as themselves. + +To use authentication in this way, simply make a request with your `ktchi_production_session` or +`btchi_production_session` cookie. + +The reason for this second authentication method is so that, when a user logs in, they can use +the cookie they were set to also interact with the API. + +## Getting Tokens + +Users may create API tokens at Settings > API Tokens. They may also revoke these tokens. + +## Permissions + +An API key does not implicitly have permission to do anything on a users behalf for security reasons. +Some endpoints require specific permissions, such as a `score:submit` permission for submitting scores. + +!!! warning + API keys **can not** have their permissions altered once set, a new key must be generated. + +!!! info + Cookie-based authentication always has *all* permissions for the user. + +### Table Of Permissions + +The table of permissions is as follows. + +| Permission | Description | +| :: | :: | +| `score:submit` | Perform requests that could submit scores for the user. | \ No newline at end of file diff --git a/docs/docs/api/overview.md b/docs/docs/api/overview.md new file mode 100644 index 000000000..3aec41aaa --- /dev/null +++ b/docs/docs/api/overview.md @@ -0,0 +1,98 @@ +# API Overview + +Tachi exposes a public API which allows programs to interact with Tachi. + +This means you could make your own applications that work off of Tachi's +datasets. + +!!! warning + This documentation assumes some basic programming knowledge, such as how to make + HTTP requests, and how to parse JSON. + +Depending on what variant of Tachi you want to interact with, the API is hosted +on `https://kamaitachi.xyz/api/v1` or `https://bokutachi.xyz/api/v1`. + +!!! note + Some API endpoints are only available on Kamaitachi or Bokutachi. If an + endpoint has this restriction, it will be documented on that endpoints' + page. + +***** + +## Abuse + +The Tachi API is provided under the assumption that it will be used to make +cool things, and used responsibly. + +Abuse of this API will result in your ability to use it being banned. + +The API has a rate limit of 100 requests every minute. This is a rather +generous rate limit, and you should not be close to hitting it. + +If you are in a scenario where you might be hitting 100 requests a minute, +please contact me at `zkldi#2965`. Otherwise, you might have your tokens revoked +for API abuse. + +## License + +[Tachi-Server](https://github.com/zkldi/tachi-server) (Where the API is wrote) is licensed under the [AGPLv3](https://www.gnu.org/licenses/agpl-3.0.en.html). + +To quote GitHub: + +!!! quote + Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available. + +This is not legal advice. + +## Requests + +Unless otherwise mentioned, all non-GET request properties should be sent to the API in `application/json` form. + +GET request properties should be sent to the API in the Query String. + +Your authentication tokens should be sent in the HTTP `Authorization` header, using `Bearer token` form. + +You should provide this token for every endpoint. Even though not all endpoints +require authentication tokens, they are still used for API analytics. + +## Response + +Unless otherwise mentioned, all responses to the API are in `application/json` form. + +The API has two schemas for JSON responses. + +### Success Response + +As the name implies, the Success Response is returned on a successful request. + +| Property | Type | Description | +| :: | :: | :: | +| success | true | Always true for a successful response. | +| description | string | Information about what happened with the request. | +| body | Endpoint Dependent | Any data that the endpoint needs to return, such as a user's document from a profile request. | + +The HTTP Status Code for any Success Response will always be of 2XX form. + +### Failed Response + +As the name implies, the Failed Response is returned when a request fails. + +| Property | Type | Description | +| :: | :: | :: | +| success | false | Always false for a failed response. | +| description | string | Information about what went wrong with the request. | + +The HTTP Status Code for any Failed Response will always be of either 4XX or 5XX form. + +!!! note + Any API request can fail for any reason. You should always account for the + case where the request fails. + +## Footnote + +That should be everything. If you have any questions about the API, you can +contact me on discord at `zkldi#2965`. You can also write an issue on the +[Issue Tracker](https://github.com/zkldi/tachi-docs). I'll get around to either. + +It's entirely possible that I might've made a typo or wrote a poor explaination +of something, so please reach out! \ No newline at end of file diff --git a/docs/docs/api/routes/auth.md b/docs/docs/api/routes/auth.md new file mode 100644 index 000000000..7c31078a6 --- /dev/null +++ b/docs/docs/api/routes/auth.md @@ -0,0 +1,108 @@ +# Internal Authentication + +***** + +!!! danger + This is **NOT** for external use. You should **NEVER** + be requesting a username and password from a user. + + Furthermore, interacting with this programmatically + is near-impossible because you need to complete a CAPTCHA. + + This is documented for completeness' sake. + +## Login with username and password. + +```POST /api/v1/login``` + +Logs a user in and returns a session cookie. + +### Parameters + +| Property | Type | Description | +| :: | :: | :: | +| username | string | The user's username. This is compared case-insensitively. +| password | string | The user's password. | +| captcha | string | | + +### Response + +| Property | Type | Description | +| :: | :: | :: | +| userID | integer | The ID of the user you authenticated as. | + +| HTTP Header | Description | +| :: | :: | +| Set-Cookie | Contains a session cookie for future authentication. | + +### Example + +#### Request + +``` +POST /api/v1/auth/login +``` + +```json +{ + "username": "zkldi", + "password": "my_password", + "captcha": "herebedragons" +} +``` + +#### Response +```json +{ + "userID": 1 +} +``` + +***** + +## Register a new account. + +`POST /api/v1/auth/register` + +### Parameters + +| Property | Type | Description | +| :: | :: | :: | +| `username` | string | A string between 3 and 20 characters. The first character must be A-Z, _ or -. The other 19 may be A-Z, 0-9, _ or -. | +| `password` | string | An 8 character or longer string. | +| `email` | string | | +| `inviteCode` (Kamaitachi Only) | string | If on Kamaitachi, this is the user's invitation code. | +| `captcha` | string | | + + +### Response + +| Property | Type | Description | +| :: | :: | :: | +| `id` | integer | The newly-created user's ID. | +| `username` | string | The newly-created user's name. | + +### Example + +#### Request +``` +POST /api/v1/auth/register +``` + +```json +{ + "username": "newguy", + "password": "my_password", + "captcha": "herebedragons", + "email": "test@example.com" +} +``` + +#### Response + +```json +{ + "id": 2, + "username": "newguy" +} +``` \ No newline at end of file diff --git a/docs/docs/api/routes/example.md b/docs/docs/api/routes/example.md new file mode 100644 index 000000000..0b8597270 --- /dev/null +++ b/docs/docs/api/routes/example.md @@ -0,0 +1,113 @@ +# Example Endpoint + +*Endpoints will be formatted like this.* + +***** + +## Greet a user. + +```GET /api/v1/greet``` + +This endpoint greets the user. + +### Permissions + +*If permissions are required, they will be listed here.* + +- example:permission + +### Parameters + +*Parameters are required unless explicitly stated to be optional.* + +!!! note + As mentioned in API Overview, GET parameters are to be sent in the query string, + and all other methods are to have their content in the request body as + `application/json`. + +| Property | Type | Description | +| :: | :: | :: | +| `name` | string | The name of the user to greet. | +| `birthday` (optional) | presence | Whether it is the user's birthday or not. | + +Not providing required parameters will result in a 400 error. + +### Response + +*Parameters are always present unless stated to be conditional/optional.* + +!!! info + The below properties correspond to keys in the `body` + property of a request. + + This means that the below table corresponds to + ```json + { + "success": true, + "description": "Greeted user.", + "body": { + "greeting": "Hello, zkldi!", + "wasBirthday": false, + } + } + ``` + + +| Property | Type | Description | +| :: | :: | :: | +| `greeting` | string | A greeting for the user. | +| `wasBirthday` | boolean | Whether today is the users birthday or not. | + +!!! info + Since the above table corresponds to keys in the `body` + property of a request, the special property name + <body> refers to the body itself. + + For example: + + | Property | Type | Description | + | :: | :: | :: | + | <body> | string | The greeting. | + + Corresponds to: + ```json + { + "success": true, + "description": "Greeted user.", + "body": "Hello, zkldi!" + } + ``` + +### Example + +#### Request + +``` +GET /greet?name=zkldi +``` + +#### Response + +```json +{ + "greeting": "Hello, zkldi!", + "wasBirthday": false +} +``` + +!!! warning + The example response is implicitly the `body` key of the API response. + That is to say that, the real response for this request is: + + ```json + { + "success": true, + "description": "Greeted user.", + "body": { + "greeting": "Hello, zkldi!", + "wasBirthday": false + } + } + ``` + + This is omitted, because it's redundant all of the time. \ No newline at end of file diff --git a/docs/docs/api/routes/import.md b/docs/docs/api/routes/import.md new file mode 100644 index 000000000..b616a9ed7 --- /dev/null +++ b/docs/docs/api/routes/import.md @@ -0,0 +1,85 @@ +# Import Scores + +***** + +## Import scores from a file. + +```POST /api/v1/import/file``` + +**Kamaitachi Only**. + +Perform a score import that depends on a file, such as a .csv import. + +### Permissions + +- `submit:score` + +### Parameters + +This endpoint expects data in `multipart/form-data` form. +This is because we're handling file imports, and is the +appropriate way to send files. + +| Property | Type | Description | +| :: | :: | :: | +| `importType` | string | The ImportType this import is for. This only accepts `file/` ImportTypes. | +| `scoreData` | file | The file to import scores from. | + +| HTTP Header | Description | +| :: | :: | +| `X-User-Intent` (optional) | If this header is present, the request is assumed to have been sent with 'User Intent'. For more on this, see [Import Types](../../codebase/import/import-types.md) | + +!!! info + It's the responsibility of the API user to use the X-User-Intent + header properly. It should only be used when the user + has explicitly requested this import (i.e. not an automated script). + +### Response + +| Property | Type | Description | +| :: | :: | :: | +| <body> | ImportDocument | The Import Document created as a result of this import. | + +### Example + +#### Request +``` +POST /api/v1/import/file +``` + +``` +// this is not actually a multipart http request example +// as those are huge. + +importType="file/eamusement-iidx-csv" +scoreData= +``` + +#### Response + +```json +{ + "importType": "file/eamusement-iidx-csv", + "idStrings": [ + "iidx:SP" + ], + "scoreIDs": [ + "R6fad5f4947454d8238b45d6a1255d63be4da1130bf91fe9d05df29765a148da8" + ], + "errors": [], + "importID": "bea63277b54a5846bab1fd3a6ce54bfd41276857", + "timeFinished": 1623352263023, + "timeStarted": 1623352260445, + "createdSessions": [ + { + "sessionID": "Qb336d6b1cc0930747f161769a13238a41dce0004", + "type": "Created" + } + ], + "userID": 1, + "classDeltas": [], + "goalInfo": [], + "milestoneInfo": [], + "userIntent": true +} +``` \ No newline at end of file diff --git a/docs/docs/api/routes/status.md b/docs/docs/api/routes/status.md new file mode 100644 index 000000000..d27618490 --- /dev/null +++ b/docs/docs/api/routes/status.md @@ -0,0 +1,70 @@ +# Status Checks + +***** + +## Check server status. + +```GET /api/v1/status``` + +This endpoint is a status check for the API and the consumer, +It's a good way of sanity checking whether your code works. + +### Parameters + +| Property | Type | Description | +| :: | :: | :: | +| `echo` | string | A string to echo. This is useful for checking whether parameters are sending over properly. | + + +### Response + +| Property | Type | Description | +| :: | :: | :: | +| `serverTime` | integer | The current time of the server in Unix Milliseconds. | +| `version` | string | The current version of Tachi-Server running. | +| `permissions` | Array<string> | The permissions this request had. | +| `echo` (Conditional) | string | If an `echo` parameter was provided, this is that exact parameter. | + +### Example + +`GET /api/v1/status?echo=helloworld` + +```json +{ + "serverTime": 1623331110661, + "version": "v2.0.0 (Mysterons)", + "permissions": ["score:submit", "example:permission"], + "echo": "helloworld" +} +``` + +***** + +## Check server status with POST. + +```POST /api/v1/status``` + +This endpoint is identical to `GET /status`, but it responds to POST requests, +and takes the `echo` parameter from the request body. + +This is a good way to check whether your code sends JSON +bodies properly. + +### Example + +```json +POST /status +--- +{ + "echo": "hello world" +} +``` + +```json +{ + "serverTime": 1623331110662, + "version": "v2.0.0 (Mysterons)", + "permissions": ["score:submit", "example:permission"], + "echo": "hello world" +} +``` \ No newline at end of file diff --git a/docs/docs/codebase/contributing.md b/docs/docs/codebase/contributing.md new file mode 100644 index 000000000..3bfd2cc40 --- /dev/null +++ b/docs/docs/codebase/contributing.md @@ -0,0 +1,118 @@ +# Contributing To Tachi + +If you want to contribute to Tachi, that's really appreciated! + +Most of the time, Tachi is a one-man operation, so +any help is really appreciated. + +***** + +## For Newcomers + +If you're looking to contribute to Tachi but aren't +really familiar with TypeScript or other things +we use, Documentation Contributions are highly +appreciated, and accessible to do without any +programming knowledge. + +If you're not familiar with git, I would recommend +downloading [GitHub Desktop](https://desktop.github.com/). + +It has a nice, intuitive UI for making changes, and +saves the trouble of having to explain how forks work. + +## Documentation Contributions + +The documentation for Tachi may have typos, slight mistakes +or inconsistencies. Fixes for those are always appreciated. + +Larger documentation contributions (such as documenting +logic in the codebase) may be subject to some more scrutiny +to make sure that they are correct, but are also significantly appreciated. + +To contribute to the documentation, you should go to +the [GitHub](https://github.com/zkldi/tachi-docs), and +fork the repository. + +## Non-Code Contributions + +Inside the Discord Servers for Bokutachi and Kamaitachi is +a `#help-wanted` channel. This channel lists things that I +currently cannot do, either for time or knowledge reasons. + +These are not code related contributions, and can range from +cool images of cabinets/setups to milestones for a game. + +These are also significantly appreciated. + +## Codebase Contributions + +`tachi-server` is the core of the logic behind tachi, and +is open sourced under AGPLv3. + +Contributions to this will be under increased scrutiny as +I am trying to keep the codebase well organised and tidy. + +### Pull Requests + +You can contribute to `tachi-server` by going to the +[GitHub](https://github.com/zkldi/tachi-server), forking, +and then making your changes. + +There are a couple of strict guidelines to follow when +writing code contributions (Pull Requests). Not following these may +result in your pull request being rejected. + +- Write tests. + +Your new code should be tested. You can read how and where +tests work in [Testing](./infrastructure/testing). + +- Follow the style guide. + +Your new code should follow the style of the repo. The linter +should not have any errors. Generally, write code that +looks like it belongs. + +- Follow the branching rules. + +Your code should follow the branching setup we have. +You can read about it [here](./infrastructure/branches) + +- Your PR should fix an issue. + +If there isn't an issue for your PR, you should make one +before making the PR. + +For larger PRs, such as making new features/new support, +you should make the issue in advance so discussion can +occur (so you don't waste your time on code that can't +be merged). + +### Issues + +Submitting issues to `tachi-server` is encouraged. Despite +the GitHub name of 'issues', issues may also be feature +requests, new support requests, and similar things. + +Guidelines for submitting issues are as follows: + +- Be Nice. + +Self-explanatory. Remember the human! + +- Make sure your issue isn't already reported as a duplicate. + +This saves me a lot of time. If there's a similar issue +but you think yours is different enough to warrant a new +issue, then that's fine. + +- If this is a bug report, be as specific as possible. + +Something like "It wont work" does not help at all. +Non-Specific bug reports will be closed immediately and marked as invalid. + +- Documentation issues go in the other repo. + +Documentation issues should go [here](https://github.com/zkldi/tachi-docs) instead. + diff --git a/docs/docs/codebase/import/conv-failures.md b/docs/docs/codebase/import/conv-failures.md new file mode 100644 index 000000000..6260a200d --- /dev/null +++ b/docs/docs/codebase/import/conv-failures.md @@ -0,0 +1,60 @@ +# Converter Failures + +This page contains the list of possible converter fail +states that can be thrown. All of them can be found in +`src/lib/score-import/common/converter-failures.ts`. + +***** + +## SkipScoreFailure + +This is a special case of a failure, in that it isn't +logged or marked as an error in the import. This call +just means we should skip the score. + +An example case of this would be something like the +eamusement IIDX CSV format, which can pass 'empty' scores +for a chart. We would want to skip over those, and they +aren't an error. + +When this is thrown, the score is skipped, and nothing is +imported. + +## KTDataNotFoundFailure + +!!! bug + KTData means 'Kamaitachi Data', but this applies to + Bokutachi just as much, it is just a legacy name + holdover. + +This failure means that the score could not match with +anything in the database, such as giving a chartID that +Tachi does not have in its database. + +This failure also takes the current data, importType and +parser context. This is so we can create [Orphan Scores](./orphans.md). + +## Invalid Score Failure + +This score provided invalid data that we cannot process. + +An example of this would be an IIDX score with -1 EX Score. + +## Internal Failure + +This is an internal failure, and thrown when something bad +has occured on our end. + +The most common case for throwing this is the "Song-Chart" +desync. This occurs when a chart has no parent song (and +a chart must have a parent song.) + +!!! info + This is generally used for states that should never happen, + but we want to handle them nicely anyway. + +## Any Exception + +If any exception is thrown from a converter that is **NOT** +an `instanceof` ConverterFailure, it is treated as an +InternalFailure. \ No newline at end of file diff --git a/docs/docs/codebase/import/goals.md b/docs/docs/codebase/import/goals.md new file mode 100644 index 000000000..20397f3bc --- /dev/null +++ b/docs/docs/codebase/import/goals.md @@ -0,0 +1,42 @@ +# Updating Goals + +If a user has goals set, we need to update their +progress on those goals as a result of this import. + +***** + +## Getting Relevant Goals + +In V1 of (Kamai)tachi, we fetched *all* goals for a user, +and recalculated all of them. + +This was fine for users with some goals, but for users with +lots of goals, it quickly became a huge performance boon. + +To avoid that, Tachi V2 fetches only the "relevant" goals +as a result of the import. This is calculated as follows: + +- Single Goals are matched if their chartID is inside the set of chartIDs affected. + +- Multi Goals are matched if their data contains any chartIDs that were affected. + +- Any Goals are always matched. + +- Folder goals are matched if the folder contains any charts inside the set of chartIDs affected. + +## Processing Goals + +For every goal matched, we iterate over it and evaluate +it using `EvaluateGoalForUser`. + +The details of how this algorithm works (and goals in general work) can be found at [Goals](todo). + +We then need to convert the returns of that function into +the expected goal format for ImportDocuments. + +That means we have to compare the old UserGoal's progress +with the newly returned one. + +If nothing has changed, we return undefined. If something +has changed, we return a `bwrite` key that contains a +MongoDB bulk-write operation to update the UserGoal, and the expected import format of goalID, old, new. diff --git a/docs/docs/codebase/import/import-doc-time.md b/docs/docs/codebase/import/import-doc-time.md new file mode 100644 index 000000000..40969ae28 --- /dev/null +++ b/docs/docs/codebase/import/import-doc-time.md @@ -0,0 +1,37 @@ +# Import Documents and Import Timings + +The final step of the process is to coalesce all the returns +of the various steps above into one analysable document +and return it. + +Details on what this document looks like can be found [here](todo). + +***** + +## Logging + +If the import had over 500 scores, we log the import at Info level. This is because those are generally rare, and we +want to know how our performance is doing at a glance. + +If the import has over 1 score, it is logged as verbose. + +Else, the import is logged as debug. + +## Timings + +An internal document - Import Timings - is created by +storing the time each step of the import process took +in miliseconds. + +This is analysed regularly to check for performance +degradation and look for potential optimisations. + +This is also stored in the database. + +## Returns + +The Import Document **is** the return from ScoreImportMain. + +This document contains all the relevant information +about the import and what it has resulted in, and +is also stored in the database under a random ID. diff --git a/docs/docs/codebase/import/import-types.md b/docs/docs/codebase/import/import-types.md new file mode 100644 index 000000000..afe5c5b5e --- /dev/null +++ b/docs/docs/codebase/import/import-types.md @@ -0,0 +1,66 @@ +# Import Types + +The score import code for Tachi uses 'Import Types' to +determine what kind of code to run on a given import, +and what converter to call (more on that later.) + +***** + +## Format + +An import type is just a string. It is formatted as follows: + +``` +type/name +``` + +## Types + +There are three types. + +| Type | Description | +| :: | :: | +| `file/` | The data is coming from a file, such as one from a multipart form submission. | +| `ir/` | The data is coming from a HTTP Request sent to us. | +| `api/` | The data is being fetched by us from another service through HTTP Requests. | + +## User Intent + +Some imports are performed with user intent, such as +the user going to the Tachi website and uploading +a file and clicking submit. + +Some imports are performed without user intent, i.e. +the user sets up a score hook to automatically import +their scores whenever they get one. The reason for the +distinction is for hiding "automatic" imports from the +users imports page. + +For certain endpoints, the special header `X-User-Intent` +is set to indicate that this was done with User Intent. + +!!! warning + As this is just a convention, you can abuse it, but you + shouldn't, because it's not nice to break things + (and you may be banned). + +## List of Import Types + +| ImportType | Description | Availability | User Intent | +| :: | :: | :: | :: | +| `file/eamusement-iidx-csv` | The E-amusement CSV format for IIDX. This type accepts both Pre-HV and Post-HV formats. | Kamaitachi | Yes | +| `file/batch-manual` | The Tachi BATCH-MANUAL format submitted through a multipart form. | Kamaitachi & Bokutachi | Yes | +| `file/solid-state-squad` | The IIDX XML output by Solid State Squad. | Kamaitachi | Yes | +| `file/mer-iidx` | The IIDX JSON output from MER. | Kamaitachi | Yes | +| `file/pli-iidx-csv` | Same as eamusement's IIDX CSV, but output from PLI. | Kamaitachi | Yes | +| `ir/direct-manual` | The Tachi BATCH-MANUAL format but submitted in a HTTP Request body as `application/json`. | Kamaitachi & Bokutachi | Depends on Header | +| `ir/barbatos` | Barbatos's format submitted in a HTTP Request Body. | Kamaitachi | No | +| `ir/fervidex` | Fervidex's score format submitted in a HTTP Request Body. | Kamaitachi | No | +| `ir/fervidex-static` | Fervidex's profile sync format submitted in a HTTP Request Body. | Kamaitachi | No | +| `ir/usc` | An implementation of the [USCIR](https://uscir.rtfd.io)'s POST /scores endpoint. | Bokutachi | No | +| `ir/beatoraja` | A handler for BokutachiIR's score format. | Bokutachi | No | +| `api/flo-iidx`, `api/eag-iidx` | Both the same IIDX format, but yielded from different APIs with different players. | Kamaitachi | Depends on Calling[^1] | +| `api/flo-sdvx`, `api/eag-sdvx` | See above, but for SDVX. | Kamaitachi | Depends on Calling[^1] | +| `api/arc-iidx`, `api/arc-sdvx`, `api/arc-ddr` | Formats yielded from ARC for that specific game. | Kamaitachi | Depends on Calling[^1] | + +[^1]: If this is synced manually by the user, I.e. they have called it to be synced, then it is with intent. If it was called for them through automatic synchronisation, then it was not done with user intent. diff --git a/docs/docs/codebase/import/importing.md b/docs/docs/codebase/import/importing.md new file mode 100644 index 000000000..81d48454a --- /dev/null +++ b/docs/docs/codebase/import/importing.md @@ -0,0 +1,108 @@ +# Importing DryScores + +This page goes over hydrating a DryScore into a fully +fledged Tachi score, and then goes over how it is imported. + +!!! warning + Performance has been squeezed out of this process + rather aggressively, as such, some of the more + 'obvious' ways to do things have been ignored + for performance gains. + +!!! info + The code that handles this logic is found in + `src/lib/score-import/framework/score-importing/score-importing.ts`. + +***** + +## Return Format + +The `ImportIterableDatapoint` returns +`ImportProcessingInfo` or `null`. The format for the former +can be found in `tachi-common`, and **should be read** before +following this document! + +As a rough outline: +```ts +{ + // whether this import worked or not + success: boolean; + // ScoreImported implies success: true, and all others imply success: false. + type: "ScoreImported" | "KTDataNotFound" // ... and more; + // An error message. + message: string | null; + // Some errors return some data about the error, or things like scoreImported returns the score that was imported. + content: see_implementation +} +``` + +## Dealing with converter returns + +As mentioned in [Parsers and Converters](./parse-conv.md), +converters will return a DryScore, and its matching chart +and song on success. + +However, failures are also an expected throw from a +converter function. We handle these throws by logging +them dependent on their severity, and returning +that this score has failed to be imported properly +in the `ImportProcessingInfo` format. + +If the converter was successful, we now have a DryScore, song and chart to work with. + +## Hydration + +Our first step is to turn that DryScore into a 'real score'. + +!!! info + The code for this is found in `HydrateScore`. + +Before anything, we calculated the ScoreID for this score. +This is used to dedupe scores, and is a checksum of +the following properties. + +- userID +- chartID +- lamp +- grade +- score +- percent + +We fill out all the properties that can be calculated +about the score, such as the aptly named `calculatedData`, +and things like `gradeIndex` and `lampIndex`, which are +just the index of the grade/lamp string in the set of grade/lamps. + +Other properties, such as `timeAdded` (the time this data was inserted into the database) can be trivially attached onto the score. + +## Insertion + +Now that we have a real Tachi score, we can insert it into +the database. + +The obvious solution here is something like: +```ts +await db.scores.insert(scoreDocument); +``` + +But this has performance implications for large sets of +scores. + +To solve these performance issues, we use a queue to insert +scores. + +ScoreDocuments are appended to a queue of 500. +If the queue hits 501 scores, the queue is bulk-written to +the database. At the end of hydrating and queueing all scores, the queue is flushed one last time. + +This simple optimisation provides large performance benefits. + +## Final Returns + +Importing each datapoint gives us an `ImportProcessingInfo` +object OR `null`. We want to return only the former, +as `null` is for skipped scores. + +We filter out all the nulls and flush the queue before +returning the array of `ImportProcessingInfo` objects +that were returned from our import process. diff --git a/docs/docs/codebase/import/main.md b/docs/docs/codebase/import/main.md new file mode 100644 index 000000000..e821bc5d2 --- /dev/null +++ b/docs/docs/codebase/import/main.md @@ -0,0 +1,156 @@ +# Score Import Main + +This page refers to the entry point function for a score +import. + +This function can be found at `src/lib/score-import/framework/score-import-main.ts`. + +## Signature + +`ScoreImportMain` takes four arguments and an optional fifth. + +| Argument | Type | Description | +| :: | :: | :: | +| `user` | PublicUserDocument | The user that is making this import request. | +| `userIntent` | boolean | Whether this import was performed with User Intent - See [Import Types](./import-types.md#User%20Intent) | +| `importType` | ImportType | What kind of import "type" this is. For more on this, see [Import Types](./import-types.md) +| `InputParser` | Function | The parser function to call. For more info, see [Parsing and Converting](./parse-conv.md) +| `providedImportObjects` (Optional) | { logger, importID } | Optionally, a logger and existing importID can be passed here. This is used for scenarios where the logger and importID have already been created before importMain was called. | + +## Import Logger + +For ease of debugging, a logger is passed between most +functions in the score importing process. This logger +contains the context of the currently importing user +and the ImportID. + +## Steps + +To make the process easier to parse, the importing process +is split into multiple steps. These steps may split into +further sub-steps, depending on how much occurs in said +step. + +!!! tip + These steps are also commented inline in + `score-import-main.ts`. + +!!! warning + These steps are only an outline of the process! + + For more information on each specific step, links + are given. + +### Setup + +If `providedImportObjects` is not given, then ScoreImportMain generates a new Logger and an ImportID. + +### Parsing + +This step calls the ParserFunction provided as an argument +to this function. The returns from that are then used +in the below steps. + +- Full Page: [Parsing and Converting](./parse-conv.md) + +### Importing + +We iterate over the `iterable` returned from the Parser Function, +calling the Converter Function for each element. + +Once we have converted the element, we can fill it in +and insert it into the database. + +This process returns an array called the "Import Info". + +This object holds information about the set of charts this +import involved, and the set of scores that were imported. + +- Full Page (Converting): [Parsing and Converting](./parse-conv.md) +- Full Page (Importing): [Importing](./importing.md) + +!!! info + This step does most of the work, and is where a + lot of our complexity lies. + +### Parse Import Info + +The next steps require certain information from our Import Info. + +This step parses that data into some values we have use for, such as the set of chartIDs impacted +by this import. + +- Full Page: [Parsing ImportProcessingInfo](./parse-ipi.md) + +### Sessions + +Now that we have imported the scores and have got +an array of scoreIDs newly created from Parse Import Info, +we can move on to creating sessions from this import. + +- Full Page: [Sessions](./sessions.md) + +### Personal Bests + +This step updates the 'personal bests' for the user. It +iterates over every unique chartID modified in this import +and conjoins the users best scores on the chart into one document. + +This is to have a single document that has the users best +lamp and score, which avoids having to join it together +whenever we want to display a users best score and lamp +at the same time. + +- Full Page: [Personal Bests](./pbs.md) + +### Game Stats + +This step updates the user's Game Stats. + +If this is the users first play on this game, then we create +game stats for them. + +The update is performed by recalculating their statistics +with the new scores inserted. + +!!! info + This section is referred to as UGS sometimes internally. + + This is short for User Game Stats. + +- Full Page: [User Game Stats](./ugs.md) + +### Goals + +This step checks the charts modified in this import and +retrieves all the relevant goals. It then checks +all of these goals for whether their status has +changed or not, and updates them accordingly. + +- Full Page: [Goals](./goals.md) + +### Milestones + +Checks the set of goals modified in the previous step, +and gets the relevant milestones. This step then re-evaluates all of those milestones and +updates accordingly. + +- Full Page: [Milestones](./milestones.md) + +### Import Document + +Finally, we take all of the returns of the above steps +and combine them into an import document. + +This is inserted to the database, and is also the return +value of this function. + +### Import Timings + +Every step in this process is timed in miliseconds. + +The time each step took is saved alongside the ImportID. + +This is used for internal debugging and degraded performance +checks. + diff --git a/docs/docs/codebase/import/milestones.md b/docs/docs/codebase/import/milestones.md new file mode 100644 index 000000000..29fe508e1 --- /dev/null +++ b/docs/docs/codebase/import/milestones.md @@ -0,0 +1,20 @@ +# Updating Milestones + +We update milestones by looking at the set of modified +goalIDs, and check the user's assigned milestones to +get those that are affected. + +***** + +## Returns + +We evaluate every milestone, and for each one, +create a bulkwrite operation to update the user's milestone +progress. + +If the user has newly achieved a milestone, a Redis Event +is emitted, which could be hooked into by our Discord Bot. + +If the progress has changed at all, the data is +pushed to an array which is then returned. This is +attached onto the Import Document. \ No newline at end of file diff --git a/docs/docs/codebase/import/orphans.md b/docs/docs/codebase/import/orphans.md new file mode 100644 index 000000000..26dce247a --- /dev/null +++ b/docs/docs/codebase/import/orphans.md @@ -0,0 +1,33 @@ +# Orphan Scores + +KTDataNotFoundFailures state that we don't have the data +to understand what chart/song this score is for. + +However, that doesn't necessarily imply we won't have that +data in the future. Orphaned scores are a way of storing +score-like data without a song or chart as a parent. + +Then, when the song and chart are made available, those +scores can be imported as if nothing had changed! + +***** + +## How do they work? + +Orphaned scores store the `data` and `context` from the parser. + +When unorphaning is attempted, the Converter Function is +called with that data and context. + +If it results in a KTDataNotFoundFailure, we do nothing. + +If it results in a success, we have a new DryScore we can import. + +!!! info + If another type of failure occurs - i.e. InvalidScoreFailure, + then we remove the orphaned score from the DB. + +!!! note + This is why Converter Functions are completely + static, so we can safely call the same Converter + Function with this data to unorphan scores. \ No newline at end of file diff --git a/docs/docs/codebase/import/overview.md b/docs/docs/codebase/import/overview.md new file mode 100644 index 000000000..01e55b4a4 --- /dev/null +++ b/docs/docs/codebase/import/overview.md @@ -0,0 +1,32 @@ +# Score Import Overview + +Score importing in Tachi is *very* complex, and involves +a lot of moving parts. For that reason, the entire process +(and my rationale behind each step) is documented here. + +The code for Score Importing can be found at +`src/lib/score-import`. + +The entry point function is `src/lib/score-import/framework/score-import-main.ts`, and everything else is located nearby. + +***** + +## Folders + +Inside `src/lib/score-import` is two folders, `framework/` +and `import-types`. + +### `framework/` + +The framework folder contains the 'moving parts' of the score importing, +such as processing pbs and inserting scores. + +This folder contains the "entry point" for the importing +mechanism, which is discussed in [Score Import Main](./main.md). + +### `import-types/` + +The Import-Types folder contains the specific parsers +and converter functions for a given ImportType. + +You can read more about import types [here](./import-types.md). diff --git a/docs/docs/codebase/import/parse-conv.md b/docs/docs/codebase/import/parse-conv.md new file mode 100644 index 000000000..3885d040f --- /dev/null +++ b/docs/docs/codebase/import/parse-conv.md @@ -0,0 +1,323 @@ +# Parsing and Converting + +Tachi faces an interesting problem with its score importing +code. We could rewrite our entire score importing process +for every possible import method, or we could try to +factor out the common parts of this process. + +The latter is what we've gone for, but it isn't without +its own traps. + +We face the first obvious problem when it comes to actually +"getting" the scores from a source and turning them into +our format. + +Our source needs to be dynamically set, sometimes we might +be drawing scores from a file, sometimes from a HTTP +request body, sometimes from another API! + +Then, we need to convert the data that document gave us +into a format the rest of our code to understand. + +To solve this problem, Tachi uses two processes, +Parsing and Converting. + +## Outline + +As a rough outline, the parser converts data from +any source into an iterable. + +The iterable is then traversed, and the Converter +is called for every element of the iterable. + +The converter function turns the element into something +the rest of the Tachi import code can work with. + +## Parsing + +Parsing is the job of drawing the data from the source into +an **iterable** (The usage of iterable and **NOT** array will be explained later). + +A parser function is intended to be created *outside* of +score-import-main, which means it can be really whatever +it wants to be, and draw data from whatever source it needs +to. + +!!! warning + Parser functions return more than just an iterable, + but in the interest of introducing complexity slowly + we're going to pretend that they only return an + iterable for now. + +### Implementation + +As an example, lets say we had a CSV being sent to us as a file. + +We could write a parser for that as follows: + +```js +// note: awful example code, this is NOT how to parse a CSV! +function ParserFunction(request) { + let csv = request.file; + + return csv.toString().split("\n"); +} +``` + +This takes the provided file, splits it on newlines, and returns +the array. This meets all our criteria, as it returns +an iterable, but we still have problems. + +Firstly, the calling signature for this involves a `request`. Score Import Main doesn't know of any HTTP requests, as it might not even be triggered by one. + +We could solve this by passing a request object, and +also any other context a parser could possibly need, +but that quickly breaks down when we start needing to +fetch from APIs. + +Instead, we define a ParserFunction is defined to be a function that +takes *one* argument, the Import Logger. + +This means that we actually have to use some +closures in order to call a parser. + +```js +function CreateParser(request) { + return (logger) => { + return request.file.toString().split("\n"); + } +} + +const ParserFunction = CreateParser(request); +``` + +Using closures, we can dynamically create the parser function +and enclose any context it needs to use inside of it. + +This approach allows ScoreImportMain to continue knowing +nothing about where the data is coming from, and +avoids passing lots of initial context, bloating up the +score import function. + +The iterable returned by the parser should be in the +most immediately sensible format. + +!!! info + Parser Functions may be asynchronous. + +### Iterable Vs. Array + +Above, we specifically mentioned that the Parser function +returns an *iterable*. There's a subtle difference here +that allows for significant performance gains when getting +data from a paginated API. + +Let's write an example where we retrieve data from an API. + +```js +function CreateParser() { + return async (logger) => { + let page = 1; + + let array = []; + while (true) { + // pretend this returns some scores + + let res = await fetch("example.com?page=" + page).then(r => r.json()); + + array.push(...res.scores); + + if (!res.moreData) { + break; + } + + page++; + } + + return array; + } +} +``` + +It's not bad, but it has some performance issues if the +user has a lot of scores, by creating a massive array +in memory and *then* passing it. + +We can improve this code significantly using generators. + +```js +function CreateParser() { + return async function* (logger) { + let page = 1; + + while (true) { + let res = await fetch("example.com?page=" + page).then(r => r.json()); + + for (const score of res.scores) { + yield score; + } + + if (!res.moreData) { + break; + } + + page++; + } + } +} +``` + +With generators, this code can now yield each score from +the data returned, instead of having to bubble it up in +one large array. + +### Other Returns + +We mentioned above that parsers return more than +just an iterable. + +In practice, a Parser returns four things in an object. Context, the Game this import is for, the Iterable discussed above and a Class Handler. + +#### `context` + +Some import formats require context to be parsed properly. + +The most obvious example is the IIDX E-Amusement CSV format, which does not declare whether the scores are from +single or double play inside the file. + +To properly interpret a score of, say, 1000 on 5.1.1. ANOTHER, we need to have some context about the score. +That is, we need data that isn't part of the 'score'. + +To solve this, parsers return a `context` object which +contains context for the converter function. + +For IIDX E-Amusement, the parser function could interpret +the playtype from user input, and then return it as context. + +#### `game` + +Sometimes, the game an import is for is not statically +known from the Import Type. This means that the parser +will have to return the `game` this import is for rather +than inferring it. + +An example of a format doing this would be the [BATCH MANUAL](foo) format, +which declares what game the import is for in a header. + +#### `classHandler` + +Sometimes, an import may result in the changing of a users' +classes. This may also sometimes be dependent on data that +is part of the import. + +An example of this would be the Fervidex Static import type, +which passes a set of the users scores, and also passes +their Dans. + +Since dans are part of the user's IIDX classes, we need to +take the information from that request and update classes +accordingly. + +The classHandler returned from the parser should be a callable function +that will update the user's classes dependent on this data. + +!!! info + If no class information is dependent on the content + known by the parser function, then `classHandler` + can be set to null, and it will not be called. + +### Final Example + +With all these other things to return, we can +write a simple example of a parser as follows: + +```js +function CreateIIDXFileParser(request) { + return (logger) => { + let csv = request.file; + + let playtype = request.body.playtype; + + if (playtype !== "SP" && playtype !== "DP") { + // todo: throw error + } + + return { + iterable: csv.toString().split("\n"), + game: "iidx", + context: { + playtype, + }, + classHandler: null, + } + } +} +``` + +## Converters + +Lets say we have a parser for SDVX that returns: +```js +[{ + score: 9000000, + lamp: "HARD CLEAR", + timestamp: 1623760037 +}, +// +] +``` + +Our converter has to take every element here and +*convert* it into the form the rest of Tachi can +work with. + +Unlike Parser Functions, Converter Functions are static, +that means that they cannot be dynamically constructed +with closures or similar. + +### Arguments +Converters are called with these four arguments: + +| Property | Type | Description | +| :: | :: | :: | +| `data` | ParserElement | The given element in the parser's iterable. | +| `context` | ParserContext | The context the parser returned. | +| `importType` | [ImportType](./import-types.md) | The import type this import is for. | +| `logger` | Logger | The import logger. | + +### Returns + +Converter functions are expected to return three values +on success. + +- `dryScore` + +A dry score is a 'partial' real Tachi Score format +with certain properties left blank (because they can be +filled elsewhere). + +In short, the Dry Score is all the properties that can +only be derived from the parser's passed data. + +Other properties, like calculated data are filled out +in a score "hydration" process (hence the name Dry Score). + +- `chart` + +The chart this score is for. + +- `song` + +The song this score is for. + +### Throws + +Converters are also expected to fail. +For this, we have a very specific throw format +that should be thrown whenever a specific error has occured. + +These are called Failures, and a full list of them can be +found [here](./conv-failures.md). + +If a non-failure is thrown, it is logged as an error and the score is ignored (because crashing the entire +import from an unexpected error is a bad idea.) diff --git a/docs/docs/codebase/import/parse-ipi.md b/docs/docs/codebase/import/parse-ipi.md new file mode 100644 index 000000000..309e315cc --- /dev/null +++ b/docs/docs/codebase/import/parse-ipi.md @@ -0,0 +1,32 @@ +# Parsing Import Processing Info + +The ImportProcessingInfo array returned from our +importing process is unwieldy to work with. We are interested +in what it has to say, though. + +This step converts that array into data we use +in successive score import steps. + +!!! info + Because the logic for this is essentially just one + loop, it does not get its own file in `score-importing/`. + + Instead, it is defined inside `score-import-main.ts`. + +***** + +## Output + +| Property | Type | Description | +| :: | :: | :: | +| `scoreIDs` | Array<string> | The array of scoreIDs imported as a result of this import. | +| `errors` | Array<{ type, message }> | An array of the failed ImportProcessingInfo's Types and Error Messages. +| `chartIDs` | Set<string> | A set of the chartIDs modified by this import. This is a set to ensure that every chartID here is unique. | +| `scorePlaytypeMap` | Record<Playtype, Array<Score>> | All of the score documents returned filtered into Playtype buckets. + +### What's with ScorePlaytypeMap? + +Some imports may import scores from multiple Playtypes. We +frequently want to only refer to the set of scores modified +under a given playtype (For importing a users game:playtype stats, as an example) +so this is a very useful data structure to have. \ No newline at end of file diff --git a/docs/docs/codebase/import/pbs.md b/docs/docs/codebase/import/pbs.md new file mode 100644 index 000000000..7d05da7e3 --- /dev/null +++ b/docs/docs/codebase/import/pbs.md @@ -0,0 +1,59 @@ +# Personal Bests + +Tachi has a lot of scenarios where it needs to reference +a users best score alongside their best lamp. + +This is a standard for most arcade games to conjoin the +best aspects of all your scores, so Tachi needs to +replicate that behaviour. + +However, not all games agree on what aspects to join about +scores. Although in most scenarios we will be joining +a user's best lamp with their best score, there are some +games (like IIDX) which have other statistics that need +to be conjoined. + +***** + +## Calculating Personal Bests + +A user is only allowed one personal best per chart, this +allows us to know what personal bests need to be modified +by looking at the set of chartIDs modified from this import. + +We can iterate over that set and recalculate the users +personal best for each chart. + +Firstly, we select the users score on the chart with the +highest percent, then we select the users score on the +chart with the highest lampIndex. + +We can then join the relevant properties of this +into a PBScore, where the best parts of the lampPB +are unioned with the best part of the scorePB. + +!!! info + A ScorePB refers to a user's best Percent/Score on a chart. + + A PBScore refers to the aforementioned conjoined score + document. + +## Game Specific Things + +Some games require specific conjoining code. The below games +have said exceptional behaviour. + +### SDVX, USC + +VF5 and VF6 depends on the users best lamp and their best score. + +That is, you cannot just select the larger volforce from +the two scores, as HC 9m -> NC 9.1M should have the +volforce of HC 9.1M. + +### IIDX, BMS + +IIDX and BMS have BP. This number is the amount of bads +plus the amount of poors a player made. The lowest non-null +value for this should be selected as that property for the +PBScore. diff --git a/docs/docs/codebase/import/sessions.md b/docs/docs/codebase/import/sessions.md new file mode 100644 index 000000000..b8e83b448 --- /dev/null +++ b/docs/docs/codebase/import/sessions.md @@ -0,0 +1,64 @@ +# Sessions + +This page documents how Sessions are constructed. +For a humanised explaination of sessions, see +TODO. + +***** + +## Creating New Sessions + +Scores are sorted on the time they were achieved. Scores +without timestamps are discarded from any session processing. + +Scores are then grouped into buckets of two hours, in pseudocode: + +```ts +let lastTimestamp = 0; + +for (score of scores) { + if (score.timestamp > lastTimestamp + TWO_HOURS) { + CloseBucket(); + CreateNewBucketWithThisScore(); + } + else { + AppendToBucket(); + } + + lastTimestamp = score.timestamp; +} +``` + +This means that if there was more than two hours between +any two successive scores, the session is marked as terminated +and a new bucket of scores is created. + +For every bucket of scores we have, we iterate over the +scores inside the bucket, and compare them against the +users' current PB. Since PBs are updated *after* sessions +are processed, this means we are processing them against +their past PBs. + +!!! bug + If a user ends up making sessions in the past, + this step will compare against their current PBs - + instead of the PBs they would've had at the time + of that session. + +Once we have compared all the scores against the PBs, we +can create a session from it. We need to generate a random +name (we have some stuff in `src/datasets` for this). + +We also need to calculate statistics for this session, +this is dependent on the game + playtype being used. + +## Appending To Sessions + +If a score within 2 hours of an existing session, +then that score is appended to the session. + +!!! bug + Sessions do not currently conjoin if a user was to + get a score between two sessions such that + there was now less than two hours between the first + sessions last score and the second sessions first score. diff --git a/docs/docs/codebase/import/ugs.md b/docs/docs/codebase/import/ugs.md new file mode 100644 index 000000000..c8d541156 --- /dev/null +++ b/docs/docs/codebase/import/ugs.md @@ -0,0 +1,116 @@ +# User Game Stats + +A user has overarching statistics for a game. + +This part of the score importing process updates those +overarching statistics, such as their profile rating +and their classes. + +!!! note + If a user has never played this game + playtype + combination before, a new user game stats + object is saved to the database as a result + of this update. + +***** + +## Format + +A User Game Stats object looks like this: +```ts +{ + userID: string; + game: "iidx" | "bms" // ... so on + playtype: "SP" | "DP" // ... the set of playtypes this game supports. + // A set of continuous statistics about a user. + ratings: { + // A record of numbers with keys determined + // by the game + playtype combination. + // such as: + VF6: number; + // for SDVX and USC. + }, + // A set of discrete statistics about a user. + classes: { + // This typically contains things like dan ranks + // or skill divisions, such as: + dan: 14, + // for IIDX, or + skillColour: 19 + // for gitadora. + + // Note that instead of remembering each number + // for each class, they are defined as constants + // in src/lib/constants/classes.ts + } +} +``` + +## Ratings + +Ratings are calculated for the given Game + Playtype combination. + +Different combinations will have different statistics, +Gitadora will have a profile skill level, whereas SDVX +would use Profile Volforce. + +All the rating functions for a game are declared +in `src/lib/score-import/framework/user-game-stats/rating.ts`. + +## Classes + +There are two types of classes in Tachi. Static classes +are classes that can always be derived from the data around +them. Things like this are typically discrete buckets +for continuous data, such as Skill Colours in GITADORA. + +The other type of classes are external classes. These +are things like Dans, where the information cannot be +derived from any other data around it, and instead must +be explicitly told to change. + +### Class Handlers + +You may recall that in [Parsers and Converters](./parse-conv.md) it was documented that parsers can return a +ClassHandler. The ClassHandler is intended to handle those +explicit changes. + +For example, lets say we have an import type that tells us: +```js +{ + scores: [{score: 1000, songID: 1, diff: "spa"}], + sp_class: "kaiden" +} +``` + +We could return a classHandler from our parser function +that encloses this updated information, such as: +```ts +function CreateClassHandler(spClass: number): ClassHandler { + return (/* classHandlers have some args but we dont need them for this example */) => { + return { + dan: spClass + } + } +} +``` + +This returned classHandler is called with 5 arguments +and is expected to return a partial record of the users +classes. This is then merged with the static classes +and returned as the users classes. + +### Static Class Handlers + +Static class handlers are built in, and always called +when a user's game stats is updated. + +## Deltas + +Once a user's classes have been calculated, they are diffed +against the current user's stats. If they have been +increased, then a redis message is sent out, and things +like the Tachi discord bot may choose to echo this +in the discord. + +The deltas are returned to be part of the ImportDocument. \ No newline at end of file diff --git a/docs/docs/codebase/infrastructure/branches.md b/docs/docs/codebase/infrastructure/branches.md new file mode 100644 index 000000000..596beec23 --- /dev/null +++ b/docs/docs/codebase/infrastructure/branches.md @@ -0,0 +1,80 @@ +# Branching Model + +`tachi-server` has a very specific branching model. + +Breaking the rules in this branching model will result +in your Pull Request being rejected. + +***** + +## Acknowledgements + +This branching model is identical to [A Successful Git Branching Model](https://nvie.com/posts/a-successful-git-branching-model/). + +You can read that post instead of this one, if you prefer, +but I will provide a simplified explaination below. + +## Static Branches + +These branches are always present, and never to be deleted. + +### `master` + +This is the current release version of `tachi-server`, and +is deployed onto production. + +This should never be committed to directly from anything other than `release-` branches. + +Every commit to master is a new release. + +### `develop` + +This is the development branch, and is where pull requests +are merged to. You should merge `issue-` branches with `develop`. + +## Ephemeral Branches + +These branches are intended to be created, merged, and then +deleted. + +### Feature Branches (`issue-`) + +These branches **may** start with the text `issue-`, and are +followed by the issue number they reference. + +You may call these branches whatever you want, +except things like `master` or `develop` or `hotfix-` etc. + +Our convention is to use `issue-` + +These are forked from `develop`, and should be merged +with `develop`. + +!!! note + Despite the name `issue-`, these branches aren't + bug fixes. They are anything that fix an Issue + on the repository. + + This means these are also feature branches, and other + things. + +### `hotfix-` + +These branches start with the text `hotfix-` and are followed by the issue number they fix. + +These are forked from `master`, and should be merged with +both `master` and `develop`. + +### `release-` + +These branches start with the text `release-` and are +followed by the Major.Minor version of the release. + +These are typically the final draft for a new release. + +These are created when all the features we want in the +new release are present in develop, and we want to +get it ready for production. + +These are forked from `develop`, and should be merged with +`master`. diff --git a/docs/docs/codebase/infrastructure/logging.md b/docs/docs/codebase/infrastructure/logging.md new file mode 100644 index 000000000..9a73394db --- /dev/null +++ b/docs/docs/codebase/infrastructure/logging.md @@ -0,0 +1,181 @@ +# Logging + +As mentioned in the [Toolchain](./toolchain), we use +WinstonJS for logging. We use a slightly modified setup +of winston, but the same basic logging principles apply. + +***** + +## Log Levels + +Tachi uses the following log levels, listed in order of +severity. + +### Crit + +```ts +logger.crit("foo"); +``` + +`crit` or Critical is the most severe log level in tachi. + +Calling this means **the process must now completely exit**. + +This fail state is triggered in things like not being able +to connect to a Mongo or Redis instance, where the application +absolutely cannot function anymore and should quit immediately. + +### Severe + +```ts +logger.severe("foo"); +``` + +`severe` is the second most severe log level in tachi. + +Calling this means an error has occured, and that error implies +it will apply to multiple parts of the codebase. + +This log level is used with things like a Song-Chart +desync -- A chart must have a song as a parent, but if it +doesn't, that's a severe-level error. + +Think of this like an error with wider-reaching implications. + +### Error + +```ts +logger.error("foo"); +``` + +`error` is the third most severe log level in tachi. + +Calling this means an error has occured, but the impact of +it is limited to the function or general area it was called in. + +Errors may be recovered from or ignored by the code, but generally that should be reserved for `warn`. + +### Warn + +```ts +logger.warn("foo"); +``` + +`warn` indicates that something has gone wrong, but is safely +recoverable from, such as a mathematical function being given NaN unexpectedly, and just returning 0. + +`warn` calls **MUST** be recoverable from, and must not +imply severe damage to the global state of the application. + +### Info + +```ts +logger.info("foo"); +``` + +!!! info + This is the default [LOG_LEVEL](./config) for tachi. + + This means that all the levels below this are + not displayed to the console or stored. + +`info` indicates that something notable has happened. This +should not be used for any errors - instead, it should be +used for notable events, such as a new user signing up. + +### Verbose + +```ts +logger.verbose("foo"); +``` + +`verbose` indicates that something has happened. This is +used for debugging, and is typically only enabled in dev +to find out what has gone wrong. + +### Debug + +```ts +logger.debug("foo"); +``` + +`debug` is the least notable logging level. This is used +exclusively for debugging, and logs typically uninteresting +things like Captcha Requests being sent, or a single score was imported. + +## Logger Usage + +In `src/lib/logger/logger.ts` we define a wrapper around winston +for our logging needs. We use two functions for this. + +The first function is `CreateLogCtx`. This spawns an +instance of the logger with "context". + +This context allows us to keep track of common information +for the logger. + +For file-level logging, you should do: + +```ts +import CreateLogCtx from "~src/lib/logger/logger"; +const logger = CreateLogCtx(__filename); + +logger.info("foo"); +// [src/file.ts] INFO: foo +``` + +!!! info + CreateLogCtx is short for Create Log Context. + +This will spawn an instance of the logger with the context +of the current filename. + +For more specific logging, you should create a logger +and pass it around as an argument to a function. + +This is used in `src/lib/score-import`, as we create a +logger with the user's name and import type as "context". + +```ts +const logger = CreateLogCtx(`${username} ${userID}`); + +logger.info("foo"); +// [zkldi 1] INFO: foo + +SomeOtherFunction(argument1, argument2, logger); +``` + +!!! info + Logger should be the last argument for a function that + takes a logger. + + The only exception to this is if it uses our `fetch` API, + which **MUST** always be the last call. + +If we have an existing logger and want to append context, +we can do that with the `AppendLogCtx` function. + +```ts +const logger = CreateLogCtx("foo"); + +logger.info("hello!"); +// [foo] INFO: hello! + +const logger2 = AppendLogCtx("bar", logger); + +logger2.info("hello!"); +// [foo | bar] INFO: hello! +``` + +## Log Files + +Logs are saved to `${pwd}/logs` in two files. The first +file logs everything above `LOG_LEVEL`. The second file +logs `error` and above calls. + +!!! bug + Unintentional behaviour occurs here if LOG_LEVEL is + above `error` - the main file will not log errors, + but the error file will log errors regardless. + + This may be fixed at some point, but is fairly low priority. diff --git a/docs/docs/codebase/infrastructure/testing.md b/docs/docs/codebase/infrastructure/testing.md new file mode 100644 index 000000000..966aaa7a9 --- /dev/null +++ b/docs/docs/codebase/infrastructure/testing.md @@ -0,0 +1,57 @@ +# Tests + +Tachi makes extensive use of tests to ensure code +is of high quality when its released. + +***** + +## Running Tests + +Tests are ran with [node-tap](https://node-tap.org). + +You can run the server test suite with the following script: + +``` +pnpm fulltest +``` + +This will execute every test, and also perform coverage +analysis. + +## Parallel Tests + +You can run tests in parallel with: +``` +pnpm paralleltest +``` + +!!! danger + **DO NOT RUN THIS ON YOUR LOCAL DEVELOPMENT SETUP**! + + This will flood your MongoDB installation with nonsense + collections. + + This is intended for things like GitHub Actions, where + the environment itself is ephemeral and fine to be + destroyed. + +This provides slight performance benefits, but as mentioned +in the warning above, is only for ephemeral environments. + +## Writing Tests + +Tests should be located in the same folder as the file they're testing, and tests should only ever test +exports from one file at a time. + +Tests should call things like `ResetDBState` and `CloseAllConnections` as lifecycle hooks to ensure +the file exits. + +Tests should use the extension `.test.ts`, and keep +the same filename as the file they are testing. + +## Coverage + +Coverage should be kept above 80%. New code should be tested. + +Pull Requests will be rejected if they contain significant +changes that are untested! \ No newline at end of file diff --git a/docs/docs/codebase/infrastructure/toolchain.md b/docs/docs/codebase/infrastructure/toolchain.md new file mode 100644 index 000000000..47b08d21c --- /dev/null +++ b/docs/docs/codebase/infrastructure/toolchain.md @@ -0,0 +1,99 @@ +# Toolchain + +This page documents all the various tools involved with +`tachi-server` and why they were chosen specifically. + +A lot of this is personal preference, but hopefully the +justifications provide enough insight into why. + +***** + +## Programming Language: TypeScript + +Everything is wrote in [TypeScript](https://www.typescriptlang.org). TypeScript provides +substantial usability improvements for JS at scale, +and integrates very nicely with IDEs. + +## Package Manager: pnpm + +`pnpm` is the preferred package manager for all `tachi-` projects. + +It can be acquired with `npm install -g pnpm`, and documentation can be found [here](https://pnpm.io). + +### Why not `npm`? + +`npm` duplicates packages across my whole system, and it takes up a lot of package space. Furthermore, each new install requires new network calls, and my internet is pretty poor. + +`pnpm` dedupes packages everywhere, and pulls packages from the local cache if possible. + +It's also way faster. The developers behind `pnpm` have seriously +put a lot of work into it, and it's a great tool. + +## Main Database: MongoDB + +We use [MongoDB](https://mongodb.com) because it works nicely with JSON, and has +good support for Javascript and TypeScript. + +!!! tip + If you're debugging MongoDB state, I cannot recommend + [MongoDB Compass](https://www.mongodb.com/products/compass) enough. It's one of the best ways to interact + with Mongo and analyse queries. + +### Why not SQL? + +We take advantage of the more dynamic nature of MongoDB documents in some places to get away with writing less code, +which means a port to SQL is non-trivial. + +The document model also meshes nicer with the rest of the codebase. + +!!! note + We also use Redis for sessions and inter-process communication. + +## Database Driver: Monk + +[Monk](https://github.com/automattic/monk) is a simple wrapper around the MongoDB native NodeJS Driver. Although it's a simple wrapper, it is a massive +UX improvement over the native wrapper. + +### Why not Mongoose? + +Mongoose mandates schemas, and works with a much more OOP-style +approach to handling documents. I personally think it's an unecessary +abstraction for this project specifically, but is a great library nonetheless. + +## Logger: Winston + +We use the [Winston](https://github.com/winstonjs/winston) framework to provide logging to the application. You can read more about this in [Logging](./logging). + +!!! note + Winston seems to be an abandoned project. Furthermore, + the level of documentation is questionable at best. + + Despite this, Winston is an incredibly mature and + stable logging framework, and is a good fit for + our project as a result. + +## HTTP Client: Express + +[Express](https://github.com/expressjs/express) is a battle tested HTTP framework for NodeJS, and +the middleware chain is incredibly useful. + +### Why Not Koa or TinyHTTP? + +It lacks a lot of the ecosystem Express has - mainly +with regards to testing. + +## Test Runner: TAP + +[Node TAP](https://node-tap.org) is an opinionated testing framework by the guy +who made NPM. + +The opinionated parts of it line up exactly with what I +want from a testing library, and it has batteries included for everything I want. + +This also handles our coverage reports by calling `nyc` +and automatically uploads to CodeCov with our CI setup. + +## CI-CD: Github Actions + +Handles our deployment and test running. We also integrate +with CodeCov for the coverage reports. diff --git a/docs/docs/codebase/overview.md b/docs/docs/codebase/overview.md new file mode 100644 index 000000000..385c3b8b5 --- /dev/null +++ b/docs/docs/codebase/overview.md @@ -0,0 +1,55 @@ +# Codebase Overview + +This part of the documentation is for the [Tachi-Server](https://github.com/zkldi/tachi-server) codebase. + +!!! info + This is *ONLY* documentation for `tachi-server`. + + As mentioned below, `tachi-client` is closed source, + and does not have public codebase documentation. + +## Codebase Documentation vs. Code Documentation + +This is documentation for the **Codebase**. **NOT** documentation for the code. + +The distinction is because we aren't writing a library here - there's no need to document function +signatures or what function calls are meant to do. That can all be done inline because no other +projects depend on our function calls! + +This documentation is more meta-level. Why things are in certain folders, what certain enums +correspond to, how `thing` works, etc. + +## Repos and Licenses + +Tachi is made up of three components: + +- `tachi-common`: Common types and values for Tachi. [GitHub](https://github.com/zkldi/tachi-common). This is licensed under MIT. +- `tachi-server`: The API, IR implementations and 'business logic' behind Tachi. [GitHub](https://github.com/zkldi/tachi-server). This is licensed under AGPLv3. +- `tachi-client`: The front-end code for Tachi. This is closed source. + +### What's with `tachi-client`? + +The original plan was to have the entire codebase be open source. However, I've decided to make the +front-end code for Tachi be closed source. This is to reduce the amount of low-effort forks Tachi +may receive by virtue of being open source. + +Other rhythm game projects as of recently have had a massive influx of low-effort forks that just +change the name of the project, rehost it, break the license, and don't upload any code back upstream. + +It's disheartening, and the reason that the client is not publically available. + +I picked the server code instead of the client code to open source because I think it's of significantly +more interest to other programmers in the scene. + +### Why even bother with open source then? + +I'm proud of the code, and I think other people will be interested in it. + +Also, it's nice to give back to the community. + +### What's with `tachi-common`? + +Some things are common between the client and server. +It's pretty much just one very large types.ts file that needs to be split up. + + diff --git a/docs/docs/codebase/setup/config.md b/docs/docs/codebase/setup/config.md new file mode 100644 index 000000000..52bf159dc --- /dev/null +++ b/docs/docs/codebase/setup/config.md @@ -0,0 +1,147 @@ +# Configuration Info + +The codebase uses a file called `conf.json5` to handle +various configurable options. + +## What is JSON5? + +JSON5 is an extension of JSON which is better suited +for configuration files. + +You can read more about it [here](https://json5.org/), but +the main benefits for us are as follows: + +- Comments +- No Quoting properties +- Trailing Commas + +## Example Config File + +```js +{ + MONGO_BASE_URL: "127.0.0.1", + LOG_LEVEL: "info", + CAPTCHA_SECRET_KEY: "google_given_secret_key", + SESSION_SECRET: "some_secret_key", + FLO_API_URL: "https://flo.example.com", + EAG_API_URL: "https://eag.example.com", + ARC_API_URL: "https://arc.example.com", + ARC_AUTH_TOKEN: "another_secret_key", + CDN_ROOT: "./local-cdn", + PORT: 8080, + TYPE: "ktchi" +} +``` + +!!! warning + **DO NOT BLINDLY COPY THIS CONFIGURATION FILE!** + + Seriously, The `SESSION_SECRET` token MUST not be + public. + +## Properties + +All properties are required. + +### MONGO_BASE_URL + +- Type: String + +Where your MongoDB server is located. For most cases, this +is `127.0.0.1` or `localhost`. + +For more creative scenarios such as running through WSL, you +might need to change this. + +### LOG_LEVEL + +- Type: Log Level + +Sets the level at which logs will be saved to disk/logged to console. + +By default, this is `"info"`. The valid values can be found at [Logging Info](../infrastructure/logging.md). + +### CAPTCHA_SECRET_KEY + +- Type: String + +Google gives us a Captcha Secret Key in order for Captcha +to work on our site. + +### SESSION_SECRET + +- Type: String + +This key is used to encrypt Session Cookies. + +!!! warning + If this key is figured out, anyone can log in as + anyone. + + Make sure this is an appropriately long string + generated from a *cryptographically secure* source. + That is, do not just mash your keyboard. + + You can generate secure random strings with something + like [KeePass](https://keepass.info/). + +### FLO_API_URL, EAG_API_URL and ARC_API_URL + +- Type: String + +The URL for the `FLO, EAG or ARC` services. This is used for integration +with the Kamaitachi version of Tachi. + +### ARC_AUTH_TOKEN + +- Type: String + +We use an ARC session token in order to pull scores from `ARC`. The session token is stored here. + +### CDN_ROOT + +- Type: File Path + +`tachi-server` expects a CDN to store things like replays +and profile pictures. This is a folder somewhere on the +system (presumably using nginx serve-static). + +### PORT + +- Type: Port (1-65536) + +What port to run the server on. + +### TYPE + +- Type: "btchi" | "ktchi" | "omni" + +What 'type' the instance of the server is. + +"btchi" will run the server under Bokutachi mode, which +will disable Kamaitachi-only routes and change the names of various things. + +"ktchi" will run the server under Kamaitachi mode, which +will disable Bokutachi-only routes and change various +names. + +"omni" will run the server without any route restrictions. +This is used for testing. + +## Environment Variables + +### TACHI_PARALLEL_TESTS + +If this is set and `tap` is invoked, this will change some +things about the environment to support running multiple +tests at once. + +!!! warning + This will ruin your MongoDB installation with lots of + ephemeral collections. Do not run this on a local + setup. + + This is intended for Github Actions' runner, which is + designed to quickly start up and then tear everything + down afterwards. + diff --git a/docs/docs/codebase/setup/setup.md b/docs/docs/codebase/setup/setup.md new file mode 100644 index 000000000..bef047128 --- /dev/null +++ b/docs/docs/codebase/setup/setup.md @@ -0,0 +1,82 @@ +# Setup + +This page documents how to get the `tachi-server` up and running. + +This guide assumes that you are running on some +flavour of Linux. +`tachi-server` is not tested on Windows, but it is likely +to work, as we don't depend on any OS specific calls. + +## Externals + +`tachi-server` depends on some external projects (mainly databases) in order to work. + +### 1. Run MongoDB + +The easiest way to do this is through `docker`. + +``` +docker pull mongo +docker run -d -p 27017:27017 --name mongodb mongo +``` + +### 2. Run Redis + +See [Redis QuickStart](https://redis.io/topics/quickstart). + +### 3. Get NodeJS and NPM. + +``` +sudo apt-get install nodejs +``` + +!!! info + `tachi-server` has not been tested on anything other than NodeJS v14, v15 and v16. + + Given that these are all the latest versions of NodeJS, + this shouldn't be an issue. + +### 4. Pull the code. + +``` +git clone https://github.com/zkldi/tachi-server +cd tachi-server +``` + +### 5. Install Dependencies + +We use `pnpm` instead of NPM. You can read why [here](../infrastructure/toolchain.md). + +Ironically, NPM is the easiest way to install PNPM. +NPM should come bundled with your install of NodeJS. + +``` +npm install -g pnpm +``` + +Then we can install our dependencies. + +``` +pnpm install +``` + +### 6. Setup the `tachi-server` config. + +We use a file called `conf.json5` in the top level +of the repository to handle configuration of the +server. + +The config file we use for running tests can be found in `.github/test.conf.json5` + +You can read more on how this config file works at [Configuration Info](./config.md) + + +### 7. Run! + +``` +pnpm buildrun +``` + +This will build the server TypeScript into JS under `/js`, +and run it. The port it runs on is determined by the config +file. diff --git a/docs/docs/codebase/structure/filesystem.md b/docs/docs/codebase/structure/filesystem.md new file mode 100644 index 000000000..65b9ee6f5 --- /dev/null +++ b/docs/docs/codebase/structure/filesystem.md @@ -0,0 +1,134 @@ +# File/Folder Organisation + +`tachi-server` has a specific setup of files and folders +to ensure that code is at where it's most sensible. + +!!! note + This documentation is a rough guide for where + to place files if you are writing a new file, + or where to look for certain functionality. + + It is not a comprehensive tutorial for every file + in the repo, as that would be a pain to keep updated. + +***** + +## Top Level + +All of these are at the root level of the project. + +### `/src` + +All of the server TypeScript code goes here. + +### `/js` + +!!! info + This folder is gitignored. + +When compiled, `tsc` will output the JS code here. + +### `/scripts` + +Various scripts for interacting with `tachi-server`, such +as single-use scripts for importing some data, or +frequently used scripts such as updating BMS tables. + +## TypeScript Source Code + +All of these are inside `/src`. + +### `/datasets` + +Some of `tachi-server`'s code interacts with datasets that +aren't worth putting into MongoDB, such as splash text. + +This is mainly for things where we want to randomly select +from the list, and not perform any serious lookups - which +is why it's a good fit for splash text/automatic session names. + +!!! info + Selecting a random element from an entire collection + in MongoDB is relatively expensive, and would quadruple + the time an import takes. + +!!! warning + TypeScript does not support copying over non-code files. + + You can use `cp` in post to move files around, or + place the data in memory, either is fine. + +### `/external` + +Code relating to the "external" applications for `tachi-server`, +such as MongoDB and Redis. + +### `/lib` + +Sets of code for `tachi-server` functionality. This is the +main important part of the codebase for handling things +like score imports, logging, and more. + +### `/server` + +This contains the express application that `tachi-server` +uses in order to be a server. + +This contains our API, IR implementations and a way of +serving our PWA. + +### `/test-utils` + +Tachi's tests need mocks and some specialised code in order +to work well. This folder contains all of those things. + +!!! warning + **NOTHING** from this folder should be ran in production. + +### `/utils` + +Small utilities for interacting with Tachi, such as +functions that retrieve a user given certain params. + +This also contains utilities for handling song/chart +database lookups - such as looking up on BMS hash. + +## Express Server + +All of the below folders are under `/src/server`. + +As mentioned above, Tachi stores the routing for our +APIs and IR implementations here. + +### `/middleware` + +This contains the middleware we use for the server, +such as authentication middleware and such. + +### `/router` + +This contains the actual 'routes' for our server. + +The folders here **MUST** be 1:1 with the endpoints +on the server. For example, the implementation of + +``` +https://bokutachi.xyz/api/v1/foo/bar +``` + +**MUST** be found at `src/server/router/api/v1/foo/bar/router.ts` + +### Router Files + +The only files allowed to declare endpoints are `router.ts` files. + +This allows us to seperate functionality from API structure +in cases where an API call needs to do a lot of things. + +## Test Files + +Test files are to be located in the same folder as the file +they're testing, and should only ever test the exports +of one file. + +They should have the filename of the file they're testing, with the extension `.test.ts`. diff --git a/docs/docs/codebase/structure/style.md b/docs/docs/codebase/structure/style.md new file mode 100644 index 000000000..b7c0b498e --- /dev/null +++ b/docs/docs/codebase/structure/style.md @@ -0,0 +1,139 @@ +# Style + +This document explains the code style rules for the repo. + +Please make sure any changes you commit adhere to the defined linting rules. + +For style, we use ESLint + Prettier. We have some options set for prettier, and a lot of ESLint rules set. + +Since it's redundant (You should just read the eslint docs), and mostly just personal preference, each individual rule isn't going to be explained here. + +ESLint is set up to automatically perform all of these changes when ran. + +!!! info + You can run ESLint in the repo any time with `pnpm lint` or `eslint ./src --ext .ts --fix`. + +***** + +## Prettier Rules + +- 4 Spaces Indenting. + +I'd prefer to use tabs, honestly, but Prettier and JSDoc like to align things with spaces and it messes with them. + +- Semicolons. + +No-Semicolons causes issues with IIFEs. + +- Try to keep things under 100 characters. + +Absolutely do not insert random line breaks to keep stuff under 100 characters. It's fine for things to go a bit over. + +Prettier has its own opinions on where these line breaks should happen, just trust them. + +- Double Quotes instead of Single Quotes + +JSON does it and that's pretty much the only reason why. + +- Line Break is LF, not CRLF + +Your editor will handle this properly. If it does not +automatically set, check the bottom right of your editor. +For Atom, VSCode and some others it will let you switch between +CRLF and LF. + +## Commenting Style + +The Tachi-Server codebase handles code comments against these rules: + +1. Do not write redundant comments. +2. Return signatures are obvious from TypeScript most of the time. If they aren't, declare them. +3. Most of the time, comments should describe *why* code does something. +4. The exception is when a line or function **needs** to do something complicated - in which case, comments can be *how*. +5. That said, code should be self-explanatory. +6. If it isn't, it should be refactored until it is. +7. If that isn't possible (The logic is complex for good reason), the documentation should be next to the code. + +### Example + +For this example, we're going to write a function for calculating +[Standard Deviation](https://en.wikipedia.org/wiki/Standard_deviation). + +```ts +function sd(arr: number[]) { + const m = arr.reduce((a, r) => a + r) / arr.length; + + return Math.sqrt(arr.reduce((a, r) => (r + m) ** 2) / arr.length); +} +``` + +This is bad code. Very bad code. It is not at all clear what +this code does from any of the variable names, and the function signature barely helps. + +Lets try and make this code more self documenting. + +We'll give everything proper variable names, and then +expand the second `reduce` call into a simpler for loop. + +```ts +function CalculateStandardDeviation(dataset: number[]) { + const mean = arr.reduce((a, r) => a + r) / dataset.length; + + let variance = 0; + + for (const value of dataset) { + variance += (value - mean) ** 2; + } + + return Math.sqrt(variance / dataset.length); +} +``` + +Now that is much cleaner, but it's still perfect. + +For example, what if someone didn't know what the standard deviation was? +Standard Deviation is *external knowledge*, and as such, we should document what this is. + +The lazy way, is just this: +```ts +/** + * @see {@link https://en.wikipedia.org/wiki/Standard_deviation} + */ +function CalculateStandardDeviation(dataset: number[]) { + // ... +} +``` + +This is a perfectly acceptable way to document code. We +don't need to describe any returns or even the function here. As always, use your head to judge whether something needs external knowledge or not. + +## Comment Directives + +Tachi uses comment directives to allow for quick searching +of certain comments. Directives are used like this: + +```ts +// @todo A bug with the foo is here. +1 + 1; +``` + +The list of directives and their meaning is here: + +| Directive | Description | +| :: | :: | +| `@todo` | This line has a bug or something unfinished that needs to be done. A GitHub issue should be created about this. | +| `@danger` | The below code is dangerous, and if called wrong could cause issues. This directive should make the programmer act cautiously around the code before making changes. | +| `@hack` | A hack is used here. | +| `@optimisable` | This code is optimisable, and how to optimise it is known, but has not been implemented yet. | +| `@inefficient` | This code is slow, and how to optimise it is not figured out yet. | + +!!! info + TypeScript also uses comment directives for things + like `@ts-expect-error`. What these do is documented + in [TypeScript's documentation](https://www.typescriptlang.org). + +## Strictness + +Don't worry about this too much, At the end of the day, as +long as the code is understandable and the linter is happy, +it's good. diff --git a/docs/docs/index.md b/docs/docs/index.md index a35d53a11..e9566bfb3 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -1 +1,45 @@ -# Documentation Name \ No newline at end of file +# Tachi Documentation + +This site serves as the documentation for Tachi. + +Tachi is a rhythm game score tracking engine, and is the name of the codebase that powers +both [Bokutachi](https://bokutachi.xyz) and [Kamaitachi](https://kamaitachi.xyz). + +## About This Documentation + +So that we don't have to write and maintain three separate sets of documentation, this site +works like a Monorepo for Tachi's documentation. + +***** + +### API Reference + +This is for people who want to make things with Tachi's API, and assumes basic knowledge of +APIs and how they work. + +!!! note + The API reference overlaps a bit with the Codebase reference, mainly in + the shape of various returned documents. +### Codebase Reference + +This is for people who want to work on Tachi's codebase. +Tachi is wrote almost entirely in [TypeScript](https://www.typescriptlang.org). + +### User Reference + +This is for end user reference, such as score importing tutorials, documentation on tachi's +statistics. It requires no programming knowledge, and is mostly used as a wiki-like reference. + +## Contributing + +If you want to contribute to this documentation, you can find the repository [here!](https://github.com/zkldi/tachi-docs) + +If you want to contribute to the Tachi backend code, you can find the repository [here!](https://github.com/zkldi/tachi-server) +!!! note + You should read the [Contributing To Tachi-Server](#) page beforehand. + +## Acknowledgements + +This documentation is built ontop of the beautiful [Material MKDocs](https://squidfunk.github.io/mkdocs-material) theme. + +--8<-- "includes/abbreviations.md" \ No newline at end of file diff --git a/docs/includes/abbreviations.md b/docs/includes/abbreviations.md new file mode 100644 index 000000000..a3307de1f --- /dev/null +++ b/docs/includes/abbreviations.md @@ -0,0 +1 @@ +*[Monorepo]: When code for many different projects is stored in the same repository. \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 62323ccbe..4427d22d5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,16 +1,79 @@ -site_name: Project Name -site_description: Project Description -site_author: Author +site_name: Tachi Documentation +site_description: Mono-documentation for Kamaitachi, Bokutachi and related things. +site_author: zkldi +site_url: https://docs.bokutachi.xyz theme: name: material + features: + - navigation.instant + - navigation.tabs palette: - scheme: slate + - scheme: slate + primary: deep purple + accent: deep purple + toggle: + icon: material/weather-sunny + name: Switch to light mode + - scheme: default + primary: deep purple + accent: deep purple + toggle: + icon: material/weather-night + name: Switch to dark mode nav: - Introduction: "index.md" + - API Reference: + - "api/overview.md" + - "api/auth.md" + + - Endpoints: + - "api/routes/example.md" + - "api/routes/status.md" + - "api/routes/import.md" + - "api/routes/auth.md" + - Codebase Reference: + - "codebase/overview.md" + - "codebase/contributing.md" + + - Setup: + - "codebase/setup/setup.md" + - "codebase/setup/config.md" + + - Infrastructure: + - "codebase/infrastructure/toolchain.md" + - "codebase/infrastructure/logging.md" + - "codebase/infrastructure/branches.md" + - "codebase/infrastructure/testing.md" + + - Structure: + - "codebase/structure/style.md" + - "codebase/structure/filesystem.md" + + - Score Importing: + - "codebase/import/overview.md" + - "codebase/import/main.md" + - "codebase/import/import-types.md" + - "codebase/import/parse-conv.md" + - "codebase/import/conv-failures.md" + - "codebase/import/importing.md" + - "codebase/import/orphans.md" + - "codebase/import/parse-ipi.md" + - "codebase/import/sessions.md" + - "codebase/import/pbs.md" + - "codebase/import/ugs.md" + - "codebase/import/goals.md" + - "codebase/import/milestones.md" + - "codebase/import/import-doc-time.md" + + + markdown_extensions: - admonition - pymdownx.highlight - - pymdownx.superfences \ No newline at end of file + - pymdownx.superfences + - abbr + - pymdownx.snippets + - footnotes