Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-05 13:28:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #863 from TNG-dev/zkldi/easy-setup

Browse Source
This commit is contained in:
zkldi
2023-02-09 01:44:36 +00:00
committed by GitHub
parent 8a26db2450 03a0932c84
commit 796804e1d0
23 changed files with 838 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+1 -2
View File
@@ -2,5 +2,4 @@
node_modules
.tsbuildinfo
**/node_modules
**/.tsbuildinfo
database-seeds
**/.tsbuildinfo
.eslintignore
+2 -1
View File
@@ -1,2 +1,3 @@
node_modules
js
js
package.json
.gitattributes
+6
View File
@@ -0,0 +1,6 @@
# if you don't do this
# a windows clone will fail
# because the .sh scripts will have their \n's turned into
# \r\n, and then bash will fail to run \r as a command.
# the future is now.
*.sh text eol=lf
.gitignore
+5 -1
View File
@@ -7,6 +7,7 @@ node_modules
# PNPM noise
.pnpm-debug.log
.pnpm-store
# Compiled output
js
@@ -36,4 +37,7 @@ coverage
out.*
failed-tests.log
.tsbuildinfo
.tsbuildinfo
# bootstrap flag
I_HAVE_BOOTSTRAPPED_OK
Dockerfile.bootstrap
+15
View File
@@ -0,0 +1,15 @@
# Yes, I've made an entire docker container just to run a bash script
# instead of rewriting the script in a ~~real~~ cross-platform scripting language.
# This docker container **depends** on an external volume being mounted
# (see how it's used in docker-compose-dev.yml)
# and *intentionally* mutates state outside of its container.
# I really do not care.
# Want it better? Do it yourself.
FROM node:16 as base
RUN npm install --silent -g pnpm
WORKDIR /app
CMD ["bash", "_scripts/bootstrap.sh"]
Dockerfile.client
+21
View File
@@ -0,0 +1,21 @@
FROM node:16-alpine as base
RUN npm install --silent -g pnpm
FROM base AS build
WORKDIR /app
COPY pnpm-lock.yaml .
COPY patches ./patches
RUN pnpm fetch
COPY client ./client
COPY common ./common
COPY *.json *.yaml ./
RUN pnpm --filter tachi-client... --filter . install --offline --frozen-lockfile
WORKDIR /app/client
CMD ["pnpm", "start"]
Dockerfile.seeds
+25
View File
@@ -0,0 +1,25 @@
# no alpine here as we want bash + general nice-to-haves
FROM node:16 as base
RUN npm install --silent -g pnpm
FROM base AS build
WORKDIR /app
COPY pnpm-lock.yaml .
COPY patches ./patches
RUN pnpm fetch
COPY database-seeds ./database-seeds
COPY common ./common
COPY *.json *.yaml ./
RUN pnpm --filter tachi-database-seeds... --filter . install --offline --frozen-lockfile
WORKDIR /app/database-seeds
# keep container alive indefinitely
# we just have this container so db-seeds maintainers have a nice "context"
# to work with
CMD ["tail", "-f", "/dev/null"]
Dockerfile.server
+21 -4
View File
@@ -2,6 +2,7 @@
# It does not spin up mongodb instances or redis instances, which the server
# does need to boot. You should consider using docker-compose for this.
# base image
FROM node:16-alpine as base
ARG COMMIT_HASH
ENV COMMIT_HASH=${COMMIT_HASH}
@@ -9,7 +10,8 @@ ENV COMMIT_HASH=${COMMIT_HASH}
RUN npm install --silent -g pnpm
RUN apk add --no-cache git curl
FROM base AS build
# install dependencies
FROM base AS install
WORKDIR /app
COPY pnpm-lock.yaml .
@@ -20,15 +22,30 @@ RUN pnpm fetch
COPY server ./server
COPY common ./common
COPY *.json *.yaml ./
RUN pnpm --filter tachi-server... --filter . install --offline --frozen-lockfile
# development image, keeps installed dependencies
FROM install AS dev
RUN git config --global --add safe.directory /app
HEALTHCHECK --interval=15s --timeout=5s CMD curl -f http://localhost:8080/api/v1/status || exit 1
WORKDIR /app/server
ENV NODE_PATH=js/
CMD ["pnpm", "dev"]
# build source and dependencies
FROM install AS build
RUN pnpm --filter tachi-server... -r build
FROM base AS app
# built source only for production
FROM base AS prod
COPY --from=build /app /app
HEALTHCHECK --interval=15s --timeout=5s CMD curl -f http://localhost:8080/api/v1/status || exit 1
WORKDIR /app/server
ENV NODE_PATH=js/
CMD ["node", "js/main.js"]
CMD ["node", "js/main.js"]
_scripts/add-repo-to-monorepo.sh
-29
View File
@@ -1,29 +0,0 @@
#! /bin/bash
set -eo pipefail
BRANCH="${2:-develop}"
PROJECT="$1"
if [ -z "$PROJECT" ]; then
echo "Please pass a tachi-X repo to meld with this repository."
exit 1
fi
git remote add "$PROJECT" gh:/tng-dev/tachi-"$PROJECT" || echo "remote already mounted. continuing anyway."
git fetch "$PROJECT" --no-tags
git branch "$PROJECT-$BRANCH" "$PROJECT/$BRANCH" || echo "branch already exists. continuing anyway."
git remote remove "$PROJECT"
git switch "$PROJECT-$BRANCH"
mkdir -p "$PROJECT"
git filter-repo --to-subdirectory-filter "$PROJECT/" --refs "$PROJECT-$BRANCH" --force
git switch master
git merge --allow-unrelated-histories "$PROJECT-$BRANCH"
_scripts/bootstrap.sh
+31 -16
View File
@@ -2,6 +2,7 @@
set -eo pipefail
# https://stackoverflow.com/questions/59895/how-can-i-get-the-directory-where-a-bash-script-is-located-from-within-the-scrip
# if you actually think bash is a good programming language you are
# *straight up delusional*
@@ -10,6 +11,11 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
cd "$SCRIPT_DIR";
cd ..;
if [ -e I_HAVE_BOOTSTRAPPED_OK ]; then
echo "Already bootstrapped."
exit 0
fi
function mvExampleFiles {
echo "Moving example config files into usable places..."
@@ -54,12 +60,26 @@ function pnpmInstall {
echo "Installed dependencies."
}
function setIndexes {
echo "Setting indexes..."
(
cd server
pnpm run set-indexes
)
echo "Set."
}
function syncDatabaseWithSeeds {
echo "Syncing database with seeds..."
cd server
(
cd server
pnpm run sync-database-local
pnpm run sync-database-local
)
echo "Synced."
}
@@ -67,23 +87,18 @@ function syncDatabaseWithSeeds {
mvExampleFiles
selfSignHTTPS
pnpmInstall
setIndexes
syncDatabaseWithSeeds
tput setaf 1
echo "Bootstrap Complete."
cat << EOF
=== READ THIS YOU MUPPET ===
cat << EOF > I_HAVE_BOOTSTRAPPED_OK
Tachi was bootstrapped here on $(date).
YOU ARE USING SELF-SIGNED HTTPS CERTIFICATES.
YOU WILL LIKELY HAVE TO GO TO HTTPS://127.0.0.1:8080 AFTER STARTING TACHI-SERVER.
TELL YOUR BROWSER THESE CERTIFICATES ARE TRUSTED!
OTHERWISE, THE CLIENT WILL FAIL TO LAUNCH, AND WILL JUST BE A WHITE PAGE!
EOF
The existence of this file stops Tachi from bootstrapping again.
There's nothing harmful about this -- you can bootstrap as much as you want!
We just don't want to necessarily bootstrap *each* time we boot Tachi.
tput sgr0
cat << EOF
Bootstrapped.
Launch the server with pnpm start-server.
Launch the client with pnpm start-client.
To bootstrap again (incase you think something has gone wrong)
Delete this file and re-run npm start.
EOF
client/package.json
+2
View File
@@ -35,6 +35,7 @@
"@fullcalendar/react": "^6.0.0",
"@mui/material": "^5.11.1",
"@nivo/bar": "0.79.1",
"@nivo/colors": "0.79.1",
"@nivo/core": "0.79.0",
"@nivo/line": "0.79.1",
"@nivo/tooltip": "0.79.0",
@@ -42,6 +43,7 @@
"@types/natural-compare": "^1.4.1",
"@types/react-google-recaptcha": "2.1.2",
"csstype": "3.1.1",
"d3-scale": "3.3.0",
"deepmerge": "4.2.2",
"dotenv": "6.2.0",
"fast-json-stable-hash": "^1.0.3",
client/src/app/pages/dashboard/import/SilentHookPage.tsx
+1 -1
View File
@@ -23,7 +23,7 @@ export default function SilentHookPage() {
Download <code>silent</code> from{" "}
<ExternalLink href="https://zkldi.xyz/stuff/silent-latest.zip">
Here
</ExternalLink>
</ExternalLink>{" "}
and place all the <code>.dll</code> files in the same folder as{" "}
<code>popn22.dll</code>.
<br />
client/src/lib/config.ts
+2 -3
View File
@@ -55,11 +55,10 @@ try {
? `
<hr />
<div><b>You're in local development mode.</b>
<ul>
<ul style="font-size: 2rem;">
<li>Have you accepted the HTTPS certificates for <a href="${ToAPIURL(
"/"
"/status"
)}">the server?</a>. If not, the site won't load.</li>
<li>Failing that, have you made sure to start the server with <code>pnpm start-server</code>?</li>
</ul>
`
: ""
client/vite.config.js
+1
View File
@@ -81,6 +81,7 @@ export default defineConfig(() => {
],
server: {
port: 3000,
host: true,
},
preview: {
port: 3000,
docker-compose-dev.yml
+80
View File
@@ -0,0 +1,80 @@
version: "3.9"
services:
mongo:
container_name: mongo
image: mongo:5
restart: always
ports:
- '27017:27017'
volumes:
- tachi-volume:/data/db
redis:
container_name: redis
command: redis-server --save 60 1 --loglevel warning
image: redis:6
restart: always
ports:
- '6379:6379'
volumes:
- tachi-redis:/data
tachi-server:
container_name: tachi-server
build:
dockerfile: Dockerfile.server
target: dev
depends_on:
bootstrap:
condition: service_completed_successfully
redis:
condition: service_started
mongo:
condition: service_started
links:
- redis
- mongo
volumes:
- ./:/app
ports:
- '8080:8080'
environment:
REDIS_URL: redis
MONGO_URL: mongo
PORT: 8080
NODE_ENV: "dev"
tachi-client:
container_name: tachi-client
depends_on:
bootstrap:
condition: service_completed_successfully
build:
dockerfile: Dockerfile.client
volumes:
- ./:/app
ports:
- '3000:3000'
tachi-seeds:
container_name: tachi-seeds
build:
dockerfile: Dockerfile.seeds
volumes:
- ./:/app
bootstrap:
container_name: tachi-bootstrap
build:
dockerfile: Dockerfile.bootstrap
volumes:
- ./:/app
links:
- redis
- mongo
environment:
REDIS_URL: redis
MONGO_URL: mongo
PORT: 8080
NODE_ENV: "dev"
volumes:
tachi-volume:
tachi-redis:
tachi-logs:
docker-compose.yml
-22
View File
@@ -1,22 +0,0 @@
version: "3.9"
services:
mongo:
container_name: mongo
image: mongo:5
restart: always
ports:
- '27017:27017'
volumes:
- tachi:/data/mongo
redis:
container_name: redis
command: redis-server --save 60 1 --loglevel warning
image: redis:6
restart: always
ports:
- '6379:6379'
volumes:
- tachi:/data/redis
volumes:
tachi:
driver: local
docs/docs/contributing/setup.md
+20 -115
View File
@@ -6,42 +6,7 @@ You don't *necessarily* need to have a working install to contribute - you could
make documentation contributions without having anything running on your machine - but
it's extremely helpful to be able to run Tachi's things while working on them.
## 0. Unix Necessary!
You *must* be on some Unix-Like operating system. That ideally means means Linux, MacOS or WSL.
!!! info
This is because of Redis, mainly. We also depend on the `sendmail` binary to send emails around, which might be even less portable.
Also - we use Linux in production, and your local
environment should be as close to production as reasonably possible.
### I'm on Windows, help!
Don't worry. You can run Linux as a subsystem inside Windows, without any virtual machine
nonsense. It's a bit of a headache to get sorted, but Microsoft provide a decent [setup guide](https://docs.microsoft.com/en-us/windows/wsl/install-win10).
!!! danger
You can pick your own Linux distro to use when setting up WSL2.
**YOU MUST PICK UBUNTU LATEST (22.04) UNLESS YOU KNOW EXACTLY WHAT YOU ARE DOING**. Anything else will likely have *ancient*
versions of software installed, and will cause some obscene headaches.
**UBUNTU 20.04 LTS HAS A SIGNIFICANTLY-TOO-OLD VERSION OF GIT. DO NOT USE IT.**
### I'm on Mac.
Everything should work out of the box.
### I'm on Linux.
Tachi has been tested on Ubuntu, Debian, Arch and Manjaro. There's no reason it won't run on any Linux variant, but
if you're running something willfully obtuse like Artix or Slackware, you'll probably hit problems. Figure it out yourself.
The happiest path for the below guide is for Arch/Manjaro users. For other distros, you
might find getting some of the services running a bit cumbersome. Ah well.
## 0.5. Get some developer tools.
## 0. Get some developer tools.
If you're an experienced programmer, you won't need this bit.
@@ -65,41 +30,22 @@ with everything we use.
You'll need a terminal to run commands in. For Linux and Mac users, you can just open
a Terminal app.
However, for Windows users we recommend installing the [Windows Terminal](https://apps.microsoft.com/store/detail/windows-terminal/9N0DX20HK701?hl=en-gb&gl=GB). It integrates well with WSL2, which is what you'll have to use.
However, for Windows users we recommend installing the [Windows Terminal](https://apps.microsoft.com/store/detail/windows-terminal/9N0DX20HK701?hl=en-gb&gl=GB).
Anyway, with a terminal open you can proceed to the next steps!
With a terminal open you can proceed to the next steps!
### Understanding the Terminal
## 1. Getting Node and Docker.
If you're completely unfamiliar with the terminal, check out our [Terminal Guide](./tools/terminal.md). We'll be assuming you know terminal basics in the below instructions.
You'll need `node` and `npm` in order to run Tachi. Install this from [the official NodeJS website](https://nodejs.org).
## 1. Getting Node, PNPM and Docker.
!!! info
If you are on linux, NodeJS will be provided by your package manager instead.
You may wish to use that instead. It honestly doesn't matter all too much.
You'll need `node` to run JavaScript code on your machine. Our codebase is wrote in JS, so this is pretty important.
Check that the install has worked by running `npm -v`. If you get an error message saying that `npm` was not found, try restarting the terminal.
Because system package managers may have outdated versions of `node` available (we want Node 16), we're going to use a separate tool called `nvm` to manage our node installs.
Follow [these instructions](https://github.com/nvm-sh/nvm#install--update-script).
Then close and re-open your shell to apply the changes. We can now install node16.
```sh
# install node 16
nvm install 16
# Check it worked
node --version
```
We use `pnpm` instead of `npm`. To get `pnpm`, you'll need to run:
```sh
# You might need `sudo` to run this command.
npm install -g pnpm
```
We use MongoDB and Redis as databases, these external databases require radically different instructions for setup depending on what OS/Linux variant you're on.
As such, we're going to take the lazy route and use something called [Docker](https://docker.com).
To set everything else up for local development, we'll use [Docker](https://docker.com).
=== "Windows, WSL Ubuntu"
You should install [Docker Desktop](https://docs.docker.com/desktop/) instead.
@@ -170,63 +116,22 @@ There's a remarkably easy way to do this, using GitHub's `gh` tool.
Once you've installed it, type `gh auth` and follow the instructions.
You should now be properly authenticated!
## 4. Start the databases.
## 4. Start Tachi!
Before we bootstrap, lets get the databases started.
With a terminal open inside the `Tachi` folder you just cloned, run `npm start`.
!!! important
**You should be inside the Tachi folder you just cloned.**
```sh
# you can write either docker compose or docker-compose. However, the former might
# require you to restart your shell if you've just installed docker compose.
docker-compose up --detach
```
## 5. Bootstrap!
!!! warning
The bootstrap script is a fairly recent addition.
If you have any issues with it, please report them!
Using a terminal, run `_scripts/bootstrap.sh`. This will "bootstrap" your install of Tachi, and load
everything you need.
The frontend will be running on `http://127.0.0.1:3000`.
The backend will be running on `https://127.0.0.1:8080`.
!!! danger
The bootstrap script you just ran created some local self-signed HTTPS certificates.
The backend **always runs on HTTPS** in local development. This is because browsers
tend to *really* hate HTTP mode nowadays, and it causes so many problems.
Browsers (rightfully) tell you that these are not actually secure HTTPS certs.
We need them for local development, though.
You **need** to navigate to your running instance of the server in a browser, and tell
You **need** to navigate to your running instance of the backend in a browser, and tell
the browser that you trust these certificates. Otherwise, all client requests to
the server will silently be chomped by the browser.
## 6. That's it!
You now have a fully working instance of Tachi on your PC. You can now start to tinker
with all of its various components.
To check everything's gone soundly, run `pnpm start-server` and `pnpm start-client`.
!!! tip
You can use the `+` inside the VSCode terminal to spawn multiple terminals.
Remember that `Ctrl-C` will kill the currently active process. You should use that
to stop the server or client.
You should then be able to navigate to https://127.0.0.1:8080, accept the certificates,
and view the client on http://127.0.0.1:3000.
!!! warning
You **MUST** accept the certificates on https://127.0.0.1:8080. The client **will not** start otherwise.
!!! help
If you're getting an `SSL_RX_RECORD_TOO_LONG` error, you're trying to access the client
via https. The client is served over HTTP - use http://127.0.0.1:3000.
## 7. Editor Plugins
## 5. Editor Plugins
If you're using VSCode as your editor (I *really* recommend it!) You'll want a couple
plugins.
@@ -243,4 +148,4 @@ Namely, Install the ESLint plugin, and enable "Format On Save" in your settings.
Now that you've got a working version of Tachi running on your local PC, you should go check out the [component-specific contribution guides](./components.md)!
[^1]: Docker is not actually a VM, it's significantly smarter and does some Linux jail cgroup nonsense. All you need to care about is that it works like having a separate Linux system on your host system.
[^1]: Docker is not actually a VM, it's significantly smarter and does some Linux jail cgroup nonsense. All you need to care about is that we're using it to spawn Linux VMs on your host system.
package.json
+24 -10
View File
@@ -4,17 +4,31 @@
"private": true,
"description": "The root of the Tachi monorepo. Contains common utils like linters.",
"scripts": {
"build": "pnpm -r build",
"start-client": "pnpm --filter ./client start",
"start-server": "pnpm --filter ./server start",
"test-server": "pnpm --filter ./server test",
"test-dbseeds": "pnpm --filter ./database-seeds/scripts test",
"sync-database": "pnpm --filter ./server sync-database",
"sync-database-local": "pnpm --filter ./server sync-database-local",
"validate-database": "pnpm --filter ./server validate-database",
"start-score-import-worker": "pnpm --filter ./server runscoreworker",
"start": "docker compose -f docker-compose-dev.yml up --build -d",
"stop": "docker compose -f docker-compose-dev.yml down",
"logs-server": "docker logs tachi-server -f",
"logs-client": "docker logs tachi-client -f",
"logs-seeds": "docker logs tachi-seeds -f",
"test-server": "docker compose -f docker-compose-dev.yml exec tachi-server pnpm test",
"test-seeds": "docker compose -f docker-compose-dev.yml exec tachi-seeds pnpm --filter ./scripts test",
"enter-seeds": "docker compose -f docker-compose-dev.yml exec tachi-seeds bash",
"sort-seeds": "docker compose -f docker-compose-dev.yml exec tachi-seeds node scripts/deterministic-collection-sort.js",
"view-seeds": "xdg-open http://127.0.0.1:3000/utils/seeds",
"load-seeds": "docker compose -f docker-compose-dev.yml exec tachi-server pnpm sync-database-local",
"validate-db": "docker compose -f docker-compose-dev.yml exec tachi-server pnpm validate-database",
"start-docs": "cd docs/ && mkdocs serve",
"make-user-admin": "pnpm --filter ./server make-user-admin $@"
"^^ DEV STUFF IS HERE ^^": "comment",
"vv PROD STUFF YOU WON'T TOUCH vv": "comment",
"build": "pnpm -r build",
"sync-database": "pnpm --filter ./server sync-database",
"start-score-import-worker": "pnpm --filter ./server runscoreworker"
},
"repository": {
"type": "git",
pnpm-lock.yaml
Generated
+555 -286
View File
File diff suppressed because it is too large Load Diff
server/nodemon.json
+10
View File
@@ -0,0 +1,10 @@
{
"watch": [
"src"
],
"ext": "ts,json",
"ignore": [
"src/**/*.spec.ts"
],
"exec": "ts-node ./src/main.ts"
}
server/package.json
+4 -1
View File
@@ -21,7 +21,9 @@
"sync-database-local": "ts-node src/scripts/sync-database --localPath '../database-seeds/collections'",
"recalc-everything": "ts-node src/scripts/state-sync/sync-state.ts",
"make-user-admin": "ts-node src/scripts/make-user-admin.ts $@",
"validate-database": "ts-node src/scripts/validate-database"
"validate-database": "ts-node src/scripts/validate-database",
"dev": "NODE_PATH=js/ nodemon",
"set-indexes": "ts-node src/scripts/set-indexes.ts"
},
"author": "zkldi",
"license": "AGPL3",
@@ -30,6 +32,7 @@
"@types/tap": "15.0.5",
"glob": "^7.2.0",
"node-mocks-http": "1.11.0",
"nodemon": "^2.0.20",
"supertest": "6.2.2",
"tap": "15.1.6",
"tsconfig-paths": "3.12.0"
server/src/scripts/set-indexes.ts
+6 -1
View File
@@ -2,13 +2,14 @@
import { SetIndexes } from "../external/mongo/indexes";
import { Command } from "commander";
import CreateLogCtx from "lib/logger/logger";
import { Environment, ServerConfig } from "lib/setup/config";
import { WrapScriptPromise } from "utils/misc";
const program = new Command();
const logger = CreateLogCtx(__filename);
program.requiredOption("-d, --db <database>", "The database to index.");
program.option("-d, --db <database>", "The database to index.");
program.option(
"-r, --reset",
"Whether to reset all indexes on this database before indexing or not."
@@ -17,6 +18,10 @@ program.option(
program.parse(process.argv);
const options: { db: string; reset?: boolean } = program.opts();
if (!options.db) {
options.db = `${Environment.mongoUrl}/${ServerConfig.MONGO_DATABASE_NAME}`;
}
if (require.main === module) {
WrapScriptPromise(SetIndexes(options.db, options.reset === true), logger);
}
server/src/server/router/api/v1/auth/auth.ts
+6 -1
View File
@@ -1,7 +1,7 @@
import bcrypt from "bcryptjs";
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import { ServerConfig } from "lib/setup/config";
import { Environment, ServerConfig } from "lib/setup/config";
import { p } from "prudence";
import { UserAuthLevels } from "tachi-common";
import nodeFetch from "utils/fetch";
@@ -97,6 +97,11 @@ export async function AddNewUser(
badges: [],
};
// all created users on a dev instance should be admins, for convenience.
if (Environment.nodeEnv === "dev") {
userDoc.authLevel = UserAuthLevels.ADMIN;
}
const res = await db.users.insert(userDoc);
const settingsRes = await InsertDefaultUserSettings(userID);