diff --git a/.scripts/make-datadumps.sh b/.scripts/make-datadumps.sh index baa2c9fd5..77ca0d161 100755 --- a/.scripts/make-datadumps.sh +++ b/.scripts/make-datadumps.sh @@ -3,28 +3,24 @@ # # For each named instance (e.g. "kamai", "boku") this script: # 1. Creates a temporary database anon-$instance on the local server -# 2. Clones the source database into it via pg_dump | pg_restore +# 2. Clones the source database (tachi_$instance) into it via pg_dump | pg_restore # 3. Runs the TypeScript anonymiser to strip PII in place # 4. Dumps the anonymised copy to a gzip-compressed SQL file # 5. Drops the temporary database # # The resulting .sql.gz files can be loaded with: -# zcat tachi-kamai-2026-05.sql.gz | psql -d +# gunzip -c tachi-kamai-2026-05.sql.gz | psql -d # # ── Required configuration ────────────────────────────────────────────────── # -# SOURCE_BASE_URL Base Postgres URL of the server to clone FROM, without a -# database name. The instance name is appended automatically. -# Example: postgresql://user:pass@prod-host:5432 -# -# LOCAL_BASE_URL Base Postgres URL of the LOCAL server where anon copies are -# created. Must not include a database name. -# Example: postgresql://tachi:tachi@tachi-postgres -# (Defaults to the value of SOURCE_BASE_URL.) +# LOCAL_BASE_URL Base Postgres URL of the server where tachi_$instance +# databases already exist, without a database name. +# Example: postgresql://tachi:tachi@tachi-postgres:5432 # # ── Optional configuration ────────────────────────────────────────────────── # -# INSTANCES Space-separated list of database names to process. +# INSTANCES Space-separated list of instance names to process. +# The source database is expected to be named tachi_$instance. # Default: "kamai boku" # # TARGET_DIR Directory to write the .sql.gz files into. @@ -32,7 +28,7 @@ # # ── Example ───────────────────────────────────────────────────────────────── # -# SOURCE_BASE_URL=postgresql://tachi:tachi@tachi-postgres \ +# LOCAL_BASE_URL=postgresql://tachi:tachi@tachi-postgres:5432 \ # INSTANCES=kamai \ # .scripts/make-datadumps.sh @@ -41,13 +37,12 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" SERVER_DIR="$SCRIPT_DIR/../typescript/server" -if [[ -z "${SOURCE_BASE_URL:-}" ]]; then - echo "make-datadumps: SOURCE_BASE_URL is required." >&2 - echo " Example: SOURCE_BASE_URL=postgresql://user:pass@host:5432" >&2 +if [[ -z "${LOCAL_BASE_URL:-}" ]]; then + echo "make-datadumps: LOCAL_BASE_URL is required." >&2 + echo " Example: LOCAL_BASE_URL=postgresql://tachi:tachi@tachi-postgres:5432" >&2 exit 1 fi -LOCAL_BASE_URL="${LOCAL_BASE_URL:-$SOURCE_BASE_URL}" INSTANCES="${INSTANCES:-kamai boku}" TARGET_DIR="${TARGET_DIR:-$SCRIPT_DIR/../datasets}" @@ -57,7 +52,7 @@ mkdir -p "$TARGET_DIR" for instance in $INSTANCES; do anon_db="anon-$instance" - source_url="${SOURCE_BASE_URL}/${instance}" + source_url="${LOCAL_BASE_URL}/tachi_${instance}" anon_url="${LOCAL_BASE_URL}/${anon_db}" maintenance_url="${LOCAL_BASE_URL}/postgres" output="$TARGET_DIR/tachi-$instance-$DATESTAMP.sql.gz" @@ -69,8 +64,8 @@ for instance in $INSTANCES; do psql "$maintenance_url" -c "DROP DATABASE IF EXISTS \"$anon_db\"" psql "$maintenance_url" -c "CREATE DATABASE \"$anon_db\"" - # 2. Clone the source database into the anon copy - echo " -> Cloning ${source_url} into ${anon_db}" + # 2. Clone the local source database into the anon copy + echo " -> Cloning tachi_${instance} into ${anon_db}" pg_dump --format=custom "$source_url" \ | pg_restore --dbname="$anon_url" --no-owner --no-acl diff --git a/Justfile-db b/Justfile-db index 65863da07..4f267a916 100644 --- a/Justfile-db +++ b/Justfile-db @@ -50,4 +50,36 @@ db-load-seeds POSTGRES_DB=DEFAULT_DB: # Reset templatedb from migrations, then dump schema and indexes for local debugging. db-schema: (db-cheap-reset "templatedb") - pg_dump "{{POSTGRES_URL}}/templatedb" --schema-only --no-owner --no-acl -f db/schema.sql \ No newline at end of file + pg_dump "{{POSTGRES_URL}}/templatedb" --schema-only --no-owner --no-acl -f db/schema.sql + +# Download (if needed) and restore an anonymised dataset dump into anon-$name on tachi-postgres. +db-load-dataset: + #!/bin/bash + set -euo pipefail + + datasets=(tachi-boku-2026-06 tachi-kamai-2026-06) + + name=$(printf '%s\n' "${datasets[@]}" | fzf --prompt="Select dataset: ") + [[ -z "$name" ]] && exit 0 + + file="datasets/$name.sql.gz" + mkdir -p datasets + + if [[ ! -f "$file" ]]; then + echo "Downloading $name.sql.gz from CDN..." + wget -O "$file" "https://cdn.tachi.ac/datasets/$name.sql.gz" + else + echo "Using cached $file" + fi + + db_name="anon-$name" + db_url="postgresql://tachi:tachi@tachi-postgres:5432/$db_name" + + psql "{{POSTGRES_URL}}/postgres" -c "DROP DATABASE IF EXISTS \"$db_name\"" + psql "{{POSTGRES_URL}}/postgres" -c "CREATE DATABASE \"$db_name\"" + + echo "Restoring $file into $db_name..." + gunzip -c "$file" | psql -d "$db_url" + + echo "Done. Change typescript/server/.env to use DATABASE_URL=$db_url." + diff --git a/README.md b/README.md index 3b3439252..909f999fd 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ properly is _just_ as important! By using Tachi, you get access to powerful, novel rhythm game score-tracking features, like automatically breaking your scores into sessions, setting goals and rivals, and more! +Full database dumps for Tachi are published semi-regularly (when I remember to); see [tachi-datasets](https://github.com/zkldi/tachi-datasets). + ## Quick Setup For Experienced Programmers - Open `Tachi` in [VSCode](https://code.visualstudio.com) or [Cursor](https://cursor.com). @@ -31,6 +33,11 @@ By using Tachi, you get access to powerful, novel rhythm game score-tracking fea **Doing Tachi dev outside of the devcontainer is unsupported.** The container is set up perfectly with the right versions of everything and is the expected way to do things. Please do not report bugs in the local setup if you are not using the devcontainer. +### Loading data + +If you want to load real data into localdev, use `just db-load-dataset` to interactively +load real datasets from the [tachi-datasets](https://github.com/zkldi/tachi-datasets). + ## Repository Info This monorepo contains the following codebases: @@ -58,4 +65,4 @@ This is also published to NPM when it hits production. - `typescript/sieglinde/`, Which contains our BMS/PMS analysis functions. (MIT) -- `java/beatoraja-ir/`, Which is the Tachi Internet Ranking implementation for the LR2oraja variant of beatoraja. (MIT) Formerly `zkldi/tachi-beatoraja-ir`, brought into the monorepo per #1515. +- `java/beatoraja-ir/`, Which is the Tachi Internet Ranking implementation for the LR2oraja variant of beatoraja. (MIT) diff --git a/dev/pgadmin/servers.json b/dev/pgadmin/servers.json new file mode 100644 index 000000000..459093f36 --- /dev/null +++ b/dev/pgadmin/servers.json @@ -0,0 +1,24 @@ +{ + "Servers": { + "1": { + "Name": "Tachi Postgres (dev)", + "Group": "Tachi", + "Host": "tachi-postgres", + "Port": 5432, + "MaintenanceDB": "tachi", + "Username": "tachi", + "Password": "tachi", + "SSLMode": "prefer" + }, + "2": { + "Name": "Tachi Postgres (test)", + "Group": "Tachi", + "Host": "tachi-postgres-test", + "Port": 5432, + "MaintenanceDB": "postgres", + "Username": "tachi", + "Password": "tachi", + "SSLMode": "prefer" + } + } +} diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 166c34f76..8f721ba52 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -103,6 +103,24 @@ services: POSTGRES_DB: postgres PGDATA: /var/lib/postgresql/data/pgdata + # PostgreSQL admin UI (http://localhost:5050). Dev + test servers are pre-registered + # on first launch via dev/pgadmin/servers.json (wipe tachi-pgadmin volume to reload). + tachi-pgadmin: + container_name: tachi-pgadmin + image: dpage/pgadmin4 + restart: unless-stopped + depends_on: + - tachi-postgres + - tachi-postgres-test + ports: + - "5050:80" + environment: + PGADMIN_DEFAULT_EMAIL: pgadmin@example.com + PGADMIN_DEFAULT_PASSWORD: password + volumes: + - tachi-pgadmin:/var/lib/pgadmin + - "./dev/pgadmin/servers.json:/pgadmin4/servers.json:ro" + tachi-s3: container_name: tachi-s3 image: quay.io/minio/minio:RELEASE.2024-10-29T16-01-48Z @@ -209,6 +227,7 @@ volumes: tachi-redis: tachi-logs: tachi-postgres: + tachi-pgadmin: tachi-minio: tachi-grafana: tachi-prometheus: \ No newline at end of file