Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-12 17:28:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: datadumps and datadump loading (#1610)

Browse Source
This commit is contained in:
zk
2026-06-01 22:19:57 +01:00
committed by GitHub
parent c48735e663
commit 1e10fb8434
5 changed files with 98 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.scripts/make-datadumps.sh
+14 -19
View File
@@ -3,28 +3,24 @@
#
# For each named instance (e.g. "kamai", "boku") this script:
# 1. Creates a temporary database anon-$instance on the local server
# 2. Clones the source database into it via pg_dump | pg_restore
# 2. Clones the source database (tachi_$instance) into it via pg_dump | pg_restore
# 3. Runs the TypeScript anonymiser to strip PII in place
# 4. Dumps the anonymised copy to a gzip-compressed SQL file
# 5. Drops the temporary database
#
# The resulting .sql.gz files can be loaded with:
# zcat tachi-kamai-2026-05.sql.gz | psql -d <target-db>
# gunzip -c tachi-kamai-2026-05.sql.gz | psql -d <target-db>
#
# ── Required configuration ──────────────────────────────────────────────────
#
# SOURCE_BASE_URL Base Postgres URL of the server to clone FROM, without a
# database name. The instance name is appended automatically.
# Example: postgresql://user:pass@prod-host:5432
#
# LOCAL_BASE_URL Base Postgres URL of the LOCAL server where anon copies are
# created. Must not include a database name.
# Example: postgresql://tachi:tachi@tachi-postgres
# (Defaults to the value of SOURCE_BASE_URL.)
# LOCAL_BASE_URL Base Postgres URL of the server where tachi_$instance
# databases already exist, without a database name.
# Example: postgresql://tachi:tachi@tachi-postgres:5432
#
# ── Optional configuration ──────────────────────────────────────────────────
#
# INSTANCES Space-separated list of database names to process.
# INSTANCES Space-separated list of instance names to process.
# The source database is expected to be named tachi_$instance.
# Default: "kamai boku"
#
# TARGET_DIR Directory to write the .sql.gz files into.
@@ -32,7 +28,7 @@
#
# ── Example ─────────────────────────────────────────────────────────────────
#
# SOURCE_BASE_URL=postgresql://tachi:tachi@tachi-postgres \
# LOCAL_BASE_URL=postgresql://tachi:tachi@tachi-postgres:5432 \
# INSTANCES=kamai \
# .scripts/make-datadumps.sh
@@ -41,13 +37,12 @@ set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
SERVER_DIR="$SCRIPT_DIR/../typescript/server"
if [[ -z "${SOURCE_BASE_URL:-}" ]]; then
echo "make-datadumps: SOURCE_BASE_URL is required." >&2
echo " Example: SOURCE_BASE_URL=postgresql://user:pass@host:5432" >&2
if [[ -z "${LOCAL_BASE_URL:-}" ]]; then
echo "make-datadumps: LOCAL_BASE_URL is required." >&2
echo " Example: LOCAL_BASE_URL=postgresql://tachi:tachi@tachi-postgres:5432" >&2
exit 1
fi
LOCAL_BASE_URL="${LOCAL_BASE_URL:-$SOURCE_BASE_URL}"
INSTANCES="${INSTANCES:-kamai boku}"
TARGET_DIR="${TARGET_DIR:-$SCRIPT_DIR/../datasets}"
@@ -57,7 +52,7 @@ mkdir -p "$TARGET_DIR"
for instance in $INSTANCES; do
anon_db="anon-$instance"
source_url="${SOURCE_BASE_URL}/${instance}"
source_url="${LOCAL_BASE_URL}/tachi_${instance}"
anon_url="${LOCAL_BASE_URL}/${anon_db}"
maintenance_url="${LOCAL_BASE_URL}/postgres"
output="$TARGET_DIR/tachi-$instance-$DATESTAMP.sql.gz"
@@ -69,8 +64,8 @@ for instance in $INSTANCES; do
psql "$maintenance_url" -c "DROP DATABASE IF EXISTS \"$anon_db\""
psql "$maintenance_url" -c "CREATE DATABASE \"$anon_db\""
# 2. Clone the source database into the anon copy
echo " -> Cloning ${source_url} into ${anon_db}"
# 2. Clone the local source database into the anon copy
echo " -> Cloning tachi_${instance} into ${anon_db}"
pg_dump --format=custom "$source_url" \
| pg_restore --dbname="$anon_url" --no-owner --no-acl
Justfile-db
+33 -1
View File
@@ -50,4 +50,36 @@ db-load-seeds POSTGRES_DB=DEFAULT_DB:
# Reset templatedb from migrations, then dump schema and indexes for local debugging.
db-schema: (db-cheap-reset "templatedb")
pg_dump "{{POSTGRES_URL}}/templatedb" --schema-only --no-owner --no-acl -f db/schema.sql
pg_dump "{{POSTGRES_URL}}/templatedb" --schema-only --no-owner --no-acl -f db/schema.sql
# Download (if needed) and restore an anonymised dataset dump into anon-$name on tachi-postgres.
db-load-dataset:
#!/bin/bash
set -euo pipefail
datasets=(tachi-boku-2026-06 tachi-kamai-2026-06)
name=$(printf '%s\n' "${datasets[@]}" | fzf --prompt="Select dataset: ")
[[ -z "$name" ]] && exit 0
file="datasets/$name.sql.gz"
mkdir -p datasets
if [[ ! -f "$file" ]]; then
echo "Downloading $name.sql.gz from CDN..."
wget -O "$file" "https://cdn.tachi.ac/datasets/$name.sql.gz"
else
echo "Using cached $file"
fi
db_name="anon-$name"
db_url="postgresql://tachi:tachi@tachi-postgres:5432/$db_name"
psql "{{POSTGRES_URL}}/postgres" -c "DROP DATABASE IF EXISTS \"$db_name\""
psql "{{POSTGRES_URL}}/postgres" -c "CREATE DATABASE \"$db_name\""
echo "Restoring $file into $db_name..."
gunzip -c "$file" | psql -d "$db_url"
echo "Done. Change typescript/server/.env to use DATABASE_URL=$db_url."
README.md
+8 -1
View File
@@ -18,6 +18,8 @@ properly is _just_ as important!
By using Tachi, you get access to powerful, novel rhythm game score-tracking features, like automatically breaking your scores into sessions, setting goals and rivals, and more!
Full database dumps for Tachi are published semi-regularly (when I remember to); see [tachi-datasets](https://github.com/zkldi/tachi-datasets).
## Quick Setup For Experienced Programmers
- Open `Tachi` in [VSCode](https://code.visualstudio.com) or [Cursor](https://cursor.com).
@@ -31,6 +33,11 @@ By using Tachi, you get access to powerful, novel rhythm game score-tracking fea
**Doing Tachi dev outside of the devcontainer is unsupported.** The container is set up perfectly with the right versions of everything and is the expected way to do things.
Please do not report bugs in the local setup if you are not using the devcontainer.
### Loading data
If you want to load real data into localdev, use `just db-load-dataset` to interactively
load real datasets from the [tachi-datasets](https://github.com/zkldi/tachi-datasets).
## Repository Info
This monorepo contains the following codebases:
@@ -58,4 +65,4 @@ This is also published to NPM when it hits production.
- `typescript/sieglinde/`, Which contains our BMS/PMS analysis functions. (MIT)
- `java/beatoraja-ir/`, Which is the Tachi Internet Ranking implementation for the LR2oraja variant of beatoraja. (MIT) Formerly `zkldi/tachi-beatoraja-ir`, brought into the monorepo per #1515.
- `java/beatoraja-ir/`, Which is the Tachi Internet Ranking implementation for the LR2oraja variant of beatoraja. (MIT)
dev/pgadmin/servers.json
+24
View File
@@ -0,0 +1,24 @@
{
"Servers": {
"1": {
"Name": "Tachi Postgres (dev)",
"Group": "Tachi",
"Host": "tachi-postgres",
"Port": 5432,
"MaintenanceDB": "tachi",
"Username": "tachi",
"Password": "tachi",
"SSLMode": "prefer"
},
"2": {
"Name": "Tachi Postgres (test)",
"Group": "Tachi",
"Host": "tachi-postgres-test",
"Port": 5432,
"MaintenanceDB": "postgres",
"Username": "tachi",
"Password": "tachi",
"SSLMode": "prefer"
}
}
}
docker-compose-dev.yml
+19
View File
@@ -103,6 +103,24 @@ services:
POSTGRES_DB: postgres
PGDATA: /var/lib/postgresql/data/pgdata
# PostgreSQL admin UI (http://localhost:5050). Dev + test servers are pre-registered
# on first launch via dev/pgadmin/servers.json (wipe tachi-pgadmin volume to reload).
tachi-pgadmin:
container_name: tachi-pgadmin
image: dpage/pgadmin4
restart: unless-stopped
depends_on:
- tachi-postgres
- tachi-postgres-test
ports:
- "5050:80"
environment:
PGADMIN_DEFAULT_EMAIL: pgadmin@example.com
PGADMIN_DEFAULT_PASSWORD: password
volumes:
- tachi-pgadmin:/var/lib/pgadmin
- "./dev/pgadmin/servers.json:/pgadmin4/servers.json:ro"
tachi-s3:
container_name: tachi-s3
image: quay.io/minio/minio:RELEASE.2024-10-29T16-01-48Z
@@ -209,6 +227,7 @@ volumes:
tachi-redis:
tachi-logs:
tachi-postgres:
tachi-pgadmin:
tachi-minio:
tachi-grafana:
tachi-prometheus: