From 1b2dd995d72e6ea15c6f9630c7eac38195e2fe13 Mon Sep 17 00:00:00 2001 From: zkldi <20380519+zkldi@users.noreply.github.com> Date: Thu, 9 May 2024 19:05:25 +0100 Subject: [PATCH] feat: datadump scripts --- _scripts/make-datadumps.sh | 28 +++ _scripts/snap-remote-ugpt.sh | 15 -- server/src/scripts/anonymise-db.ts | 170 +++++++++++++++++++ server/src/scripts/clone-and-anonymise-db.ts | 134 --------------- 4 files changed, 198 insertions(+), 149 deletions(-) create mode 100755 _scripts/make-datadumps.sh delete mode 100755 _scripts/snap-remote-ugpt.sh create mode 100644 server/src/scripts/anonymise-db.ts delete mode 100644 server/src/scripts/clone-and-anonymise-db.ts diff --git a/_scripts/make-datadumps.sh b/_scripts/make-datadumps.sh new file mode 100755 index 000000000..b76e58d70 --- /dev/null +++ b/_scripts/make-datadumps.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# a script for making tachi's anonymised datasets. +# designed to run on my machine +# +# you are expected to have pnpm, mongo etc. in path. + +set -eox pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "$SCRIPT_DIR"; +cd ../server; + +# stuff based on my local system, modify as wished +remote_port=12345 +target=../../tachi-datasets/datasets + +for kind in "kamai" "boku"; do + mongosh --eval "use anon-$kind" --eval "db.dropDatabase()" + + mongodump --archive --port=$remote_port --db=$kind | mongorestore --archive --nsFrom="$kind.*" --nsTo="anon-$kind.*" + + pnpm ts-node src/scripts/anonymise-db 127.0.0.1:27017/anon-$kind + + TCHIS_CONF_LOCATION=$kind.dataset.conf.json5 pnpm set-indexes + + mongodump --db=anon-$kind --archive="$target/anon-$kind-$(date +%Y-%m).dump" --gzip +done; \ No newline at end of file diff --git a/_scripts/snap-remote-ugpt.sh b/_scripts/snap-remote-ugpt.sh deleted file mode 100755 index 8bad21edc..000000000 --- a/_scripts/snap-remote-ugpt.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Copy a user from a remote Tachi instance and import it locally. -# This script is intended for users who have MongoDB access to said remote Tachi instance. - -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage 'snap-remote-ugpt.sh " - exit 1 -fi - -mongoexport mongodb://127.0.0.1:12345/ktchidb --jsonArray -c users -q "{ \"userID\": $1 }" | jq 'map(del(._id))' | mongoimport mongodb://127.0.0.1:27017/localdb --jsonArray -c users - -for collection in "game-stats" "scores" "sessions" "personal-bests"; do - mongoexport mongodb://127.0.0.1:12345/ktchidb --jsonArray -c "$collection" -q "{ \"userID\": $1, \"game\": \"$2\", \"playtype\": \"$3\" }" | jq 'map(del(._id))' | mongoimport mongodb://127.0.0.1:27017/localdb --jsonArray -c "$collection" -done diff --git a/server/src/scripts/anonymise-db.ts b/server/src/scripts/anonymise-db.ts new file mode 100644 index 000000000..56b1a5470 --- /dev/null +++ b/server/src/scripts/anonymise-db.ts @@ -0,0 +1,170 @@ +import { Command } from "commander"; +import CreateLogCtx from "lib/logger/logger"; +import monk from "monk"; +import { execSync } from "child_process"; + +// anonymise a database so it can be shared around and distributed. + +const program = new Command(); + +program.argument(""); + +program.parse(process.argv); +const args = program.args; + +const path = args[0]; + +if (!path) { + throw new Error("expected path!"); +} + +const logger = CreateLogCtx(__filename); + +/** + * Strips all name and private information from the database. + * For use for things like future tachi-db-dumps? + * + * @param to The database to anonymise. DO NOT PASS THE PRODUCTION DATABASE TO THIS! + */ +async function AnonymiseDB(to: string) { + logger.info(`Connecting to ${to}.`); + + const clonedDB = monk(to); + + logger.info(`Connected to ${to}.`); + + const r1 = await clonedDB.get("user-private-information").update( + {}, + [ + { + $set: { + // This is "password" encrypted in bcrypt12. + password: { + $const: "$2b$12$QRFCAxvFoNI2spszFPgt/e.qLy55GvYWlSHioa0AujRbFpChLwHmu", + }, + + // emails are uniquely indexed. We need to anonymise these in + // a way that they dont duplicate. + email: { $concat: [{ $toString: "$userID" }, "@example.com"] }, + }, + }, + ], + { + multi: true, + } + ); + + logger.info(`Stripped private info.`, { r1 }); + + const r2 = await clonedDB.get("users").update( + {}, + [ + { + $set: { + socialMedia: { $const: {} }, + customBannerLocation: null, + customPfpLocation: null, + about: "Example About Me", + status: null, + username: { $concat: ["user", { $toString: "$id" }] }, + usernameLowercase: { $concat: ["user", { $toString: "$id" }] }, + isSupporter: { $const: false }, + }, + }, + ], + { + multi: true, + } + ); + + logger.info(`Stripped username info.`, { r2 }); + + const r3 = await clonedDB.get("sessions").update( + {}, + [ + { + $set: { + name: "Untitled Session", + desc: null, + }, + }, + ], + { + multi: true, + } + ); + + logger.info(`Stripped session info.`, { r3 }); + + const whitelist = [ + "bms-course-lookup", + "class-achievements", + "counters", + "folders", + "folder-chart-lookup", + "game-settings", + "game-stats", + "game-stats-snapshots", + "goal-subs", + "goals", + "iidx-bpi-data", + "import-locks", + "import-timings", + "import-trackers", + "imports", + "migrations", + "personal-bests", + "quest-subs", + "quests", + "questlines", + "recent-folder-views", + "score-blacklist", + "scores", + "sessions", + "tables", + "user-private-information", + "user-settings", + "users", + ]; + + const collections = await clonedDB.listCollections(); + + for (const coll of collections) { + if (coll.name.startsWith("charts-") || coll.name.startsWith("songs-")) { + continue; + } + + if (!whitelist.includes(coll.name)) { + await coll.drop(); + logger.info(`Removed collection ${coll.name}`); + } + } + + logger.info(`Done! Closing.`); + + process.exit(0); +} + +if (require.main === module) { + // Don't run any risks -- there's no way we're ever accidentally anonymising the production database. + // any nsTo argument MUST start with anon- + if (!path.includes("/anon-")) { + logger.error( + `Tried to clone to and anonymise ${path}, which is illegal. Anonymised DBs must start with anon-.`, + () => { + process.exit(1); + } + ); + } else { + AnonymiseDB(path) + .then(() => { + logger.info(`Anonymised database successfully. Saved to ${path}.`); + process.exit(0); + }) + .catch((err: unknown) => { + logger.error(`Failed to anonymise database.`, { err }, () => { + process.exit(1); + }); + }); + } +} diff --git a/server/src/scripts/clone-and-anonymise-db.ts b/server/src/scripts/clone-and-anonymise-db.ts deleted file mode 100644 index ab1ec6c71..000000000 --- a/server/src/scripts/clone-and-anonymise-db.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { Command } from "commander"; -import CreateLogCtx from "lib/logger/logger"; -import monk from "monk"; -import { execSync } from "child_process"; - -const program = new Command(); - -program - .option("--nsFrom ") - .option("--nsTo "); - -program.parse(process.argv); -const options: { nsTo?: string; nsFrom?: string } = program.opts(); - -if (!options.nsTo || !options.nsFrom) { - throw new Error(`Expected --nsFrom and --nsTo options.`); -} - -const logger = CreateLogCtx(__filename); - -/** - * Strips all name and private information from the database. - * For use for things like future tachi-db-dumps? - * - * @param nsTo The database to anonymise. DO NOT PASS THE PRODUCTION DATABASE TO THIS! - */ -async function AnonymiseDB(nsTo: string) { - logger.info(`Connecting to ${nsTo}.`); - - const clonedDB = monk(`127.0.0.1/${nsTo}`); - - logger.info(`Connected to ${nsTo}.`); - - const r1 = await clonedDB.get("user-private-information").update( - {}, - [ - { - $set: { - // This is "password" encrypted in bcrypt12. - password: "$2b$12$QRFCAxvFoNI2spszFPgt/e.qLy55GvYWlSHioa0AujRbFpChLwHmu", - - // emails are uniquely indexed. We need to anonymise these in - // a way that they dont duplicate. - email: { $concat: [{ $toString: "$userID" }, "@example.com"] }, - }, - }, - ], - { - multi: true, - } - ); - - logger.info(`Stripped private info.`, { r1 }); - - const r2 = await clonedDB.get("users").update( - {}, - [ - { - $set: { - socialMedia: { $const: {} }, - }, - }, - { - $set: { - username: { $concat: ["u", { $toString: "$id" }] }, - usernameLowercase: { $concat: ["u", { $toString: "$id" }] }, - }, - }, - ], - { - multi: true, - } - ); - - logger.info(`Stripped username info.`, { r2 }); - - for (const collection of [ - "api-clients", - "oauth2-auth-codes", - "password-reset-codes", - "api-tokens", - "kai-auth-tokens", - ]) { - // eslint-disable-next-line no-await-in-loop - const r3 = await clonedDB.get("collection").remove({}); - - logger.info(`Removed all ${collection} documents.`, { r3 }); - } - - logger.info(`Done! Closing.`); - - process.exit(0); -} - -function CloneDB(nsFrom: string, nsTo: string) { - logger.info(`Cloning Database. This will take a while.`); - execSync(`mongodump --gzip --archive=temp-dump --db=${nsFrom}`); - execSync(`mongorestore --archive=temp-dump --gzip --nsFrom='${nsFrom}.*' --nsTo='${nsTo}.*'`); - execSync(`rm temp-dump`); - logger.info(`Cloned!`); -} - -if (require.main === module) { - if (!options.nsTo.startsWith("anon-")) { - logger.error( - `Tried to clone to and anonymise ${options.nsTo}, which is illegal. Anonymised DBs must start with anon-.`, - () => { - process.exit(1); - } - ); - } else { - // else guard here is important, as the above statement doesn't immediately halt - // execution - // -- return isn't valid at top level, and i can't be bothered wrapping this in - // a useless fn. - CloneDB(options.nsFrom, options.nsTo); - - // Don't run any risks -- there's no way we're ever accidentally anonymising the production database. - // any nsTo argument MUST start with anon- - - if (options.nsTo.startsWith("anon-")) { - AnonymiseDB(options.nsTo) - .then(() => { - logger.info(`Anonymised database successfully. Saved to ${options.nsTo}.`); - process.exit(0); - }) - .catch((err: unknown) => { - logger.error(`Failed to anonymise database.`, { err }, () => { - process.exit(1); - }); - }); - } - } -}