mirror of
https://github.com/zkldi/Tachi.git
synced 2026-10-09 07:09:57 +03:00
feat: datadump scripts
This commit is contained in:
zkldi
@@ -0,0 +1,170 @@
|
||||
import { Command } from "commander";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import monk from "monk";
|
||||
import { execSync } from "child_process";
|
||||
|
||||
// anonymise a database so it can be shared around and distributed.
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program.argument("<url to anonymise>");
|
||||
|
||||
program.parse(process.argv);
|
||||
const args = program.args;
|
||||
|
||||
const path = args[0];
|
||||
|
||||
if (!path) {
|
||||
throw new Error("expected path!");
|
||||
}
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
/**
|
||||
* Strips all name and private information from the database.
|
||||
* For use for things like future tachi-db-dumps?
|
||||
*
|
||||
* @param to The database to anonymise. DO NOT PASS THE PRODUCTION DATABASE TO THIS!
|
||||
*/
|
||||
async function AnonymiseDB(to: string) {
|
||||
logger.info(`Connecting to ${to}.`);
|
||||
|
||||
const clonedDB = monk(to);
|
||||
|
||||
logger.info(`Connected to ${to}.`);
|
||||
|
||||
const r1 = await clonedDB.get("user-private-information").update(
|
||||
{},
|
||||
[
|
||||
{
|
||||
$set: {
|
||||
// This is "password" encrypted in bcrypt12.
|
||||
password: {
|
||||
$const: "$2b$12$QRFCAxvFoNI2spszFPgt/e.qLy55GvYWlSHioa0AujRbFpChLwHmu",
|
||||
},
|
||||
|
||||
// emails are uniquely indexed. We need to anonymise these in
|
||||
// a way that they dont duplicate.
|
||||
email: { $concat: [{ $toString: "$userID" }, "@example.com"] },
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
multi: true,
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Stripped private info.`, { r1 });
|
||||
|
||||
const r2 = await clonedDB.get("users").update(
|
||||
{},
|
||||
[
|
||||
{
|
||||
$set: {
|
||||
socialMedia: { $const: {} },
|
||||
customBannerLocation: null,
|
||||
customPfpLocation: null,
|
||||
about: "Example About Me",
|
||||
status: null,
|
||||
username: { $concat: ["user", { $toString: "$id" }] },
|
||||
usernameLowercase: { $concat: ["user", { $toString: "$id" }] },
|
||||
isSupporter: { $const: false },
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
multi: true,
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Stripped username info.`, { r2 });
|
||||
|
||||
const r3 = await clonedDB.get("sessions").update(
|
||||
{},
|
||||
[
|
||||
{
|
||||
$set: {
|
||||
name: "Untitled Session",
|
||||
desc: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
multi: true,
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Stripped session info.`, { r3 });
|
||||
|
||||
const whitelist = [
|
||||
"bms-course-lookup",
|
||||
"class-achievements",
|
||||
"counters",
|
||||
"folders",
|
||||
"folder-chart-lookup",
|
||||
"game-settings",
|
||||
"game-stats",
|
||||
"game-stats-snapshots",
|
||||
"goal-subs",
|
||||
"goals",
|
||||
"iidx-bpi-data",
|
||||
"import-locks",
|
||||
"import-timings",
|
||||
"import-trackers",
|
||||
"imports",
|
||||
"migrations",
|
||||
"personal-bests",
|
||||
"quest-subs",
|
||||
"quests",
|
||||
"questlines",
|
||||
"recent-folder-views",
|
||||
"score-blacklist",
|
||||
"scores",
|
||||
"sessions",
|
||||
"tables",
|
||||
"user-private-information",
|
||||
"user-settings",
|
||||
"users",
|
||||
];
|
||||
|
||||
const collections = await clonedDB.listCollections();
|
||||
|
||||
for (const coll of collections) {
|
||||
if (coll.name.startsWith("charts-") || coll.name.startsWith("songs-")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!whitelist.includes(coll.name)) {
|
||||
await coll.drop();
|
||||
logger.info(`Removed collection ${coll.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Done! Closing.`);
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
// Don't run any risks -- there's no way we're ever accidentally anonymising the production database.
|
||||
// any nsTo argument MUST start with anon-
|
||||
if (!path.includes("/anon-")) {
|
||||
logger.error(
|
||||
`Tried to clone to and anonymise ${path}, which is illegal. Anonymised DBs must start with anon-.`,
|
||||
() => {
|
||||
process.exit(1);
|
||||
}
|
||||
);
|
||||
} else {
|
||||
AnonymiseDB(path)
|
||||
.then(() => {
|
||||
logger.info(`Anonymised database successfully. Saved to ${path}.`);
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
logger.error(`Failed to anonymise database.`, { err }, () => {
|
||||
process.exit(1);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
import { Command } from "commander";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import monk from "monk";
|
||||
import { execSync } from "child_process";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.option("--nsFrom <Database to clone from.>")
|
||||
.option("--nsTo <Database to clone and anonymise to. Must begin with anon->");
|
||||
|
||||
program.parse(process.argv);
|
||||
const options: { nsTo?: string; nsFrom?: string } = program.opts();
|
||||
|
||||
if (!options.nsTo || !options.nsFrom) {
|
||||
throw new Error(`Expected --nsFrom and --nsTo options.`);
|
||||
}
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
/**
|
||||
* Strips all name and private information from the database.
|
||||
* For use for things like future tachi-db-dumps?
|
||||
*
|
||||
* @param nsTo The database to anonymise. DO NOT PASS THE PRODUCTION DATABASE TO THIS!
|
||||
*/
|
||||
async function AnonymiseDB(nsTo: string) {
|
||||
logger.info(`Connecting to ${nsTo}.`);
|
||||
|
||||
const clonedDB = monk(`127.0.0.1/${nsTo}`);
|
||||
|
||||
logger.info(`Connected to ${nsTo}.`);
|
||||
|
||||
const r1 = await clonedDB.get("user-private-information").update(
|
||||
{},
|
||||
[
|
||||
{
|
||||
$set: {
|
||||
// This is "password" encrypted in bcrypt12.
|
||||
password: "$2b$12$QRFCAxvFoNI2spszFPgt/e.qLy55GvYWlSHioa0AujRbFpChLwHmu",
|
||||
|
||||
// emails are uniquely indexed. We need to anonymise these in
|
||||
// a way that they dont duplicate.
|
||||
email: { $concat: [{ $toString: "$userID" }, "@example.com"] },
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
multi: true,
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Stripped private info.`, { r1 });
|
||||
|
||||
const r2 = await clonedDB.get("users").update(
|
||||
{},
|
||||
[
|
||||
{
|
||||
$set: {
|
||||
socialMedia: { $const: {} },
|
||||
},
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
username: { $concat: ["u", { $toString: "$id" }] },
|
||||
usernameLowercase: { $concat: ["u", { $toString: "$id" }] },
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
multi: true,
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Stripped username info.`, { r2 });
|
||||
|
||||
for (const collection of [
|
||||
"api-clients",
|
||||
"oauth2-auth-codes",
|
||||
"password-reset-codes",
|
||||
"api-tokens",
|
||||
"kai-auth-tokens",
|
||||
]) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const r3 = await clonedDB.get("collection").remove({});
|
||||
|
||||
logger.info(`Removed all ${collection} documents.`, { r3 });
|
||||
}
|
||||
|
||||
logger.info(`Done! Closing.`);
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
function CloneDB(nsFrom: string, nsTo: string) {
|
||||
logger.info(`Cloning Database. This will take a while.`);
|
||||
execSync(`mongodump --gzip --archive=temp-dump --db=${nsFrom}`);
|
||||
execSync(`mongorestore --archive=temp-dump --gzip --nsFrom='${nsFrom}.*' --nsTo='${nsTo}.*'`);
|
||||
execSync(`rm temp-dump`);
|
||||
logger.info(`Cloned!`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
if (!options.nsTo.startsWith("anon-")) {
|
||||
logger.error(
|
||||
`Tried to clone to and anonymise ${options.nsTo}, which is illegal. Anonymised DBs must start with anon-.`,
|
||||
() => {
|
||||
process.exit(1);
|
||||
}
|
||||
);
|
||||
} else {
|
||||
// else guard here is important, as the above statement doesn't immediately halt
|
||||
// execution
|
||||
// -- return isn't valid at top level, and i can't be bothered wrapping this in
|
||||
// a useless fn.
|
||||
CloneDB(options.nsFrom, options.nsTo);
|
||||
|
||||
// Don't run any risks -- there's no way we're ever accidentally anonymising the production database.
|
||||
// any nsTo argument MUST start with anon-
|
||||
|
||||
if (options.nsTo.startsWith("anon-")) {
|
||||
AnonymiseDB(options.nsTo)
|
||||
.then(() => {
|
||||
logger.info(`Anonymised database successfully. Saved to ${options.nsTo}.`);
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
logger.error(`Failed to anonymise database.`, { err }, () => {
|
||||
process.exit(1);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user