mirror of
https://github.com/zkldi/Tachi.git
synced 2026-10-06 13:58:05 +03:00
Groundwork for DB Validators
This commit is contained in:
zkldi
@@ -0,0 +1,255 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
// This script syncs this tachi instances database up with the tachi-database-seeds.
|
||||
|
||||
import { execSync } from "child_process";
|
||||
import db, { monkDB } from "external/mongo/db";
|
||||
import fs from "fs";
|
||||
import CreateLogCtx, { KtLogger } from "lib/logger/logger";
|
||||
import { ServerTypeInfo } from "lib/setup/config";
|
||||
import { BulkWriteOperation, FilterQuery } from "mongodb";
|
||||
import { ICollection } from "monk";
|
||||
import path from "path";
|
||||
import { ChartDocument, FolderDocument, SongDocument, TableDocument } from "tachi-common";
|
||||
import deepEqual from "deep-equal";
|
||||
|
||||
interface SyncInstructions {
|
||||
pattern: RegExp;
|
||||
handler: (c: any[], collection: ICollection<any>, logger: KtLogger) => Promise<void>;
|
||||
}
|
||||
|
||||
async function RemoveNotPresent<T>(
|
||||
documents: T[],
|
||||
collection: ICollection<T>,
|
||||
field: keyof T,
|
||||
logger: KtLogger
|
||||
) {
|
||||
logger.verbose(`Removing all documents that are no longer present.`);
|
||||
// Remove anything no longer present.
|
||||
// Note that $nin is incredibly slow.
|
||||
// @ts-expect-error generic system failing
|
||||
const r = await collection.remove({
|
||||
[field]: { $nin: documents.map((e) => e[field]) },
|
||||
});
|
||||
|
||||
if (r.deletedCount) {
|
||||
logger.info(`Removed ${r.deletedCount} documents.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function GenericUpsert<T>(
|
||||
documents: T[],
|
||||
collection: ICollection<T>,
|
||||
field: keyof T,
|
||||
logger: KtLogger,
|
||||
remove = false
|
||||
) {
|
||||
logger.verbose(`Running bulkwrite.`);
|
||||
|
||||
const bwriteOps: BulkWriteOperation<T>[] = [];
|
||||
|
||||
const allExistingDocs = await collection.find({});
|
||||
|
||||
const map = new Map();
|
||||
for (const doc of allExistingDocs) {
|
||||
map.set(doc[field], doc);
|
||||
}
|
||||
|
||||
let i = 0;
|
||||
for (const document of documents) {
|
||||
i++;
|
||||
if (i % 10_000 === 0) {
|
||||
logger.info(`On document ${i}/${documents.length}.`);
|
||||
}
|
||||
|
||||
const exists = map.get(document[field]);
|
||||
|
||||
if (!exists) {
|
||||
bwriteOps.push({
|
||||
// @ts-expect-error Actually, T is assignable to OptionalId<T>.
|
||||
insertOne: { document },
|
||||
});
|
||||
} else if (!deepEqual(document, exists, { strict: true })) {
|
||||
bwriteOps.push({
|
||||
updateOne: {
|
||||
// @ts-expect-error Known X->Y generic issue.
|
||||
filter: {
|
||||
[field]: document[field],
|
||||
},
|
||||
update: {
|
||||
$set: document,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// free some memory.
|
||||
map.delete(document[field]);
|
||||
}
|
||||
|
||||
if (bwriteOps.length === 0) {
|
||||
logger.info(`No differences. Not performing any update.`);
|
||||
} else {
|
||||
const result = await collection.bulkWrite(bwriteOps);
|
||||
logger.info(`Performed bulkWrite.`, result);
|
||||
}
|
||||
|
||||
if (remove) {
|
||||
await RemoveNotPresent(documents, collection, field, logger);
|
||||
}
|
||||
}
|
||||
|
||||
const syncInstructions: SyncInstructions[] = [
|
||||
{
|
||||
pattern: /^charts-/u,
|
||||
handler: (charts: ChartDocument[], collection: ICollection<ChartDocument>, logger) =>
|
||||
GenericUpsert(charts, collection, "chartID", logger),
|
||||
},
|
||||
{
|
||||
pattern: /^songs-/u,
|
||||
handler: (songs: SongDocument[], collection: ICollection<SongDocument>, logger) =>
|
||||
GenericUpsert(songs, collection, "id", logger),
|
||||
},
|
||||
{
|
||||
pattern: /^folders$/u,
|
||||
handler: async (
|
||||
folders: FolderDocument[],
|
||||
collection: ICollection<FolderDocument>,
|
||||
logger
|
||||
) => {
|
||||
const bwriteOps: BulkWriteOperation<FolderDocument>[] = [];
|
||||
|
||||
for (const folder of folders) {
|
||||
if (!ServerTypeInfo.supportedGames.includes(folder.game)) {
|
||||
continue; // Skip things for games we don't care about.
|
||||
}
|
||||
|
||||
const exists = await collection.findOne({
|
||||
folderID: folder.folderID,
|
||||
});
|
||||
|
||||
if (!exists) {
|
||||
bwriteOps.push({
|
||||
// @ts-expect-error faulty monk types
|
||||
insertOne: { document: folder },
|
||||
});
|
||||
} else if (!deepEqual(folder, exists, { strict: true })) {
|
||||
bwriteOps.push({
|
||||
updateOne: {
|
||||
filter: {
|
||||
folderID: folder.folderID,
|
||||
},
|
||||
update: {
|
||||
$set: folder,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await collection.bulkWrite(bwriteOps);
|
||||
},
|
||||
},
|
||||
{
|
||||
pattern: /^tables$/u,
|
||||
handler: async (
|
||||
tables: TableDocument[],
|
||||
collection: ICollection<TableDocument>,
|
||||
logger
|
||||
) => {
|
||||
const bwriteOps: BulkWriteOperation<TableDocument>[] = [];
|
||||
|
||||
for (const table of tables) {
|
||||
if (!ServerTypeInfo.supportedGames.includes(table.game)) {
|
||||
continue; // Skip things for games we don't care about.
|
||||
}
|
||||
|
||||
const exists = await collection.findOne({
|
||||
tableID: table.tableID,
|
||||
});
|
||||
|
||||
if (!exists) {
|
||||
bwriteOps.push({
|
||||
// @ts-expect-error faulty monk types
|
||||
insertOne: { document: table },
|
||||
});
|
||||
} else if (!deepEqual(table, exists, { strict: true })) {
|
||||
bwriteOps.push({
|
||||
updateOne: {
|
||||
filter: {
|
||||
tableID: table.tableID,
|
||||
},
|
||||
update: {
|
||||
$set: table,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await collection.bulkWrite(bwriteOps);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const logger = CreateLogCtx("Database Sync");
|
||||
|
||||
async function SynchroniseDBWithSeeds() {
|
||||
const seedsDir = path.join(__dirname, "tachi-database-seeds");
|
||||
|
||||
fs.rmSync(seedsDir, { recursive: true, force: true });
|
||||
// This will create a directory called tachi-database-seeds in this folder.
|
||||
execSync(`cd ${__dirname} && git clone https://github.com/TNG-dev/tachi-database-seeds`, {
|
||||
stdio: "inherit",
|
||||
});
|
||||
|
||||
const collections = fs.readdirSync(path.join(seedsDir, "collections"));
|
||||
|
||||
for (const jsonName of collections) {
|
||||
const collectionName = path.parse(jsonName).name;
|
||||
const spawnLogger = CreateLogCtx(`${collectionName} Sync`);
|
||||
|
||||
if (collectionName.startsWith("songs-") || collectionName.startsWith("charts-")) {
|
||||
const game = collectionName.split("-")[1];
|
||||
|
||||
if (!ServerTypeInfo.supportedGames.includes(game as any)) {
|
||||
spawnLogger.info(
|
||||
`Skipping ${collectionName} (${game}) as it isn't for ${ServerTypeInfo.name}.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
spawnLogger.verbose(`Getting data.`);
|
||||
|
||||
let data = JSON.parse(
|
||||
fs.readFileSync(path.join(seedsDir, "collections", jsonName), "utf-8")
|
||||
);
|
||||
|
||||
spawnLogger.info(`Found ${data.length} documents.`);
|
||||
|
||||
let matchedSomething = false;
|
||||
for (const syncInst of syncInstructions) {
|
||||
if (jsonName.match(syncInst.pattern)) {
|
||||
spawnLogger.verbose(`Starting handler...`);
|
||||
await syncInst.handler(data, monkDB.get(collectionName), spawnLogger);
|
||||
matchedSomething = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// free memory forcefully
|
||||
data = null;
|
||||
|
||||
if (!matchedSomething) {
|
||||
spawnLogger.warn(
|
||||
`Collection ${collectionName} didn't match any sync instructions. Skipping.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Done.`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
SynchroniseDBWithSeeds().then(() => process.exit(0));
|
||||
}
|
||||
Reference in New Issue
Block a user