mirror of
https://github.com/zkldi/Tachi.git
synced 2026-10-01 19:38:13 +03:00
experimental: initial autofix blast
This commit is contained in:
@@ -1,9 +1,10 @@
|
||||
import { execSync } from "child_process";
|
||||
import { Command } from "commander";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import monk from "monk";
|
||||
import { execSync } from "child_process";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.option("--nsFrom <Database to clone from.>")
|
||||
.option("--nsTo <Database to clone and anonymise to.>");
|
||||
@@ -37,6 +38,7 @@ async function AnonymiseDB(nsTo: string) {
|
||||
$set: {
|
||||
// This is "password" encrypted in bcrypt12.
|
||||
password: "$2b$12$QRFCAxvFoNI2spszFPgt/e.qLy55GvYWlSHioa0AujRbFpChLwHmu",
|
||||
|
||||
// emails are uniquely indexed. We need to anonymise these in
|
||||
// a way that they dont duplicate.
|
||||
email: { $concat: [{ $toString: "$userID" }, "@example.com"] },
|
||||
|
||||
@@ -4,6 +4,7 @@ import { MainHTMLWrapper } from "lib/email/formats";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program.option("-e, --email <Email to send to>");
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
import db from "external/mongo/db";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import fetch from "node-fetch";
|
||||
import { Difficulties, integer } from "tachi-common";
|
||||
import { FindChartWithPTDF } from "utils/queries/charts";
|
||||
import { FindSongOnTitle } from "utils/queries/songs";
|
||||
import type { Difficulties, integer } from "tachi-common";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
@@ -23,7 +23,7 @@ interface SP12Data {
|
||||
}
|
||||
|
||||
async function FetchSP12Data() {
|
||||
const rj: { sheets: SP12Data[] } = await fetch("https://sp12.iidx.app/api/v1/sheets").then(
|
||||
const rj: { sheets: Array<SP12Data> } = await fetch("https://sp12.iidx.app/api/v1/sheets").then(
|
||||
(r) => r.json()
|
||||
); // will throw if somethings wrong, anyway
|
||||
|
||||
@@ -33,6 +33,7 @@ async function FetchSP12Data() {
|
||||
|
||||
for (const sh of rj.sheets) {
|
||||
let chart;
|
||||
|
||||
try {
|
||||
chart = await HumanisedTitleLookup(sh.title);
|
||||
} catch (err) {
|
||||
@@ -48,7 +49,7 @@ async function FetchSP12Data() {
|
||||
let val;
|
||||
|
||||
switch (key) {
|
||||
case "n_clear":
|
||||
case "n_clear": {
|
||||
const v = Math.floor(sh[key] / 2);
|
||||
|
||||
if (v === 9) {
|
||||
@@ -68,7 +69,9 @@ async function FetchSP12Data() {
|
||||
}
|
||||
|
||||
break;
|
||||
case "hard":
|
||||
}
|
||||
|
||||
case "hard": {
|
||||
const v2 = Math.floor(sh[key] / 2);
|
||||
|
||||
if (v2 === 9) {
|
||||
@@ -88,8 +91,11 @@ async function FetchSP12Data() {
|
||||
}
|
||||
|
||||
break;
|
||||
case "exh":
|
||||
}
|
||||
|
||||
case "exh": {
|
||||
const v3 = sh[key];
|
||||
|
||||
if (v3 >= 12 || v3 <= 0) {
|
||||
continue;
|
||||
}
|
||||
@@ -97,6 +103,8 @@ async function FetchSP12Data() {
|
||||
val = 12.4 + (12 - v3) * 0.1;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
throw new Error("??");
|
||||
}
|
||||
@@ -147,13 +155,13 @@ async function HumanisedTitleLookup(originalTitle: string) {
|
||||
|
||||
let title = originalTitle;
|
||||
|
||||
if (title.match(/(†|†LEGGENDARIA)$/u)) {
|
||||
if (/(†|†LEGGENDARIA)$/u.exec(title)) {
|
||||
difficulty = "LEGGENDARIA";
|
||||
title = title.split(/(†|†LEGGENDARIA)$/u)[0];
|
||||
} else if (title.match(/\[H\]$/u)) {
|
||||
} else if (/\[H\]$/u.exec(title)) {
|
||||
difficulty = "HYPER";
|
||||
title = title.split("[")[0];
|
||||
} else if (title.match(/\[A\]$/u)) {
|
||||
} else if (/\[A\]$/u.exec(title)) {
|
||||
difficulty = "ANOTHER"; // lmao
|
||||
title = title.split("[")[0];
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import deepmerge from "deepmerge";
|
||||
import { ICollection } from "monk";
|
||||
import { ChartDocument, IDStrings } from "tachi-common";
|
||||
import { Random20Hex } from "utils/misc";
|
||||
import type { ICollection } from "monk";
|
||||
import type { ChartDocument, IDStrings } from "tachi-common";
|
||||
|
||||
export async function QuickInsert<I extends IDStrings>(
|
||||
base: Omit<ChartDocument<I>, "chartID">,
|
||||
content: Partial<ChartDocument<I>>[],
|
||||
content: Array<Partial<ChartDocument<I>>>,
|
||||
coll: ICollection
|
||||
) {
|
||||
const docs = [];
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/* eslint-disable no-await-in-loop */
|
||||
import { Command } from "commander";
|
||||
import { SetIndexes } from "../external/mongo/indexes";
|
||||
import { Command } from "commander";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ export async function CreateGameProfiles() {
|
||||
|
||||
for (const gpt of gpts) {
|
||||
const { game, playtype } = gpt._id;
|
||||
|
||||
logger.info(`Updating ${FormatUserDoc(user)}'s ${game} ${playtype} stats.`);
|
||||
await UpdateUsersGamePlaytypeStats(game, playtype, user.id, null, logger);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { CreateGameProfiles } from "./recalc-game-profiles";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import { RecalcAllScores, UpdateAllPBs } from "utils/calculations/recalc-scores";
|
||||
import { RecalcSessions } from "utils/calculations/recalc-sessions";
|
||||
import { CreateGameProfiles } from "./recalc-game-profiles";
|
||||
|
||||
const logger = CreateLogCtx(__filename);
|
||||
|
||||
|
||||
@@ -4,14 +4,19 @@
|
||||
import db, { monkDB } from "external/mongo/db";
|
||||
import fjsh from "fast-json-stable-hash";
|
||||
import { PullDatabaseSeeds } from "lib/database-seeds/repo";
|
||||
import CreateLogCtx, { KtLogger } from "lib/logger/logger";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import UpdateIsPrimaryStatus from "lib/score-mutation/update-isprimary";
|
||||
import { TachiConfig } from "lib/setup/config";
|
||||
import { RemoveStaleFolderShowcaseStats } from "lib/showcase/showcase";
|
||||
import { UpdateMilestoneSubscriptions } from "lib/targets/milestones";
|
||||
import { BulkWriteOperation } from "mongodb";
|
||||
import { ICollection } from "monk";
|
||||
import {
|
||||
import { RecalcAllScores } from "utils/calculations/recalc-scores";
|
||||
import { UpdateGameSongIDCounter } from "utils/db";
|
||||
import { InitaliseFolderChartLookup } from "utils/folder";
|
||||
import { ArrayDiff, IsSupported } from "utils/misc";
|
||||
import type { KtLogger } from "lib/logger/logger";
|
||||
import type { BulkWriteOperation } from "mongodb";
|
||||
import type { ICollection } from "monk";
|
||||
import type {
|
||||
BMSCourseDocument,
|
||||
ChartDocument,
|
||||
FolderDocument,
|
||||
@@ -22,31 +27,28 @@ import {
|
||||
SongDocument,
|
||||
TableDocument,
|
||||
} from "tachi-common";
|
||||
import { RecalcAllScores } from "utils/calculations/recalc-scores";
|
||||
import { UpdateGameSongIDCounter } from "utils/db";
|
||||
import { InitaliseFolderChartLookup } from "utils/folder";
|
||||
import { ArrayDiff, IsSupported } from "utils/misc";
|
||||
|
||||
interface SyncInstructions {
|
||||
pattern: RegExp;
|
||||
handler: (
|
||||
// These 'any's are necessary because generifying here kinda sucks.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
c: any[],
|
||||
c: Array<any>,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
collection: ICollection<any>,
|
||||
collection: ICollection,
|
||||
logger: KtLogger,
|
||||
collectionName: string
|
||||
) => Promise<unknown>;
|
||||
}
|
||||
|
||||
async function RemoveNotPresent<T>(
|
||||
documents: T[],
|
||||
documents: Array<T>,
|
||||
collection: ICollection<T>,
|
||||
field: keyof T,
|
||||
logger: KtLogger
|
||||
) {
|
||||
logger.verbose(`Removing all documents that are no longer present.`);
|
||||
|
||||
// Remove anything no longer present.
|
||||
// Note that $nin is incredibly slow.
|
||||
// @ts-expect-error generic system failing
|
||||
@@ -62,7 +64,7 @@ async function RemoveNotPresent<T>(
|
||||
}
|
||||
|
||||
async function GenericUpsert<T>(
|
||||
documents: T[],
|
||||
documents: Array<T>,
|
||||
collection: ICollection<T>,
|
||||
field: keyof T,
|
||||
logger: KtLogger,
|
||||
@@ -71,11 +73,12 @@ async function GenericUpsert<T>(
|
||||
) {
|
||||
logger.verbose(`Running bulkwrite.`);
|
||||
|
||||
const bwriteOps: BulkWriteOperation<T>[] = [];
|
||||
const bwriteOps: Array<BulkWriteOperation<T>> = [];
|
||||
|
||||
const allExistingDocs = await collection.find({});
|
||||
|
||||
const map = new Map();
|
||||
|
||||
for (const doc of allExistingDocs) {
|
||||
map.set(doc[field], doc);
|
||||
}
|
||||
@@ -83,6 +86,7 @@ async function GenericUpsert<T>(
|
||||
const changedFields = [];
|
||||
|
||||
let i = 0;
|
||||
|
||||
for (const document of documents) {
|
||||
i++;
|
||||
if (i % 10_000 === 0) {
|
||||
@@ -139,11 +143,11 @@ async function GenericUpsert<T>(
|
||||
};
|
||||
}
|
||||
|
||||
const syncInstructions: SyncInstructions[] = [
|
||||
const syncInstructions: Array<SyncInstructions> = [
|
||||
{
|
||||
pattern: /^charts-(b|p)ms/u,
|
||||
handler: async (
|
||||
charts: ChartDocument[],
|
||||
charts: Array<ChartDocument>,
|
||||
collection: ICollection<ChartDocument>,
|
||||
logger,
|
||||
collectionName
|
||||
@@ -165,7 +169,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
{
|
||||
pattern: /^charts-/u,
|
||||
handler: async (
|
||||
charts: ChartDocument[],
|
||||
charts: Array<ChartDocument>,
|
||||
collection: ICollection<ChartDocument>,
|
||||
logger
|
||||
) => {
|
||||
@@ -183,7 +187,11 @@ const syncInstructions: SyncInstructions[] = [
|
||||
},
|
||||
{
|
||||
pattern: /^songs-(b|p)ms/u,
|
||||
handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
|
||||
handler: async (
|
||||
songs: Array<SongDocument>,
|
||||
collection: ICollection<SongDocument>,
|
||||
logger
|
||||
) => {
|
||||
const r = await GenericUpsert(songs, collection, "id", logger, false);
|
||||
|
||||
if (r.thingsChanged) {
|
||||
@@ -195,7 +203,11 @@ const syncInstructions: SyncInstructions[] = [
|
||||
},
|
||||
{
|
||||
pattern: /^songs-/u,
|
||||
handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
|
||||
handler: async (
|
||||
songs: Array<SongDocument>,
|
||||
collection: ICollection<SongDocument>,
|
||||
logger
|
||||
) => {
|
||||
const r = await GenericUpsert(songs, collection, "id", logger, true);
|
||||
|
||||
if (r.thingsChanged) {
|
||||
@@ -208,7 +220,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
{
|
||||
pattern: /^folders/u,
|
||||
handler: async (
|
||||
folders: FolderDocument[],
|
||||
folders: Array<FolderDocument>,
|
||||
collection: ICollection<FolderDocument>,
|
||||
logger
|
||||
) => {
|
||||
@@ -223,7 +235,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
if (r) {
|
||||
await InitaliseFolderChartLookup();
|
||||
|
||||
const allModifiedFolderIDs = r.changedFields as string[];
|
||||
const allModifiedFolderIDs = r.changedFields as Array<string>;
|
||||
|
||||
const keptFolderIDs = await db.folders.find(
|
||||
{
|
||||
@@ -245,7 +257,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
},
|
||||
{
|
||||
pattern: /^tables/u,
|
||||
handler: (tables: TableDocument[], collection: ICollection<TableDocument>, logger) =>
|
||||
handler: (tables: Array<TableDocument>, collection: ICollection<TableDocument>, logger) =>
|
||||
GenericUpsert(
|
||||
tables.filter((e) => TachiConfig.GAMES.includes(e.game)),
|
||||
collection,
|
||||
@@ -257,7 +269,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
{
|
||||
pattern: /^bms-course-lookup/u,
|
||||
handler: async (
|
||||
bmsCourseDocuments: BMSCourseDocument[],
|
||||
bmsCourseDocuments: Array<BMSCourseDocument>,
|
||||
collection: ICollection<BMSCourseDocument>,
|
||||
logger
|
||||
) => {
|
||||
@@ -270,7 +282,11 @@ const syncInstructions: SyncInstructions[] = [
|
||||
},
|
||||
{
|
||||
pattern: /^goals/u,
|
||||
handler: async (goals: GoalDocument[], collection: ICollection<GoalDocument>, logger) => {
|
||||
handler: async (
|
||||
goals: Array<GoalDocument>,
|
||||
collection: ICollection<GoalDocument>,
|
||||
logger
|
||||
) => {
|
||||
// never remove goals. Never update goals either. Only insert new ones as
|
||||
// they come in.
|
||||
await GenericUpsert(
|
||||
@@ -286,7 +302,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
{
|
||||
pattern: /^milestone-sets/u,
|
||||
handler: async (
|
||||
milestoneSets: MilestoneSetDocument[],
|
||||
milestoneSets: Array<MilestoneSetDocument>,
|
||||
collection: ICollection<MilestoneSetDocument>,
|
||||
logger
|
||||
) => {
|
||||
@@ -302,7 +318,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
{
|
||||
pattern: /^milestones/u,
|
||||
handler: async (
|
||||
milestones: MilestoneDocument[],
|
||||
milestones: Array<MilestoneDocument>,
|
||||
collection: ICollection<MilestoneDocument>,
|
||||
logger
|
||||
) => {
|
||||
@@ -315,7 +331,7 @@ const syncInstructions: SyncInstructions[] = [
|
||||
);
|
||||
|
||||
if (r.thingsChanged) {
|
||||
const affectedMilestoneIDs = r.changedFields as string[];
|
||||
const affectedMilestoneIDs = r.changedFields as Array<string>;
|
||||
|
||||
await Promise.all(affectedMilestoneIDs.map((e) => UpdateMilestoneSubscriptions(e)));
|
||||
}
|
||||
@@ -348,8 +364,9 @@ async function SynchroniseDBWithSeeds() {
|
||||
spawnLogger.verbose(`Found ${data.length} documents.`);
|
||||
|
||||
let matchedSomething = false;
|
||||
|
||||
for (const syncInst of syncInstructions) {
|
||||
if (collectionName.match(syncInst.pattern)) {
|
||||
if (syncInst.pattern.exec(collectionName)) {
|
||||
spawnLogger.info(`Starting handler...`);
|
||||
await syncInst.handler(
|
||||
data,
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
// note: a hell of a lot of this code is copy pasted
|
||||
// this is just a script, so, feel free to refactor it.
|
||||
import { Command } from "commander";
|
||||
import { Databases, monkDB } from "external/mongo/db";
|
||||
import { monkDB } from "external/mongo/db";
|
||||
import { DatabaseSchemas } from "external/mongo/schemas";
|
||||
import CreateLogCtx from "lib/logger/logger";
|
||||
import { PrudenceError } from "prudence";
|
||||
import { FormatPrError } from "utils/prudence";
|
||||
import type { Databases } from "external/mongo/db";
|
||||
import type { PrudenceError } from "prudence";
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program.option("-c, --collection <collectionName>");
|
||||
|
||||
program.parse(process.argv);
|
||||
@@ -22,7 +24,7 @@ export async function ValidateCollection(collectionName: Databases): Promise<voi
|
||||
}
|
||||
|
||||
// collectionName is in database schemas.
|
||||
const schemaRunner = DatabaseSchemas[collectionName as keyof typeof DatabaseSchemas];
|
||||
const schemaRunner = DatabaseSchemas[collectionName];
|
||||
|
||||
const documents = await monkDB.get(collectionName).count({});
|
||||
|
||||
@@ -34,6 +36,7 @@ export async function ValidateCollection(collectionName: Databases): Promise<voi
|
||||
await monkDB
|
||||
.get(collectionName)
|
||||
.find({})
|
||||
|
||||
// @ts-expect-error faulty monk types
|
||||
.each((c) => {
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user