experimental: initial autofix blast

This commit is contained in:
zkldi
2022-05-06 03:39:03 +01:00
parent 61ea7a4bf7
commit 0adc39b5cb
275 changed files with 2207 additions and 1877 deletions
+3 -1
View File
@@ -1,9 +1,10 @@
import { execSync } from "child_process";
import { Command } from "commander";
import CreateLogCtx from "lib/logger/logger";
import monk from "monk";
import { execSync } from "child_process";
const program = new Command();
program
.option("--nsFrom <Database to clone from.>")
.option("--nsTo <Database to clone and anonymise to.>");
@@ -37,6 +38,7 @@ async function AnonymiseDB(nsTo: string) {
$set: {
// This is "password" encrypted in bcrypt12.
password: "$2b$12$QRFCAxvFoNI2spszFPgt/e.qLy55GvYWlSHioa0AujRbFpChLwHmu",
// emails are uniquely indexed. We need to anonymise these in
// a way that they dont duplicate.
email: { $concat: [{ $toString: "$userID" }, "@example.com"] },
@@ -4,6 +4,7 @@ import { MainHTMLWrapper } from "lib/email/formats";
import CreateLogCtx from "lib/logger/logger";
const program = new Command();
program.option("-e, --email <Email to send to>");
const logger = CreateLogCtx(__filename);
@@ -4,9 +4,9 @@
import db from "external/mongo/db";
import CreateLogCtx from "lib/logger/logger";
import fetch from "node-fetch";
import { Difficulties, integer } from "tachi-common";
import { FindChartWithPTDF } from "utils/queries/charts";
import { FindSongOnTitle } from "utils/queries/songs";
import type { Difficulties, integer } from "tachi-common";
const logger = CreateLogCtx(__filename);
@@ -23,7 +23,7 @@ interface SP12Data {
}
async function FetchSP12Data() {
const rj: { sheets: SP12Data[] } = await fetch("https://sp12.iidx.app/api/v1/sheets").then(
const rj: { sheets: Array<SP12Data> } = await fetch("https://sp12.iidx.app/api/v1/sheets").then(
(r) => r.json()
); // will throw if somethings wrong, anyway
@@ -33,6 +33,7 @@ async function FetchSP12Data() {
for (const sh of rj.sheets) {
let chart;
try {
chart = await HumanisedTitleLookup(sh.title);
} catch (err) {
@@ -48,7 +49,7 @@ async function FetchSP12Data() {
let val;
switch (key) {
case "n_clear":
case "n_clear": {
const v = Math.floor(sh[key] / 2);
if (v === 9) {
@@ -68,7 +69,9 @@ async function FetchSP12Data() {
}
break;
case "hard":
}
case "hard": {
const v2 = Math.floor(sh[key] / 2);
if (v2 === 9) {
@@ -88,8 +91,11 @@ async function FetchSP12Data() {
}
break;
case "exh":
}
case "exh": {
const v3 = sh[key];
if (v3 >= 12 || v3 <= 0) {
continue;
}
@@ -97,6 +103,8 @@ async function FetchSP12Data() {
val = 12.4 + (12 - v3) * 0.1;
break;
}
default:
throw new Error("??");
}
@@ -147,13 +155,13 @@ async function HumanisedTitleLookup(originalTitle: string) {
let title = originalTitle;
if (title.match(/(†|†LEGGENDARIA)$/u)) {
if (/(†|†LEGGENDARIA)$/u.exec(title)) {
difficulty = "LEGGENDARIA";
title = title.split(/(†|†LEGGENDARIA)$/u)[0];
} else if (title.match(/\[H\]$/u)) {
} else if (/\[H\]$/u.exec(title)) {
difficulty = "HYPER";
title = title.split("[")[0];
} else if (title.match(/\[A\]$/u)) {
} else if (/\[A\]$/u.exec(title)) {
difficulty = "ANOTHER"; // lmao
title = title.split("[")[0];
}
@@ -1,11 +1,11 @@
import deepmerge from "deepmerge";
import { ICollection } from "monk";
import { ChartDocument, IDStrings } from "tachi-common";
import { Random20Hex } from "utils/misc";
import type { ICollection } from "monk";
import type { ChartDocument, IDStrings } from "tachi-common";
export async function QuickInsert<I extends IDStrings>(
base: Omit<ChartDocument<I>, "chartID">,
content: Partial<ChartDocument<I>>[],
content: Array<Partial<ChartDocument<I>>>,
coll: ICollection
) {
const docs = [];
+1 -1
View File
@@ -1,6 +1,6 @@
/* eslint-disable no-await-in-loop */
import { Command } from "commander";
import { SetIndexes } from "../external/mongo/indexes";
import { Command } from "commander";
const program = new Command();
@@ -30,6 +30,7 @@ export async function CreateGameProfiles() {
for (const gpt of gpts) {
const { game, playtype } = gpt._id;
logger.info(`Updating ${FormatUserDoc(user)}'s ${game} ${playtype} stats.`);
await UpdateUsersGamePlaytypeStats(game, playtype, user.id, null, logger);
}
+1 -1
View File
@@ -1,7 +1,7 @@
import { CreateGameProfiles } from "./recalc-game-profiles";
import CreateLogCtx from "lib/logger/logger";
import { RecalcAllScores, UpdateAllPBs } from "utils/calculations/recalc-scores";
import { RecalcSessions } from "utils/calculations/recalc-sessions";
import { CreateGameProfiles } from "./recalc-game-profiles";
const logger = CreateLogCtx(__filename);
+44 -27
View File
@@ -4,14 +4,19 @@
import db, { monkDB } from "external/mongo/db";
import fjsh from "fast-json-stable-hash";
import { PullDatabaseSeeds } from "lib/database-seeds/repo";
import CreateLogCtx, { KtLogger } from "lib/logger/logger";
import CreateLogCtx from "lib/logger/logger";
import UpdateIsPrimaryStatus from "lib/score-mutation/update-isprimary";
import { TachiConfig } from "lib/setup/config";
import { RemoveStaleFolderShowcaseStats } from "lib/showcase/showcase";
import { UpdateMilestoneSubscriptions } from "lib/targets/milestones";
import { BulkWriteOperation } from "mongodb";
import { ICollection } from "monk";
import {
import { RecalcAllScores } from "utils/calculations/recalc-scores";
import { UpdateGameSongIDCounter } from "utils/db";
import { InitaliseFolderChartLookup } from "utils/folder";
import { ArrayDiff, IsSupported } from "utils/misc";
import type { KtLogger } from "lib/logger/logger";
import type { BulkWriteOperation } from "mongodb";
import type { ICollection } from "monk";
import type {
BMSCourseDocument,
ChartDocument,
FolderDocument,
@@ -22,31 +27,28 @@ import {
SongDocument,
TableDocument,
} from "tachi-common";
import { RecalcAllScores } from "utils/calculations/recalc-scores";
import { UpdateGameSongIDCounter } from "utils/db";
import { InitaliseFolderChartLookup } from "utils/folder";
import { ArrayDiff, IsSupported } from "utils/misc";
interface SyncInstructions {
pattern: RegExp;
handler: (
// These 'any's are necessary because generifying here kinda sucks.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
c: any[],
c: Array<any>,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
collection: ICollection<any>,
collection: ICollection,
logger: KtLogger,
collectionName: string
) => Promise<unknown>;
}
async function RemoveNotPresent<T>(
documents: T[],
documents: Array<T>,
collection: ICollection<T>,
field: keyof T,
logger: KtLogger
) {
logger.verbose(`Removing all documents that are no longer present.`);
// Remove anything no longer present.
// Note that $nin is incredibly slow.
// @ts-expect-error generic system failing
@@ -62,7 +64,7 @@ async function RemoveNotPresent<T>(
}
async function GenericUpsert<T>(
documents: T[],
documents: Array<T>,
collection: ICollection<T>,
field: keyof T,
logger: KtLogger,
@@ -71,11 +73,12 @@ async function GenericUpsert<T>(
) {
logger.verbose(`Running bulkwrite.`);
const bwriteOps: BulkWriteOperation<T>[] = [];
const bwriteOps: Array<BulkWriteOperation<T>> = [];
const allExistingDocs = await collection.find({});
const map = new Map();
for (const doc of allExistingDocs) {
map.set(doc[field], doc);
}
@@ -83,6 +86,7 @@ async function GenericUpsert<T>(
const changedFields = [];
let i = 0;
for (const document of documents) {
i++;
if (i % 10_000 === 0) {
@@ -139,11 +143,11 @@ async function GenericUpsert<T>(
};
}
const syncInstructions: SyncInstructions[] = [
const syncInstructions: Array<SyncInstructions> = [
{
pattern: /^charts-(b|p)ms/u,
handler: async (
charts: ChartDocument[],
charts: Array<ChartDocument>,
collection: ICollection<ChartDocument>,
logger,
collectionName
@@ -165,7 +169,7 @@ const syncInstructions: SyncInstructions[] = [
{
pattern: /^charts-/u,
handler: async (
charts: ChartDocument[],
charts: Array<ChartDocument>,
collection: ICollection<ChartDocument>,
logger
) => {
@@ -183,7 +187,11 @@ const syncInstructions: SyncInstructions[] = [
},
{
pattern: /^songs-(b|p)ms/u,
handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
handler: async (
songs: Array<SongDocument>,
collection: ICollection<SongDocument>,
logger
) => {
const r = await GenericUpsert(songs, collection, "id", logger, false);
if (r.thingsChanged) {
@@ -195,7 +203,11 @@ const syncInstructions: SyncInstructions[] = [
},
{
pattern: /^songs-/u,
handler: async (songs: SongDocument[], collection: ICollection<SongDocument>, logger) => {
handler: async (
songs: Array<SongDocument>,
collection: ICollection<SongDocument>,
logger
) => {
const r = await GenericUpsert(songs, collection, "id", logger, true);
if (r.thingsChanged) {
@@ -208,7 +220,7 @@ const syncInstructions: SyncInstructions[] = [
{
pattern: /^folders/u,
handler: async (
folders: FolderDocument[],
folders: Array<FolderDocument>,
collection: ICollection<FolderDocument>,
logger
) => {
@@ -223,7 +235,7 @@ const syncInstructions: SyncInstructions[] = [
if (r) {
await InitaliseFolderChartLookup();
const allModifiedFolderIDs = r.changedFields as string[];
const allModifiedFolderIDs = r.changedFields as Array<string>;
const keptFolderIDs = await db.folders.find(
{
@@ -245,7 +257,7 @@ const syncInstructions: SyncInstructions[] = [
},
{
pattern: /^tables/u,
handler: (tables: TableDocument[], collection: ICollection<TableDocument>, logger) =>
handler: (tables: Array<TableDocument>, collection: ICollection<TableDocument>, logger) =>
GenericUpsert(
tables.filter((e) => TachiConfig.GAMES.includes(e.game)),
collection,
@@ -257,7 +269,7 @@ const syncInstructions: SyncInstructions[] = [
{
pattern: /^bms-course-lookup/u,
handler: async (
bmsCourseDocuments: BMSCourseDocument[],
bmsCourseDocuments: Array<BMSCourseDocument>,
collection: ICollection<BMSCourseDocument>,
logger
) => {
@@ -270,7 +282,11 @@ const syncInstructions: SyncInstructions[] = [
},
{
pattern: /^goals/u,
handler: async (goals: GoalDocument[], collection: ICollection<GoalDocument>, logger) => {
handler: async (
goals: Array<GoalDocument>,
collection: ICollection<GoalDocument>,
logger
) => {
// never remove goals. Never update goals either. Only insert new ones as
// they come in.
await GenericUpsert(
@@ -286,7 +302,7 @@ const syncInstructions: SyncInstructions[] = [
{
pattern: /^milestone-sets/u,
handler: async (
milestoneSets: MilestoneSetDocument[],
milestoneSets: Array<MilestoneSetDocument>,
collection: ICollection<MilestoneSetDocument>,
logger
) => {
@@ -302,7 +318,7 @@ const syncInstructions: SyncInstructions[] = [
{
pattern: /^milestones/u,
handler: async (
milestones: MilestoneDocument[],
milestones: Array<MilestoneDocument>,
collection: ICollection<MilestoneDocument>,
logger
) => {
@@ -315,7 +331,7 @@ const syncInstructions: SyncInstructions[] = [
);
if (r.thingsChanged) {
const affectedMilestoneIDs = r.changedFields as string[];
const affectedMilestoneIDs = r.changedFields as Array<string>;
await Promise.all(affectedMilestoneIDs.map((e) => UpdateMilestoneSubscriptions(e)));
}
@@ -348,8 +364,9 @@ async function SynchroniseDBWithSeeds() {
spawnLogger.verbose(`Found ${data.length} documents.`);
let matchedSomething = false;
for (const syncInst of syncInstructions) {
if (collectionName.match(syncInst.pattern)) {
if (syncInst.pattern.exec(collectionName)) {
spawnLogger.info(`Starting handler...`);
await syncInst.handler(
data,
+6 -3
View File
@@ -1,13 +1,15 @@
// note: a hell of a lot of this code is copy pasted
// this is just a script, so, feel free to refactor it.
import { Command } from "commander";
import { Databases, monkDB } from "external/mongo/db";
import { monkDB } from "external/mongo/db";
import { DatabaseSchemas } from "external/mongo/schemas";
import CreateLogCtx from "lib/logger/logger";
import { PrudenceError } from "prudence";
import { FormatPrError } from "utils/prudence";
import type { Databases } from "external/mongo/db";
import type { PrudenceError } from "prudence";
const program = new Command();
program.option("-c, --collection <collectionName>");
program.parse(process.argv);
@@ -22,7 +24,7 @@ export async function ValidateCollection(collectionName: Databases): Promise<voi
}
// collectionName is in database schemas.
const schemaRunner = DatabaseSchemas[collectionName as keyof typeof DatabaseSchemas];
const schemaRunner = DatabaseSchemas[collectionName];
const documents = await monkDB.get(collectionName).count({});
@@ -34,6 +36,7 @@ export async function ValidateCollection(collectionName: Databases): Promise<voi
await monkDB
.get(collectionName)
.find({})
// @ts-expect-error faulty monk types
.each((c) => {
try {