Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-10 08:18:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

Assorted Fixes for database validation.

Browse Source
This commit is contained in:
zkldi
2021-10-27 16:14:18 +01:00
parent 3e8f87d98b
commit 1ee98480ef
8 changed files with 151 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/package.json
+2 -2
View File
@@ -72,12 +72,12 @@
"node-cache": "^5.1.2",
"node-fetch": "2.6.1",
"nodemailer": "^6.6.3",
"prudence": "0.9.6",
"prudence": "0.9.7",
"rate-limit-redis": "2.1.0",
"redis": "3.1.2",
"rimraf": "3.0.2",
"safe-json-stringify": "1.2.0",
"tachi-common": "0.2.28",
"tachi-common": "0.2.34",
"ts-node": "10.0.0",
"typescript": "4.3.4",
"winston": "3.3.3",
server/pnpm-lock.yaml
Generated
+8 -8
View File
@@ -53,13 +53,13 @@ specifiers:
node-mocks-http: 1.10.1
nodemailer: ^6.6.3
prettier: 2.3.1
prudence: 0.9.6
prudence: 0.9.7
rate-limit-redis: 2.1.0
redis: 3.1.2
rimraf: 3.0.2
safe-json-stringify: 1.2.0
supertest: 6.1.3
tachi-common: 0.2.28
tachi-common: 0.2.34
tap: 15.0.9
ts-node: 10.0.0
tsconfig-paths: 3.10.1
@@ -95,12 +95,12 @@ dependencies:
node-cache: 5.1.2
node-fetch: 2.6.1
nodemailer: 6.6.3
prudence: 0.9.6
prudence: 0.9.7
rate-limit-redis: 2.1.0
redis: 3.1.2
rimraf: 3.0.2
safe-json-stringify: 1.2.0
tachi-common: 0.2.28
tachi-common: 0.2.34
ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b
typescript: 4.3.4
winston: 3.3.3
@@ -3554,8 +3554,8 @@ packages:
ipaddr.js: 1.9.1
dev: false
/prudence/0.9.6:
resolution: {integrity: sha512-kCPaLDVfVovahwd+SXlszt4y+av3VJL5HXD4II2+m8+zm6J6wt8vFU+kDFeZvaycaGJemB8VUv39hFplHbxTIw==}
/prudence/0.9.7:
resolution: {integrity: sha512-7214GHTBThBI/PGlKWXUSJ++ca0ld54VpF4k98U1EDAFXai5mnvWbb4speVOnWkilqfN25x8biWUAj4gHNu5Og==}
dev: false
/psl/1.8.0:
@@ -4189,8 +4189,8 @@ packages:
strip-ansi: 6.0.0
dev: true
/tachi-common/0.2.28:
resolution: {integrity: sha512-9iqO/2y7VNbpyJBf4CDffw6E9PZuiib7nkK4Vk6Ov4i6a2zn787rBzQ8XgoQGRZKlHV5tfFd+CFbbiTRPM3ZBA==}
/tachi-common/0.2.34:
resolution: {integrity: sha512-qKXIqzMTuT4eNL0Eo3N00fGYDF63bLxvZbf1/7jI+i8pDqJ8Xd1kmSuzNENBr+cp75Krh3qt279xf5oejUOiGw==}
dependencies:
monk: 7.3.4
transitivePeerDependencies:
server/scripts/__KT_DATABASE_MIGRATION/sessions.ts
+14 -4
View File
@@ -1,11 +1,11 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { integer, SessionDocument, SessionScoreInfo } from "tachi-common";
import db from "external/mongo/db";
import MigrateRecords from "./migrate";
import { CreateSessionID } from "lib/score-import/framework/sessions/sessions";
import { oldKTDB } from "./old-db";
import CreateLogCtx from "lib/logger/logger";
import { CreateSessionID } from "lib/score-import/framework/sessions/sessions";
import { GetGameConfig, integer, SessionDocument, SessionScoreInfo } from "tachi-common";
import { allSupportedGames } from "tachi-common/js/config/static-config";
import MigrateRecords from "./migrate";
import { oldKTDB } from "./old-db";
const logger = CreateLogCtx(__filename);
@@ -44,6 +44,10 @@ async function ConvertScoreToScoreInfo(
};
}
if (score.pbInfo.scoreDelta === null) {
return null;
}
return {
scoreID: scoreIDs.new,
isNewScore: false,
@@ -59,6 +63,12 @@ async function ConvertFn(c: any): Promise<SessionDocument | null> {
return null;
}
const gameConfig = GetGameConfig(c.game);
if (!gameConfig.validPlaytypes.includes(c.playtype)) {
return null;
}
const scores = [];
for (const sc of c.scores) {
server/scripts/arc-sync/arc-db-sync.ts
+1 -1
View File
@@ -179,7 +179,7 @@ async function MergeIDs() {
// eslint-disable-next-line no-await-in-loop
await db.charts[options.game as "sdvx" | "iidx"].update(
{
_id: tachiChart._id,
chartID: tachiChart.chartID,
},
{
$set: {
server/scripts/single-use/fix-user-goals.ts
+32
View File
@@ -0,0 +1,32 @@
import db from "external/mongo/db";
if (require.main === module) {
(async () => {
const ugs = await db["user-goals"].find(
{},
{
projectID: true,
}
);
for (const ug of ugs) {
await db["user-goals"].update(
{
_id: ug._id,
},
{
$set: {
progressHuman: ug.progressHuman.toString(),
outOfHuman: ug.outOfHuman.toString(),
lastInteraction: null,
},
$unset: {
note: 1,
},
}
);
}
process.exit(0);
})();
}
server/scripts/single-use/remove-null-scoreInfo-sessions.ts
+30
View File
@@ -0,0 +1,30 @@
import db from "external/mongo/db";
if (require.main === module) {
(async () => {
const badScoreInfos = await db.sessions.find({
"scoreInfo.isNewScore": false,
"scoreInfo.scoreDelta": null,
});
for (const session of badScoreInfos) {
const scoreInfo = session.scoreInfo.filter(
(e) => !(!e.isNewScore && e.scoreDelta === null)
);
// eslint-disable-next-line no-await-in-loop
await db.sessions.update(
{
sessionID: session.sessionID,
},
{
$set: {
scoreInfo,
},
}
);
}
process.exit(0);
})();
}
server/scripts/sync-database/sync-database.ts
+28 -15
View File
@@ -11,6 +11,13 @@ import { ICollection } from "monk";
import path from "path";
import { ChartDocument, FolderDocument, SongDocument, TableDocument } from "tachi-common";
import deepEqual from "deep-equal";
import { Command } from "commander";
const program = new Command();
program.option("-c, --cache");
program.parse(process.argv);
const options = program.opts();
interface SyncInstructions {
pattern: RegExp;
@@ -70,14 +77,12 @@ async function GenericUpsert<T>(
});
} else if (!deepEqual(document, exists, { strict: true })) {
bwriteOps.push({
updateOne: {
replaceOne: {
// @ts-expect-error Known X->Y generic issue.
filter: {
[field]: document[field],
},
update: {
$set: document,
},
replacement: document,
},
});
}
@@ -102,12 +107,12 @@ const syncInstructions: SyncInstructions[] = [
{
pattern: /^charts-/u,
handler: (charts: ChartDocument[], collection: ICollection<ChartDocument>, logger) =>
GenericUpsert(charts, collection, "chartID", logger),
GenericUpsert(charts, collection, "chartID", logger, true),
},
{
pattern: /^songs-/u,
handler: (songs: SongDocument[], collection: ICollection<SongDocument>, logger) =>
GenericUpsert(songs, collection, "id", logger),
GenericUpsert(songs, collection, "id", logger, true),
},
{
pattern: /^folders$/u,
@@ -146,7 +151,9 @@ const syncInstructions: SyncInstructions[] = [
}
}
await collection.bulkWrite(bwriteOps);
if (bwriteOps.length) {
await collection.bulkWrite(bwriteOps);
}
},
},
{
@@ -186,7 +193,9 @@ const syncInstructions: SyncInstructions[] = [
}
}
await collection.bulkWrite(bwriteOps);
if (bwriteOps.length) {
await collection.bulkWrite(bwriteOps);
}
},
},
];
@@ -196,11 +205,15 @@ const logger = CreateLogCtx("Database Sync");
async function SynchroniseDBWithSeeds() {
const seedsDir = path.join(__dirname, "tachi-database-seeds");
fs.rmSync(seedsDir, { recursive: true, force: true });
// This will create a directory called tachi-database-seeds in this folder.
execSync(`cd ${__dirname} && git clone https://github.com/TNG-dev/tachi-database-seeds`, {
stdio: "inherit",
});
if (!options.cache) {
fs.rmSync(seedsDir, { recursive: true, force: true });
logger.info(`--cache not provided, fetching from git.`);
// This will create a directory called tachi-database-seeds in this folder.
execSync(`cd ${__dirname} && git clone https://github.com/TNG-dev/tachi-database-seeds`, {
stdio: "inherit",
});
}
const collections = fs.readdirSync(path.join(seedsDir, "collections"));
@@ -225,11 +238,11 @@ async function SynchroniseDBWithSeeds() {
fs.readFileSync(path.join(seedsDir, "collections", jsonName), "utf-8")
);
spawnLogger.info(`Found ${data.length} documents.`);
spawnLogger.verbose(`Found ${data.length} documents.`);
let matchedSomething = false;
for (const syncInst of syncInstructions) {
if (jsonName.match(syncInst.pattern)) {
if (collectionName.match(syncInst.pattern)) {
spawnLogger.verbose(`Starting handler...`);
await syncInst.handler(data, monkDB.get(collectionName), spawnLogger);
matchedSomething = true;
server/src/external/mongo/schemas.ts
Vendored
+36 -22
View File
@@ -151,10 +151,9 @@ const getPlaytype = (game: Game, self: unknown): Playtypes[Game] => {
};
const PR_SongDocument = (data: PrudenceSchema): PrudenceSchema => ({
id: p.isPositiveNonZeroInteger,
id: p.isPositiveInteger,
title: "string",
artist: "string",
firstVersion: "?string",
searchTerms: ["string"],
altTitles: ["string"],
data,
@@ -168,7 +167,7 @@ const PR_ChartDocument = (
const gptConfig = GetGamePTConfig(game, playtype);
return {
songID: p.isPositiveNonZeroInteger,
songID: p.isPositiveInteger,
chartID: "string",
rgcID: "?string",
level: "string",
@@ -247,7 +246,7 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
},
joinDate: p.isPositiveInteger,
about: p.isBoundedString(0, 4000),
status: p.isBoundedString(3, 140),
status: p.nullable(p.isBoundedString(3, 140)),
customPfp: "boolean",
customBanner: "boolean",
clan: p.nullable(p.isBoundedString(2, 4)),
@@ -268,7 +267,7 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
const s = self as Record<string, unknown>;
for (const key in AllPermissions) {
if (s[key] !== undefined || typeof s[key] !== "boolean") {
if (s[key] !== undefined && typeof s[key] !== "boolean") {
return `Invalid permission value of ${s[key]} at ${key}.`;
}
}
@@ -354,7 +353,7 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
preferences: {
preferredScoreAlg: p.nullable(p.isIn(gptConfig.scoreRatingAlgs)),
preferredSessionAlg: p.nullable(p.isIn(gptConfig.sessionRatingAlgs)),
preferredProfileAlgs: p.nullable(p.isIn(gptConfig.profileRatingAlgs)),
preferredProfileAlg: p.nullable(p.isIn(gptConfig.profileRatingAlgs)),
// ouch
stats: p.and(
[
@@ -462,7 +461,7 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
timestamp: p.isPositive,
}),
"user-private-information": prSchemaify({
userInfo: p.isPositiveNonZeroInteger,
userID: p.isPositiveNonZeroInteger,
password: "string",
email: "string",
}),
@@ -706,30 +705,45 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
"songs-chunithm": prSchemaify(
PR_SongDocument({
genre: "string",
displayVersion: "string",
})
),
"songs-ddr": prSchemaify(
PR_SongDocument({
displayVersion: "string",
})
),
"songs-sdvx": prSchemaify(
PR_SongDocument({
displayVersion: "string",
})
),
"songs-ddr": prSchemaify(PR_SongDocument({})),
"songs-sdvx": prSchemaify(PR_SongDocument({})),
"songs-usc": prSchemaify(PR_SongDocument({})),
"songs-maimai": prSchemaify(
PR_SongDocument({
titleJP: "string",
artistJP: "string",
genre: "string",
displayVersion: "string",
})
),
"songs-museca": prSchemaify(
PR_SongDocument({
titleJP: "string",
artistJP: "string",
displayVersion: "string",
})
),
"songs-gitadora": prSchemaify(
PR_SongDocument({
isHot: "boolean",
})
),
"songs-iidx": prSchemaify(
PR_SongDocument({
genre: "string",
displayVersion: "string",
})
),
"songs-gitadora": prSchemaify({
isHot: "boolean",
}),
"songs-iidx": prSchemaify({
genre: "string",
}),
"charts-iidx": (self) => {
const playtype = getPlaytype("iidx", self);
@@ -799,7 +813,7 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
"charts-sdvx": prSchemaify(
PR_ChartDocument("sdvx", "Single", {
inGameID: p.isPositiveInteger,
arcChartID: "string",
arcChartID: "?string",
})
),
"charts-usc": prSchemaify(
@@ -817,7 +831,7 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
goalID: "string",
criteria: p.or(
{
mode: "single",
mode: p.is("single"),
key: p.isIn(
"scoreData.percent",
"scoreData.lampIndex",
@@ -840,18 +854,18 @@ export const DatabaseSchemas: Record<Databases, ValidatorFunction> = {
),
charts: p.or(
{
type: "any",
type: p.is("any"),
},
{
type: "folder",
type: p.is("folder"),
data: "string",
},
{
type: "multi",
type: p.is("multi"),
data: ["string"],
},
{
type: "single",
type: p.is("single"),
data: "string",
}
),