diff --git a/server/package.json b/server/package.json index 3d0e87c73..627343e69 100644 --- a/server/package.json +++ b/server/package.json @@ -72,12 +72,12 @@ "node-cache": "^5.1.2", "node-fetch": "2.6.1", "nodemailer": "^6.6.3", - "prudence": "0.9.6", + "prudence": "0.9.7", "rate-limit-redis": "2.1.0", "redis": "3.1.2", "rimraf": "3.0.2", "safe-json-stringify": "1.2.0", - "tachi-common": "0.2.28", + "tachi-common": "0.2.34", "ts-node": "10.0.0", "typescript": "4.3.4", "winston": "3.3.3", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 3c6b9c32d..ba065a90f 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -53,13 +53,13 @@ specifiers: node-mocks-http: 1.10.1 nodemailer: ^6.6.3 prettier: 2.3.1 - prudence: 0.9.6 + prudence: 0.9.7 rate-limit-redis: 2.1.0 redis: 3.1.2 rimraf: 3.0.2 safe-json-stringify: 1.2.0 supertest: 6.1.3 - tachi-common: 0.2.28 + tachi-common: 0.2.34 tap: 15.0.9 ts-node: 10.0.0 tsconfig-paths: 3.10.1 @@ -95,12 +95,12 @@ dependencies: node-cache: 5.1.2 node-fetch: 2.6.1 nodemailer: 6.6.3 - prudence: 0.9.6 + prudence: 0.9.7 rate-limit-redis: 2.1.0 redis: 3.1.2 rimraf: 3.0.2 safe-json-stringify: 1.2.0 - tachi-common: 0.2.28 + tachi-common: 0.2.34 ts-node: 10.0.0_83f53b0a0c5616d3fa00ed4e30b9ce1b typescript: 4.3.4 winston: 3.3.3 @@ -3554,8 +3554,8 @@ packages: ipaddr.js: 1.9.1 dev: false - /prudence/0.9.6: - resolution: {integrity: sha512-kCPaLDVfVovahwd+SXlszt4y+av3VJL5HXD4II2+m8+zm6J6wt8vFU+kDFeZvaycaGJemB8VUv39hFplHbxTIw==} + /prudence/0.9.7: + resolution: {integrity: sha512-7214GHTBThBI/PGlKWXUSJ++ca0ld54VpF4k98U1EDAFXai5mnvWbb4speVOnWkilqfN25x8biWUAj4gHNu5Og==} dev: false /psl/1.8.0: @@ -4189,8 +4189,8 @@ packages: strip-ansi: 6.0.0 dev: true - /tachi-common/0.2.28: - resolution: {integrity: sha512-9iqO/2y7VNbpyJBf4CDffw6E9PZuiib7nkK4Vk6Ov4i6a2zn787rBzQ8XgoQGRZKlHV5tfFd+CFbbiTRPM3ZBA==} + /tachi-common/0.2.34: + resolution: {integrity: sha512-qKXIqzMTuT4eNL0Eo3N00fGYDF63bLxvZbf1/7jI+i8pDqJ8Xd1kmSuzNENBr+cp75Krh3qt279xf5oejUOiGw==} dependencies: monk: 7.3.4 transitivePeerDependencies: diff --git a/server/scripts/__KT_DATABASE_MIGRATION/sessions.ts b/server/scripts/__KT_DATABASE_MIGRATION/sessions.ts index 0d748d44a..04700d6f8 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/sessions.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/sessions.ts @@ -1,11 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { integer, SessionDocument, SessionScoreInfo } from "tachi-common"; import db from "external/mongo/db"; -import MigrateRecords from "./migrate"; -import { CreateSessionID } from "lib/score-import/framework/sessions/sessions"; -import { oldKTDB } from "./old-db"; import CreateLogCtx from "lib/logger/logger"; +import { CreateSessionID } from "lib/score-import/framework/sessions/sessions"; +import { GetGameConfig, integer, SessionDocument, SessionScoreInfo } from "tachi-common"; import { allSupportedGames } from "tachi-common/js/config/static-config"; +import MigrateRecords from "./migrate"; +import { oldKTDB } from "./old-db"; const logger = CreateLogCtx(__filename); @@ -44,6 +44,10 @@ async function ConvertScoreToScoreInfo( }; } + if (score.pbInfo.scoreDelta === null) { + return null; + } + return { scoreID: scoreIDs.new, isNewScore: false, @@ -59,6 +63,12 @@ async function ConvertFn(c: any): Promise { return null; } + const gameConfig = GetGameConfig(c.game); + + if (!gameConfig.validPlaytypes.includes(c.playtype)) { + return null; + } + const scores = []; for (const sc of c.scores) { diff --git a/server/scripts/arc-sync/arc-db-sync.ts b/server/scripts/arc-sync/arc-db-sync.ts index 7575662b8..370e6bb8c 100644 --- a/server/scripts/arc-sync/arc-db-sync.ts +++ b/server/scripts/arc-sync/arc-db-sync.ts @@ -179,7 +179,7 @@ async function MergeIDs() { // eslint-disable-next-line no-await-in-loop await db.charts[options.game as "sdvx" | "iidx"].update( { - _id: tachiChart._id, + chartID: tachiChart.chartID, }, { $set: { diff --git a/server/scripts/single-use/fix-user-goals.ts b/server/scripts/single-use/fix-user-goals.ts new file mode 100644 index 000000000..bf56542fa --- /dev/null +++ b/server/scripts/single-use/fix-user-goals.ts @@ -0,0 +1,32 @@ +import db from "external/mongo/db"; + +if (require.main === module) { + (async () => { + const ugs = await db["user-goals"].find( + {}, + { + projectID: true, + } + ); + + for (const ug of ugs) { + await db["user-goals"].update( + { + _id: ug._id, + }, + { + $set: { + progressHuman: ug.progressHuman.toString(), + outOfHuman: ug.outOfHuman.toString(), + lastInteraction: null, + }, + $unset: { + note: 1, + }, + } + ); + } + + process.exit(0); + })(); +} diff --git a/server/scripts/single-use/remove-null-scoreInfo-sessions.ts b/server/scripts/single-use/remove-null-scoreInfo-sessions.ts new file mode 100644 index 000000000..8095cede1 --- /dev/null +++ b/server/scripts/single-use/remove-null-scoreInfo-sessions.ts @@ -0,0 +1,30 @@ +import db from "external/mongo/db"; + +if (require.main === module) { + (async () => { + const badScoreInfos = await db.sessions.find({ + "scoreInfo.isNewScore": false, + "scoreInfo.scoreDelta": null, + }); + + for (const session of badScoreInfos) { + const scoreInfo = session.scoreInfo.filter( + (e) => !(!e.isNewScore && e.scoreDelta === null) + ); + + // eslint-disable-next-line no-await-in-loop + await db.sessions.update( + { + sessionID: session.sessionID, + }, + { + $set: { + scoreInfo, + }, + } + ); + } + + process.exit(0); + })(); +} diff --git a/server/scripts/sync-database/sync-database.ts b/server/scripts/sync-database/sync-database.ts index 39b8dceb4..d020462da 100644 --- a/server/scripts/sync-database/sync-database.ts +++ b/server/scripts/sync-database/sync-database.ts @@ -11,6 +11,13 @@ import { ICollection } from "monk"; import path from "path"; import { ChartDocument, FolderDocument, SongDocument, TableDocument } from "tachi-common"; import deepEqual from "deep-equal"; +import { Command } from "commander"; + +const program = new Command(); +program.option("-c, --cache"); + +program.parse(process.argv); +const options = program.opts(); interface SyncInstructions { pattern: RegExp; @@ -70,14 +77,12 @@ async function GenericUpsert( }); } else if (!deepEqual(document, exists, { strict: true })) { bwriteOps.push({ - updateOne: { + replaceOne: { // @ts-expect-error Known X->Y generic issue. filter: { [field]: document[field], }, - update: { - $set: document, - }, + replacement: document, }, }); } @@ -102,12 +107,12 @@ const syncInstructions: SyncInstructions[] = [ { pattern: /^charts-/u, handler: (charts: ChartDocument[], collection: ICollection, logger) => - GenericUpsert(charts, collection, "chartID", logger), + GenericUpsert(charts, collection, "chartID", logger, true), }, { pattern: /^songs-/u, handler: (songs: SongDocument[], collection: ICollection, logger) => - GenericUpsert(songs, collection, "id", logger), + GenericUpsert(songs, collection, "id", logger, true), }, { pattern: /^folders$/u, @@ -146,7 +151,9 @@ const syncInstructions: SyncInstructions[] = [ } } - await collection.bulkWrite(bwriteOps); + if (bwriteOps.length) { + await collection.bulkWrite(bwriteOps); + } }, }, { @@ -186,7 +193,9 @@ const syncInstructions: SyncInstructions[] = [ } } - await collection.bulkWrite(bwriteOps); + if (bwriteOps.length) { + await collection.bulkWrite(bwriteOps); + } }, }, ]; @@ -196,11 +205,15 @@ const logger = CreateLogCtx("Database Sync"); async function SynchroniseDBWithSeeds() { const seedsDir = path.join(__dirname, "tachi-database-seeds"); - fs.rmSync(seedsDir, { recursive: true, force: true }); - // This will create a directory called tachi-database-seeds in this folder. - execSync(`cd ${__dirname} && git clone https://github.com/TNG-dev/tachi-database-seeds`, { - stdio: "inherit", - }); + if (!options.cache) { + fs.rmSync(seedsDir, { recursive: true, force: true }); + + logger.info(`--cache not provided, fetching from git.`); + // This will create a directory called tachi-database-seeds in this folder. + execSync(`cd ${__dirname} && git clone https://github.com/TNG-dev/tachi-database-seeds`, { + stdio: "inherit", + }); + } const collections = fs.readdirSync(path.join(seedsDir, "collections")); @@ -225,11 +238,11 @@ async function SynchroniseDBWithSeeds() { fs.readFileSync(path.join(seedsDir, "collections", jsonName), "utf-8") ); - spawnLogger.info(`Found ${data.length} documents.`); + spawnLogger.verbose(`Found ${data.length} documents.`); let matchedSomething = false; for (const syncInst of syncInstructions) { - if (jsonName.match(syncInst.pattern)) { + if (collectionName.match(syncInst.pattern)) { spawnLogger.verbose(`Starting handler...`); await syncInst.handler(data, monkDB.get(collectionName), spawnLogger); matchedSomething = true; diff --git a/server/src/external/mongo/schemas.ts b/server/src/external/mongo/schemas.ts index 985e9e77b..65aabc371 100644 --- a/server/src/external/mongo/schemas.ts +++ b/server/src/external/mongo/schemas.ts @@ -151,10 +151,9 @@ const getPlaytype = (game: Game, self: unknown): Playtypes[Game] => { }; const PR_SongDocument = (data: PrudenceSchema): PrudenceSchema => ({ - id: p.isPositiveNonZeroInteger, + id: p.isPositiveInteger, title: "string", artist: "string", - firstVersion: "?string", searchTerms: ["string"], altTitles: ["string"], data, @@ -168,7 +167,7 @@ const PR_ChartDocument = ( const gptConfig = GetGamePTConfig(game, playtype); return { - songID: p.isPositiveNonZeroInteger, + songID: p.isPositiveInteger, chartID: "string", rgcID: "?string", level: "string", @@ -247,7 +246,7 @@ export const DatabaseSchemas: Record = { }, joinDate: p.isPositiveInteger, about: p.isBoundedString(0, 4000), - status: p.isBoundedString(3, 140), + status: p.nullable(p.isBoundedString(3, 140)), customPfp: "boolean", customBanner: "boolean", clan: p.nullable(p.isBoundedString(2, 4)), @@ -268,7 +267,7 @@ export const DatabaseSchemas: Record = { const s = self as Record; for (const key in AllPermissions) { - if (s[key] !== undefined || typeof s[key] !== "boolean") { + if (s[key] !== undefined && typeof s[key] !== "boolean") { return `Invalid permission value of ${s[key]} at ${key}.`; } } @@ -354,7 +353,7 @@ export const DatabaseSchemas: Record = { preferences: { preferredScoreAlg: p.nullable(p.isIn(gptConfig.scoreRatingAlgs)), preferredSessionAlg: p.nullable(p.isIn(gptConfig.sessionRatingAlgs)), - preferredProfileAlgs: p.nullable(p.isIn(gptConfig.profileRatingAlgs)), + preferredProfileAlg: p.nullable(p.isIn(gptConfig.profileRatingAlgs)), // ouch stats: p.and( [ @@ -462,7 +461,7 @@ export const DatabaseSchemas: Record = { timestamp: p.isPositive, }), "user-private-information": prSchemaify({ - userInfo: p.isPositiveNonZeroInteger, + userID: p.isPositiveNonZeroInteger, password: "string", email: "string", }), @@ -706,30 +705,45 @@ export const DatabaseSchemas: Record = { "songs-chunithm": prSchemaify( PR_SongDocument({ genre: "string", + displayVersion: "string", + }) + ), + "songs-ddr": prSchemaify( + PR_SongDocument({ + displayVersion: "string", + }) + ), + "songs-sdvx": prSchemaify( + PR_SongDocument({ + displayVersion: "string", }) ), - "songs-ddr": prSchemaify(PR_SongDocument({})), - "songs-sdvx": prSchemaify(PR_SongDocument({})), "songs-usc": prSchemaify(PR_SongDocument({})), "songs-maimai": prSchemaify( PR_SongDocument({ titleJP: "string", artistJP: "string", - genre: "string", + displayVersion: "string", }) ), "songs-museca": prSchemaify( PR_SongDocument({ titleJP: "string", artistJP: "string", + displayVersion: "string", + }) + ), + "songs-gitadora": prSchemaify( + PR_SongDocument({ + isHot: "boolean", + }) + ), + "songs-iidx": prSchemaify( + PR_SongDocument({ + genre: "string", + displayVersion: "string", }) ), - "songs-gitadora": prSchemaify({ - isHot: "boolean", - }), - "songs-iidx": prSchemaify({ - genre: "string", - }), "charts-iidx": (self) => { const playtype = getPlaytype("iidx", self); @@ -799,7 +813,7 @@ export const DatabaseSchemas: Record = { "charts-sdvx": prSchemaify( PR_ChartDocument("sdvx", "Single", { inGameID: p.isPositiveInteger, - arcChartID: "string", + arcChartID: "?string", }) ), "charts-usc": prSchemaify( @@ -817,7 +831,7 @@ export const DatabaseSchemas: Record = { goalID: "string", criteria: p.or( { - mode: "single", + mode: p.is("single"), key: p.isIn( "scoreData.percent", "scoreData.lampIndex", @@ -840,18 +854,18 @@ export const DatabaseSchemas: Record = { ), charts: p.or( { - type: "any", + type: p.is("any"), }, { - type: "folder", + type: p.is("folder"), data: "string", }, { - type: "multi", + type: p.is("multi"), data: ["string"], }, { - type: "single", + type: p.is("single"), data: "string", } ),