diff --git a/database-seeds/scripts/package.json b/database-seeds/scripts/package.json index cae0a216b..147df95c6 100644 --- a/database-seeds/scripts/package.json +++ b/database-seeds/scripts/package.json @@ -38,7 +38,7 @@ "monk": "^7.3.4", "node-fetch": "2.6.7", "prudence": "^0.9.7", - "tachi-common": "0.7.11", + "tachi-common": "0.11.3", "ts-node": "^10.4.0", "xml2js": "^0.4.23" }, diff --git a/database-seeds/scripts/pnpm-lock.yaml b/database-seeds/scripts/pnpm-lock.yaml index d8706cc03..1f8d5936c 100644 --- a/database-seeds/scripts/pnpm-lock.yaml +++ b/database-seeds/scripts/pnpm-lock.yaml @@ -23,7 +23,7 @@ specifiers: node-fetch: 2.6.7 prettier: ^2.5.1 prudence: ^0.9.7 - tachi-common: 0.7.11 + tachi-common: 0.11.3 ts-node: ^10.4.0 typescript: ^4.5.5 xml2js: ^0.4.23 @@ -44,7 +44,7 @@ dependencies: monk: 7.3.4 node-fetch: 2.6.7 prudence: 0.9.7 - tachi-common: 0.7.11 + tachi-common: 0.11.3 ts-node: 10.4.0_40146b36d18138e3202fbb722e5f65e4 xml2js: 0.4.23 @@ -385,6 +385,14 @@ packages: bson: 4.5.2 dev: false + /@types/express-serve-static-core/4.17.28: + resolution: {integrity: sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==} + dependencies: + '@types/node': 17.0.13 + '@types/qs': 6.9.7 + '@types/range-parser': 1.2.4 + dev: false + /@types/json-schema/7.0.9: resolution: {integrity: sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==} dev: true @@ -411,6 +419,14 @@ packages: resolution: {integrity: sha512-Y86MAxASe25hNzlDbsviXl8jQHb0RDvKt4c40ZJQ1Don0AAL0STLZSs4N+6gLEO55pedy7r2cLwS+ZDxPm/2Bw==} dev: false + /@types/qs/6.9.7: + resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==} + dev: false + + /@types/range-parser/1.2.4: + resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==} + dev: false + /@typescript-eslint/eslint-plugin/5.10.2_2595c2126aec4d4b6e944b931dabb4c2: resolution: {integrity: sha512-4W/9lLuE+v27O/oe7hXJKjNtBLnZE8tQAFpapdxwSVHqtmIoPB1gph3+ahNwVuNL37BX7YQHyGF9Xv6XCnIX2Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -2674,6 +2690,10 @@ packages: resolution: {integrity: sha512-7214GHTBThBI/PGlKWXUSJ++ca0ld54VpF4k98U1EDAFXai5mnvWbb4speVOnWkilqfN25x8biWUAj4gHNu5Og==} dev: false + /prudence/0.9.8: + resolution: {integrity: sha512-my6HEhcTW3aKKjAx8j6jgUj3x5Kc71NEv3DySRXun2Zspw6Fdo5zDnbpXv/j4tyk9f9XXrpu1grWVBX4WU+1Tg==} + dev: false + /psl/1.8.0: resolution: {integrity: sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==} dev: false @@ -3075,10 +3095,12 @@ packages: engines: {node: '>= 0.4'} dev: true - /tachi-common/0.7.11: - resolution: {integrity: sha512-rUApEoAFae2lKFIDhxtCRqXIDixMeQwKjID9/BW07DqPj5PhB9SdvqPZFs8X6RxJ07G72khzAbYcAktlMBHLcQ==} + /tachi-common/0.11.3: + resolution: {integrity: sha512-McFWWxbrzLvbo4M7p69xhvlC7FZL2c4b+/nID+PZtJpf1YfqtxvzYkX3ZWEfC+SOL9rHriffdk5G6MvcDP87cQ==} dependencies: + '@types/express-serve-static-core': 4.17.28 '@types/mongodb': 3.6.18 + prudence: 0.9.8 dev: false /tap-mocha-reporter/5.0.1: diff --git a/database-seeds/scripts/test/collections.test.ts b/database-seeds/scripts/test/collections.test.ts index a311c55ac..0e7bd75ea 100644 --- a/database-seeds/scripts/test/collections.test.ts +++ b/database-seeds/scripts/test/collections.test.ts @@ -1,7 +1,9 @@ import chalk from "chalk"; +import { SCHEMAS } from "tachi-common/js/lib/schemas"; import { ReadCollection } from "../util"; -import { SCHEMAS } from "./schemas"; import { FormatFunctions } from "./test-utils"; +import fs from "fs"; +import path from "path"; function FormatPrError(err, foreword = "Error") { const receivedText = @@ -15,7 +17,11 @@ function FormatPrError(err, foreword = "Error") { let exitCode = 0; const suites = []; -for (const [collection, validator] of Object.entries(SCHEMAS)) { +const collections = fs + .readdirSync(path.join(__dirname, "../../collections")) + .map((e) => path.basename(e).replace(/\.json$/u, "")); + +for (const collection of collections) { console.log(`[VALIDATING] ${collection}`); let success = 0; @@ -26,7 +32,10 @@ for (const [collection, validator] of Object.entries(SCHEMAS)) { const data = ReadCollection(collectionName, true); + const validator = SCHEMAS[collection]; + for (const d of data) { + // Will throw if formatFn is undefined -- that's a test failure in my book. const pretty = formatFn(d); try { @@ -57,11 +66,7 @@ for (const [collection, validator] of Object.entries(SCHEMAS)) { console.log(`=== Suite Overview ===`); for (const suite of suites) { - console.log( - chalk[suite.good ? "green" : "red"]( - `${suite.name}: ${suite.report} (Objects here don't match the schema.)` - ) - ); + console.log(chalk[suite.good ? "green" : "red"](`[SCHEMAS] ${suite.name}: ${suite.report}`)); } process.exit(exitCode); diff --git a/database-seeds/scripts/test/id.test.ts b/database-seeds/scripts/test/id.test.ts index c7e035c78..fe537f055 100644 --- a/database-seeds/scripts/test/id.test.ts +++ b/database-seeds/scripts/test/id.test.ts @@ -1,8 +1,8 @@ import chalk from "chalk"; import { Game } from "tachi-common"; import { allSupportedGames } from "tachi-common/js/config/static-config"; +import { SCHEMAS } from "tachi-common/js/lib/schemas"; import { ReadCollection } from "../util"; -import { SCHEMAS } from "./schemas"; import { FormatFunctions } from "./test-utils"; // @ts-expect-error filled out dynamically. @@ -18,7 +18,7 @@ for (const game of allSupportedGames) { SongChartKeys[`charts-${game}`] = "chartID"; } -const UniqueKeys: Record = { +const UniqueKeys: Partial> = { "bms-course-lookup": "md5sums", folders: "folderID", tables: "tableID", @@ -75,11 +75,7 @@ for (const [collection, uniqueID] of Object.entries(UniqueKeys)) { console.log(`=== Suite Overview ===`); for (const suite of suites) { - console.log( - chalk[suite.good ? "green" : "red"]( - `${suite.name}: ${suite.report} (Objects here share unique IDs, which is illegal.)` - ) - ); + console.log(chalk[suite.good ? "green" : "red"](`[DUPES] ${suite.name}: ${suite.report}`)); } process.exit(exitCode); diff --git a/database-seeds/scripts/test/schemas.ts b/database-seeds/scripts/test/schemas.ts deleted file mode 100644 index 77f040082..000000000 --- a/database-seeds/scripts/test/schemas.ts +++ /dev/null @@ -1,354 +0,0 @@ -import p, { PrudenceSchema } from "prudence"; -import { Game, Playtypes, GetGamePTConfig, GetGameConfig } from "tachi-common"; -import { allSupportedGames } from "tachi-common/js/config/static-config"; - -function prSchemaify(schema: PrudenceSchema) { - return (s: unknown): true => { - const err = p(s, schema); - - if (err) { - throw err; - } - - return true; - }; -} - -function IsValidPlaytype(game: Game, str: string): str is Playtypes[Game] { - return GetGameConfig(game).validPlaytypes.includes(str as Playtypes[Game]); -} - -function IsValidGame(str: string): str is Game { - return allSupportedGames.includes(str as Game); -} - -const getPlaytype = (game: Game, self: unknown): Playtypes[Game] => { - if (!self || typeof self !== "object") { - throw new Error("Expected an object."); - } - - const gameConfig = GetGameConfig(game); - - const s = self as Record; - - const playtype = s.playtype as string; - if (!IsValidPlaytype(game, playtype)) { - throw new Error(`Expected any of ${gameConfig.validPlaytypes.join(", ")}`); - } - - return playtype; -}; - -const games = allSupportedGames; - -const isValidPlaytype = (self: unknown, parent: Record) => { - if (!parent.game || typeof parent.game !== "string" || !IsValidGame(parent.game)) { - throw new Error(`Invalid Schema, need game to base IsValidPlaytype off of.`); - } - - if (typeof self !== "string") { - return "Expected a string."; - } - - if (!IsValidPlaytype(parent.game, self)) { - return `Expected a valid playtype for ${parent.game}`; - } - - return true; -}; - -const PR_SongDocument = (data: PrudenceSchema): PrudenceSchema => ({ - id: p.isPositiveInteger, - title: "string", - artist: "string", - searchTerms: ["string"], - altTitles: ["string"], - data, -}); - -const PR_ChartDocument = ( - game: Game, - playtype: Playtypes[Game], - data: PrudenceSchema -): PrudenceSchema => { - const gptConfig = GetGamePTConfig(game, playtype); - - return { - songID: p.isPositiveInteger, - chartID: "string", - rgcID: "?string", - level: "string", - levelNum: "number", - isPrimary: "boolean", - difficulty: p.isIn(gptConfig.difficulties), - playtype: p.is(playtype), - data, - tierlistInfo: Object.fromEntries( - gptConfig.tierlists.map((t) => [ - t, - p.optional({ text: "string", value: "number", individualDifference: "*boolean" }), - ]) - ), - versions: [p.isIn(gptConfig.supportedVersions)], - }; -}; - -export const SCHEMAS = { - "bms-course-lookup": prSchemaify({ - title: "string", - // Must be comprised of 4 md5 hashes, which are 32 chars long. - md5sums: (self) => { - if (typeof self !== "string") { - return "Expected a string"; - } - - if (self.length !== 32 * 4) { - return "Expected 32 * 4 characters (4 md5 hashes long)."; - } - - if (!self.match(/^[a-z0-9]*$/u)) { - return "Expected all chars to be in the range of a-z0-9."; - } - - return true; - }, - set: p.isIn("genocideDan", "stslDan"), - playtype: p.isIn("7K", "14K"), - value: p.isInteger, - }), - folders: prSchemaify({ - title: "string", - game: p.isIn(games), - playtype: isValidPlaytype, - folderID: "string", - inactive: "boolean", - searchTerms: ["string"], - type: p.isIn("songs", "charts", "static"), - data: (self, parent) => { - if (parent.type === "songs") { - return true; //temp. should be a song. - } else if (parent.type === "charts") { - return true; //temp. should be a chart. - } - - return ( - (Array.isArray(self) && self.every((r) => typeof r === "string")) || - "Expected an array of only strings." - ); - }, - }), - tables: prSchemaify({ - tableID: "string", - game: p.isIn(games), - playtype: isValidPlaytype, - title: "string", - default: "boolean", - description: "string", - folders: ["string"], - inactive: "boolean", - }), - "songs-bms": prSchemaify( - PR_SongDocument({ - genre: "?string", - subtitle: "?string", - subartist: "?string", - tableString: "?string", - }) - ), - "songs-chunithm": prSchemaify( - PR_SongDocument({ - genre: "string", - displayVersion: "string", - }) - ), - // "songs-ddr": prSchemaify( - // PR_SongDocument({ - // displayVersion: "string", - // }) - // ), - "songs-sdvx": prSchemaify( - PR_SongDocument({ - displayVersion: "string", - }) - ), - "songs-usc": prSchemaify(PR_SongDocument({})), - "songs-maimai": prSchemaify( - PR_SongDocument({ - titleJP: "string", - artistJP: "string", - displayVersion: "string", - }) - ), - "songs-museca": prSchemaify( - PR_SongDocument({ - titleJP: "string", - artistJP: "string", - displayVersion: "string", - }) - ), - // "songs-gitadora": prSchemaify( - // PR_SongDocument({ - // isHot: "boolean", - // }) - // ), - "songs-iidx": prSchemaify( - PR_SongDocument({ - genre: "string", - displayVersion: "string", - }) - ), - "songs-wacca": prSchemaify( - PR_SongDocument({ - genre: "string", - displayVersion: "?string", - titleJP: "string", - artistJP: "string", - }) - ), - "songs-pms": prSchemaify( - PR_SongDocument({ - genre: "?string", - subtitle: "?string", - subartist: "?string", - tableString: "?string", - }) - ), - "songs-popn": prSchemaify( - PR_SongDocument({ - genre: "string", - genreEN: "?string", - displayVersion: "?string", - }) - ), - "songs-jubeat": prSchemaify( - PR_SongDocument({ - displayVersion: "string", - }) - ), - "charts-iidx": (self) => { - const playtype = getPlaytype("iidx", self); - - return prSchemaify( - PR_ChartDocument("iidx", playtype, { - notecount: p.isPositiveNonZeroInteger, - inGameID: p.or(p.isPositiveNonZeroInteger, [p.isPositiveNonZeroInteger]), - arcChartID: "?string", - hashSHA256: "?string", - "2dxtraSet": "?string", - kaidenAverage: "?number", - worldRecord: "?number", - bpiCoefficient: "?number", - }) - )(self); - }, - "charts-bms": (self) => { - const playtype = getPlaytype("bms", self); - - return prSchemaify( - PR_ChartDocument("bms", playtype, { - notecount: p.isPositiveNonZeroInteger, - hashSHA256: "?string", - hashMD5: "?string", - tableFolders: [ - { - table: "string", - level: "string", - }, - ], - }) - )(self); - }, - "charts-chunithm": prSchemaify( - PR_ChartDocument("chunithm", "Single", { - inGameID: p.isPositiveInteger, - }) - ), - "charts-jubeat": prSchemaify( - PR_ChartDocument("jubeat", "Single", { - inGameID: p.isPositiveInteger, - isHardMode: "boolean", - }) - ), - // "charts-gitadora": (self) => { - // const playtype = getPlaytype("gitadora", self); - - // return prSchemaify( - // PR_ChartDocument("gitadora", playtype, { - // inGameID: p.isPositiveInteger, - // }) - // )(self); - // }, - // "charts-ddr": (self) => { - // const playtype = getPlaytype("ddr", self); - - // return prSchemaify( - // PR_ChartDocument("ddr", playtype, { - // inGameID: "string", - // songHash: "string", - // }) - // )(self); - // }, - "charts-maimai": prSchemaify( - PR_ChartDocument("maimai", "Single", { - maxPercent: p.gt(0), - inGameID: p.isPositiveInteger, - inGameStrID: "string", - }) - ), - "charts-museca": prSchemaify( - PR_ChartDocument("museca", "Single", { - inGameID: p.isPositiveInteger, - }) - ), - "charts-sdvx": prSchemaify( - PR_ChartDocument("sdvx", "Single", { - inGameID: p.isPositiveInteger, - arcChartID: "?string", - }) - ), - "charts-usc": (self) => { - const playtype = getPlaytype("usc", self); - - return prSchemaify( - PR_ChartDocument("usc", playtype, { - hashSHA1: p.or("string", ["string"]), - isOfficial: "boolean", - effector: "string", - tableFolders: [ - { - table: "string", - level: "string", - }, - ], - }) - )(self); - }, - "charts-wacca": prSchemaify( - PR_ChartDocument("wacca", "Single", { - isHot: "boolean", - }) - ), - "charts-pms": (self) => { - const playtype = getPlaytype("pms", self); - - return prSchemaify( - PR_ChartDocument("pms", playtype, { - notecount: p.isPositiveNonZeroInteger, - hashSHA256: "?string", - hashMD5: "?string", - tableFolders: [ - { - table: "string", - level: "string", - }, - ], - }) - )(self); - }, - "charts-popn": prSchemaify( - PR_ChartDocument("popn", "9B", { - hashSHA256: "?string", - inGameID: p.isPositiveInteger, - }) - ), -}; diff --git a/database-seeds/scripts/test/test-utils.ts b/database-seeds/scripts/test/test-utils.ts index fae27c5b7..607dd1585 100644 --- a/database-seeds/scripts/test/test-utils.ts +++ b/database-seeds/scripts/test/test-utils.ts @@ -1,16 +1,14 @@ -import { SCHEMAS } from "./schemas"; +import { SCHEMAS } from "tachi-common/js/lib/schemas"; const songFormat = (s) => `${s.artist} - ${s.title} (${s.id})`; const chartFormat = (s) => `${s.id} - ${s.playtype} ${s.difficulty} (${s.chartID})`; -export const FormatFunctions: Record string> = { +export const FormatFunctions: Partial string>> = { "bms-course-lookup": (d) => d.title, folders: (d) => d.title, tables: (d) => d.name, "songs-bms": songFormat, "songs-chunithm": songFormat, - // "songs-ddr": songFormat, - // "songs-gitadora": songFormat, "songs-iidx": songFormat, "songs-jubeat": songFormat, "songs-maimai": songFormat, @@ -22,8 +20,6 @@ export const FormatFunctions: Record string> = { "songs-wacca": songFormat, "charts-bms": chartFormat, "charts-chunithm": chartFormat, - // "charts-ddr": chartFormat, - // "charts-gitadora": chartFormat, "charts-iidx": chartFormat, "charts-jubeat": chartFormat, "charts-maimai": chartFormat, diff --git a/database-seeds/scripts/util.js b/database-seeds/scripts/util.js index f45465174..062986f9e 100644 --- a/database-seeds/scripts/util.js +++ b/database-seeds/scripts/util.js @@ -3,7 +3,6 @@ const path = require("path"); const DeterministicCollectionSort = require("./deterministic-collection-sort"); const crypto = require("crypto"); const fjsh = require("fast-json-stable-hash"); -const glob = require("glob"); function IterateCollections(cb) { for (const collection of fs.readdirSync(COLLECTIONS_DIR)) {