refactor: read schemas from tachi-common instead

database-seeds/scripts/package.json

@@ -38,7 +38,7 @@
 		"monk": "^7.3.4",
 		"node-fetch": "2.6.7",
 		"prudence": "^0.9.7",
-		"tachi-common": "0.7.11",
+		"tachi-common": "0.11.3",
 		"ts-node": "^10.4.0",
 		"xml2js": "^0.4.23"
 	},

database-seeds/scripts/pnpm-lock.yaml

@@ -23,7 +23,7 @@ specifiers:
   node-fetch: 2.6.7
   prettier: ^2.5.1
   prudence: ^0.9.7
-  tachi-common: 0.7.11
+  tachi-common: 0.11.3
   ts-node: ^10.4.0
   typescript: ^4.5.5
   xml2js: ^0.4.23
@@ -44,7 +44,7 @@ dependencies:
   monk: 7.3.4
   node-fetch: 2.6.7
   prudence: 0.9.7
-  tachi-common: 0.7.11
+  tachi-common: 0.11.3
   ts-node: 10.4.0_40146b36d18138e3202fbb722e5f65e4
   xml2js: 0.4.23
 
@@ -385,6 +385,14 @@ packages:
       bson: 4.5.2
     dev: false
 
+  /@types/express-serve-static-core/4.17.28:
+    resolution: {integrity: sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==}
+    dependencies:
+      '@types/node': 17.0.13
+      '@types/qs': 6.9.7
+      '@types/range-parser': 1.2.4
+    dev: false
+
   /@types/json-schema/7.0.9:
     resolution: {integrity: sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==}
     dev: true
@@ -411,6 +419,14 @@ packages:
     resolution: {integrity: sha512-Y86MAxASe25hNzlDbsviXl8jQHb0RDvKt4c40ZJQ1Don0AAL0STLZSs4N+6gLEO55pedy7r2cLwS+ZDxPm/2Bw==}
     dev: false
 
+  /@types/qs/6.9.7:
+    resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==}
+    dev: false
+
+  /@types/range-parser/1.2.4:
+    resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==}
+    dev: false
+
   /@typescript-eslint/eslint-plugin/5.10.2_2595c2126aec4d4b6e944b931dabb4c2:
     resolution: {integrity: sha512-4W/9lLuE+v27O/oe7hXJKjNtBLnZE8tQAFpapdxwSVHqtmIoPB1gph3+ahNwVuNL37BX7YQHyGF9Xv6XCnIX2Q==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
@@ -2674,6 +2690,10 @@ packages:
     resolution: {integrity: sha512-7214GHTBThBI/PGlKWXUSJ++ca0ld54VpF4k98U1EDAFXai5mnvWbb4speVOnWkilqfN25x8biWUAj4gHNu5Og==}
     dev: false
 
+  /prudence/0.9.8:
+    resolution: {integrity: sha512-my6HEhcTW3aKKjAx8j6jgUj3x5Kc71NEv3DySRXun2Zspw6Fdo5zDnbpXv/j4tyk9f9XXrpu1grWVBX4WU+1Tg==}
+    dev: false
+
   /psl/1.8.0:
     resolution: {integrity: sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==}
     dev: false
@@ -3075,10 +3095,12 @@ packages:
     engines: {node: '>= 0.4'}
     dev: true
 
-  /tachi-common/0.7.11:
-    resolution: {integrity: sha512-rUApEoAFae2lKFIDhxtCRqXIDixMeQwKjID9/BW07DqPj5PhB9SdvqPZFs8X6RxJ07G72khzAbYcAktlMBHLcQ==}
+  /tachi-common/0.11.3:
+    resolution: {integrity: sha512-McFWWxbrzLvbo4M7p69xhvlC7FZL2c4b+/nID+PZtJpf1YfqtxvzYkX3ZWEfC+SOL9rHriffdk5G6MvcDP87cQ==}
     dependencies:
+      '@types/express-serve-static-core': 4.17.28
       '@types/mongodb': 3.6.18
+      prudence: 0.9.8
     dev: false
 
   /tap-mocha-reporter/5.0.1:

database-seeds/scripts/test/collections.test.ts

@@ -1,7 +1,9 @@
 import chalk from "chalk";
+import { SCHEMAS } from "tachi-common/js/lib/schemas";
 import { ReadCollection } from "../util";
-import { SCHEMAS } from "./schemas";
 import { FormatFunctions } from "./test-utils";
+import fs from "fs";
+import path from "path";
 
 function FormatPrError(err, foreword = "Error") {
 	const receivedText =
@@ -15,7 +17,11 @@ function FormatPrError(err, foreword = "Error") {
 let exitCode = 0;
 const suites = [];
 
-for (const [collection, validator] of Object.entries(SCHEMAS)) {
+const collections = fs
+	.readdirSync(path.join(__dirname, "../../collections"))
+	.map((e) => path.basename(e).replace(/\.json$/u, ""));
+
+for (const collection of collections) {
 	console.log(`[VALIDATING] ${collection}`);
 
 	let success = 0;
@@ -26,7 +32,10 @@ for (const [collection, validator] of Object.entries(SCHEMAS)) {
 
 	const data = ReadCollection(collectionName, true);
 
+	const validator = SCHEMAS[collection];
+
 	for (const d of data) {
+		// Will throw if formatFn is undefined -- that's a test failure in my book.
 		const pretty = formatFn(d);
 
 		try {
@@ -57,11 +66,7 @@ for (const [collection, validator] of Object.entries(SCHEMAS)) {
 
 console.log(`=== Suite Overview ===`);
 for (const suite of suites) {
-	console.log(
-		chalk[suite.good ? "green" : "red"](
-			`${suite.name}: ${suite.report} (Objects here don't match the schema.)`
-		)
-	);
+	console.log(chalk[suite.good ? "green" : "red"](`[SCHEMAS] ${suite.name}: ${suite.report}`));
 }
 
 process.exit(exitCode);

database-seeds/scripts/test/id.test.ts

@@ -1,8 +1,8 @@
 import chalk from "chalk";
 import { Game } from "tachi-common";
 import { allSupportedGames } from "tachi-common/js/config/static-config";
+import { SCHEMAS } from "tachi-common/js/lib/schemas";
 import { ReadCollection } from "../util";
-import { SCHEMAS } from "./schemas";
 import { FormatFunctions } from "./test-utils";
 
 // @ts-expect-error filled out dynamically.
@@ -18,7 +18,7 @@ for (const game of allSupportedGames) {
 	SongChartKeys[`charts-${game}`] = "chartID";
 }
 
-const UniqueKeys: Record<keyof typeof SCHEMAS, string> = {
+const UniqueKeys: Partial<Record<keyof typeof SCHEMAS, string>> = {
 	"bms-course-lookup": "md5sums",
 	folders: "folderID",
 	tables: "tableID",
@@ -75,11 +75,7 @@ for (const [collection, uniqueID] of Object.entries(UniqueKeys)) {
 
 console.log(`=== Suite Overview ===`);
 for (const suite of suites) {
-	console.log(
-		chalk[suite.good ? "green" : "red"](
-			`${suite.name}: ${suite.report} (Objects here share unique IDs, which is illegal.)`
-		)
-	);
+	console.log(chalk[suite.good ? "green" : "red"](`[DUPES] ${suite.name}: ${suite.report}`));
 }
 
 process.exit(exitCode);

database-seeds/scripts/test/schemas.ts

@@ -1,354 +0,0 @@
-import p, { PrudenceSchema } from "prudence";
-import { Game, Playtypes, GetGamePTConfig, GetGameConfig } from "tachi-common";
-import { allSupportedGames } from "tachi-common/js/config/static-config";
-
-function prSchemaify(schema: PrudenceSchema) {
-	return (s: unknown): true => {
-		const err = p(s, schema);
-
-		if (err) {
-			throw err;
-		}
-
-		return true;
-	};
-}
-
-function IsValidPlaytype(game: Game, str: string): str is Playtypes[Game] {
-	return GetGameConfig(game).validPlaytypes.includes(str as Playtypes[Game]);
-}
-
-function IsValidGame(str: string): str is Game {
-	return allSupportedGames.includes(str as Game);
-}
-
-const getPlaytype = (game: Game, self: unknown): Playtypes[Game] => {
-	if (!self || typeof self !== "object") {
-		throw new Error("Expected an object.");
-	}
-
-	const gameConfig = GetGameConfig(game);
-
-	const s = self as Record<string, unknown>;
-
-	const playtype = s.playtype as string;
-	if (!IsValidPlaytype(game, playtype)) {
-		throw new Error(`Expected any of ${gameConfig.validPlaytypes.join(", ")}`);
-	}
-
-	return playtype;
-};
-
-const games = allSupportedGames;
-
-const isValidPlaytype = (self: unknown, parent: Record<string, unknown>) => {
-	if (!parent.game || typeof parent.game !== "string" || !IsValidGame(parent.game)) {
-		throw new Error(`Invalid Schema, need game to base IsValidPlaytype off of.`);
-	}
-
-	if (typeof self !== "string") {
-		return "Expected a string.";
-	}
-
-	if (!IsValidPlaytype(parent.game, self)) {
-		return `Expected a valid playtype for ${parent.game}`;
-	}
-
-	return true;
-};
-
-const PR_SongDocument = (data: PrudenceSchema): PrudenceSchema => ({
-	id: p.isPositiveInteger,
-	title: "string",
-	artist: "string",
-	searchTerms: ["string"],
-	altTitles: ["string"],
-	data,
-});
-
-const PR_ChartDocument = (
-	game: Game,
-	playtype: Playtypes[Game],
-	data: PrudenceSchema
-): PrudenceSchema => {
-	const gptConfig = GetGamePTConfig(game, playtype);
-
-	return {
-		songID: p.isPositiveInteger,
-		chartID: "string",
-		rgcID: "?string",
-		level: "string",
-		levelNum: "number",
-		isPrimary: "boolean",
-		difficulty: p.isIn(gptConfig.difficulties),
-		playtype: p.is(playtype),
-		data,
-		tierlistInfo: Object.fromEntries(
-			gptConfig.tierlists.map((t) => [
-				t,
-				p.optional({ text: "string", value: "number", individualDifference: "*boolean" }),
-			])
-		),
-		versions: [p.isIn(gptConfig.supportedVersions)],
-	};
-};
-
-export const SCHEMAS = {
-	"bms-course-lookup": prSchemaify({
-		title: "string",
-		// Must be comprised of 4 md5 hashes, which are 32 chars long.
-		md5sums: (self) => {
-			if (typeof self !== "string") {
-				return "Expected a string";
-			}
-
-			if (self.length !== 32 * 4) {
-				return "Expected 32 * 4 characters (4 md5 hashes long).";
-			}
-
-			if (!self.match(/^[a-z0-9]*$/u)) {
-				return "Expected all chars to be in the range of a-z0-9.";
-			}
-
-			return true;
-		},
-		set: p.isIn("genocideDan", "stslDan"),
-		playtype: p.isIn("7K", "14K"),
-		value: p.isInteger,
-	}),
-	folders: prSchemaify({
-		title: "string",
-		game: p.isIn(games),
-		playtype: isValidPlaytype,
-		folderID: "string",
-		inactive: "boolean",
-		searchTerms: ["string"],
-		type: p.isIn("songs", "charts", "static"),
-		data: (self, parent) => {
-			if (parent.type === "songs") {
-				return true; //temp. should be a song.
-			} else if (parent.type === "charts") {
-				return true; //temp. should be a chart.
-			}
-
-			return (
-				(Array.isArray(self) && self.every((r) => typeof r === "string")) ||
-				"Expected an array of only strings."
-			);
-		},
-	}),
-	tables: prSchemaify({
-		tableID: "string",
-		game: p.isIn(games),
-		playtype: isValidPlaytype,
-		title: "string",
-		default: "boolean",
-		description: "string",
-		folders: ["string"],
-		inactive: "boolean",
-	}),
-	"songs-bms": prSchemaify(
-		PR_SongDocument({
-			genre: "?string",
-			subtitle: "?string",
-			subartist: "?string",
-			tableString: "?string",
-		})
-	),
-	"songs-chunithm": prSchemaify(
-		PR_SongDocument({
-			genre: "string",
-			displayVersion: "string",
-		})
-	),
-	// "songs-ddr": prSchemaify(
-	// 	PR_SongDocument({
-	// 		displayVersion: "string",
-	// 	})
-	// ),
-	"songs-sdvx": prSchemaify(
-		PR_SongDocument({
-			displayVersion: "string",
-		})
-	),
-	"songs-usc": prSchemaify(PR_SongDocument({})),
-	"songs-maimai": prSchemaify(
-		PR_SongDocument({
-			titleJP: "string",
-			artistJP: "string",
-			displayVersion: "string",
-		})
-	),
-	"songs-museca": prSchemaify(
-		PR_SongDocument({
-			titleJP: "string",
-			artistJP: "string",
-			displayVersion: "string",
-		})
-	),
-	// "songs-gitadora": prSchemaify(
-	// 	PR_SongDocument({
-	// 		isHot: "boolean",
-	// 	})
-	// ),
-	"songs-iidx": prSchemaify(
-		PR_SongDocument({
-			genre: "string",
-			displayVersion: "string",
-		})
-	),
-	"songs-wacca": prSchemaify(
-		PR_SongDocument({
-			genre: "string",
-			displayVersion: "?string",
-			titleJP: "string",
-			artistJP: "string",
-		})
-	),
-	"songs-pms": prSchemaify(
-		PR_SongDocument({
-			genre: "?string",
-			subtitle: "?string",
-			subartist: "?string",
-			tableString: "?string",
-		})
-	),
-	"songs-popn": prSchemaify(
-		PR_SongDocument({
-			genre: "string",
-			genreEN: "?string",
-			displayVersion: "?string",
-		})
-	),
-	"songs-jubeat": prSchemaify(
-		PR_SongDocument({
-			displayVersion: "string",
-		})
-	),
-	"charts-iidx": (self) => {
-		const playtype = getPlaytype("iidx", self);
-
-		return prSchemaify(
-			PR_ChartDocument("iidx", playtype, {
-				notecount: p.isPositiveNonZeroInteger,
-				inGameID: p.or(p.isPositiveNonZeroInteger, [p.isPositiveNonZeroInteger]),
-				arcChartID: "?string",
-				hashSHA256: "?string",
-				"2dxtraSet": "?string",
-				kaidenAverage: "?number",
-				worldRecord: "?number",
-				bpiCoefficient: "?number",
-			})
-		)(self);
-	},
-	"charts-bms": (self) => {
-		const playtype = getPlaytype("bms", self);
-
-		return prSchemaify(
-			PR_ChartDocument("bms", playtype, {
-				notecount: p.isPositiveNonZeroInteger,
-				hashSHA256: "?string",
-				hashMD5: "?string",
-				tableFolders: [
-					{
-						table: "string",
-						level: "string",
-					},
-				],
-			})
-		)(self);
-	},
-	"charts-chunithm": prSchemaify(
-		PR_ChartDocument("chunithm", "Single", {
-			inGameID: p.isPositiveInteger,
-		})
-	),
-	"charts-jubeat": prSchemaify(
-		PR_ChartDocument("jubeat", "Single", {
-			inGameID: p.isPositiveInteger,
-			isHardMode: "boolean",
-		})
-	),
-	// "charts-gitadora": (self) => {
-	// 	const playtype = getPlaytype("gitadora", self);
-
-	// 	return prSchemaify(
-	// 		PR_ChartDocument("gitadora", playtype, {
-	// 			inGameID: p.isPositiveInteger,
-	// 		})
-	// 	)(self);
-	// },
-	// "charts-ddr": (self) => {
-	// 	const playtype = getPlaytype("ddr", self);
-
-	// 	return prSchemaify(
-	// 		PR_ChartDocument("ddr", playtype, {
-	// 			inGameID: "string",
-	// 			songHash: "string",
-	// 		})
-	// 	)(self);
-	// },
-	"charts-maimai": prSchemaify(
-		PR_ChartDocument("maimai", "Single", {
-			maxPercent: p.gt(0),
-			inGameID: p.isPositiveInteger,
-			inGameStrID: "string",
-		})
-	),
-	"charts-museca": prSchemaify(
-		PR_ChartDocument("museca", "Single", {
-			inGameID: p.isPositiveInteger,
-		})
-	),
-	"charts-sdvx": prSchemaify(
-		PR_ChartDocument("sdvx", "Single", {
-			inGameID: p.isPositiveInteger,
-			arcChartID: "?string",
-		})
-	),
-	"charts-usc": (self) => {
-		const playtype = getPlaytype("usc", self);
-
-		return prSchemaify(
-			PR_ChartDocument("usc", playtype, {
-				hashSHA1: p.or("string", ["string"]),
-				isOfficial: "boolean",
-				effector: "string",
-				tableFolders: [
-					{
-						table: "string",
-						level: "string",
-					},
-				],
-			})
-		)(self);
-	},
-	"charts-wacca": prSchemaify(
-		PR_ChartDocument("wacca", "Single", {
-			isHot: "boolean",
-		})
-	),
-	"charts-pms": (self) => {
-		const playtype = getPlaytype("pms", self);
-
-		return prSchemaify(
-			PR_ChartDocument("pms", playtype, {
-				notecount: p.isPositiveNonZeroInteger,
-				hashSHA256: "?string",
-				hashMD5: "?string",
-				tableFolders: [
-					{
-						table: "string",
-						level: "string",
-					},
-				],
-			})
-		)(self);
-	},
-	"charts-popn": prSchemaify(
-		PR_ChartDocument("popn", "9B", {
-			hashSHA256: "?string",
-			inGameID: p.isPositiveInteger,
-		})
-	),
-};

database-seeds/scripts/test/test-utils.ts

@@ -1,16 +1,14 @@
-import { SCHEMAS } from "./schemas";
+import { SCHEMAS } from "tachi-common/js/lib/schemas";
 
 const songFormat = (s) => `${s.artist} - ${s.title} (${s.id})`;
 const chartFormat = (s) => `${s.id} - ${s.playtype} ${s.difficulty} (${s.chartID})`;
 
-export const FormatFunctions: Record<keyof typeof SCHEMAS, (d) => string> = {
+export const FormatFunctions: Partial<Record<keyof typeof SCHEMAS, (d) => string>> = {
 	"bms-course-lookup": (d) => d.title,
 	folders: (d) => d.title,
 	tables: (d) => d.name,
 	"songs-bms": songFormat,
 	"songs-chunithm": songFormat,
-	// "songs-ddr": songFormat,
-	// "songs-gitadora": songFormat,
 	"songs-iidx": songFormat,
 	"songs-jubeat": songFormat,
 	"songs-maimai": songFormat,
@@ -22,8 +20,6 @@ export const FormatFunctions: Record<keyof typeof SCHEMAS, (d) => string> = {
 	"songs-wacca": songFormat,
 	"charts-bms": chartFormat,
 	"charts-chunithm": chartFormat,
-	// "charts-ddr": chartFormat,
-	// "charts-gitadora": chartFormat,
 	"charts-iidx": chartFormat,
 	"charts-jubeat": chartFormat,
 	"charts-maimai": chartFormat,

database-seeds/scripts/util.js

@@ -3,7 +3,6 @@ const path = require("path");
 const DeterministicCollectionSort = require("./deterministic-collection-sort");
 const crypto = require("crypto");
 const fjsh = require("fast-json-stable-hash");
-const glob = require("glob");
 
 function IterateCollections(cb) {
 	for (const collection of fs.readdirSync(COLLECTIONS_DIR)) {