From 4f92c5c2cf48c060385404ca3f1c01b5f3f329ce Mon Sep 17 00:00:00 2001 From: zkldi Date: Sat, 10 Apr 2021 22:18:02 +0100 Subject: [PATCH] Bigger patches --- server/src/core/auth-core.test.ts | 2 +- server/src/core/auth-core.ts | 2 +- server/src/core/db-core.test.ts | 2 +- server/src/core/db-core.ts | 2 +- server/src/core/format-user.test.ts | 13 ++ server/src/core/user-core.test.ts | 16 ++ server/src/core/user-core.ts | 2 +- server/src/{ => db}/db.ts | 196 +++++++++--------- server/src/db/schemas.ts | 17 ++ server/src/internal-api/auth/auth.ts | 2 +- .../database-lookup/chart-ptdf.ts | 2 +- .../database-lookup/song-title.ts | 2 +- .../calculated-data/game-specific-stats.ts | 2 +- .../core/calculated-data/game-specific.ts | 2 +- .../framework/core/import-score.test.ts | 2 +- .../framework/core/insert-score.ts | 2 +- .../score-import/framework/core/score-id.ts | 2 +- .../framework/importing/score-importing.ts | 2 +- server/src/scripts/update-bpi-poyashi.ts | 2 +- server/src/test-utils/mock-db/users.json | 4 +- server/src/test-utils/reset-db-state.ts | 2 +- 21 files changed, 161 insertions(+), 117 deletions(-) create mode 100644 server/src/core/format-user.test.ts create mode 100644 server/src/core/user-core.test.ts rename server/src/{ => db}/db.ts (95%) create mode 100644 server/src/db/schemas.ts diff --git a/server/src/core/auth-core.test.ts b/server/src/core/auth-core.test.ts index d5eaa4318..56b5fa190 100644 --- a/server/src/core/auth-core.test.ts +++ b/server/src/core/auth-core.test.ts @@ -6,7 +6,7 @@ import { ReinstateInvite, } from "./auth-core"; import t from "tap"; -import db, { CloseConnection } from "../db"; +import db, { CloseConnection } from "../db/db"; import { PrivateUserDocument } from "kamaitachi-common"; import prAssert from "../test-utils/prassert"; import Prudence from "prudence"; diff --git a/server/src/core/auth-core.ts b/server/src/core/auth-core.ts index b0da95f21..8c5928bd4 100644 --- a/server/src/core/auth-core.ts +++ b/server/src/core/auth-core.ts @@ -6,7 +6,7 @@ import { PublicAPIKeyDocument, PublicUserDocument, } from "kamaitachi-common"; -import db from "../db"; +import db from "../db/db"; import { GetNextCounterValue } from "./db-core"; import { InsertResult } from "monk"; import CreateLogCtx from "../logger"; diff --git a/server/src/core/db-core.test.ts b/server/src/core/db-core.test.ts index 4e0352422..72783f7bd 100644 --- a/server/src/core/db-core.test.ts +++ b/server/src/core/db-core.test.ts @@ -1,6 +1,6 @@ import { GetNextCounterValue } from "./db-core"; import t from "tap"; -import db, { CloseConnection } from "../db"; +import db, { CloseConnection } from "../db/db"; import ResetDBState from "../test-utils/reset-db-state"; t.test("#GetNextCounterValue", (t) => { diff --git a/server/src/core/db-core.ts b/server/src/core/db-core.ts index e6ad8da11..121e5543d 100644 --- a/server/src/core/db-core.ts +++ b/server/src/core/db-core.ts @@ -1,4 +1,4 @@ -import db from "../db"; +import db from "../db/db"; import CreateLogCtx from "../logger"; const logger = CreateLogCtx("db-core.ts"); import { integer } from "../types"; diff --git a/server/src/core/format-user.test.ts b/server/src/core/format-user.test.ts new file mode 100644 index 000000000..75bc1749b --- /dev/null +++ b/server/src/core/format-user.test.ts @@ -0,0 +1,13 @@ +import { PublicUserDocument } from "kamaitachi-common"; +import t from "tap"; +import { FormatUserDoc } from "./format-user"; + +t.test("#FormatUserDoc", (t) => { + t.is( + FormatUserDoc({ username: "zkldi", id: 123 } as PublicUserDocument), + "zkldi (~123)", + "Should format a user document into username ~id format." + ); + + t.end(); +}); diff --git a/server/src/core/user-core.test.ts b/server/src/core/user-core.test.ts new file mode 100644 index 000000000..8c264e122 --- /dev/null +++ b/server/src/core/user-core.test.ts @@ -0,0 +1,16 @@ +import t from "tap"; +import prAssert from "../test-utils/prassert"; +import { GetUserCaseInsensitive } from "./user-core"; + +t.test("#GetUserCaseInsensitive", async (t) => { + t.test("Should return the user for an exact username", async (t) => { + let result = await GetUserCaseInsensitive("zkldi"); + + t.isNot(result, null, "Should not return null"); + + prAssert(result,) + }) + + + t.end() +}); \ No newline at end of file diff --git a/server/src/core/user-core.ts b/server/src/core/user-core.ts index b30c602f3..dbdd049db 100644 --- a/server/src/core/user-core.ts +++ b/server/src/core/user-core.ts @@ -1,5 +1,5 @@ import { integer, PublicUserDocument } from "kamaitachi-common"; -import db from "../db"; +import db from "../db/db"; const OMIT_PRIVATE_USER_RETURNS = { password: 0, diff --git a/server/src/db.ts b/server/src/db/db.ts similarity index 95% rename from server/src/db.ts rename to server/src/db/db.ts index 348bfd8a4..79621137f 100644 --- a/server/src/db.ts +++ b/server/src/db/db.ts @@ -1,98 +1,98 @@ -import { - ChartDocument, - config, - CounterDocument, - FolderDocument, - Game, - GoalDocument, - IIDXBPIData, - IIDXEamusementScoreDocument, - ImportDocument, - InviteCodeDocument, - MilestoneDocument, - NotificationDocument, - PrivateUserDocument, - PublicAPIKeyDocument, - ScoreDocument, - SessionDocument, - SongDocument, - UserGoalDocument, - UserMilestoneDocument, -} from "kamaitachi-common"; -import monk, { ICollection } from "monk"; -import CreateLogCtx from "./logger"; - -const logger = CreateLogCtx("db.ts"); - -const url = - process.env.NODE_ENV === "test" ? "localhost:27017/testingdb" : "localhost:27017/ktblackdb"; - -let dbtime: [number, number] = [0, 0]; -if (process.env.NODE_ENV !== "test") { - logger.info(`Connecting to database ${url}...`); - dbtime = process.hrtime(); -} - -export let monkDB = monk(url); - -if (process.env.NODE_ENV !== "test") { - monkDB.then(() => { - let time = process.hrtime(dbtime); - let elapsed = time[0] + time[1] / 1000000; - logger.info(`Database connection successful: took ${elapsed}ms`); - }); -} - -export async function ReOpenConnection() { - monkDB = monk(url); - await monkDB.then(); -} - -export async function CloseConnection() { - await monkDB.close(); -} - -const StaticCollections = { - scores: monkDB.get("scores"), - folders: monkDB.get("folders"), - goals: monkDB.get("goals"), - "user-goals": monkDB.get("user-goals"), - milestones: monkDB.get("milestones"), - "user-milestones": monkDB.get("user-milestones"), - users: monkDB.get("users"), - imports: monkDB.get("imports"), - notifications: monkDB.get("notifications"), - sessions: monkDB.get("sessions"), - "iidx-bpi-data": monkDB.get("iidx-bpi-data"), - "public-api-keys": monkDB.get("public-api-keys"), - invites: monkDB.get("invites"), - counters: monkDB.get("counters"), - "iidx-eam-scores": monkDB.get("iidx-eam-scores"), -}; - -type GameCollections = Record; - -interface GCPartial { - songs: Partial; - charts: Partial; -} - -interface GameCollectionFull { - songs: GameCollections; - charts: GameCollections; -} - -let GameCollectionPartial: GCPartial = { - songs: {}, - charts: {}, -}; - -for (const game of config.supportedGames) { - GameCollectionPartial.songs[game] = monkDB.get(`songs-${game}`); - GameCollectionPartial.charts[game] = monkDB.get(`charts-${game}`); -} - -// a typescript-friendly interface for the database. -const db = Object.assign(StaticCollections, GameCollectionPartial as GameCollectionFull); - -export default db; +import { + ChartDocument, + config, + CounterDocument, + FolderDocument, + Game, + GoalDocument, + IIDXBPIData, + IIDXEamusementScoreDocument, + ImportDocument, + InviteCodeDocument, + MilestoneDocument, + NotificationDocument, + PrivateUserDocument, + PublicAPIKeyDocument, + ScoreDocument, + SessionDocument, + SongDocument, + UserGoalDocument, + UserMilestoneDocument, +} from "kamaitachi-common"; +import monk, { ICollection } from "monk"; +import CreateLogCtx from "../logger"; + +const logger = CreateLogCtx("db.ts"); + +const url = + process.env.NODE_ENV === "test" ? "localhost:27017/testingdb" : "localhost:27017/ktblackdb"; + +let dbtime: [number, number] = [0, 0]; +if (process.env.NODE_ENV !== "test") { + logger.info(`Connecting to database ${url}...`); + dbtime = process.hrtime(); +} + +export let monkDB = monk(url); + +if (process.env.NODE_ENV !== "test") { + monkDB.then(() => { + let time = process.hrtime(dbtime); + let elapsed = time[0] + time[1] / 1000000; + logger.info(`Database connection successful: took ${elapsed}ms`); + }); +} + +export async function ReOpenConnection() { + monkDB = monk(url); + await monkDB.then(); +} + +export async function CloseConnection() { + await monkDB.close(); +} + +const StaticCollections = { + scores: monkDB.get("scores"), + folders: monkDB.get("folders"), + goals: monkDB.get("goals"), + "user-goals": monkDB.get("user-goals"), + milestones: monkDB.get("milestones"), + "user-milestones": monkDB.get("user-milestones"), + users: monkDB.get("users"), + imports: monkDB.get("imports"), + notifications: monkDB.get("notifications"), + sessions: monkDB.get("sessions"), + "iidx-bpi-data": monkDB.get("iidx-bpi-data"), + "public-api-keys": monkDB.get("public-api-keys"), + invites: monkDB.get("invites"), + counters: monkDB.get("counters"), + "iidx-eam-scores": monkDB.get("iidx-eam-scores"), +}; + +type GameCollections = Record; + +interface GCPartial { + songs: Partial; + charts: Partial; +} + +interface GameCollectionFull { + songs: GameCollections; + charts: GameCollections; +} + +let GameCollectionPartial: GCPartial = { + songs: {}, + charts: {}, +}; + +for (const game of config.supportedGames) { + GameCollectionPartial.songs[game] = monkDB.get(`songs-${game}`); + GameCollectionPartial.charts[game] = monkDB.get(`charts-${game}`); +} + +// a typescript-friendly interface for the database. +const db = Object.assign(StaticCollections, GameCollectionPartial as GameCollectionFull); + +export default db; diff --git a/server/src/db/schemas.ts b/server/src/db/schemas.ts new file mode 100644 index 000000000..fb6422401 --- /dev/null +++ b/server/src/db/schemas.ts @@ -0,0 +1,17 @@ +import Prudence, { PrudenceSchema } from "prudence"; + +// eslint-disable-next-line no-useless-escape +const LAZY_EMAIL_REGEX = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; + +export const PRUDENCE_PUBLIC_USER: PrudenceSchema = { + username: Prudence.regex(/^[a-zA-Z_-][a-zA-Z0-9_-]{2,20}$/), + usernameLowercase: (self, parent) => self === (parent!.username as string).toLowerCase(), + password: "string", // could be a tighter fit related to bcrypt? + email: Prudence.regex(LAZY_EMAIL_REGEX), + id: Prudence.isPositiveInteger, + settings: { + nsfwsplashes: "boolean", + invisible: "boolean", + useSimpleLadderColours: "boolean", + } +}; diff --git a/server/src/internal-api/auth/auth.ts b/server/src/internal-api/auth/auth.ts index 56aff7ca7..3f9d49f6d 100644 --- a/server/src/internal-api/auth/auth.ts +++ b/server/src/internal-api/auth/auth.ts @@ -10,7 +10,7 @@ import { import { ValidateCaptcha } from "../../core/captcha-core"; import { FormatUserDoc } from "../../core/format-user"; import { GetUserCaseInsensitive, PRIVATEINFO_GetUserCaseInsensitive } from "../../core/user-core"; -import db from "../../db"; +import db from "../../db/db"; import CreateLogCtx from "../../logger"; import prValidate from "../../middleware/prudence-validate"; import { RequireLoggedIn } from "../../middleware/require-logged-in"; diff --git a/server/src/score-import/database-lookup/chart-ptdf.ts b/server/src/score-import/database-lookup/chart-ptdf.ts index 3971286ff..aedf5210e 100644 --- a/server/src/score-import/database-lookup/chart-ptdf.ts +++ b/server/src/score-import/database-lookup/chart-ptdf.ts @@ -1,7 +1,7 @@ // Chart "Playtype+Difficulty" lookup. This is a common way of resolving Kamaitachi charts. import { Difficulties, Game, integer, Playtypes, IDStrings } from "kamaitachi-common"; -import db from "../../db"; +import db from "../../db/db"; export function FindChartWithPTDF< G extends Game = Game, diff --git a/server/src/score-import/database-lookup/song-title.ts b/server/src/score-import/database-lookup/song-title.ts index 2c12df9a9..6e2c4f2d6 100644 --- a/server/src/score-import/database-lookup/song-title.ts +++ b/server/src/score-import/database-lookup/song-title.ts @@ -1,5 +1,5 @@ import { Game } from "kamaitachi-common"; -import db from "../../db"; +import db from "../../db/db"; /** * Finds a song document for the given game with the given title (or alt-title) diff --git a/server/src/score-import/framework/core/calculated-data/game-specific-stats.ts b/server/src/score-import/framework/core/calculated-data/game-specific-stats.ts index acd396d2e..b5738353a 100644 --- a/server/src/score-import/framework/core/calculated-data/game-specific-stats.ts +++ b/server/src/score-import/framework/core/calculated-data/game-specific-stats.ts @@ -1,6 +1,6 @@ import { ChartDocument, ESDCore, integer } from "kamaitachi-common"; import { Logger } from "winston"; -import db from "../../../../db"; +import db from "../../../../db/db"; import { DryScore } from "../../../../types"; /** diff --git a/server/src/score-import/framework/core/calculated-data/game-specific.ts b/server/src/score-import/framework/core/calculated-data/game-specific.ts index 21888f846..42ff372f3 100644 --- a/server/src/score-import/framework/core/calculated-data/game-specific.ts +++ b/server/src/score-import/framework/core/calculated-data/game-specific.ts @@ -1,6 +1,6 @@ import { ChartDocument, Game, Playtypes } from "kamaitachi-common"; import { Logger } from "winston"; -import db from "../../../../db"; +import db from "../../../../db/db"; import { DryScore } from "../../../../types"; import { CalculateBPI, diff --git a/server/src/score-import/framework/core/import-score.test.ts b/server/src/score-import/framework/core/import-score.test.ts index 5c5e0f650..b5a492089 100644 --- a/server/src/score-import/framework/core/import-score.test.ts +++ b/server/src/score-import/framework/core/import-score.test.ts @@ -1,6 +1,6 @@ import { ScoreDocument } from "kamaitachi-common"; import t from "tap"; -import db, { CloseConnection } from "../../../db"; +import db, { CloseConnection } from "../../../db/db"; import ResetDBState from "../../../test-utils/reset-db-state"; import { InsertQueue, QueueScoreInsert } from "./insert-score"; diff --git a/server/src/score-import/framework/core/insert-score.ts b/server/src/score-import/framework/core/insert-score.ts index ea4307fa3..71b71bf9c 100644 --- a/server/src/score-import/framework/core/insert-score.ts +++ b/server/src/score-import/framework/core/insert-score.ts @@ -1,5 +1,5 @@ import { ScoreDocument } from "kamaitachi-common"; -import db from "../../../db"; +import db from "../../../db/db"; let ScoreQueue: ScoreDocument[] = []; const MAX_PIPELINE_LENGTH = 500; diff --git a/server/src/score-import/framework/core/score-id.ts b/server/src/score-import/framework/core/score-id.ts index 767f97f76..152eabf69 100644 --- a/server/src/score-import/framework/core/score-id.ts +++ b/server/src/score-import/framework/core/score-id.ts @@ -1,6 +1,6 @@ import { integer, DryScore } from "../../../types"; import crypto from "crypto"; -import db from "../../../db"; +import db from "../../../db/db"; function CreateScoreIDString(userID: integer, partialScore: DryScore, chartID: string) { const { lamp, grade } = partialScore.scoreData; diff --git a/server/src/score-import/framework/importing/score-importing.ts b/server/src/score-import/framework/importing/score-importing.ts index 21b4ffb43..6b3852b37 100644 --- a/server/src/score-import/framework/importing/score-importing.ts +++ b/server/src/score-import/framework/importing/score-importing.ts @@ -14,7 +14,7 @@ import { KTDataNotFoundFailure, } from "../core/converter-errors"; import { CreateScoreID } from "../core/score-id"; -import db from "../../../db"; +import db from "../../../db/db"; import { Logger } from "winston"; import { AppendLogCtx } from "../../../logger"; diff --git a/server/src/scripts/update-bpi-poyashi.ts b/server/src/scripts/update-bpi-poyashi.ts index f94645fd5..b9178255c 100644 --- a/server/src/scripts/update-bpi-poyashi.ts +++ b/server/src/scripts/update-bpi-poyashi.ts @@ -6,7 +6,7 @@ import path from "path"; import { config, Difficulties, ESDCore, IIDXBPIData } from "kamaitachi-common"; import { FindSongOnTitleVersion } from "../score-import/database-lookup/song-title"; import { FindChartWithPTDF } from "../score-import/database-lookup/chart-ptdf"; -import db from "../db"; +import db from "../db/db"; import CreateLogCtx from "../logger"; const program = new Command(); diff --git a/server/src/test-utils/mock-db/users.json b/server/src/test-utils/mock-db/users.json index 436017794..5026550f2 100644 --- a/server/src/test-utils/mock-db/users.json +++ b/server/src/test-utils/mock-db/users.json @@ -7,9 +7,7 @@ "id": 1, "settings": { "nsfwsplashes": true, - "invisible": false, - "useSimpleLadderColours": true, - "trustEamIIDXTimestamps": false + "invisible": false }, "friends": [], "socialmedia": {}, diff --git a/server/src/test-utils/reset-db-state.ts b/server/src/test-utils/reset-db-state.ts index 1d0497f96..de7b140f3 100644 --- a/server/src/test-utils/reset-db-state.ts +++ b/server/src/test-utils/reset-db-state.ts @@ -1,7 +1,7 @@ /** * Resets the state of the database. */ -import db from "../db"; +import db from "../db/db"; import fs from "fs"; import path from "path";