diff --git a/server/package.json b/server/package.json index 08344d6f3..afa678b5c 100644 --- a/server/package.json +++ b/server/package.json @@ -18,10 +18,12 @@ "@types/connect-redis": "^0.0.16", "@types/express": "^4.17.12", "@types/express-session": "^1.17.3", + "@types/mkdirp": "^1.0.1", "@types/multer": "^1.4.5", "@types/node": "^15.12.0", "@types/node-fetch": "^2.5.10", "@types/range-parser": "^1.2.3", + "@types/rimraf": "^3.0.0", "@types/supertest": "^2.0.11", "@types/tap": "^15.0.2", "@types/uuid": "^8.3.0", @@ -53,12 +55,14 @@ "fast-json-stable-hash": "^1.0.2", "fast-xml-parser": "^3.19.0", "kamaitachi-common": "github:zkldi/Kamaitachi-common", + "mkdirp": "^1.0.4", "mongodb": "3.6.3", "monk": "^7.3.4", "multer": "^1.4.2", "node-fetch": "^2.6.1", "prudence": "^0.9.6", "redis": "^3.1.2", + "rimraf": "^3.0.2", "typescript": "^4.3.2", "winston": "^3.3.3" }, diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index e55d64764..6f200beff 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -5,10 +5,12 @@ specifiers: '@types/connect-redis': ^0.0.16 '@types/express': ^4.17.12 '@types/express-session': ^1.17.3 + '@types/mkdirp': ^1.0.1 '@types/multer': ^1.4.5 '@types/node': ^15.12.0 '@types/node-fetch': ^2.5.10 '@types/range-parser': ^1.2.3 + '@types/rimraf': ^3.0.0 '@types/supertest': ^2.0.11 '@types/tap': ^15.0.2 '@types/uuid': ^8.3.0 @@ -32,6 +34,7 @@ specifiers: fast-json-stable-hash: ^1.0.2 fast-xml-parser: ^3.19.0 kamaitachi-common: github:zkldi/Kamaitachi-common + mkdirp: ^1.0.4 mongodb: 3.6.3 monk: ^7.3.4 multer: ^1.4.2 @@ -40,6 +43,7 @@ specifiers: prettier: ^2.3.0 prudence: ^0.9.6 redis: ^3.1.2 + rimraf: ^3.0.2 supertest: ^6.1.3 tap: ^15.0.9 ts-node: ^10.0.0 @@ -61,12 +65,14 @@ dependencies: fast-json-stable-hash: 1.0.2 fast-xml-parser: 3.19.0 kamaitachi-common: github.com/zkldi/Kamaitachi-common/ceaccaf38a718a78257d48a36a0c524faa1c5132 + mkdirp: 1.0.4 mongodb: 3.6.3 monk: 7.3.4 multer: 1.4.2 node-fetch: 2.6.1 prudence: 0.9.6 redis: 3.1.2 + rimraf: 3.0.2 typescript: 4.3.2 winston: 3.3.3 @@ -75,10 +81,12 @@ devDependencies: '@types/connect-redis': 0.0.16 '@types/express': 4.17.12 '@types/express-session': 1.17.3 + '@types/mkdirp': 1.0.1 '@types/multer': 1.4.5 '@types/node': 15.12.0 '@types/node-fetch': 2.5.10 '@types/range-parser': 1.2.3 + '@types/rimraf': 3.0.0 '@types/supertest': 2.0.11 '@types/tap': 15.0.2 '@types/uuid': 8.3.0 @@ -523,6 +531,13 @@ packages: '@types/serve-static': 1.13.9 dev: true + /@types/glob/7.1.3: + resolution: {integrity: sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==} + dependencies: + '@types/minimatch': 3.0.4 + '@types/node': 15.12.1 + dev: true + /@types/ioredis/4.26.4: resolution: {integrity: sha512-QFbjNq7EnOGw6d1gZZt2h26OFXjx7z+eqEnbCHSrDI1OOLEgOHMKdtIajJbuCr9uO+X9kQQRe7Lz6uxqxl5XKg==} dependencies: @@ -541,6 +556,16 @@ packages: resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} dev: true + /@types/minimatch/3.0.4: + resolution: {integrity: sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==} + dev: true + + /@types/mkdirp/1.0.1: + resolution: {integrity: sha512-HkGSK7CGAXncr8Qn/0VqNtExEE+PHMWb+qlR1faHMao7ng6P3tAaoWWBMdva0gL5h4zprjIO89GJOLXsMcDm1Q==} + dependencies: + '@types/node': 15.12.1 + dev: true + /@types/mongodb/3.6.17: resolution: {integrity: sha512-9hhgvYPdC5iHyyksPcKCu45gfaAIPQHKHGdvNXu4582DmOZX3wrUJIJPT40o4G1oTKPgpMMFqZglOTjhnYoF+A==} dependencies: @@ -567,7 +592,6 @@ packages: /@types/node/15.12.1: resolution: {integrity: sha512-zyxJM8I1c9q5sRMtVF+zdd13Jt6RU4r4qfhTd7lQubyThvLfx6yYekWSQjGCGV2Tkecgxnlpl/DNlb6Hg+dmEw==} - dev: false /@types/qs/6.9.6: resolution: {integrity: sha512-0/HnwIfW4ki2D8L8c9GVcG5I72s9jP5GSLVF0VIXDW00kmIpA6O33G7a8n59Tmh7Nz0WUC3rSb7PTY/sdW2JzA==} @@ -583,6 +607,13 @@ packages: '@types/node': 15.12.0 dev: true + /@types/rimraf/3.0.0: + resolution: {integrity: sha512-7WhJ0MdpFgYQPXlF4Dx+DhgvlPCfz/x5mHaeDQAKhcenvQP1KCpLQ18JklAqeGMYSAT2PxLpzd0g2/HE7fj7hQ==} + dependencies: + '@types/glob': 7.1.3 + '@types/node': 15.12.1 + dev: true + /@types/serve-static/1.13.9: resolution: {integrity: sha512-ZFqF6qa48XsPdjXV5Gsz0Zqmux2PerNd3a/ktL45mHpa19cuMi/cL8tcxdAx497yRh+QtYPuofjT9oWw9P7nkA==} dependencies: diff --git a/server/src/lib/cdn/cdn.ts b/server/src/lib/cdn/cdn.ts index 9a34a37d6..826bd482f 100644 --- a/server/src/lib/cdn/cdn.ts +++ b/server/src/lib/cdn/cdn.ts @@ -3,6 +3,7 @@ import path from "path"; import { KTCDN_ROOT } from "../env/env"; import CreateLogCtx from "../logger/logger"; import { promisify } from "util"; +import mkdirp from "mkdirp"; const readFilePromise = promisify(fs.readFile); const writeFilePromise = promisify(fs.writeFile); @@ -34,8 +35,13 @@ const WRITE_NO_OVERWRITE = "wx"; * Stores the provided buffer or string as a file at the given location. * @returns Nothing on success. Throws on error. */ -export function StoreCDN(fileLoc: string, data: Buffer | string) { +export async function StoreCDN(fileLoc: string, data: Buffer | string) { logger.debug(`Storing path ${fileLoc}.`); - return writeFilePromise(CDNRoot(fileLoc), data, { flag: WRITE_NO_OVERWRITE }); + const loc = CDNRoot(fileLoc); + + // make the parent folders if they dont exist. else, mkdirp is a no-op. + await mkdirp(path.dirname(loc)); + + return writeFilePromise(loc, data, { flag: WRITE_NO_OVERWRITE }); } diff --git a/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts b/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts index 7a9bfa5f3..99eb8bb2f 100644 --- a/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts +++ b/server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts @@ -12,7 +12,7 @@ import { } from "../../../../test-utils/test-data"; import { CreateCalculatedData, CalculateDataForGamePT } from "./calculated-data"; import deepmerge from "deepmerge"; -import ResetDBState from "../../../../test-utils/reset-db-state"; +import ResetDBState from "../../../../test-utils/resets"; import { CloseAllConnections } from "../../../../test-utils/close-connections"; const logger = CreateLogCtx(__filename); diff --git a/server/src/lib/score-import/framework/calculated-data/stats.test.ts b/server/src/lib/score-import/framework/calculated-data/stats.test.ts index 410c84561..06e182abf 100644 --- a/server/src/lib/score-import/framework/calculated-data/stats.test.ts +++ b/server/src/lib/score-import/framework/calculated-data/stats.test.ts @@ -22,7 +22,7 @@ import { } from "../../../../test-utils/test-data"; import { CloseAllConnections } from "../../../../test-utils/close-connections"; import deepmerge from "deepmerge"; -import ResetDBState from "../../../../test-utils/reset-db-state"; +import ResetDBState from "../../../../test-utils/resets"; t.test("#CalculateBPI", (t) => { t.test("AA BPI tests", (t) => { diff --git a/server/src/lib/score-import/framework/goals/goals.test.ts b/server/src/lib/score-import/framework/goals/goals.test.ts index 86d22101f..f04fbadad 100644 --- a/server/src/lib/score-import/framework/goals/goals.test.ts +++ b/server/src/lib/score-import/framework/goals/goals.test.ts @@ -1,6 +1,6 @@ import t from "tap"; import db from "../../../../external/mongo/db"; -import ResetDBState from "../../../../test-utils/reset-db-state"; +import ResetDBState from "../../../../test-utils/resets"; import { GetRelevantFolderGoals, GetRelevantGoals, UpdateGoalsForUser, ProcessGoal } from "./goals"; import { GoalDocument, UserGoalDocument } from "kamaitachi-common"; import { CreateFolderChartLookup } from "../../../../utils/folder"; @@ -253,10 +253,10 @@ t.test("#UpdateGoalsForUser", (t) => { const goal = deepmerge(baseGoalDocument, { criteria: { value: 2 } }); await db.goals.insert(goal); - const userGoal = (deepmerge(baseUserGoalDocument, { + const userGoal = deepmerge(baseUserGoalDocument, { outOf: 2, outOfHuman: "2", - }) as unknown) as UserGoalDocument; + }) as unknown as UserGoalDocument; await db["user-goals"].insert(userGoal); // we dont delete _id here because updategoalsforuser diff --git a/server/src/server/router/api/v1/import/router.ts b/server/src/server/router/api/v1/import/router.ts index 080f314cf..90ce3da33 100644 --- a/server/src/server/router/api/v1/import/router.ts +++ b/server/src/server/router/api/v1/import/router.ts @@ -7,10 +7,16 @@ import CreateLogCtx, { KtLogger } from "../../../../../lib/logger/logger"; import prValidate from "../../../../middleware/prudence-validate"; import { RequireLoggedIn } from "../../../../middleware/require-logged-in"; import ScoreImportFatalError from "../../../../../lib/score-import/framework/score-importing/score-import-error"; - +import { SIXTEEN_MEGABTYES } from "../../../../../lib/constants/filesize"; import { ExpressWrappedScoreImportMain } from "../../../../../lib/score-import/framework/express-wrapper"; import { CreateMulterSingleUploadMiddleware } from "../../../../middleware/multer-upload"; +import ParseEamusementIIDXCSV from "../../../../../lib/score-import/import-types/file/eamusement-iidx-csv/parser"; +import ParseBatchManual from "../../../../../lib/score-import/import-types/file/batch-manual/parser"; +import { ParseSolidStateXML } from "../../../../../lib/score-import/import-types/file/solid-state-squad/parser"; +import { ParseMerIIDX } from "../../../../../lib/score-import/import-types/file/mer-iidx/parser"; +import ParsePLIIIDXCSV from "../../../../../lib/score-import/import-types/file/pli-iidx-csv/parser"; + const logger = CreateLogCtx(__filename); const router: Router = Router({ mergeParams: true }); @@ -68,13 +74,6 @@ router.post( } ); -import ParseEamusementIIDXCSV from "../../../../../lib/score-import/import-types/file/eamusement-iidx-csv/parser"; -import ParseBatchManual from "../../../../../lib/score-import/import-types/file/batch-manual/parser"; -import { ParseSolidStateXML } from "../../../../../lib/score-import/import-types/file/solid-state-squad/parser"; -import { ParseMerIIDX } from "../../../../../lib/score-import/import-types/file/mer-iidx/parser"; -import ParsePLIIIDXCSV from "../../../../../lib/score-import/import-types/file/pli-iidx-csv/parser"; -import { SIXTEEN_MEGABTYES } from "../../../../../lib/constants/filesize"; - /** * Resolves the data from a file upload into an iterable, * The appropriate processing function to map that iterable over, diff --git a/server/src/server/router/ir/usc/router.test.ts b/server/src/server/router/ir/usc/router.test.ts index 4e083b583..b49043b3c 100644 --- a/server/src/server/router/ir/usc/router.test.ts +++ b/server/src/server/router/ir/usc/router.test.ts @@ -2,7 +2,7 @@ import t from "tap"; import db from "../../../../external/mongo/db"; import { CloseAllConnections } from "../../../../test-utils/close-connections"; import mockApi from "../../../../test-utils/mock-api"; -import ResetDBState from "../../../../test-utils/resets"; +import ResetDBState, { ResetCDN } from "../../../../test-utils/resets"; import deepmerge from "deepmerge"; import { PBScoreDocument, ScoreDocument } from "kamaitachi-common"; import { GetKTDataBuffer } from "../../../../test-utils/test-data"; @@ -337,6 +337,7 @@ t.test("GET /charts/:chartHash/leaderboard", (t) => { t.test("POST /replays", (t) => { t.beforeEach(ResetDBState); + t.beforeEach(ResetCDN); t.test("Should successfully upload a file where an identifier matches", async (t) => { await db.scores.insert({ @@ -357,7 +358,7 @@ t.test("POST /replays", (t) => { t.strictSame(res.body, { statusCode: 20, - description: "Uploaded replay.", + description: "Saved replay.", body: null, }); @@ -368,6 +369,80 @@ t.test("POST /replays", (t) => { t.end(); }); + t.test("Should reject a request with no identifier", async (t) => { + await db.scores.insert({ + game: "usc", + userID: 1, + scoreID: "MOCK_IDENTIFIER", + } as ScoreDocument); + + const replayFile = GetKTDataBuffer("./usc/replayfile.urf"); + + const res = await mockApi + .post("/ir/usc/replays") + .attach("replay", replayFile, "replay.urf") + .set("Authorization", "Bearer foo"); + + t.equal(res.status, 200); + + t.strictSame(res.body, { + statusCode: 40, + description: "No Identifier Provided.", + }); + + t.end(); + }); + + t.test("Should reject a request with no file", async (t) => { + await db.scores.insert({ + game: "usc", + userID: 1, + scoreID: "MOCK_IDENTIFIER", + } as ScoreDocument); + + // const replayFile = GetKTDataBuffer("./usc/replayfile.urf"); + + const res = await mockApi + .post("/ir/usc/replays") + .field("identifier", "MOCK_IDENTIFIER") + // .attach("replay", replayFile, "replay.urf") + .set("Authorization", "Bearer foo"); + + t.equal(res.status, 200); + + t.strictSame(res.body, { + statusCode: 40, + description: "No File Provided.", + }); + + t.end(); + }); + + t.test("Should reject a request with an invalid identifier", async (t) => { + await db.scores.insert({ + game: "usc", + userID: 1, + scoreID: "MOCK_IDENTIFIER", + } as ScoreDocument); + + const replayFile = GetKTDataBuffer("./usc/replayfile.urf"); + + const res = await mockApi + .post("/ir/usc/replays") + .field("identifier", "INVALID_IDENTIFIER") + .attach("replay", replayFile, "replay.urf") + .set("Authorization", "Bearer foo"); + + t.equal(res.status, 200); + + t.strictSame(res.body, { + statusCode: 44, + description: "No score corresponds to this identifier.", + }); + + t.end(); + }); + t.end(); }); diff --git a/server/src/server/router/ir/usc/router.ts b/server/src/server/router/ir/usc/router.ts index 187f0e09d..bf2db9de1 100644 --- a/server/src/server/router/ir/usc/router.ts +++ b/server/src/server/router/ir/usc/router.ts @@ -326,6 +326,13 @@ router.post( }); } + if (!req.file) { + return res.status(200).json({ + statusCode: STATUS_CODES.BAD_REQ, + description: "No File Provided.", + }); + } + const correspondingScore = await db.scores.findOne({ userID: req[SYMBOL_KtchiData]!.uscAuthDoc!.userID, game: "usc", @@ -348,8 +355,11 @@ router.post( body: null, }); } catch (err) { + // impossible to test pretty much. + /* istanbul ignore next */ logger.error(`USCIR Replay Store error.`, { err }); - return res.status(500).json({ + /* istanbul ignore next */ + return res.status(200).json({ statusCode: STATUS_CODES.SERVER_ERROR, description: "An error has occured in storing the replay.", }); diff --git a/server/src/test-utils/fake-session.ts b/server/src/test-utils/fake-session.ts index 43e2e00f2..7667f0983 100644 --- a/server/src/test-utils/fake-session.ts +++ b/server/src/test-utils/fake-session.ts @@ -1,4 +1,4 @@ -import ResetDBState from "./reset-db-state"; +import ResetDBState from "./resets"; import CreateLogCtx from "../lib/logger/logger"; import supertest from "supertest"; diff --git a/server/src/test-utils/resets.ts b/server/src/test-utils/resets.ts index 68b9ce056..583e53cc3 100644 --- a/server/src/test-utils/resets.ts +++ b/server/src/test-utils/resets.ts @@ -5,6 +5,9 @@ import db from "../external/mongo/db"; import fs from "fs"; import path from "path"; import CreateLogCtx from "../lib/logger/logger"; +// im installing an entire library for rm rf... +import rimraf from "rimraf"; +import { KTCDN_ROOT } from "../lib/env/env"; const logger = CreateLogCtx(__filename); @@ -70,3 +73,14 @@ export default async function ResetDBState() { await Promise.all(promises); } + +export function ResetCDN() { + return new Promise((resolve, reject) => + rimraf(KTCDN_ROOT, (err) => { + if (err) { + reject(err); + } + resolve(); + }) + ); +} diff --git a/server/src/test-utils/test-data.ts b/server/src/test-utils/test-data.ts index 745581942..fdfd6b87c 100644 --- a/server/src/test-utils/test-data.ts +++ b/server/src/test-utils/test-data.ts @@ -20,7 +20,7 @@ import { } from "../lib/constants/usc-ir"; import { DryScore } from "../lib/score-import/framework/common/types"; import { BarbatosScore } from "../lib/score-import/import-types/ir/barbatos/types"; -import { USCClientScore } from "../server/router/ir/usc/usc"; +import { USCClientScore } from "../server/router/ir/usc/types"; const file = (name: string) => path.join(__dirname, "/test-data", name); diff --git a/server/src/test-utils/test-data/usc/replayfile.urf b/server/src/test-utils/test-data/usc/replayfile.urf new file mode 100644 index 000000000..b69c3524d Binary files /dev/null and b/server/src/test-utils/test-data/usc/replayfile.urf differ