Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-10-05 21:38:06 +03:00
Code Issues Packages Projects Releases Wiki Activity

Properly store replays sent with POST /replays. Fixes #113

Browse Source
This commit is contained in:
zkldi
2021-06-08 10:19:42 +01:00
parent 40bbf4eae5
commit 3947154e37
13 changed files with 160 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/package.json
+4
View File
@@ -18,10 +18,12 @@
"@types/connect-redis": "^0.0.16",
"@types/express": "^4.17.12",
"@types/express-session": "^1.17.3",
"@types/mkdirp": "^1.0.1",
"@types/multer": "^1.4.5",
"@types/node": "^15.12.0",
"@types/node-fetch": "^2.5.10",
"@types/range-parser": "^1.2.3",
"@types/rimraf": "^3.0.0",
"@types/supertest": "^2.0.11",
"@types/tap": "^15.0.2",
"@types/uuid": "^8.3.0",
@@ -53,12 +55,14 @@
"fast-json-stable-hash": "^1.0.2",
"fast-xml-parser": "^3.19.0",
"kamaitachi-common": "github:zkldi/Kamaitachi-common",
"mkdirp": "^1.0.4",
"mongodb": "3.6.3",
"monk": "^7.3.4",
"multer": "^1.4.2",
"node-fetch": "^2.6.1",
"prudence": "^0.9.6",
"redis": "^3.1.2",
"rimraf": "^3.0.2",
"typescript": "^4.3.2",
"winston": "^3.3.3"
},
server/pnpm-lock.yaml
Generated
+32 -1
View File
@@ -5,10 +5,12 @@ specifiers:
'@types/connect-redis': ^0.0.16
'@types/express': ^4.17.12
'@types/express-session': ^1.17.3
'@types/mkdirp': ^1.0.1
'@types/multer': ^1.4.5
'@types/node': ^15.12.0
'@types/node-fetch': ^2.5.10
'@types/range-parser': ^1.2.3
'@types/rimraf': ^3.0.0
'@types/supertest': ^2.0.11
'@types/tap': ^15.0.2
'@types/uuid': ^8.3.0
@@ -32,6 +34,7 @@ specifiers:
fast-json-stable-hash: ^1.0.2
fast-xml-parser: ^3.19.0
kamaitachi-common: github:zkldi/Kamaitachi-common
mkdirp: ^1.0.4
mongodb: 3.6.3
monk: ^7.3.4
multer: ^1.4.2
@@ -40,6 +43,7 @@ specifiers:
prettier: ^2.3.0
prudence: ^0.9.6
redis: ^3.1.2
rimraf: ^3.0.2
supertest: ^6.1.3
tap: ^15.0.9
ts-node: ^10.0.0
@@ -61,12 +65,14 @@ dependencies:
fast-json-stable-hash: 1.0.2
fast-xml-parser: 3.19.0
kamaitachi-common: github.com/zkldi/Kamaitachi-common/ceaccaf38a718a78257d48a36a0c524faa1c5132
mkdirp: 1.0.4
mongodb: 3.6.3
monk: 7.3.4
multer: 1.4.2
node-fetch: 2.6.1
prudence: 0.9.6
redis: 3.1.2
rimraf: 3.0.2
typescript: 4.3.2
winston: 3.3.3
@@ -75,10 +81,12 @@ devDependencies:
'@types/connect-redis': 0.0.16
'@types/express': 4.17.12
'@types/express-session': 1.17.3
'@types/mkdirp': 1.0.1
'@types/multer': 1.4.5
'@types/node': 15.12.0
'@types/node-fetch': 2.5.10
'@types/range-parser': 1.2.3
'@types/rimraf': 3.0.0
'@types/supertest': 2.0.11
'@types/tap': 15.0.2
'@types/uuid': 8.3.0
@@ -523,6 +531,13 @@ packages:
'@types/serve-static': 1.13.9
dev: true
/@types/glob/7.1.3:
resolution: {integrity: sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==}
dependencies:
'@types/minimatch': 3.0.4
'@types/node': 15.12.1
dev: true
/@types/ioredis/4.26.4:
resolution: {integrity: sha512-QFbjNq7EnOGw6d1gZZt2h26OFXjx7z+eqEnbCHSrDI1OOLEgOHMKdtIajJbuCr9uO+X9kQQRe7Lz6uxqxl5XKg==}
dependencies:
@@ -541,6 +556,16 @@ packages:
resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==}
dev: true
/@types/minimatch/3.0.4:
resolution: {integrity: sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==}
dev: true
/@types/mkdirp/1.0.1:
resolution: {integrity: sha512-HkGSK7CGAXncr8Qn/0VqNtExEE+PHMWb+qlR1faHMao7ng6P3tAaoWWBMdva0gL5h4zprjIO89GJOLXsMcDm1Q==}
dependencies:
'@types/node': 15.12.1
dev: true
/@types/mongodb/3.6.17:
resolution: {integrity: sha512-9hhgvYPdC5iHyyksPcKCu45gfaAIPQHKHGdvNXu4582DmOZX3wrUJIJPT40o4G1oTKPgpMMFqZglOTjhnYoF+A==}
dependencies:
@@ -567,7 +592,6 @@ packages:
/@types/node/15.12.1:
resolution: {integrity: sha512-zyxJM8I1c9q5sRMtVF+zdd13Jt6RU4r4qfhTd7lQubyThvLfx6yYekWSQjGCGV2Tkecgxnlpl/DNlb6Hg+dmEw==}
dev: false
/@types/qs/6.9.6:
resolution: {integrity: sha512-0/HnwIfW4ki2D8L8c9GVcG5I72s9jP5GSLVF0VIXDW00kmIpA6O33G7a8n59Tmh7Nz0WUC3rSb7PTY/sdW2JzA==}
@@ -583,6 +607,13 @@ packages:
'@types/node': 15.12.0
dev: true
/@types/rimraf/3.0.0:
resolution: {integrity: sha512-7WhJ0MdpFgYQPXlF4Dx+DhgvlPCfz/x5mHaeDQAKhcenvQP1KCpLQ18JklAqeGMYSAT2PxLpzd0g2/HE7fj7hQ==}
dependencies:
'@types/glob': 7.1.3
'@types/node': 15.12.1
dev: true
/@types/serve-static/1.13.9:
resolution: {integrity: sha512-ZFqF6qa48XsPdjXV5Gsz0Zqmux2PerNd3a/ktL45mHpa19cuMi/cL8tcxdAx497yRh+QtYPuofjT9oWw9P7nkA==}
dependencies:
server/src/lib/cdn/cdn.ts
+8 -2
View File
@@ -3,6 +3,7 @@ import path from "path";
import { KTCDN_ROOT } from "../env/env";
import CreateLogCtx from "../logger/logger";
import { promisify } from "util";
import mkdirp from "mkdirp";
const readFilePromise = promisify(fs.readFile);
const writeFilePromise = promisify(fs.writeFile);
@@ -34,8 +35,13 @@ const WRITE_NO_OVERWRITE = "wx";
* Stores the provided buffer or string as a file at the given location.
* @returns Nothing on success. Throws on error.
*/
export function StoreCDN(fileLoc: string, data: Buffer | string) {
export async function StoreCDN(fileLoc: string, data: Buffer | string) {
logger.debug(`Storing path ${fileLoc}.`);
return writeFilePromise(CDNRoot(fileLoc), data, { flag: WRITE_NO_OVERWRITE });
const loc = CDNRoot(fileLoc);
// make the parent folders if they dont exist. else, mkdirp is a no-op.
await mkdirp(path.dirname(loc));
return writeFilePromise(loc, data, { flag: WRITE_NO_OVERWRITE });
}
server/src/lib/score-import/framework/calculated-data/calculated-data.test.ts
+1 -1
View File
@@ -12,7 +12,7 @@ import {
} from "../../../../test-utils/test-data";
import { CreateCalculatedData, CalculateDataForGamePT } from "./calculated-data";
import deepmerge from "deepmerge";
import ResetDBState from "../../../../test-utils/reset-db-state";
import ResetDBState from "../../../../test-utils/resets";
import { CloseAllConnections } from "../../../../test-utils/close-connections";
const logger = CreateLogCtx(__filename);
server/src/lib/score-import/framework/calculated-data/stats.test.ts
+1 -1
View File
@@ -22,7 +22,7 @@ import {
} from "../../../../test-utils/test-data";
import { CloseAllConnections } from "../../../../test-utils/close-connections";
import deepmerge from "deepmerge";
import ResetDBState from "../../../../test-utils/reset-db-state";
import ResetDBState from "../../../../test-utils/resets";
t.test("#CalculateBPI", (t) => {
t.test("AA BPI tests", (t) => {
server/src/lib/score-import/framework/goals/goals.test.ts
+3 -3
View File
@@ -1,6 +1,6 @@
import t from "tap";
import db from "../../../../external/mongo/db";
import ResetDBState from "../../../../test-utils/reset-db-state";
import ResetDBState from "../../../../test-utils/resets";
import { GetRelevantFolderGoals, GetRelevantGoals, UpdateGoalsForUser, ProcessGoal } from "./goals";
import { GoalDocument, UserGoalDocument } from "kamaitachi-common";
import { CreateFolderChartLookup } from "../../../../utils/folder";
@@ -253,10 +253,10 @@ t.test("#UpdateGoalsForUser", (t) => {
const goal = deepmerge(baseGoalDocument, { criteria: { value: 2 } });
await db.goals.insert(goal);
const userGoal = (deepmerge(baseUserGoalDocument, {
const userGoal = deepmerge(baseUserGoalDocument, {
outOf: 2,
outOfHuman: "2",
}) as unknown) as UserGoalDocument;
}) as unknown as UserGoalDocument;
await db["user-goals"].insert(userGoal);
// we dont delete _id here because updategoalsforuser
server/src/server/router/api/v1/import/router.ts
+7 -8
View File
@@ -7,10 +7,16 @@ import CreateLogCtx, { KtLogger } from "../../../../../lib/logger/logger";
import prValidate from "../../../../middleware/prudence-validate";
import { RequireLoggedIn } from "../../../../middleware/require-logged-in";
import ScoreImportFatalError from "../../../../../lib/score-import/framework/score-importing/score-import-error";
import { SIXTEEN_MEGABTYES } from "../../../../../lib/constants/filesize";
import { ExpressWrappedScoreImportMain } from "../../../../../lib/score-import/framework/express-wrapper";
import { CreateMulterSingleUploadMiddleware } from "../../../../middleware/multer-upload";
import ParseEamusementIIDXCSV from "../../../../../lib/score-import/import-types/file/eamusement-iidx-csv/parser";
import ParseBatchManual from "../../../../../lib/score-import/import-types/file/batch-manual/parser";
import { ParseSolidStateXML } from "../../../../../lib/score-import/import-types/file/solid-state-squad/parser";
import { ParseMerIIDX } from "../../../../../lib/score-import/import-types/file/mer-iidx/parser";
import ParsePLIIIDXCSV from "../../../../../lib/score-import/import-types/file/pli-iidx-csv/parser";
const logger = CreateLogCtx(__filename);
const router: Router = Router({ mergeParams: true });
@@ -68,13 +74,6 @@ router.post(
}
);
import ParseEamusementIIDXCSV from "../../../../../lib/score-import/import-types/file/eamusement-iidx-csv/parser";
import ParseBatchManual from "../../../../../lib/score-import/import-types/file/batch-manual/parser";
import { ParseSolidStateXML } from "../../../../../lib/score-import/import-types/file/solid-state-squad/parser";
import { ParseMerIIDX } from "../../../../../lib/score-import/import-types/file/mer-iidx/parser";
import ParsePLIIIDXCSV from "../../../../../lib/score-import/import-types/file/pli-iidx-csv/parser";
import { SIXTEEN_MEGABTYES } from "../../../../../lib/constants/filesize";
/**
* Resolves the data from a file upload into an iterable,
* The appropriate processing function to map that iterable over,
server/src/server/router/ir/usc/router.test.ts
+77 -2
View File
@@ -2,7 +2,7 @@ import t from "tap";
import db from "../../../../external/mongo/db";
import { CloseAllConnections } from "../../../../test-utils/close-connections";
import mockApi from "../../../../test-utils/mock-api";
import ResetDBState from "../../../../test-utils/resets";
import ResetDBState, { ResetCDN } from "../../../../test-utils/resets";
import deepmerge from "deepmerge";
import { PBScoreDocument, ScoreDocument } from "kamaitachi-common";
import { GetKTDataBuffer } from "../../../../test-utils/test-data";
@@ -337,6 +337,7 @@ t.test("GET /charts/:chartHash/leaderboard", (t) => {
t.test("POST /replays", (t) => {
t.beforeEach(ResetDBState);
t.beforeEach(ResetCDN);
t.test("Should successfully upload a file where an identifier matches", async (t) => {
await db.scores.insert({
@@ -357,7 +358,7 @@ t.test("POST /replays", (t) => {
t.strictSame(res.body, {
statusCode: 20,
description: "Uploaded replay.",
description: "Saved replay.",
body: null,
});
@@ -368,6 +369,80 @@ t.test("POST /replays", (t) => {
t.end();
});
t.test("Should reject a request with no identifier", async (t) => {
await db.scores.insert({
game: "usc",
userID: 1,
scoreID: "MOCK_IDENTIFIER",
} as ScoreDocument);
const replayFile = GetKTDataBuffer("./usc/replayfile.urf");
const res = await mockApi
.post("/ir/usc/replays")
.attach("replay", replayFile, "replay.urf")
.set("Authorization", "Bearer foo");
t.equal(res.status, 200);
t.strictSame(res.body, {
statusCode: 40,
description: "No Identifier Provided.",
});
t.end();
});
t.test("Should reject a request with no file", async (t) => {
await db.scores.insert({
game: "usc",
userID: 1,
scoreID: "MOCK_IDENTIFIER",
} as ScoreDocument);
// const replayFile = GetKTDataBuffer("./usc/replayfile.urf");
const res = await mockApi
.post("/ir/usc/replays")
.field("identifier", "MOCK_IDENTIFIER")
// .attach("replay", replayFile, "replay.urf")
.set("Authorization", "Bearer foo");
t.equal(res.status, 200);
t.strictSame(res.body, {
statusCode: 40,
description: "No File Provided.",
});
t.end();
});
t.test("Should reject a request with an invalid identifier", async (t) => {
await db.scores.insert({
game: "usc",
userID: 1,
scoreID: "MOCK_IDENTIFIER",
} as ScoreDocument);
const replayFile = GetKTDataBuffer("./usc/replayfile.urf");
const res = await mockApi
.post("/ir/usc/replays")
.field("identifier", "INVALID_IDENTIFIER")
.attach("replay", replayFile, "replay.urf")
.set("Authorization", "Bearer foo");
t.equal(res.status, 200);
t.strictSame(res.body, {
statusCode: 44,
description: "No score corresponds to this identifier.",
});
t.end();
});
t.end();
});
server/src/server/router/ir/usc/router.ts
+11 -1
View File
@@ -326,6 +326,13 @@ router.post(
});
}
if (!req.file) {
return res.status(200).json({
statusCode: STATUS_CODES.BAD_REQ,
description: "No File Provided.",
});
}
const correspondingScore = await db.scores.findOne({
userID: req[SYMBOL_KtchiData]!.uscAuthDoc!.userID,
game: "usc",
@@ -348,8 +355,11 @@ router.post(
body: null,
});
} catch (err) {
// impossible to test pretty much.
/* istanbul ignore next */
logger.error(`USCIR Replay Store error.`, { err });
return res.status(500).json({
/* istanbul ignore next */
return res.status(200).json({
statusCode: STATUS_CODES.SERVER_ERROR,
description: "An error has occured in storing the replay.",
});
server/src/test-utils/fake-session.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import ResetDBState from "./reset-db-state";
import ResetDBState from "./resets";
import CreateLogCtx from "../lib/logger/logger";
import supertest from "supertest";
server/src/test-utils/resets.ts
+14
View File
@@ -5,6 +5,9 @@ import db from "../external/mongo/db";
import fs from "fs";
import path from "path";
import CreateLogCtx from "../lib/logger/logger";
// im installing an entire library for rm rf...
import rimraf from "rimraf";
import { KTCDN_ROOT } from "../lib/env/env";
const logger = CreateLogCtx(__filename);
@@ -70,3 +73,14 @@ export default async function ResetDBState() {
await Promise.all(promises);
}
export function ResetCDN() {
return new Promise<void>((resolve, reject) =>
rimraf(KTCDN_ROOT, (err) => {
if (err) {
reject(err);
}
resolve();
})
);
}
server/src/test-utils/test-data.ts
+1 -1
View File
@@ -20,7 +20,7 @@ import {
} from "../lib/constants/usc-ir";
import { DryScore } from "../lib/score-import/framework/common/types";
import { BarbatosScore } from "../lib/score-import/import-types/ir/barbatos/types";
import { USCClientScore } from "../server/router/ir/usc/usc";
import { USCClientScore } from "../server/router/ir/usc/types";
const file = (name: string) => path.join(__dirname, "/test-data", name);
server/src/test-utils/test-data/usc/replayfile.urf
BIN
View File
Binary file not shown.