From faf8dc0cb391f52814dbdf36278146e9bfc0fce4 Mon Sep 17 00:00:00 2001 From: zkldi Date: Mon, 6 Sep 2021 10:01:54 +0100 Subject: [PATCH] Add calculations and bugfix to session porting scripts --- server/README.md | 8 ++-- .../__KT_DATABASE_MIGRATION/scores/bms.ts | 8 +--- .../__KT_DATABASE_MIGRATION/scores/iidx.ts | 8 +--- .../scores/migrate-all.ts | 15 +++++++ .../__KT_DATABASE_MIGRATION/scores/museca.ts | 8 +--- .../scores/reference.ts | 8 +--- .../__KT_DATABASE_MIGRATION/scores/sdvx.ts | 8 +--- .../state-sync/recalc-sessions.ts | 40 ++++++++++++++++++ .../state-sync/sync-state.ts | 2 + .../user-private-info.ts | 2 +- .../scripts/__KT_DATABASE_MIGRATION/users.ts | 2 +- .../api/v1/sessions/_sessionID/middleware.ts | 41 ++++++++++++------- 12 files changed, 95 insertions(+), 55 deletions(-) create mode 100644 server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-sessions.ts diff --git a/server/README.md b/server/README.md index 91ca9cdef..bb53ac9ed 100644 --- a/server/README.md +++ b/server/README.md @@ -17,11 +17,13 @@ Tachi itself is a supercharged Rhythm Game Score Tracker, powering multiple game `tachi-server` is one part of the larger Tachi framework. Other repositories include: -[tachi-common](https://github.com/zkldi/tachi-common): Common Types and Utilities for Tachi. (Stuff that's shared between the client and the server, basically. +[tachi-common](https://github.com/TeamNewGuys/tachi-common): Common Types and Utilities for Tachi. (Stuff that's shared between the client and the server, basically. -[tachi-docs](https://github.com/zkldi/tachi-docs): Documentation for Tachi. +[tachi-docs](https://github.com/TeamNewGuys/tachi-docs): Documentation for Tachi. -[tachi-db-importer](https://github.com/zkldi/tachi-db-importer): A UI for importing local database files to Tachi. +[tachi-db-importer](https://github.com/TeamNewGuys/tachi-db-importer): A UI for importing local database files to Tachi. + +[tachi-bot](https://github.com/TeamNewGuys/tachi-db-importer): A discord bot that integrates with the Tachi API. ## Where's the client? diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts index 502940089..3e0030865 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/bms.ts @@ -1,12 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { - PrivateUserDocument, - ScoreDocument, - GetGameConfig, - GetGamePTConfig, - StaticConfig, -} from "tachi-common"; +import { ScoreDocument, GetGamePTConfig, StaticConfig } from "tachi-common"; import db from "external/mongo/db"; import MigrateRecords from "../migrate"; import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id"; diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts index a6e0dbecf..9f8c328bd 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/iidx.ts @@ -1,12 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { - PrivateUserDocument, - ScoreDocument, - GetGameConfig, - GetGamePTConfig, - StaticConfig, -} from "tachi-common"; +import { ScoreDocument, GetGameConfig, GetGamePTConfig, StaticConfig } from "tachi-common"; import db from "external/mongo/db"; import MigrateRecords from "../migrate"; import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id"; diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts index 1435cffee..937e0500b 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/migrate-all.ts @@ -1,6 +1,9 @@ import CreateLogCtx from "lib/logger/logger"; import { MigrateBMSScores } from "./bms"; +import { MigrateDDRScores } from "./ddr"; import { MigrateIIDXScores } from "./iidx"; +import { MigrateMusecaScores } from "./museca"; +import { MigrateSDVXScores } from "./sdvx"; const logger = CreateLogCtx(__filename); @@ -12,4 +15,16 @@ const logger = CreateLogCtx(__filename); logger.info(`Migrating BMS Scores.`); await MigrateBMSScores(); + + logger.info(`Migrating DDR Scores.`); + + await MigrateDDRScores(); + + logger.info(`Migrating MUSECA Scores.`); + + await MigrateMusecaScores(); + + logger.info(`Migrating SDVX Scores.`); + + await MigrateSDVXScores(); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts index e9ab0e0c3..f755f9790 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/museca.ts @@ -1,12 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { - PrivateUserDocument, - ScoreDocument, - GetGameConfig, - GetGamePTConfig, - StaticConfig, -} from "tachi-common"; +import { ScoreDocument, GetGameConfig, GetGamePTConfig, StaticConfig } from "tachi-common"; import db from "external/mongo/db"; import MigrateRecords from "../migrate"; import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id"; diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts index 97ce9904b..6851b2522 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/reference.ts @@ -1,12 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { - PrivateUserDocument, - ScoreDocument, - GetGameConfig, - GetGamePTConfig, - StaticConfig, -} from "tachi-common"; +import { ScoreDocument, GetGameConfig, GetGamePTConfig, StaticConfig } from "tachi-common"; import db from "external/mongo/db"; import MigrateRecords from "../migrate"; import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id"; diff --git a/server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts b/server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts index 6aea3d6fa..086750558 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/scores/sdvx.ts @@ -1,12 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { - PrivateUserDocument, - ScoreDocument, - GetGameConfig, - GetGamePTConfig, - StaticConfig, -} from "tachi-common"; +import { ScoreDocument, GetGamePTConfig, StaticConfig } from "tachi-common"; import db from "external/mongo/db"; import MigrateRecords from "../migrate"; import { CreateScoreID } from "lib/score-import/framework/score-importing/score-id"; diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-sessions.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-sessions.ts new file mode 100644 index 000000000..0d32dce9f --- /dev/null +++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/recalc-sessions.ts @@ -0,0 +1,40 @@ +/* eslint-disable no-await-in-loop */ +import db from "external/mongo/db"; +import CreateLogCtx from "lib/logger/logger"; +import { ProcessPBs } from "lib/score-import/framework/pb/process-pbs"; +import { FormatUserDoc } from "utils/user"; +import deepmerge from "deepmerge"; +import { CreateSessionCalcData } from "lib/score-import/framework/sessions/calculated-data"; + +const logger = CreateLogCtx(__filename); + +export async function RecalcSessions() { + const allSessions = await db.sessions.find({}); + + for (const session of allSessions) { + const scores = await db.scores.find( + { scoreID: { $in: session.scoreInfo.map((e) => e.scoreID) } }, + { + projection: { calculatedData: 1 }, + } + ); + + let c; + try { + c = CreateSessionCalcData(session.game, session.playtype, scores); + } catch (err) { + logger.error(`${session.game} (${session.playtype}) failed.`); + logger.warn(`Destroying session.`); + await db.sessions.remove({ sessionID: session.sessionID }); + continue; + } + + await db.sessions.update({ sessionID: session.sessionID }, { $set: { calculatedData: c } }); + } + + logger.info(`Done!`); +} + +if (require.main === module) { + RecalcSessions(); +} diff --git a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts index 0def9c8fd..56800c5d5 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/state-sync/sync-state.ts @@ -2,6 +2,7 @@ import { CreateGameProfiles } from "./create-game-profiles"; import { UpdateAllPBs } from "./update-all-pbs"; import { RecalcAllScores } from "./recalc-all-scores"; import CreateLogCtx from "lib/logger/logger"; +import { RecalcSessions } from "./recalc-sessions"; const logger = CreateLogCtx(__filename); @@ -9,6 +10,7 @@ const logger = CreateLogCtx(__filename); await RecalcAllScores(); await UpdateAllPBs(); await CreateGameProfiles(); + await RecalcSessions(); logger.info(`Completely done!`); process.exit(0); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/user-private-info.ts b/server/scripts/__KT_DATABASE_MIGRATION/user-private-info.ts index 532527dc6..e974a3a78 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/user-private-info.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/user-private-info.ts @@ -13,7 +13,7 @@ function ConvertFn(c: any): PrivateUserInfoDocument { } (async () => { - await MigrateRecords(db.users, "user-private-information", ConvertFn); + await MigrateRecords(db["user-private-information"], "users", ConvertFn); process.exit(0); })(); diff --git a/server/scripts/__KT_DATABASE_MIGRATION/users.ts b/server/scripts/__KT_DATABASE_MIGRATION/users.ts index bbdca2c0c..fb8a94084 100644 --- a/server/scripts/__KT_DATABASE_MIGRATION/users.ts +++ b/server/scripts/__KT_DATABASE_MIGRATION/users.ts @@ -30,7 +30,7 @@ function ConvertFn(c: any): PublicUserDocument { } (async () => { - await MigrateRecords(db.users, "users", ConvertFn); + await MigrateRecords(db, "users", ConvertFn); process.exit(0); })(); diff --git a/server/src/server/router/api/v1/sessions/_sessionID/middleware.ts b/server/src/server/router/api/v1/sessions/_sessionID/middleware.ts index d1ce43343..6832922d4 100644 --- a/server/src/server/router/api/v1/sessions/_sessionID/middleware.ts +++ b/server/src/server/router/api/v1/sessions/_sessionID/middleware.ts @@ -1,8 +1,11 @@ import { RequestHandler } from "express"; import db from "external/mongo/db"; import { SYMBOL_TachiAPIAuth, SYMBOL_TachiData } from "lib/constants/tachi"; +import CreateLogCtx from "lib/logger/logger"; import { AssignToReqTachiData } from "utils/req-tachi-data"; +const logger = CreateLogCtx(__filename); + export const GetSessionFromParam: RequestHandler = async (req, res, next) => { const session = await db.sessions.findOne({ sessionID: req.params.sessionID, @@ -37,6 +40,7 @@ export const RequireOwnershipOfSession: RequestHandler = (req, res, next) => { export const UpdateSessionViewcount: RequestHandler = async (req, res, next) => { const session = req[SYMBOL_TachiData]!.sessionDoc!; + // Maybe we could improve how this cache works using redis. Maybe. // we don't need to actually check timestamp - this collection expires documents every 24hours. const hasViewedRecently = await db["session-view-cache"].findOne({ sessionID: session.sessionID, @@ -47,27 +51,34 @@ export const UpdateSessionViewcount: RequestHandler = async (req, res, next) => return next(); } - await db["session-view-cache"].insert([ - { + // This sometimes fails due to a race condition in the above statement. + // @hack I've just lazily wrapped this in a try catch -> continue flow. + // Maybe there'll be a neater solution. + try { + await db["session-view-cache"].insert({ sessionID: session.sessionID, ip: req.ip, timestamp: Date.now(), - }, - ]); + }); - await db.sessions.update( - { - sessionID: session.sessionID, - }, - { - $inc: { - views: 1, + await db.sessions.update( + { + sessionID: session.sessionID, }, - } - ); + { + $inc: { + views: 1, + }, + } + ); - // increment locally so that the right state is shown to the end user. - session.views++; + // increment locally so that the right state is shown to the end user. + session.views++; + } catch (err) { + logger.warn( + `Race condition protection triggered in UpdateSessionViewcount for ${session.sessionID}. Ignoring.` + ); + } return next(); };