From ee4eddd63962993e17d4efe89cc7bf038478a5b3 Mon Sep 17 00:00:00 2001 From: ppc Date: Sun, 15 Sep 2024 19:22:39 +0000 Subject: [PATCH 01/45] add buddies plus support --- titles/mai2/buddiesplus.py | 36 ++++++++++++++++++++++++++++++++++++ titles/mai2/const.py | 4 +++- titles/mai2/index.py | 8 ++++++-- 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 titles/mai2/buddiesplus.py diff --git a/titles/mai2/buddiesplus.py b/titles/mai2/buddiesplus.py new file mode 100644 index 0000000..0467b03 --- /dev/null +++ b/titles/mai2/buddiesplus.py @@ -0,0 +1,36 @@ +from typing import Dict + +from core.config import CoreConfig +from titles.mai2.buddies import Mai2Buddies +from titles.mai2.const import Mai2Constants +from titles.mai2.config import Mai2Config + + +class Mai2BuddiesPlus(Mai2Buddies): + def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None: + super().__init__(cfg, game_cfg) + self.version = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS + + async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: + user_data = await super().handle_cm_get_user_preview_api_request(data) + + # hardcode lastDataVersion for CardMaker + user_data["lastDataVersion"] = "1.45.00" + return user_data + + async def handle_get_game_weekly_data_api_request(self, data: Dict) -> Dict: + return { + "gameWeeklyData": { + "missionCategory": 0, + "updateDate": "2024-03-21 09:00:00.0", + "beforeDate": "2099-12-31 00:00:00.0" + } + } + + async def handle_create_token_api_request(self, data: Dict) -> Dict: + return { + "Bearer": "ARTEMiSTOKEN" # duplicate of handle_user_login_api_request from Mai2Festival + } + + async def handle_remove_token_api_request(self, data: Dict) -> Dict: + return {} diff --git a/titles/mai2/const.py b/titles/mai2/const.py index 4dc10ce..0d13a0d 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -55,6 +55,7 @@ class Mai2Constants: VER_MAIMAI_DX_FESTIVAL = 19 VER_MAIMAI_DX_FESTIVAL_PLUS = 20 VER_MAIMAI_DX_BUDDIES = 21 + VER_MAIMAI_DX_BUDDIES_PLUS = 22 VERSION_STRING = ( "maimai", @@ -78,7 +79,8 @@ class Mai2Constants: "maimai DX UNiVERSE PLUS", "maimai DX FESTiVAL", "maimai DX FESTiVAL PLUS", - "maimai DX BUDDiES" + "maimai DX BUDDiES", + "maimai DX BUDDiES PLUS" ) @classmethod diff --git a/titles/mai2/index.py b/titles/mai2/index.py index ad02648..e8b88ec 100644 --- a/titles/mai2/index.py +++ b/titles/mai2/index.py @@ -30,6 +30,7 @@ from .universeplus import Mai2UniversePlus from .festival import Mai2Festival from .festivalplus import Mai2FestivalPlus from .buddies import Mai2Buddies +from .buddiesplus import Mai2BuddiesPlus class Mai2Servlet(BaseServlet): @@ -64,7 +65,8 @@ class Mai2Servlet(BaseServlet): Mai2UniversePlus, Mai2Festival, Mai2FestivalPlus, - Mai2Buddies + Mai2Buddies, + Mai2BuddiesPlus ] self.logger = logging.getLogger("mai2") @@ -302,8 +304,10 @@ class Mai2Servlet(BaseServlet): internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL elif version >= 135 and version < 140: # FESTiVAL PLUS internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS - elif version >= 140: # BUDDiES + elif version >= 140 and version < 145: # BUDDiES internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES + elif version >= 145: # BUDDiES PLUS + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS elif game_code == "SDGA": # Int if version < 105: # 1.0 internal_ver = Mai2Constants.VER_MAIMAI_DX From cc302b6e56e5eb43b761ccc6a5a6a7f5b7de7178 Mon Sep 17 00:00:00 2001 From: ppc Date: Sun, 15 Sep 2024 19:22:47 +0000 Subject: [PATCH 02/45] update cm reader --- titles/cm/read.py | 1 + 1 file changed, 1 insertion(+) diff --git a/titles/cm/read.py b/titles/cm/read.py index 2b5ec8a..b4b3b5e 100644 --- a/titles/cm/read.py +++ b/titles/cm/read.py @@ -207,6 +207,7 @@ class CardMakerReader(BaseReader): "1.30": Mai2Constants.VER_MAIMAI_DX_FESTIVAL, "1.35": Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS, "1.40": Mai2Constants.VER_MAIMAI_DX_BUDDIES, + "1.45": Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS } for root, dirs, files in os.walk(base_dir): From 01dad267b9962ee87edabdd2bb91b5a518e46445 Mon Sep 17 00:00:00 2001 From: ppc Date: Sun, 15 Sep 2024 20:48:24 +0000 Subject: [PATCH 03/45] add mai2 database upgrade --- .../28443e2da5b8_mai2_buddies_plus.py | 28 +++++++++++++++++++ titles/mai2/schema/profile.py | 5 +++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py diff --git a/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py b/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py new file mode 100644 index 0000000..42fcdde --- /dev/null +++ b/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py @@ -0,0 +1,28 @@ +"""mai2_buddies_plus + +Revision ID: 28443e2da5b8 +Revises: 5ea73f89d982 +Create Date: 2024-09-15 20:44:02.351819 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '28443e2da5b8' +down_revision = '5ea73f89d982' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('mai2_profile_detail', sa.Column('point', sa.Integer())) + op.add_column('mai2_profile_detail', sa.Column('totalPoint', sa.Integer())) + op.add_column('mai2_profile_detail', sa.Column('friendRegistSkip', sa.SmallInteger())) + + +def downgrade(): + op.drop_column('mai2_profile_detail', 'point') + op.drop_column('mai2_profile_detail', 'totalPoint') + op.drop_column('mai2_profile_detail', 'friendRegistSkip') diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index c191a1a..988dcb0 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -3,7 +3,7 @@ from titles.mai2.const import Mai2Constants from typing import Optional, Dict, List from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row @@ -43,6 +43,8 @@ detail = Table( Column("currentPlayCount", Integer), # new with buddies Column("renameCredit", Integer), # new with buddies Column("mapStock", Integer), # new with fes+ + Column("point", Integer), # new with buddies+ + Column("totalPoint", Integer), # new with buddies+ Column("eventWatchedDate", String(25)), Column("lastGameId", String(25)), Column("lastRomVersion", String(25)), @@ -97,6 +99,7 @@ detail = Table( Column("playerOldRating", BigInteger), Column("playerNewRating", BigInteger), Column("dateTime", BigInteger), + Column("friendRegistSkip", SmallInteger), # new with buddies+ Column("banState", Integer), # new with uni+ UniqueConstraint("user", "version", name="mai2_profile_detail_uk"), mysql_charset="utf8mb4", From b01ac24799cf98cd0cebd2184f8cf1b3329d598b Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 09:54:39 +0000 Subject: [PATCH 04/45] add shop stock/friend bonus handlers --- titles/mai2/buddiesplus.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/titles/mai2/buddiesplus.py b/titles/mai2/buddiesplus.py index 0467b03..a3784d5 100644 --- a/titles/mai2/buddiesplus.py +++ b/titles/mai2/buddiesplus.py @@ -22,15 +22,28 @@ class Mai2BuddiesPlus(Mai2Buddies): return { "gameWeeklyData": { "missionCategory": 0, - "updateDate": "2024-03-21 09:00:00.0", - "beforeDate": "2099-12-31 00:00:00.0" + "updateDate": "2024-03-21 09:00:00", + "beforeDate": "2099-12-31 00:00:00" } } - + async def handle_create_token_api_request(self, data: Dict) -> Dict: return { "Bearer": "ARTEMiSTOKEN" # duplicate of handle_user_login_api_request from Mai2Festival } - + async def handle_remove_token_api_request(self, data: Dict) -> Dict: return {} + + async def handle_get_user_friend_bonus_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "returnCode": 1, + "getMiles": 0 + } + + async def handle_get_user_shop_stock_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "userShopStockList": [] + } From e128631e8fe83f0d0a14568d3ad57db27ec8b9e9 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 12:39:15 +0100 Subject: [PATCH 05/45] fix upsert failures --- titles/mai2/dx.py | 3 ++- titles/mai2/schema/item.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 7a067d7..53abc61 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -242,7 +242,8 @@ class Mai2DX(Mai2Base): if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0: for fav in upsert["userFavoriteList"]: - await self.data.item.put_favorite(user_id, fav["kind"], fav["itemIdList"]) + kind_id = fav.get("kind", fav.get("itemKind")) # itemKind key used in BUDDiES+ + await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) if ( "userFriendSeasonRankingList" in upsert diff --git a/titles/mai2/schema/item.py b/titles/mai2/schema/item.py index d53ebbc..13afbe4 100644 --- a/titles/mai2/schema/item.py +++ b/titles/mai2/schema/item.py @@ -453,10 +453,10 @@ class Mai2ItemData(BaseData): self, user_id: int, kind: int, item_id_list: List[int] ) -> Optional[int]: sql = insert(favorite).values( - user=user_id, kind=kind, item_id_list=item_id_list + user=user_id, itemKind=kind, itemIdList=item_id_list ) - conflict = sql.on_duplicate_key_update(item_id_list=item_id_list) + conflict = sql.on_duplicate_key_update(itemIdList=item_id_list) result = await self.execute(conflict) if result is None: From 77aa1afaa0f14883ddce3c5f4243b4bce1e94b04 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 16:55:09 +0100 Subject: [PATCH 06/45] add mai2 favorite music support --- ...c91c1206dca_mai2_favorite_song_ordering.py | 24 +++++++++++++++++++ titles/mai2/base.py | 2 +- titles/mai2/dx.py | 4 ++++ titles/mai2/schema/item.py | 8 ++++--- 4 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py diff --git a/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py new file mode 100644 index 0000000..ff4a62f --- /dev/null +++ b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py @@ -0,0 +1,24 @@ +"""mai2_favorite_song_ordering + +Revision ID: bc91c1206dca +Revises: 28443e2da5b8 +Create Date: 2024-09-16 14:24:56.714066 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'bc91c1206dca' +down_revision = '28443e2da5b8' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('mai2_item_favorite_music', sa.Column('orderId', sa.Integer(nullable=True))) + + +def downgrade(): + op.drop_column('mai2_item_favorite_music', 'orderId') diff --git a/titles/mai2/base.py b/titles/mai2/base.py index bbd074f..b041028 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -922,7 +922,7 @@ class Mai2Base: fav_music = await self.data.item.get_fav_music(user_id) if fav_music: for fav in fav_music: - id_list.append({"orderId": 0, "id": fav["musicId"]}) + id_list.append({"orderId": fav["orderId"] or 0, "id": fav["musicId"]}) if len(id_list) >= 100: # Lazy but whatever break diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 53abc61..0e86acd 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -245,6 +245,10 @@ class Mai2DX(Mai2Base): kind_id = fav.get("kind", fav.get("itemKind")) # itemKind key used in BUDDiES+ await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) + if "userFavoritemusicList" in upsert and len(upsert["userFavoritemusicList"]) > 0: + for fav in upsert["userFavoritemusicList"]: + await self.data.item.add_fav_music(user_id, fav["id"], fav["orderId"]) + if ( "userFriendSeasonRankingList" in upsert and len(upsert["userFriendSeasonRankingList"]) > 0 diff --git a/titles/mai2/schema/item.py b/titles/mai2/schema/item.py index 13afbe4..87ddca4 100644 --- a/titles/mai2/schema/item.py +++ b/titles/mai2/schema/item.py @@ -144,6 +144,7 @@ fav_music = Table( nullable=False, ), Column("musicId", Integer, nullable=False), + Column("orderId", Integer, nullable=True), UniqueConstraint("user", "musicId", name="mai2_item_favorite_music_uk"), mysql_charset="utf8mb4", ) @@ -484,13 +485,14 @@ class Mai2ItemData(BaseData): if result: return result.fetchall() - async def add_fav_music(self, user_id: int, music_id: int) -> Optional[int]: + async def add_fav_music(self, user_id: int, music_id: int, order_id: Optional[int] = None) -> Optional[int]: sql = insert(fav_music).values( user = user_id, - musicId = music_id + musicId = music_id, + orderId = order_id ) - conflict = sql.on_duplicate_key_update(musicId = music_id) + conflict = sql.on_duplicate_key_update(orderId = order_id) result = await self.execute(conflict) if result: From d8169e37cc88b2041b4a4fec2151486037459ae6 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 17:56:22 +0100 Subject: [PATCH 07/45] add mai2 UserIntimateApi --- .../versions/54a84103b84e_mai2_intimacy.py | 43 +++++++++++++++++++ titles/mai2/dx.py | 23 ++++++++++ titles/mai2/schema/profile.py | 37 ++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 core/data/alembic/versions/54a84103b84e_mai2_intimacy.py diff --git a/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py b/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py new file mode 100644 index 0000000..a180bbb --- /dev/null +++ b/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py @@ -0,0 +1,43 @@ +"""mai2_intimacy + +Revision ID: 54a84103b84e +Revises: bc91c1206dca +Create Date: 2024-09-16 17:47:49.164546 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import Column, Integer, UniqueConstraint + +# revision identifiers, used by Alembic. +revision = '54a84103b84e' +down_revision = 'bc91c1206dca' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "mai2_user_intimate", + Column("id", Integer, primary_key=True, nullable=False), + Column("user", Integer, nullable=False), + Column("partnerId", Integer, nullable=False), + Column("intimateLevel", Integer, nullable=False), + Column("intimateCountRewarded", Integer, nullable=False), + UniqueConstraint("user", "partnerId", name="mai2_user_intimate_uk"), + mysql_charset="utf8mb4", + ) + + op.create_foreign_key( + None, + "mai2_user_intimate", + "aime_user", + ["user"], + ["id"], + ondelete="cascade", + onupdate="cascade", + ) + + +def downgrade(): + op.drop_table("mai2_user_intimate") diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 0e86acd..a31f98c 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -264,6 +264,11 @@ class Mai2DX(Mai2Base): if "user2pPlaylog" in upsert: await self.data.score.put_playlog_2p(user_id, upsert["user2pPlaylog"]) + # added in BUDDiES+ + if "userIntimateList" in upsert and len(upsert["userIntimateList"]) > 0: + for intimate in upsert["userIntimateList"]: + await self.data.profile.put_intimacy(user_id, intimate["partnerId"], intimate["intimateLevel"], intimate["intimateCountRewarded"]) + return {"returnCode": 1, "apiName": "UpsertUserAllApi"} async def handle_get_user_data_api_request(self, data: Dict) -> Dict: @@ -713,6 +718,24 @@ class Mai2DX(Mai2Base): ret['loginId'] = ret.get('loginCount', 0) return ret + # Intimate api added in BUDDiES+ + async def handle_get_user_intimate_api_request(self, data: Dict) -> Dict: + intimate = await self.data.profile.get_intimacy(data["userId"]) + if intimate is None: + return {} + + partner_list = [{ + "partnerId": i["partnerId"], + "intimateLevel": i["intimateLevel"], + "intimateCountRewarded": i["intimateCountRewarded"] + } for i in intimate] + + return { + "userId": data["userId"], + "length": len(partner_list), + "userIntimateList": partner_list + } + # CardMaker support added in Universe async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: p = await self.data.profile.get_profile_detail(data["userId"], self.version) diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index 988dcb0..3ff85d2 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -513,6 +513,22 @@ rival = Table( mysql_charset="utf8mb4", ) +intimacy = Table( +"mai2_user_intimate", + metadata, + Column("id", Integer, primary_key=True, nullable=False), + Column( + "user", + ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("partnerId", Integer, nullable=False), + Column("intimateLevel", Integer, nullable=False), + Column("intimateCountRewarded", Integer, nullable=False), + UniqueConstraint("user", "partnerId", name="mai2_user_intimate_uk"), + mysql_charset="utf8mb4", +) + class Mai2ProfileData(BaseData): async def get_all_profile_versions(self, user_id: int) -> Optional[List[Row]]: result = await self.execute(detail.select(detail.c.user == user_id)) @@ -908,6 +924,27 @@ class Mai2ProfileData(BaseData): if not result: self.logger.error(f"Failed to remove rival {rival_id} for user {user_id}!") + async def get_intimacy(self, user_id: int) -> Optional[List[Row]]: + result = await self.execute(intimacy.select(intimacy.c.user == user_id)) + if result: + return result.fetchall() + + async def put_intimacy(self, user_id: int, partner_id: int, level: int, count_rewarded: int) -> Optional[int]: + sql = insert(intimacy).values( + user = user_id, + partnerId = partner_id, + intimateLevel = level, + intimateCountRewarded = count_rewarded + ) + + conflict = sql.on_duplicate_key_update(intimateLevel = level, intimateCountRewarded = count_rewarded) + + result = await self.execute(conflict) + if result: + return result.lastrowid + + self.logger.error(f"Failed to update intimacy for user {user_id} and partner {partner_id}!") + async def update_name(self, user_id: int, new_name: str) -> bool: sql = detail.update(detail.c.user == user_id).values( userName=new_name From 196aa601f3e461329af095b0e3b5120d30e858c3 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 18:01:06 +0100 Subject: [PATCH 08/45] handle GetUserMissionDataApi --- titles/mai2/buddiesplus.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/titles/mai2/buddiesplus.py b/titles/mai2/buddiesplus.py index a3784d5..e87fae6 100644 --- a/titles/mai2/buddiesplus.py +++ b/titles/mai2/buddiesplus.py @@ -47,3 +47,14 @@ class Mai2BuddiesPlus(Mai2Buddies): "userId": data["userId"], "userShopStockList": [] } + + async def handle_get_user_mission_data_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "userMissionDataList": [], + "userWeeklyData": { + "lastLoginWeek": "2024-03-21 09:00:00", + "beforeLoginWeek": "2099-12-31 00:00:00", + "friendBonusFlag": False + } + } From d85c575c61006e340722e69eae217f38a8d2b3e7 Mon Sep 17 00:00:00 2001 From: ppc Date: Wed, 18 Sep 2024 11:29:14 +0100 Subject: [PATCH 09/45] add nullcheck --- titles/mai2/dx.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index a31f98c..66bf914 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -243,7 +243,8 @@ class Mai2DX(Mai2Base): if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0: for fav in upsert["userFavoriteList"]: kind_id = fav.get("kind", fav.get("itemKind")) # itemKind key used in BUDDiES+ - await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) + if kind_id is not None: + await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) if "userFavoritemusicList" in upsert and len(upsert["userFavoritemusicList"]) > 0: for fav in upsert["userFavoritemusicList"]: From 8d04d74f523faca71b3b7552124c9f660aa041e9 Mon Sep 17 00:00:00 2001 From: ppc Date: Wed, 18 Sep 2024 17:59:24 +0100 Subject: [PATCH 10/45] migration consistency --- .../versions/bc91c1206dca_mai2_favorite_song_ordering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py index ff4a62f..abf6357 100644 --- a/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py +++ b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py @@ -17,7 +17,7 @@ depends_on = None def upgrade(): - op.add_column('mai2_item_favorite_music', sa.Column('orderId', sa.Integer(nullable=True))) + op.add_column('mai2_item_favorite_music', sa.Column('orderId', sa.Integer(), nullable=True)) def downgrade(): From 5c60cde14f89b9db2212f8963bcec20ade88c75e Mon Sep 17 00:00:00 2001 From: ppc Date: Thu, 19 Sep 2024 23:10:54 +0100 Subject: [PATCH 11/45] update docs --- docs/game_specific_info.md | 1 + readme.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 37561fc..a0aed71 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -218,6 +218,7 @@ Presents are items given to the user when they login, with a little animation (f | SDEZ | 19 | maimai DX FESTiVAL | | SDEZ | 20 | maimai DX FESTiVAL PLUS | | SDEZ | 21 | maimai DX BUDDiES | +| SDEZ | 22 | maimai DX BUDDiES PLUS | ### Importer diff --git a/readme.md b/readme.md index c792928..0564c70 100644 --- a/readme.md +++ b/readme.md @@ -50,6 +50,7 @@ Games listed below have been tested and confirmed working. Only game versions ol + FESTiVAL + FESTiVAL PLUS + BUDDiES + + BUDDiES PLUS + O.N.G.E.K.I. + SUMMER From 3843ac6eb14b130cb8819c318c41e110344906ee Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Thu, 3 Oct 2024 19:32:17 +0000 Subject: [PATCH 12/45] mai2: calc GetGameRanking result --- titles/mai2/base.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/titles/mai2/base.py b/titles/mai2/base.py index b041028..9d85857 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -1,3 +1,4 @@ +import pymysql from datetime import datetime, timedelta from typing import Any, Dict, List import logging @@ -76,7 +77,40 @@ class Mai2Base: } async def handle_get_game_ranking_api_request(self, data: Dict) -> Dict: - return {"length": 0, "gameRankingList": []} + conn = pymysql.connect( + host=self.core_config.database.host, + port=self.core_config.database.port, + user=self.core_config.database.username, + password=self.core_config.database.password, + database=self.core_config.database.name, + charset='utf8mb4' + ) + try: + cursor = conn.cursor() + + query = """ + SELECT musicid AS id, COUNT(*) AS point + FROM mai2_playlog + GROUP BY musicid + ORDER BY point DESC + LIMIT 100 + """ + cursor.execute(query) + + results = cursor.fetchall() + ranking_list = [{"id": row[0], "point": row[1], "userName": ""} for row in results] + output = { + "type": 1, + "gameRankingList": ranking_list, + "gameRankingInstantList": None + } + + cursor.close() + conn.close() + return output + + except Exception as e: + return {'length': 0, 'gameRankingList': []} async def handle_get_game_tournament_info_api_request(self, data: Dict) -> Dict: # TODO: Tournament support From 58ae491a8ccd76cae5682b150b2da89bdc3b7295 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Thu, 3 Oct 2024 19:47:36 +0000 Subject: [PATCH 13/45] add pymysql to requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fe5b4ef..72d9844 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,5 @@ starlette asyncio uvicorn alembic -python-multipart \ No newline at end of file +python-multipart +pymysql \ No newline at end of file From 0cef797a8a74c6a895fbfaab8035e0bd57a6b65c Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Sun, 6 Oct 2024 03:47:10 -0400 Subject: [PATCH 14/45] mai2: rework photo uploads, relates to #67 --- .../versions/d8cd1fa04c2a_mai2_add_photos.py | 38 +++++ titles/mai2/base.py | 61 ++++---- titles/mai2/frontend.py | 130 +++++++++++++++++- titles/mai2/schema/profile.py | 54 +++++++- titles/mai2/templates/mai2_header.jinja | 3 + titles/mai2/templates/mai2_photos.jinja | 28 ++++ 6 files changed, 272 insertions(+), 42 deletions(-) create mode 100644 core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py create mode 100644 titles/mai2/templates/mai2_photos.jinja diff --git a/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py b/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py new file mode 100644 index 0000000..312a127 --- /dev/null +++ b/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py @@ -0,0 +1,38 @@ +"""mai2_add_photos + +Revision ID: d8cd1fa04c2a +Revises: 54a84103b84e +Create Date: 2024-10-06 03:09:15.959817 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'd8cd1fa04c2a' +down_revision = '54a84103b84e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('mai2_user_photo', + sa.Column('id', sa.VARCHAR(length=36), nullable=False), + sa.Column('user', sa.Integer(), nullable=False), + sa.Column('playlog_num', sa.INTEGER(), nullable=False), + sa.Column('track_num', sa.INTEGER(), nullable=False), + sa.Column('when_upload', sa.TIMESTAMP(), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user'], ['aime_user.id'], onupdate='cascade', ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user', 'playlog_num', 'track_num', name='mai2_user_photo_uk'), + mysql_charset='utf8mb4' + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('mai2_user_photo') + # ### end Alembic commands ### diff --git a/titles/mai2/base.py b/titles/mai2/base.py index b041028..cb74206 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from typing import Any, Dict, List import logging from base64 import b64decode -from os import path, stat, remove +from os import path, stat, remove, mkdir, access, W_OK from PIL import ImageFile from random import randint @@ -866,46 +866,33 @@ class Mai2Base: self.logger.warning(f"Incorrect data size after decoding (Expected 10240, got {len(photo_chunk)})") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - out_name = f"{self.game_config.uploads.photos_dir}/{user_id}_{playlog_id}_{track_num}" + photo_data = await self.data.profile.get_user_photo_by_user_playlog_track(user_id, playlog_id, track_num) + + if not photo_data: + photo_id = await self.data.profile.put_user_photo(user_id, playlog_id, track_num) + else: + photo_id = photo_data['id'] - if not path.exists(f"{out_name}.bin") and div_num != 0: - self.logger.warning(f"Out of order photo upload (div_num {div_num})") - return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - if path.exists(f"{out_name}.bin") and div_num == 0: - self.logger.warning(f"Duplicate file upload") + out_folder = f"{self.game_config.uploads.photos_dir}/{photo_id}" + out_file = f"{out_folder}/{div_num}_{div_len - 1}.bin" + + if not path.exists(out_folder): + mkdir(out_folder) + + if not access(out_folder, W_OK): + self.logger.error(f"Cannot access {out_folder}") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - elif path.exists(f"{out_name}.bin"): - fstats = stat(f"{out_name}.bin") - if fstats.st_size != 10240 * div_num: - self.logger.warning(f"Out of order photo upload (trying to upload div {div_num}, expected div {fstats.st_size / 10240} for file sized {fstats.st_size} bytes)") + if path.exists(out_file): + self.logger.warning(f"Photo chunk {out_file} already exists, skipping") + + else: + with open(out_file, "wb") as f: + written = f.write(photo_chunk) + + if written != len(photo_chunk): + self.logger.error(f"Writing {out_file} failed! Wrote {written} bytes, expected {photo_chunk} bytes") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - try: - with open(f"{out_name}.bin", "ab") as f: - f.write(photo_chunk) - - except Exception: - self.logger.error(f"Failed writing to {out_name}.bin") - return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - if div_num + 1 == div_len and path.exists(f"{out_name}.bin"): - try: - p = ImageFile.Parser() - with open(f"{out_name}.bin", "rb") as f: - p.feed(f.read()) - - im = p.close() - im.save(f"{out_name}.jpeg") - except Exception: - self.logger.error(f"File {out_name}.bin failed image validation") - - try: - remove(f"{out_name}.bin") - - except Exception: - self.logger.error(f"Failed to delete {out_name}.bin, please remove it manually") return {'returnCode': ret_code, 'apiName': 'UploadUserPhotoApi'} diff --git a/titles/mai2/frontend.py b/titles/mai2/frontend.py index 976e2c4..f1f961a 100644 --- a/titles/mai2/frontend.py +++ b/titles/mai2/frontend.py @@ -1,11 +1,14 @@ from typing import List from starlette.routing import Route, Mount from starlette.requests import Request -from starlette.responses import Response, RedirectResponse -from os import path +from starlette.responses import Response, RedirectResponse, FileResponse +from os import path, walk, remove import yaml import jinja2 -from datetime import datetime +from datetime import datetime, timedelta +from PIL import ImageFile +import re +import shutil from core.frontend import FE_Base, UserSession, PermissionOffset from core.config import CoreConfig @@ -31,7 +34,8 @@ class Mai2Frontend(FE_Base): Route("/", self.render_GET, methods=['GET']), Mount("/playlog", routes=[ Route("/", self.render_GET_playlog, methods=['GET']), - Route("/{index}", self.render_GET_playlog, methods=['GET']), + Route("/{index:int}", self.render_GET_playlog, methods=['GET']), + Route("/photos", self.render_GET_photos, methods=['GET']), ]), Mount("/events", routes=[ Route("/", self.render_events, methods=['GET']), @@ -41,6 +45,7 @@ class Mai2Frontend(FE_Base): ]), Route("/update.name", self.update_name, methods=['POST']), Route("/version.change", self.version_change, methods=['POST']), + Route("/photo/{photo_id}", self.get_photo, methods=['GET']), ] async def render_GET(self, request: Request) -> bytes: @@ -140,6 +145,50 @@ class Mai2Frontend(FE_Base): else: return RedirectResponse("/gate/", 303) + async def render_GET_photos(self, request: Request) -> bytes: + template = self.environment.get_template( + "titles/mai2/templates/mai2_photos.jinja" + ) + usr_sesh = self.validate_session(request) + if not usr_sesh: + usr_sesh = UserSession() + + if usr_sesh.user_id > 0: + if usr_sesh.maimai_version < 0: + return RedirectResponse("/game/mai2/", 303) + + photos = await self.data.profile.get_user_photos_by_user(usr_sesh.user_id) + + photos_fixed = [] + for photo in photos: + if datetime.now().timestamp() > (photo['when_upload'] + timedelta(days=7)).timestamp(): + await self.data.profile.delete_user_photo_by_id(photo['id']) + + if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg"): + remove(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg") + + if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}"): + shutil.rmtree(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}") + + continue + + photos_fixed.append({ + "id": photo['id'], + "playlog_num": photo['playlog_num'], + "track_num": photo['track_num'], + "when_upload": photo['when_upload'], + }) + + return Response(template.render( + title=f"{self.core_config.server.name} | {self.nav_name}", + game_list=self.environment.globals["game_list"], + sesh=vars(usr_sesh), + photos=photos_fixed, + expire_days=7, + ), media_type="text/html; charset=utf-8") + else: + return RedirectResponse("/gate/", 303) + async def update_name(self, request: Request) -> bytes: usr_sesh = self.validate_session(request) if not usr_sesh: @@ -299,3 +348,76 @@ class Mai2Frontend(FE_Base): await self.data.static.update_event_by_id(int(event_id), new_enabled, new_start_date) return RedirectResponse("/game/mai2/events/?s=1", 303) + + async def get_photo(self, request: Request) -> RedirectResponse: + usr_sesh = self.validate_session(request) + if not usr_sesh: + return RedirectResponse("/gate/", 303) + + photo_jpeg = request.path_params.get("photo_id", None) + if not photo_jpeg: + return Response(status_code=400) + + matcher = re.match(r"^([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}).jpeg$", photo_jpeg) + if not matcher: + return Response(status_code=400) + + photo_id = matcher.groups()[0] + photo_info = await self.data.profile.get_user_photo_by_id(photo_id) + if not photo_info: + return Response(status_code=404) + + if photo_info["user"] != usr_sesh.user_id: + return Response(status_code=403) + + out_folder = f"{self.game_cfg.uploads.photos_dir}/{photo_id}" + + if datetime.now().timestamp() > (photo_info['when_upload'] + timedelta(days=7)).timestamp(): + await self.data.profile.delete_user_photo_by_id(photo_info['id']) + if path.exists(f"{out_folder}.jpeg"): + remove(f"{out_folder}.jpeg") + + if path.exists(f"{out_folder}"): + shutil.rmtree(out_folder) + + return Response(status_code=404) + + if path.exists(f"{out_folder}"): + print("path exists") + max_idx = 0 + p = ImageFile.Parser() + for _, _, files in walk("out_folder"): + if not files: + break + + matcher = re.match("^(\d+)_(\d+)$", files[0]) + if not matcher: + break + + max_idx = int(matcher.groups()[1]) + + if max_idx + 1 != len(files): + self.logger.error(f"Expected {max_idx + 1} files, found {len(files)}") + max_idx = 0 + break + + if max_idx == 0: + return Response(status_code=500) + + for i in range(max_idx + 1): + with open(f"{out_folder}/{i}_{max_idx}", "rb") as f: + p.feed(f.read()) + try: + im = p.close() + im.save(f"{out_folder}.jpeg") + + except Exception as e: + self.logger.error(f"{photo_id} failed PIL validation! - {e}") + + shutil.rmtree(out_folder) + + if path.exists(f"{out_folder}.jpeg"): + print(f"{out_folder}.jpeg exists") + return FileResponse(f"{out_folder}.jpeg") + + return Response(status_code=404) diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index 3ff85d2..ede0adf 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -1,9 +1,10 @@ from core.data.schema import BaseData, metadata from titles.mai2.const import Mai2Constants +from uuid import uuid4 from typing import Optional, Dict, List from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger, VARCHAR, INTEGER from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row @@ -529,6 +530,22 @@ intimacy = Table( mysql_charset="utf8mb4", ) +photo = Table( # end-of-credit memorial photos, NOT user portraits + "mai2_user_photo", + metadata, + Column("id", VARCHAR(36), primary_key=True, nullable=False), + Column( + "user", + ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("playlog_num", INTEGER, nullable=False), + Column("track_num", INTEGER, nullable=False), + Column("when_upload", TIMESTAMP, nullable=False, server_default=func.now()), + UniqueConstraint("user", "playlog_num", "track_num", name="mai2_user_photo_uk"), + mysql_charset="utf8mb4", +) + class Mai2ProfileData(BaseData): async def get_all_profile_versions(self, user_id: int) -> Optional[List[Row]]: result = await self.execute(detail.select(detail.c.user == user_id)) @@ -945,6 +962,41 @@ class Mai2ProfileData(BaseData): self.logger.error(f"Failed to update intimacy for user {user_id} and partner {partner_id}!") + async def put_user_photo(self, user_id: int, playlog_num: int, track_num: int) -> Optional[str]: + photo_id = str(uuid4()) + sql = insert(photo).values( + id = photo_id, + user = user_id, + playlog_num = playlog_num, + track_num = track_num, + ) + + conflict = sql.on_duplicate_key_update(user = user_id) + + result = await self.execute(conflict) + if result: + return photo_id + + async def get_user_photo_by_id(self, photo_id: str) -> Optional[Row]: + result = await self.execute(photo.select(photo.c.id.like(photo_id))) + if result: + return result.fetchone() + + async def get_user_photo_by_user_playlog_track(self, user_id: int, playlog_num: int, track_num: int) -> Optional[Row]: + result = await self.execute(photo.select(and_(and_(photo.c.user == user_id, photo.c.playlog_num == playlog_num), photo.c.track_num == track_num))) + if result: + return result.fetchone() + + async def get_user_photos_by_user(self, user_id: int) -> Optional[List[Row]]: + result = await self.execute(photo.select(photo.c.user == user_id)) + if result: + return result.fetchall() + + async def delete_user_photo_by_id(self, photo_id: str) -> Optional[List[Row]]: + result = await self.execute(photo.delete(photo.c.id.like(photo_id))) + if not result: + self.logger.error(f"Failed to delete photo {photo_id}") + async def update_name(self, user_id: int, new_name: str) -> bool: sql = detail.update(detail.c.user == user_id).values( userName=new_name diff --git a/titles/mai2/templates/mai2_header.jinja b/titles/mai2/templates/mai2_header.jinja index f226fbe..7e6757f 100644 --- a/titles/mai2/templates/mai2_header.jinja +++ b/titles/mai2/templates/mai2_header.jinja @@ -3,6 +3,7 @@