Implement bridging visual_media, link and all share items

ROADMAP.md

@@ -17,14 +17,13 @@
 * Instagram → Matrix
   * [ ] Message content
     * [x] Text
-    * [ ] Media
+    * [x] Media
       * [x] Images
       * [x] Videos
       * [x] Gifs
       * [x] Voice messages
       * [x] Locations
-      * [ ] Media share
-      * [ ] Other random types
+      * [x] Story/reel share
   * [x] Message unsend
   * [x] Message reactions
   * [x] Message history

mauigpapi/types/__init__.py

@@ -13,7 +13,8 @@ from .thread_item import (ThreadItemType, ThreadItemActionLog, ViewMode, Creativ
                           RegularMediaItem, MediaShareItem, ReplayableMediaItem, VideoVersion,
                           AudioInfo, VoiceMediaItem, AnimatedMediaImage, AnimatedMediaImages,
                           AnimatedMediaItem, ThreadItem, VoiceMediaData, Reaction, Reactions,
-                          Location)
+                          Location, ExpiredMediaItem, ReelMediaShareItem, ReelShareItem, LinkItem,
+                          ReelShareType, ReelShareReactionInfo, SharingFrictionInfo, LinkContext)
 from .thread import Thread, ThreadUser, ThreadItem, ThreadUserLastSeenAt, ThreadTheme
 from .mqtt import (Operation, ThreadAction, ReactionStatus, TypingStatus, CommandResponsePayload,
                    CommandResponse, IrisPayloadData, IrisPayload, MessageSyncMessage,

mauigpapi/types/account.py

@@ -41,7 +41,7 @@ class UserIdentifier(SerializableAttrs['UserIdentifier']):
 class BaseResponseUser(UserIdentifier, SerializableAttrs['BaseResponseUser']):
     full_name: str
     is_private: bool
-    is_verified: bool
+    is_verified: bool = False
     profile_pic_url: str
     # When this doesn't exist, the profile picture is probably the default one
     profile_pic_id: Optional[str] = None

mauigpapi/types/thread_item.py

@@ -13,11 +13,12 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Any, Optional
+from typing import List, Optional, Union, Any
 
 import attr
 from attr import dataclass
-from mautrix.types import SerializableAttrs, SerializableEnum
+from mautrix.types import SerializableAttrs, SerializableEnum, JSON
+from mautrix.types.util.serializable_attrs import _dict_to_attrs
 
 from .account import BaseResponseUser, UserIdentifier
 
@@ -122,19 +123,26 @@ class MediaType(SerializableEnum):
     SHOWREEL_NATIVE = 12
 
 
+@dataclass(kw_only=True)
+class ExpiredMediaItem(SerializableAttrs['ExpiredMediaItem']):
+    media_type: MediaType
+
+
 @dataclass(kw_only=True)
 class RegularMediaItem(SerializableAttrs['RegularMediaItem']):
     id: str
     image_versions2: Optional[ImageVersions] = None
     video_versions: Optional[List[VideoVersion]] = None
-    original_width: int
-    original_height: int
+    original_width: Optional[int] = None
+    original_height: Optional[int] = None
     media_type: MediaType
     media_id: Optional[int] = None
     organic_tracking_token: Optional[str] = None
     creative_config: Optional[CreativeConfig] = None
     create_mode_attribution: Optional[CreateModeAttribution] = None
 
+    # TODO carousel_media shares
+
     @property
     def best_image(self) -> Optional[ImageVersion]:
         if not self.image_versions2:
@@ -179,27 +187,37 @@ class Caption(SerializableAttrs['Caption']):
     media_id: int
 
 
+@dataclass
+class Location(SerializableAttrs['Location']):
+    pk: int
+    short_name: str
+    facebook_places_id: int
+    # TODO enum?
+    external_source: str  # facebook_places
+    name: str
+    address: str
+    city: str
+    lng: float
+    lat: float
+
+
 @dataclass(kw_only=True)
-class MediaShareItem(SerializableAttrs['MediaShareItem']):
+class MediaShareItem(RegularMediaItem, SerializableAttrs['MediaShareItem']):
     taken_at: int
     pk: int
-    id: str
     device_timestamp: int
-    media_type: MediaType
     code: str
     client_cache_key: str
     filter_type: int
-    image_versions2: ImageVersions
-    video_versions: VideoVersion
-    original_width: int
-    original_height: int
     user: BaseResponseUser
-    can_viewer_reshare: bool
+    # Not present in reel shares
+    can_viewer_reshare: Optional[bool] = None
     caption_is_edited: bool
     comment_likes_enabled: bool
     comment_threading_enabled: bool
     has_more_comments: bool
     max_num_visible_preview_comments: int
+    # preview_comments: List[TODO]
     can_view_more_preview_comments: bool
     comment_count: int
     like_count: int
@@ -207,7 +225,35 @@ class MediaShareItem(SerializableAttrs['MediaShareItem']):
     photo_of_you: bool
     caption: Caption
     can_viewer_save: bool
-    organic_tracking_token: str
+    location: Optional[Location] = None
+
+
+@dataclass
+class SharingFrictionInfo(SerializableAttrs['SharingFrictionInfo']):
+    should_have_sharing_friction: bool
+    bloks_app_url: Optional[str]
+
+
+# The fields in this class have been observed in reel share items, but may exist elsewhere too.
+# If they're observed in other types, they should be moved to MediaShareItem.
+@dataclass
+class ReelMediaShareItem(MediaShareItem, SerializableAttrs['ReelMediaShareItem']):
+    # TODO enum?
+    caption_position: int
+    is_reel_media: bool
+    timezone_offset: int
+    # likers: List[TODO]
+    can_see_insights_as_brand: bool
+    expiring_at: int
+    sharing_friction_info: SharingFrictionInfo
+    is_in_profile_grid: bool
+    profile_grid_control_enabled: bool
+    is_shop_the_look_eligible: bool
+    # TODO enum?
+    deleted_reason: int
+    integrity_review_decision: str
+    # Not present in story_share, only reel_share
+    story_is_saved_to_archive: Optional[bool] = None
 
 
 @dataclass(kw_only=True)
@@ -215,14 +261,23 @@ class ReplayableMediaItem(SerializableAttrs['ReplayableMediaItem']):
     view_mode: ViewMode
     seen_count: int
     seen_user_ids: List[int]
-    replay_expiring_at_us: Optional[Any] = None
+    replay_expiring_at_us: Optional[int] = None
 
 
 @dataclass(kw_only=True)
 class VisualMedia(ReplayableMediaItem, SerializableAttrs['VisualMedia']):
-    url_expire_at_secs: int
-    playback_duration_secs: int
-    media: RegularMediaItem
+    url_expire_at_secs: Optional[int] = None
+    playback_duration_secs: Optional[int] = None
+    media: Union[RegularMediaItem, ExpiredMediaItem]
+
+    @classmethod
+    def deserialize(cls, data: JSON) -> 'VisualMedia':
+        data = {**data}
+        if "id" not in data["media"]:
+            data["media"] = ExpiredMediaItem.deserialize(data["media"])
+        else:
+            data["media"] = RegularMediaItem.deserialize(data["media"])
+        return _dict_to_attrs(cls, data)
 
 
 @dataclass(kw_only=True)
@@ -291,17 +346,54 @@ class Reactions(SerializableAttrs['Reactions']):
 
 
 @dataclass
-class Location(SerializableAttrs['Location']):
-    pk: int
-    short_name: str
-    facebook_places_id: int
+class LinkContext(SerializableAttrs['LinkContext']):
+    link_url: str
+    link_title: str
+    link_summary: str
+    link_image_url: str
+
+
+@dataclass
+class LinkItem(SerializableAttrs['LinkItem']):
+    text: str
+    link_context: LinkContext
+    client_context: str
+    mutation_token: str
+
+
+class ReelShareType(SerializableEnum):
+    REPLY = "reply"
+    REACTION = "reaction"
+
+
+@dataclass
+class ReelShareReactionInfo(SerializableAttrs['ReelShareReactionInfo']):
+    emoji: str
+    # TODO find type
+    # intensity: Any
+
+
+@dataclass
+class ReelShareItem(SerializableAttrs['ReelShareItem']):
+    text: str
+    type: ReelShareType
+    reel_owner_id: int
+    is_reel_persisted: int
+    reel_type: str
+    media: ReelMediaShareItem
+    reaction_info: Optional[ReelShareReactionInfo] = None
+
+
+@dataclass
+class StoryShareItem(SerializableAttrs['StoryShareItem']):
+    text: str
+    is_reel_persisted: bool
     # TODO enum?
-    external_source: str  # facebook_places
-    name: str
-    address: str
-    city: str
-    lng: float
-    lat: float
+    reel_type: str  # user_reel
+    reel_id: str
+    # TODO enum?
+    story_share_type: str  # default
+    media: ReelMediaShareItem
 
 
 @dataclass(kw_only=True)
@@ -322,6 +414,9 @@ class ThreadItem(SerializableAttrs['ThreadItem']):
     animated_media: Optional[AnimatedMediaItem] = None
     visual_media: Optional[VisualMedia] = None
     media_share: Optional[MediaShareItem] = None
+    reel_share: Optional[ReelShareItem] = None
+    story_share: Optional[StoryShareItem] = None
     location: Optional[Location] = None
     reactions: Optional[Reactions] = None
     like: Optional[str] = None
+    link: Optional[LinkItem] = None

mautrix_instagram/db/message.py

@@ -62,3 +62,7 @@ class Message:
         if not row:
             return None
         return cls(**row)
+
+    @property
+    def is_internal(self) -> bool:
+        return self.item_id.startswith("fi.mau.instagram.")

mautrix_instagram/matrix.py

@@ -99,7 +99,7 @@ class MatrixHandler(BaseMatrixHandler):
     async def handle_read_receipt(self, user: 'u.User', portal: 'po.Portal', event_id: EventID,
                                   data: SingleReceiptEventContent) -> None:
         message = await DBMessage.get_by_mxid(event_id, portal.mxid)
-        if not message:
+        if not message or message.is_internal:
             return
         # TODO implement
         # user.log.debug(f"Marking messages in {portal.thread_id} read up to {message.item_id}")

mautrix_instagram/portal.py

@@ -24,7 +24,7 @@ import magic
 
 from mauigpapi.types import (Thread, ThreadUser, ThreadItem, RegularMediaItem, MediaType,
                              ReactionStatus, Reaction, AnimatedMediaItem, ThreadItemType,
-                             VoiceMediaItem, Location)
+                             VoiceMediaItem, ExpiredMediaItem, MediaShareItem, ReelShareType)
 from mautrix.appservice import AppService, IntentAPI
 from mautrix.bridge import BasePortal, NotificationDisabler
 from mautrix.types import (EventID, MessageEventContent, RoomID, EventType, MessageType, ImageInfo,
@@ -189,7 +189,7 @@ class Portal(DBPortal, BasePortal):
     async def handle_matrix_reaction(self, sender: 'u.User', event_id: EventID,
                                      reacting_to: EventID, emoji: str) -> None:
         message = await DBMessage.get_by_mxid(reacting_to, self.mxid)
-        if not message:
+        if not message or message.is_internal:
             self.log.debug(f"Ignoring reaction to unknown event {reacting_to}")
             return
 
@@ -226,7 +226,7 @@ class Portal(DBPortal, BasePortal):
             return
 
         message = await DBMessage.get_by_mxid(event_id, self.mxid)
-        if message:
+        if message and not message.is_internal:
             try:
                 await message.delete()
                 await sender.client.delete_item(self.thread_id, message.item_id)
@@ -252,11 +252,15 @@ class Portal(DBPortal, BasePortal):
                                         intent: IntentAPI) -> Optional[ReuploadedMediaInfo]:
         if media.media_type == MediaType.IMAGE:
             image = media.best_image
+            if not image:
+                return None
             url = image.url
             msgtype = MessageType.IMAGE
             info = ImageInfo(height=image.height, width=image.width)
         elif media.media_type == MediaType.VIDEO:
             video = media.best_video
+            if not video:
+                return None
             url = video.url
             msgtype = MessageType.VIDEO
             info = VideoInfo(height=video.height, width=video.width)
@@ -315,11 +319,25 @@ class Portal(DBPortal, BasePortal):
                                       ) -> Optional[EventID]:
         if item.media:
             reuploaded = await self._reupload_instagram_media(source, item.media, intent)
+        elif item.visual_media:
+            if isinstance(item.visual_media.media, ExpiredMediaItem):
+                # TODO send error message instead
+                return None
+            reuploaded = await self._reupload_instagram_media(source, item.visual_media.media,
+                                                              intent)
         elif item.animated_media:
             reuploaded = await self._reupload_instagram_animated(source, item.animated_media,
                                                                  intent)
         elif item.voice_media:
             reuploaded = await self._reupload_instagram_voice(source, item.voice_media, intent)
+        elif item.reel_share:
+            reuploaded = await self._reupload_instagram_media(source, item.reel_share.media,
+                                                              intent)
+        elif item.story_share:
+            reuploaded = await self._reupload_instagram_media(source, item.story_share.media,
+                                                              intent)
+        elif item.media_share:
+            reuploaded = await self._reupload_instagram_media(source, item.media_share, intent)
         else:
             reuploaded = None
         if not reuploaded:
@@ -330,9 +348,56 @@ class Portal(DBPortal, BasePortal):
                                            info=reuploaded.info, msgtype=reuploaded.msgtype)
         return await self._send_message(intent, content, timestamp=item.timestamp // 1000)
 
-    async def _handle_instagram_text(self, intent: IntentAPI, item: ThreadItem) -> EventID:
-        content = TextMessageEventContent(msgtype=MessageType.TEXT, body=item.text or item.like)
-        return await self._send_message(intent, content, timestamp=item.timestamp // 1000)
+    async def _handle_instagram_media_share(self, source: 'u.User', intent: IntentAPI,
+                                            item: ThreadItem) -> Optional[EventID]:
+        share_item = item.media_share or item.story_share.media
+        if share_item == item.media_share:
+            prefix_text = f"Sent {share_item.user.username}'s photo"
+        else:
+            prefix_text = f"Shared {share_item.user.username}'s story"
+        prefix = TextMessageEventContent(msgtype=MessageType.NOTICE, body=prefix_text)
+        await self._send_message(intent, prefix)
+        event_id = await self._handle_instagram_media(source, intent, item)
+        if share_item.caption:
+            body = f"> {share_item.caption.user.username}: {share_item.caption.text}"
+            formatted_body = (f"<blockquote><strong>{share_item.caption.user.username}</strong>"
+                              f" {share_item.caption.text}</blockquote>")
+            caption = TextMessageEventContent(msgtype=MessageType.TEXT, body=body,
+                                              formatted_body=formatted_body, format=Format.HTML)
+            await self._send_message(intent, caption)
+        return event_id
+
+    async def _handle_instagram_reel_share(self, source: 'u.User', intent: IntentAPI,
+                                           item: ThreadItem) -> Optional[EventID]:
+        if item.reel_share.type == ReelShareType.REPLY:
+            if item.reel_share.reel_owner_id == source.igpk:
+                prefix = "Replied to your story"
+            else:
+                prefix = f"Sent {item.reel_share.media.user.username}'s story"
+        elif item.reel_share.type == ReelShareType.REACTION:
+            prefix = "Reacted to your story"
+        else:
+            self.log.debug(f"Unsupported reel share type {item.reel_share.type}")
+            return None
+        prefix = TextMessageEventContent(msgtype=MessageType.NOTICE, body=prefix)
+        content = TextMessageEventContent(msgtype=MessageType.TEXT, body=item.text)
+        await self._send_message(intent, prefix)
+        fake_item_id = f"fi.mau.instagram.reel_share_item.{item.reel_share.media.pk}"
+        existing = await DBMessage.get_by_item_id(fake_item_id, self.receiver)
+        if existing:
+            # If the user already reacted or replied to the same reel share item,
+            # use a Matrix reply instead of reposting the image.
+            content.set_reply(existing.mxid)
+        else:
+            media_event_id = await self._handle_instagram_media(source, intent, item)
+            await DBMessage(mxid=media_event_id, mx_room=self.mxid, item_id=fake_item_id,
+                            receiver=self.receiver, sender=item.reel_share.media.user.pk).insert()
+        return await self._send_message(intent, content)
+
+    async def _handle_instagram_text(self, intent: IntentAPI, text: str, timestamp: int
+                                     ) -> EventID:
+        content = TextMessageEventContent(msgtype=MessageType.TEXT, body=text)
+        return await self._send_message(intent, content, timestamp=timestamp // 1000)
 
     async def _handle_instagram_location(self, intent: IntentAPI, item: ThreadItem) -> EventID:
         loc = item.location
@@ -378,13 +443,22 @@ class Portal(DBPortal, BasePortal):
             else:
                 intent = sender.intent_for(self)
             event_id = None
-            if item.media or item.animated_media or item.voice_media:
+            if item.media or item.animated_media or item.voice_media or item.visual_media:
                 event_id = await self._handle_instagram_media(source, intent, item)
             elif item.location:
                 event_id = await self._handle_instagram_location(intent, item)
-            # We handle likes as text because Matrix clients do big emoji on their own.
-            if item.text or item.like:
-                event_id = await self._handle_instagram_text(intent, item)
+            elif item.reel_share:
+                event_id = await self._handle_instagram_reel_share(source, intent, item)
+            elif item.media_share or item.story_share:
+                event_id = await self._handle_instagram_media_share(source, intent, item)
+            if item.text:
+                event_id = await self._handle_instagram_text(intent, item.text, item.timestamp)
+            elif item.like:
+                # We handle likes as text because Matrix clients do big emoji on their own.
+                event_id = await self._handle_instagram_text(intent, item.like, item.timestamp)
+            elif item.link:
+                event_id = await self._handle_instagram_text(intent, item.link.text,
+                                                             item.timestamp)
             if event_id:
                 msg = DBMessage(mxid=event_id, mx_room=self.mxid, item_id=item.item_id,
                                 receiver=self.receiver, sender=sender.pk)