Add support for Matrix->Instagram audio and video files

ROADMAP.md

@@ -5,8 +5,8 @@
     * [x] Text
     * [ ] Media
       * [x] Images
-      * [ ] Videos
-      * [ ] Voice messages
+      * [x] Videos
+      * [x] Voice messages
       * [ ] Locations
       * [ ] †Files
     * [ ] Replies
@@ -42,9 +42,7 @@
     * [x] At startup
     * [x] When receiving message
   * [ ] Private chat creation by inviting Matrix puppet of Instagram user to new room
-  * [ ] Option to use own Matrix account for messages sent from other Instagram clients
-    * [x] Automatic login with shared secret
-    * [ ] Manual login with `login-matrix`
+  * [x] Option to use own Matrix account for messages sent from other Instagram clients
   * [x] End-to-bridge encryption in Matrix rooms
 
 † Not supported on Instagram

mauigpapi/http/base.py

@@ -107,11 +107,13 @@ class BaseAndroidAPI:
 
     async def std_http_post(self, path: str, data: Optional[JSON] = None, raw: bool = False,
                             filter_nulls: bool = False, headers: Optional[Dict[str, str]] = None,
+                            query: Optional[Dict[str, str]] = None,
                             response_type: Optional[Type[T]] = JSON) -> T:
         headers = {**self._headers, **headers} if headers else self._headers
         if not raw:
             data = self.sign(data, filter_nulls=filter_nulls)
-        resp = await self.http.post(url=self.url.with_path(path), headers=headers, data=data)
+        url = self.url.with_path(path).with_query(query or {})
+        resp = await self.http.post(url=url, headers=headers, data=data)
         self.log.trace(f"{path} response: {await resp.text()}")
         if response_type is str or response_type is None:
             self._handle_response_headers(resp)

mauigpapi/http/thread.py

@@ -1,5 +1,5 @@
 # mautrix-instagram - A Matrix-Instagram puppeting bridge.
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,12 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, AsyncIterable
-from uuid import uuid4
+from typing import Optional, AsyncIterable, Type, Union
 
-from .base import BaseAndroidAPI
+from .base import BaseAndroidAPI, T
 from ..types import (DMInboxResponse, DMThreadResponse, Thread, ThreadItem, ThreadAction,
-                     ThreadItemType, CommandResponse)
+                     ThreadItemType, CommandResponse, ShareVoiceResponse)
 
 
 class ThreadAPI(BaseAndroidAPI):
@@ -86,13 +85,15 @@ class ThreadAPI(BaseAndroidAPI):
                                  data={"_csrftoken": self.state.cookies.csrf_token,
                                        "_uuid": self.state.device.uuid})
 
-    async def broadcast(self, thread_id: str, item_type: ThreadItemType, signed: bool = False,
-                        client_context: Optional[str] = None, **kwargs) -> CommandResponse:
+    async def _broadcast(self, thread_id: str, item_type: str, response_type: Type[T],
+                         signed: bool = False, client_context: Optional[str] = None, **kwargs
+                         ) -> T:
         client_context = client_context or self.state.gen_client_context()
         form = {
             "action": ThreadAction.SEND_ITEM.value,
-            "send_attribution": "inbox",
-            "thread_id": thread_id,
+            "send_attribution": "direct_thread",
+            "thread_ids": f"[{thread_id}]",
+            "is_shh_mode": "0",
             "client_context": client_context,
             "_csrftoken": self.state.cookies.csrf_token,
             "device_id": self.state.device.id,
@@ -101,5 +102,17 @@ class ThreadAPI(BaseAndroidAPI):
             **kwargs,
             "offline_threading_id": client_context,
         }
-        return await self.std_http_post(f"/api/v1/direct_v2/threads/broadcast/{item_type.value}/",
-                                        data=form, raw=not signed, response_type=CommandResponse)
+        return await self.std_http_post(f"/api/v1/direct_v2/threads/broadcast/{item_type}/",
+                                        data=form, raw=not signed, response_type=response_type)
+
+    async def broadcast(self, thread_id: str, item_type: ThreadItemType, signed: bool = False,
+                        client_context: Optional[str] = None, **kwargs) -> CommandResponse:
+        return await self._broadcast(thread_id, item_type.value, CommandResponse, signed,
+                                     client_context, **kwargs)
+
+    async def broadcast_audio(self, thread_id: str, is_direct: bool,
+                              client_context: Optional[str] = None, **kwargs
+                              ) -> Union[ShareVoiceResponse, CommandResponse]:
+        response_type = ShareVoiceResponse if is_direct else CommandResponse
+        return await self._broadcast(thread_id, "share_voice", response_type, False,
+                                     client_context, **kwargs)

mauigpapi/http/upload.py

@@ -1,5 +1,5 @@
 # mautrix-instagram - A Matrix-Instagram puppeting bridge.
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,20 +13,26 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Dict, Any
+from __future__ import annotations
+
 from uuid import uuid4
 import random
 import time
 import json
 
 from .base import BaseAndroidAPI
-from ..types import UploadPhotoResponse, MediaType
+from ..types import UploadPhotoResponse, UploadVideoResponse, FinishUploadResponse, MediaType
 
 
 class UploadAPI(BaseAndroidAPI):
-    async def upload_jpeg_photo(self, data: bytes, upload_id: Optional[str] = None,
-                                is_sidecar: bool = False, waterfall_id: Optional[str] = None,
-                                media_type: MediaType = MediaType.IMAGE) -> UploadPhotoResponse:
+    async def upload_photo(
+        self,
+        data: bytes,
+        mime: str,
+        upload_id: str | None = None,
+        width: int | None = None,
+        height: int | None = None,
+    ) -> UploadPhotoResponse:
         upload_id = upload_id or str(int(time.time() * 1000))
         name = f"{upload_id}_0_{random.randint(1000000000, 9999999999)}"
         params = {
@@ -35,30 +41,85 @@ class UploadAPI(BaseAndroidAPI):
                 "num_reupload": 0,
                 "num_step_manual_retry": 0,
             }),
-            "media_type": str(media_type.value),
+            "media_type": str(MediaType.IMAGE.value),
             "upload_id": upload_id,
             "xsharing_user_ids": json.dumps([]),
-            "image_compression": json.dumps({
+        }
+        if mime == "image/jpeg":
+            params["image_compression"] = json.dumps({
                 "lib_name": "moz",
                 "lib_version": "3.1.m",
                 "quality": 80
-            }),
-        }
-        if is_sidecar:
-            params["is_sidecar"] = "1"
+            })
+        if width and height:
+            params["original_width"] = str(width)
+            params["original_height"] = str(height)
         headers = {
-            "X_FB_PHOTO_WATERFALL_ID": waterfall_id or str(uuid4()),
-            "X-Entity-Type": "image/jpeg",
+            "X_FB_PHOTO_WATERFALL_ID": str(uuid4()),
+            "X-Entity-Type": mime,
             "Offset": "0",
             "X-Instagram-Rupload-Params": json.dumps(params),
             "X-Entity-Name": name,
             "X-Entity-Length": str(len(data)),
             "Content-Type": "application/octet-stream",
+            "priority": "u=6, i",
         }
         return await self.std_http_post(f"/rupload_igphoto/{name}", headers=headers, data=data,
                                         raw=True, response_type=UploadPhotoResponse)
 
-    async def finish_upload(self, upload_id: str, source_type: str):
+    async def upload_mp4(
+        self,
+        data: bytes,
+        upload_id: str | None = None,
+        audio: bool = False,
+        duration_ms: int | None = None,
+        width: int | None = None,
+        height: int | None = None,
+    ) -> tuple[UploadVideoResponse, str]:
+        upload_id = upload_id or str(int(time.time() * 1000))
+        name = f"{upload_id}_0_{random.randint(1000000000, 9999999999)}"
+        media_type = MediaType.AUDIO if audio else MediaType.VIDEO
+        params: dict[str, str] = {
+            "retry_context": json.dumps({
+                "num_step_auto_retry": 0,
+                "num_reupload": 0,
+                "num_step_manual_retry": 0,
+            }),
+            "media_type": str(media_type.value),
+            "upload_id": upload_id,
+            "xsharing_user_ids": json.dumps([]),
+        }
+        if duration_ms:
+            params["upload_media_duration_ms"] = str(duration_ms)
+        if audio:
+            params["is_direct_voice"] = "1"
+        else:
+            params["direct_v2"] = "1"
+            params["for_direct_story"] = "1"
+            params["content_tags"] = "use_default_cover"
+            params["extract_cover_frame"] = "1"
+            if width and height:
+                params["upload_media_width"] = str(width)
+                params["upload_media_height"] = str(height)
+        headers = {
+            "X_FB_VIDEO_WATERFALL_ID": str(uuid4()),
+            "X-Entity-Type": "audio/mp4" if audio else "video/mp4",
+            "Offset": "0",
+            "X-Instagram-Rupload-Params": json.dumps(params),
+            "X-Entity-Name": name,
+            "X-Entity-Length": str(len(data)),
+            "Content-Type": "application/octet-stream",
+            "priority": "u=6, i",
+        }
+        if not audio:
+            headers["segment-type"] = "3"
+            headers["segment-start-offset"] = "0"
+        return await self.std_http_post(f"/rupload_igvideo/{name}", headers=headers, data=data,
+                                        raw=True, response_type=UploadVideoResponse), upload_id
+
+    async def finish_upload(
+        self, upload_id: str, source_type: str, video: bool = False
+    ) -> FinishUploadResponse:
         headers = {
             "retry_context": json.dumps({
                 "num_step_auto_retry": 0,
@@ -76,4 +137,8 @@ class UploadAPI(BaseAndroidAPI):
             "upload_id": upload_id,
             "device": self.state.device.payload,
         }
-        return await self.std_http_post("/api/v1/media/upload_finish/", headers=headers, data=req)
+        query = {}
+        if video:
+            query["video"] = "1"
+        return await self.std_http_post("/api/v1/media/upload_finish/", headers=headers, data=req,
+                                        query=query, response_type=FinishUploadResponse)

mauigpapi/types/__init__.py

@@ -7,7 +7,8 @@ from .account import (CurrentUser, EntityText, HDProfilePictureVersion, CurrentU
                       FriendshipStatus, UserIdentifier, BaseFullResponseUser, BaseResponseUser,
                       ProfileEditParams)
 from .direct_inbox import DMInboxResponse, DMInboxCursor, DMInbox, DMThreadResponse
-from .upload import UploadPhotoResponse
+from .upload import (UploadPhotoResponse, UploadVideoResponse, FinishUploadResponse,
+                     ShareVoiceResponse, ShareVoiceResponseMessage)
 from .thread_item import (ThreadItemType, ThreadItemActionLog, ViewMode, CreativeConfig, MediaType,
                           CreateModeAttribution, ImageVersion, ImageVersions, VisualMedia, Caption,
                           RegularMediaItem, MediaShareItem, ReplayableMediaItem, VideoVersion,

mauigpapi/types/thread_item.py

@@ -1,5 +1,5 @@
 # mautrix-instagram - A Matrix-Instagram puppeting bridge.
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -35,6 +35,8 @@ class ThreadItemType(SerializableEnum):
     PROFILE = "profile"
     MEDIA_SHARE = "media_share"
     CONFIGURE_PHOTO = "configure_photo"
+    CONFIGURE_VIDEO = "configure_video"
+    SHARE_VOICE = "share_voice"
     LOCATION = "location"
     ACTION_LOG = "action_log"
     TITLE = "title"

mauigpapi/types/upload.py

@@ -1,5 +1,5 @@
 # mautrix-instagram - A Matrix-Instagram puppeting bridge.
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,15 +13,47 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Any
+from typing import Any, List, Optional
 
 from attr import dataclass
 
 from mautrix.types import SerializableAttrs
 
 
+@dataclass
+class FinishUploadResponse(SerializableAttrs):
+    status: str
+
+
 @dataclass
 class UploadPhotoResponse(SerializableAttrs):
     upload_id: str
     status: str
     xsharing_nonces: Any
+
+
+@dataclass
+class UploadVideoResponse(SerializableAttrs):
+    status: str
+    xsharing_nonces: Any
+
+
+@dataclass(kw_only=True)
+class ShareVoiceResponseMessage(SerializableAttrs):
+    thread_id: Optional[str] = None
+    item_id: Optional[str] = None
+    timestamp: Optional[str] = None
+    client_context: Optional[str] = None
+    participant_ids: Optional[List[int]] = None
+    message: Optional[str] = None
+
+
+@dataclass
+class ShareVoiceResponse(SerializableAttrs):
+    message_metadata: List[ShareVoiceResponseMessage]
+    status: str
+    upload_id: str
+
+    @property
+    def payload(self) -> ShareVoiceResponseMessage:
+        return self.message_metadata[0]

mautrix_instagram/portal.py

@@ -1,5 +1,5 @@
 # mautrix-instagram - A Matrix-Instagram puppeting bridge.
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,6 +19,7 @@ from collections import deque
 from io import BytesIO
 import mimetypes
 import asyncio
+import json
 
 import asyncpg
 import magic
@@ -27,7 +28,8 @@ from mautrix.util.message_send_checkpoint import MessageSendCheckpointStatus
 from mauigpapi.types import (Thread, ThreadUser, ThreadItem, RegularMediaItem, MediaType,
                              ReactionStatus, Reaction, AnimatedMediaItem, ThreadItemType,
                              VoiceMediaItem, ExpiredMediaItem, MessageSyncMessage, ReelShareType,
-                             TypingStatus, ThreadUserLastSeenAt, MediaShareItem, ReelMediaShareItem)
+                             TypingStatus, ThreadUserLastSeenAt, MediaShareItem,
+                             ReelMediaShareItem, CommandResponse, ShareVoiceResponse)
 from mautrix.appservice import AppService, IntentAPI
 from mautrix.bridge import BasePortal, NotificationDisabler, async_getter_lock
 from mautrix.types import (EventID, MessageEventContent, RoomID, EventType, MessageType, ImageInfo,
@@ -35,7 +37,7 @@ from mautrix.types import (EventID, MessageEventContent, RoomID, EventType, Mess
                            ContentURI, LocationMessageEventContent, Format, UserID)
 from mautrix.errors import MatrixError, MForbidden, MNotFound, SessionNotFound
 from mautrix.util.simple_lock import SimpleLock
-from mautrix.util.ffmpeg import convert_bytes
+from mautrix.util import ffmpeg
 
 from .db import Portal as DBPortal, Message as DBMessage, Reaction as DBReaction
 from .config import Config
@@ -204,6 +206,80 @@ class Portal(DBPortal, BasePortal):
                 confirmed=True,
             )
 
+    async def _handle_matrix_image(
+        self, sender: 'u.User', event_id: EventID, request_id: str, data: bytes, mime_type: str,
+        width: Optional[int] = None, height: Optional[int] = None
+    ) -> CommandResponse:
+        if mime_type not in ("image/jpeg", "image/webp"):
+            with BytesIO(data) as inp, BytesIO() as out:
+                img = Image.open(inp)
+                img.convert("RGBA").save(out, format="WEBP")
+                data = out.getvalue()
+                mime_type = "image/webp"
+
+        self.log.trace(f"Uploading photo from {event_id} (mime: {mime_type})")
+        upload_resp = await sender.client.upload_photo(
+            data, mime=mime_type, width=width, height=height
+        )
+        self.log.trace(f"Broadcasting uploaded photo with request ID {request_id}")
+        return await sender.client.broadcast(
+            self.thread_id,
+            ThreadItemType.CONFIGURE_PHOTO,
+            client_context=request_id,
+            upload_id=upload_resp.upload_id,
+            allow_full_aspect_ratio="1",
+        )
+
+    async def _handle_matrix_video(
+        self, sender: 'u.User', event_id: EventID, request_id: str, data: bytes, mime_type: str,
+        duration: Optional[int] = None, width: Optional[int] = None, height: Optional[int] = None,
+    ) -> CommandResponse:
+        if mime_type != "video/mp4":
+            data = await ffmpeg.convert_bytes(
+                data,
+                output_extension=".mp4",
+                output_args=("-c:v", "libx264", "-c:a", "aac"),
+                input_mime=mime_type
+            )
+
+        self.log.trace(f"Uploading video from {event_id}")
+        _, upload_id = await sender.client.upload_mp4(
+            data, duration_ms=duration, width=width, height=height
+        )
+        self.log.trace(f"Broadcasting uploaded video with request ID {request_id}")
+        return await sender.client.broadcast(
+            self.thread_id,
+            ThreadItemType.CONFIGURE_VIDEO,
+            client_context=request_id,
+            upload_id=upload_id,
+            video_result="",
+        )
+
+    async def _handle_matrix_audio(
+        self, sender: 'u.User', event_id: EventID, request_id: str, data: bytes, mime_type: str,
+        waveform: List[int], duration: Optional[int] = None,
+    ) -> CommandResponse:
+        if mime_type != "audio/mp4":
+            data = await ffmpeg.convert_bytes(
+                data,
+                output_extension=".m4a",
+                output_args=("-c:a", "aac"),
+                input_mime=mime_type
+            )
+
+        self.log.trace(f"Uploading audio from {event_id}")
+        _, upload_id = await sender.client.upload_mp4(data, audio=True, duration_ms=duration)
+        self.log.trace(f"Broadcasting uploaded audio with request ID {request_id}")
+        return await sender.client.broadcast_audio(
+            self.thread_id,
+            is_direct=self.is_direct,
+            client_context=request_id,
+            upload_id=upload_id,
+            waveform=json.dumps([(part or 0) / 1024 for part in waveform],
+                                separators=(",", ":")),
+            waveform_sampling_frequency_hz="10",
+        )
+
     async def _handle_matrix_message(self, orig_sender: 'u.User', message: MessageEventContent,
                                      event_id: EventID) -> None:
         sender, is_relay = await self.get_relay_sender(orig_sender, f"message {event_id}")
@@ -232,25 +308,26 @@ class Portal(DBPortal, BasePortal):
             else:
                 data = await self.main_intent.download_media(message.url)
             mime_type = message.info.mimetype or magic.from_buffer(data, mime=True)
-            if mime_type != "image/jpeg" and mime_type.startswith("image/"):
-                with BytesIO(data) as inp:
-                    img = Image.open(inp)
-                    with BytesIO() as out:
-                        img.convert("RGB").save(out, format="JPEG", quality=80)
-                        data = out.getvalue()
-                mime_type = "image/jpeg"
-            if mime_type == "image/jpeg":
-                self.log.trace(f"Uploading photo from {event_id}")
-                upload_resp = await sender.client.upload_jpeg_photo(data)
-                self.log.trace(f"Broadcasting uploaded photo with request ID {request_id}")
-                # TODO is it possible to do this with MQTT?
-                resp = await sender.client.broadcast(self.thread_id,
-                                                     ThreadItemType.CONFIGURE_PHOTO,
-                                                     client_context=request_id,
-                                                     upload_id=upload_resp.upload_id,
-                                                     allow_full_aspect_ratio="1")
+            if message.msgtype == MessageType.IMAGE:
+                resp = await self._handle_matrix_image(
+                    sender, event_id, request_id, data, mime_type,
+                    width=message.info.width, height=message.info.height,
+                )
+            elif message.msgtype == MessageType.AUDIO:
+                waveform = message.get("org.matrix.msc1767.audio", {}).get("waveform", [0] * 30)
+                resp = await self._handle_matrix_audio(
+                    sender, event_id, request_id, data, mime_type,
+                    waveform, duration=message.info.duration,
+                )
+            elif message.msgtype == MessageType.VIDEO:
+                resp = await self._handle_matrix_video(
+                    sender, event_id, request_id, data, mime_type, duration=message.info.duration,
+                    width=message.info.width, height=message.info.height,
+                )
             else:
-                raise NotImplementedError("Non-image files are currently not supported")
+                raise NotImplementedError(
+                    "Non-image/video/audio files are currently not supported"
+                )
         else:
             raise NotImplementedError(f"Unknown message type {message.msgtype}")
 
@@ -466,13 +543,13 @@ class Portal(DBPortal, BasePortal):
     async def _reupload_instagram_voice(self, source: 'u.User', media: VoiceMediaItem,
                                         intent: IntentAPI) -> MediaMessageEventContent:
         async def convert_to_ogg(data, mimetype):
-            converted = await convert_bytes(data, ".ogg", output_args=('-c:a', 'libvorbis'),
-                                            input_mime=mimetype)
+            converted = await ffmpeg.convert_bytes(data, ".ogg", output_args=('-c:a', 'libvorbis'),
+                                                   input_mime=mimetype)
             return converted, "audio/ogg"
 
         url = media.media.audio.audio_src
         info = AudioInfo(duration=media.media.audio.duration)
-        waveform = [int(p * 1000) for p in media.media.audio.waveform_data]
+        waveform = [int(p * 1024) for p in media.media.audio.waveform_data]
         content = await self._reupload_instagram_file(
             source, url, MessageType.AUDIO, info, intent, convert_to_ogg
         )

optional-requirements.txt

@@ -9,4 +9,4 @@ unpaddedbase64>=1,<3
 prometheus_client>=0.6,<0.13
 
 #/imageconvert
-pillow>=4,<9
+pillow>=4,<10