Add support for converting lottie stickers

CHANGELOG.md

@@ -3,6 +3,8 @@
 * Started automatically subscribing to bridged guilds. This fixes two problems:
   * Typing notifications should now work automatically in guilds.
   * Huge guilds now actually get messages bridged.
+* Added support for converting animated lottie stickers to raster formats using
+  [lottieconverter](https://github.com/sot-tech/LottieConverter).
 * Improved markdown parsing to disable more features that don't exist on Discord.
 * Removed width from inline images (e.g. in the `guilds status` output) to
   handle non-square images properly.

Dockerfile

@@ -1,3 +1,5 @@
+FROM dock.mau.dev/tulir/lottieconverter AS lottie
+
 FROM golang:1-alpine3.17 AS builder
 
 RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev
@@ -11,8 +13,11 @@ FROM alpine:3.17
 ENV UID=1337 \
     GID=1337
 
-RUN apk add --no-cache ffmpeg su-exec ca-certificates olm bash jq yq curl
+RUN apk add --no-cache ffmpeg su-exec ca-certificates olm bash jq yq curl \
+    zlib libpng giflib libstdc++ libgcc
 
+COPY --from=lottie /usr/lib/librlottie.so* /usr/lib/
+COPY --from=lottie /usr/local/bin/lottieconverter /usr/local/bin/lottieconverter
 COPY --from=builder /usr/bin/mautrix-discord /usr/bin/mautrix-discord
 COPY --from=builder /build/example-config.yaml /opt/mautrix-discord/example-config.yaml
 COPY --from=builder /build/docker-run.sh /docker-run.sh

Dockerfile.ci

@@ -1,10 +1,15 @@
+FROM dock.mau.dev/tulir/lottieconverter AS lottie
+
 FROM alpine:3.17
 
 ENV UID=1337 \
     GID=1337
 
-RUN apk add --no-cache ffmpeg su-exec ca-certificates bash jq curl yq
+RUN apk add --no-cache ffmpeg su-exec ca-certificates bash jq curl yq \
+    zlib libpng giflib libstdc++ libgcc
 
+COPY --from=lottie /usr/lib/librlottie.so* /usr/lib/
+COPY --from=lottie /usr/local/bin/lottieconverter /usr/local/bin/lottieconverter
 ARG EXECUTABLE=./mautrix-discord
 COPY $EXECUTABLE /usr/bin/mautrix-discord
 COPY ./example-config.yaml /opt/mautrix-discord/example-config.yaml

Dockerfile.dev

@@ -1,7 +1,12 @@
+FROM dock.mau.dev/tulir/lottieconverter AS lottie
+
 FROM golang:1-alpine3.17 AS builder
 
-RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev bash jq yq curl
+RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev bash jq yq curl \
+    zlib libpng giflib libstdc++ libgcc
 
+COPY --from=lottie /usr/lib/librlottie.so* /usr/lib/
+COPY --from=lottie /usr/local/bin/lottieconverter /usr/local/bin/lottieconverter
 COPY . /build
 WORKDIR /build
 RUN go build -o /usr/bin/mautrix-discord

attachments.go

@@ -2,10 +2,15 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"image"
 	"io"
 	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -18,6 +23,7 @@ import (
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 	"maunium.net/go/mautrix/util"
+	"maunium.net/go/mautrix/util/ffmpeg"
 
 	"go.mau.fi/mautrix-discord/database"
 )
@@ -151,6 +157,7 @@ type AttachmentMeta struct {
 	MimeType      string
 	EmojiName     string
 	CopyIfMissing bool
+	Converter     func([]byte) ([]byte, string, error)
 }
 
 var NoMeta = AttachmentMeta{}
@@ -160,6 +167,78 @@ type attachmentKey struct {
 	Encrypt bool
 }
 
+func (br *DiscordBridge) convertLottie(data []byte) ([]byte, string, error) {
+	fps := br.Config.Bridge.AnimatedSticker.Args.FPS
+	width := br.Config.Bridge.AnimatedSticker.Args.Width
+	height := br.Config.Bridge.AnimatedSticker.Args.Height
+	target := br.Config.Bridge.AnimatedSticker.Target
+	var lottieTarget, outputMime string
+	switch target {
+	case "png":
+		lottieTarget = "png"
+		outputMime = "image/png"
+		fps = 1
+	case "gif":
+		lottieTarget = "gif"
+		outputMime = "image/gif"
+	case "webm":
+		lottieTarget = "pngs"
+		outputMime = "video/webm"
+	case "webp":
+		lottieTarget = "pngs"
+		outputMime = "image/webp"
+	case "disable":
+		return data, "application/json", nil
+	default:
+		return nil, "", fmt.Errorf("invalid animated sticker target %q in bridge config", br.Config.Bridge.AnimatedSticker.Target)
+	}
+
+	ctx := context.Background()
+	tempdir, err := os.MkdirTemp("", "mautrix_discord_lottie_")
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to create temp dir: %w", err)
+	}
+
+	lottieOutput := filepath.Join(tempdir, "out_")
+	if lottieTarget != "pngs" {
+		lottieOutput = filepath.Join(tempdir, "output."+lottieTarget)
+	}
+	cmd := exec.CommandContext(ctx, "lottieconverter", "-", lottieOutput, lottieTarget, fmt.Sprintf("%dx%d", width, height), strconv.Itoa(fps))
+	cmd.Stdin = bytes.NewReader(data)
+	err = cmd.Run()
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to run lottieconverter: %w", err)
+	}
+	var path string
+	if lottieTarget == "pngs" {
+		var videoCodec string
+		outputExtension := "." + target
+		if target == "webm" {
+			videoCodec = "libvpx-vp9"
+		} else if target == "webp" {
+			videoCodec = "libwebp_anim"
+		} else {
+			panic(fmt.Errorf("impossible case: unknown target %q", target))
+		}
+		path, err = ffmpeg.ConvertPath(
+			ctx, lottieOutput+"*.png", outputExtension,
+			[]string{"-framerate", strconv.Itoa(fps), "-pattern_type", "glob"},
+			[]string{"-c:v", videoCodec, "-pix_fmt", "yuva420p", "-f", target},
+			false,
+		)
+		if err != nil {
+			return nil, "", fmt.Errorf("failed to run ffmpeg: %w", err)
+		}
+	} else {
+		path = lottieOutput
+	}
+	data, err = os.ReadFile(path)
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to read converted file: %w", err)
+	}
+	return data, outputMime, nil
+}
+
 func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, url string, encrypt bool, meta AttachmentMeta) (returnDBFile *database.File, returnErr error) {
 	isCacheable := !encrypt
 	returnDBFile = br.DB.File.Get(url, encrypt)
@@ -180,6 +259,14 @@ func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, ur
 				return
 			}
 
+			if meta.Converter != nil {
+				data, meta.MimeType, onceErr = meta.Converter(data)
+				if onceErr != nil {
+					onceErr = fmt.Errorf("failed to convert attachment: %w", onceErr)
+					return
+				}
+			}
+
 			onceDBFile, onceErr = br.uploadMatrixAttachment(intent, data, url, encrypt, meta)
 			if onceErr != nil {
 				return

config/bridge.go

@@ -50,6 +50,14 @@ type BridgeConfig struct {
 	DeletePortalOnChannelDelete bool `yaml:"delete_portal_on_channel_delete"`
 	DeleteGuildOnLeave          bool `yaml:"delete_guild_on_leave"`
 	FederateRooms               bool `yaml:"federate_rooms"`
+	AnimatedSticker             struct {
+		Target string `yaml:"target"`
+		Args   struct {
+			Width  int `yaml:"width"`
+			Height int `yaml:"height"`
+			FPS    int `yaml:"fps"`
+		} `yaml:"args"`
+	} `yaml:"animated_sticker"`
 
 	DoublePuppetServerMap      map[string]string `yaml:"double_puppet_server_map"`
 	DoublePuppetAllowDiscovery bool              `yaml:"double_puppet_allow_discovery"`

config/upgrade.go

@@ -45,6 +45,10 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Bool, "bridge", "delete_portal_on_channel_delete")
 	helper.Copy(up.Bool, "bridge", "delete_guild_on_leave")
 	helper.Copy(up.Bool, "bridge", "federate_rooms")
+	helper.Copy(up.Str, "bridge", "animated_sticker", "target")
+	helper.Copy(up.Int, "bridge", "animated_sticker", "args", "width")
+	helper.Copy(up.Int, "bridge", "animated_sticker", "args", "height")
+	helper.Copy(up.Int, "bridge", "animated_sticker", "args", "fps")
 	helper.Copy(up.Map, "bridge", "double_puppet_server_map")
 	helper.Copy(up.Bool, "bridge", "double_puppet_allow_discovery")
 	helper.Copy(up.Map, "bridge", "login_shared_secret_map")

example-config.yaml

@@ -140,6 +140,20 @@ bridge:
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
     federate_rooms: true
+    # Settings for converting animated stickers.
+    animated_sticker:
+        # Format to which animated stickers should be converted.
+        # disable - No conversion, send as-is (lottie JSON)
+        # png - converts to non-animated png (fastest)
+        # gif - converts to animated gif
+        # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
+        # webp - converts to animated webp, requires ffmpeg executable with webp codec/container support
+        target: webp
+        # Arguments for converter. All converters take width and height.
+        args:
+            width: 256
+            height: 256
+            fps: 25 # only for webm, webp and gif (2, 5, 10, 20 or 25 recommended)
     # Servers to always allow double puppeting from
     double_puppet_server_map:
         example.com: https://example.com

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/stretchr/testify v1.8.1
 	github.com/yuin/goldmark v1.5.4
 	maunium.net/go/maulogger/v2 v2.3.2
-	maunium.net/go/mautrix v0.13.1-0.20230204122701-3c0e64060114
+	maunium.net/go/mautrix v0.13.1-0.20230204140716-485b4f376dc6
 )
 
 require (

go.sum

@@ -77,5 +77,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
 maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
 maunium.net/go/maulogger/v2 v2.3.2 h1:1XmIYmMd3PoQfp9J+PaHhpt80zpfmMqaShzUTC7FwY0=
 maunium.net/go/maulogger/v2 v2.3.2/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
-maunium.net/go/mautrix v0.13.1-0.20230204122701-3c0e64060114 h1:H6/OwVn9Z5PNhwzeWSvYCU/Cw3nvTbLTcvJo5HS/lyU=
-maunium.net/go/mautrix v0.13.1-0.20230204122701-3c0e64060114/go.mod h1:3u2Fz3JY/eXVLOzxn2ODVL4rzCIjkkt0jlygEx4Qnaw=
+maunium.net/go/mautrix v0.13.1-0.20230204140716-485b4f376dc6 h1:mSq0HwzhpM5XOk+YRgOsEx62AVG6N/lonmz/3iBwf+A=
+maunium.net/go/mautrix v0.13.1-0.20230204140716-485b4f376dc6/go.mod h1:3u2Fz3JY/eXVLOzxn2ODVL4rzCIjkkt0jlygEx4Qnaw=

portal.go

@@ -555,7 +555,11 @@ func (portal *Portal) sendMediaFailedMessage(intent *appservice.IntentAPI, bridg
 const DiscordStickerSize = 160
 
 func (portal *Portal) handleDiscordFile(typeName string, intent *appservice.IntentAPI, id, url string, content *event.MessageEventContent, ts time.Time, threadRelation *event.RelatesTo) *database.MessagePart {
-	dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, url, portal.Encrypted, AttachmentMeta{AttachmentID: id, MimeType: content.Info.MimeType})
+	meta := AttachmentMeta{AttachmentID: id, MimeType: content.Info.MimeType}
+	if typeName == "sticker" && content.Info.MimeType == "application/json" {
+		meta.Converter = portal.bridge.convertLottie
+	}
+	dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, url, portal.Encrypted, meta)
 	if err != nil {
 		errorEventID := portal.sendMediaFailedMessage(intent, err)
 		if errorEventID != "" {
@@ -566,6 +570,9 @@ func (portal *Portal) handleDiscordFile(typeName string, intent *appservice.Inte
 		}
 		return nil
 	}
+	if typeName == "sticker" && content.Info.MimeType == "application/json" {
+		content.Info.MimeType = dbFile.MimeType
+	}
 	content.Info.Size = dbFile.Size
 	if content.Info.Width == 0 && content.Info.Height == 0 {
 		content.Info.Width = dbFile.Width