Remove content from message table. Fixes #320

database/message.go

@@ -17,14 +17,11 @@
 package database
 
 import (
-	"bytes"
 	"database/sql"
-	"encoding/json"
 	"strings"
 	"time"
 
 	"github.com/Rhymen/go-whatsapp"
-	waProto "github.com/Rhymen/go-whatsapp/binary/proto"
 
 	log "maunium.net/go/maulogger/v2"
 
@@ -44,7 +41,7 @@ func (mq *MessageQuery) New() *Message {
 }
 
 func (mq *MessageQuery) GetAll(chat PortalKey) (messages []*Message) {
-	rows, err := mq.db.Query("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content FROM message WHERE chat_jid=$1 AND chat_receiver=$2", chat.JID, chat.Receiver)
+	rows, err := mq.db.Query("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent FROM message WHERE chat_jid=$1 AND chat_receiver=$2", chat.JID, chat.Receiver)
 	if err != nil || rows == nil {
 		return nil
 	}
@@ -56,12 +53,12 @@ func (mq *MessageQuery) GetAll(chat PortalKey) (messages []*Message) {
 }
 
 func (mq *MessageQuery) GetByJID(chat PortalKey, jid whatsapp.MessageID) *Message {
-	return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+
+	return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+
 		"FROM message WHERE chat_jid=$1 AND chat_receiver=$2 AND jid=$3", chat.JID, chat.Receiver, jid)
 }
 
 func (mq *MessageQuery) GetByMXID(mxid id.EventID) *Message {
-	return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+
+	return mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+
 		"FROM message WHERE mxid=$1", mxid)
 }
 
@@ -70,7 +67,7 @@ func (mq *MessageQuery) GetLastInChat(chat PortalKey) *Message {
 }
 
 func (mq *MessageQuery) GetLastInChatBefore(chat PortalKey, maxTimestamp int64) *Message {
-	msg := mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content "+
+	msg := mq.get("SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent "+
 		"FROM message WHERE chat_jid=$1 AND chat_receiver=$2 AND timestamp<=$3 AND sent=true ORDER BY timestamp DESC LIMIT 1",
 		chat.JID, chat.Receiver, maxTimestamp)
 	if msg == nil || msg.Timestamp == 0 {
@@ -98,7 +95,6 @@ type Message struct {
 	Sender    whatsapp.JID
 	Timestamp int64
 	Sent      bool
-	Content   *waProto.Message
 }
 
 func (msg *Message) IsFakeMXID() bool {
@@ -106,8 +102,7 @@ func (msg *Message) IsFakeMXID() bool {
 }
 
 func (msg *Message) Scan(row Scannable) *Message {
-	var content []byte
-	err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.Timestamp, &msg.Sent, &content)
+	err := row.Scan(&msg.Chat.JID, &msg.Chat.Receiver, &msg.JID, &msg.MXID, &msg.Sender, &msg.Timestamp, &msg.Sent)
 	if err != nil {
 		if err != sql.ErrNoRows {
 			msg.log.Errorln("Database scan failed:", err)
@@ -115,36 +110,14 @@ func (msg *Message) Scan(row Scannable) *Message {
 		return nil
 	}
 
-	msg.decodeBinaryContent(content)
-
 	return msg
 }
 
-func (msg *Message) decodeBinaryContent(content []byte) {
-	msg.Content = &waProto.Message{}
-	reader := bytes.NewReader(content)
-	dec := json.NewDecoder(reader)
-	err := dec.Decode(msg.Content)
-	if err != nil {
-		msg.log.Warnln("Failed to decode message content:", err)
-	}
-}
-
-func (msg *Message) encodeBinaryContent() []byte {
-	var buf bytes.Buffer
-	enc := json.NewEncoder(&buf)
-	err := enc.Encode(msg.Content)
-	if err != nil {
-		msg.log.Warnln("Failed to encode message content:", err)
-	}
-	return buf.Bytes()
-}
-
 func (msg *Message) Insert() {
 	_, err := msg.db.Exec(`INSERT INTO message
-			(chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent, content)
-			VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
-		msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, msg.Sender, msg.Timestamp, msg.Sent, msg.encodeBinaryContent())
+			(chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent)
+			VALUES ($1, $2, $3, $4, $5, $6, $7)`,
+		msg.Chat.JID, msg.Chat.Receiver, msg.JID, msg.MXID, msg.Sender, msg.Timestamp, msg.Sent)
 	if err != nil {
 		msg.log.Warnfln("Failed to insert %s@%s: %v", msg.Chat, msg.JID, err)
 	}

database/upgrades/2021-08-19-remove-message-content.go

@@ -0,0 +1,44 @@
+package upgrades
+
+import (
+	"database/sql"
+)
+
+func init() {
+	upgrades[21] = upgrade{"Remove message content from local database", func(tx *sql.Tx, ctx context) error {
+		if ctx.dialect == SQLite {
+			_, err := tx.Exec("ALTER TABLE message RENAME TO old_message")
+			if err != nil {
+				return err
+			}
+			_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS message (
+				chat_jid      TEXT,
+				chat_receiver TEXT,
+				jid           TEXT,
+				mxid          TEXT    NOT NULL UNIQUE,
+				sender        TEXT    NOT NULL,
+				timestamp     BIGINT  NOT NULL,
+				sent          BOOLEAN NOT NULL,
+
+				PRIMARY KEY (chat_jid, chat_receiver, jid),
+				FOREIGN KEY (chat_jid, chat_receiver) REFERENCES portal(jid, receiver) ON DELETE CASCADE
+			)`)
+			if err != nil {
+				return err
+			}
+			_, err = tx.Exec("INSERT INTO message SELECT chat_jid, chat_receiver, jid, mxid, sender, timestamp, sent FROM old_message")
+			return err
+		} else {
+			_, err := tx.Exec(`ALTER TABLE message DROP COLUMN content`)
+			if err != nil {
+				return err
+			}
+			_, err = tx.Exec(`ALTER TABLE message ALTER COLUMN timestamp DROP DEFAULT`)
+			if err != nil {
+				return err
+			}
+			_, err = tx.Exec(`ALTER TABLE message ALTER COLUMN sent DROP DEFAULT`)
+			return err
+		}
+	}}
+}

database/upgrades/2021-08-19-varchar-to-text-crypto.go

@@ -0,0 +1,13 @@
+package upgrades
+
+import (
+	"database/sql"
+
+	"maunium.net/go/mautrix/crypto/sql_store_upgrade"
+)
+
+func init() {
+	upgrades[23] = upgrade{"Replace VARCHAR(255) with TEXT in the crypto database", func(tx *sql.Tx, ctx context) error {
+		return sql_store_upgrade.Upgrades[4](tx, ctx.dialect.String())
+	}}
+}

database/upgrades/2021-08-19-varchar-to-text.go

@@ -0,0 +1,48 @@
+package upgrades
+
+import (
+	"database/sql"
+)
+
+func init() {
+	upgrades[22] = upgrade{"Replace VARCHAR(255) with TEXT in the database", func(tx *sql.Tx, ctx context) error {
+		if ctx.dialect == SQLite {
+			// SQLite doesn't enforce varchar sizes anyway
+			return nil
+		}
+		return execMany(tx,
+			`ALTER TABLE message ALTER COLUMN chat_jid TYPE TEXT`,
+			`ALTER TABLE message ALTER COLUMN chat_receiver TYPE TEXT`,
+			`ALTER TABLE message ALTER COLUMN jid TYPE TEXT`,
+			`ALTER TABLE message ALTER COLUMN mxid TYPE TEXT`,
+			`ALTER TABLE message ALTER COLUMN sender TYPE TEXT`,
+
+			`ALTER TABLE portal ALTER COLUMN jid TYPE TEXT`,
+			`ALTER TABLE portal ALTER COLUMN receiver TYPE TEXT`,
+			`ALTER TABLE portal ALTER COLUMN mxid TYPE TEXT`,
+			`ALTER TABLE portal ALTER COLUMN name TYPE TEXT`,
+			`ALTER TABLE portal ALTER COLUMN topic TYPE TEXT`,
+			`ALTER TABLE portal ALTER COLUMN avatar TYPE TEXT`,
+			`ALTER TABLE portal ALTER COLUMN avatar_url TYPE TEXT`,
+
+			`ALTER TABLE puppet ALTER COLUMN jid TYPE TEXT`,
+			`ALTER TABLE puppet ALTER COLUMN avatar TYPE TEXT`,
+			`ALTER TABLE puppet ALTER COLUMN displayname TYPE TEXT`,
+			`ALTER TABLE puppet ALTER COLUMN custom_mxid TYPE TEXT`,
+			`ALTER TABLE puppet ALTER COLUMN access_token TYPE TEXT`,
+			`ALTER TABLE puppet ALTER COLUMN next_batch TYPE TEXT`,
+			`ALTER TABLE puppet ALTER COLUMN avatar_url TYPE TEXT`,
+
+			`ALTER TABLE "user" ALTER COLUMN mxid TYPE TEXT`,
+			`ALTER TABLE "user" ALTER COLUMN jid TYPE TEXT`,
+			`ALTER TABLE "user" ALTER COLUMN management_room TYPE TEXT`,
+			`ALTER TABLE "user" ALTER COLUMN client_id TYPE TEXT`,
+			`ALTER TABLE "user" ALTER COLUMN client_token TYPE TEXT`,
+			`ALTER TABLE "user" ALTER COLUMN server_token TYPE TEXT`,
+
+			`ALTER TABLE user_portal ALTER COLUMN user_jid TYPE TEXT`,
+			`ALTER TABLE user_portal ALTER COLUMN portal_jid TYPE TEXT`,
+			`ALTER TABLE user_portal ALTER COLUMN portal_receiver TYPE TEXT`,
+		)
+	}}
+}

database/upgrades/upgrades.go

@@ -39,7 +39,7 @@ type upgrade struct {
 	fn      upgradeFunc
 }
 
-const NumberOfUpgrades = 21
+const NumberOfUpgrades = 24
 
 var upgrades [NumberOfUpgrades]upgrade
 
@@ -68,6 +68,16 @@ func SetVersion(tx *sql.Tx, version int) error {
 	return err
 }
 
+func execMany(tx *sql.Tx, queries ...string) error {
+	for _, query := range queries {
+		_, err := tx.Exec(query)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func Run(log log.Logger, dialectName string, db *sql.DB) error {
 	var dialect Dialect
 	switch strings.ToLower(dialectName) {
@@ -89,13 +99,14 @@ func Run(log log.Logger, dialectName string, db *sql.DB) error {
 	}
 
 	log.Infofln("Database currently on v%d, latest: v%d", version, NumberOfUpgrades)
-	for i, upgrade := range upgrades[version:] {
-		log.Infofln("Upgrading database to v%d: %s", version+i+1, upgrade.message)
-		tx, err := db.Begin()
+	for i, upgradeItem := range upgrades[version:] {
+		log.Infofln("Upgrading database to v%d: %s", version+i+1, upgradeItem.message)
+		var tx *sql.Tx
+		tx, err = db.Begin()
 		if err != nil {
 			return err
 		}
-		err = upgrade.fn(tx, context{dialect, db, log})
+		err = upgradeItem.fn(tx, context{dialect, db, log})
 		if err != nil {
 			return err
 		}

go.mod

@@ -6,13 +6,13 @@ require (
 	github.com/Rhymen/go-whatsapp v0.1.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/lib/pq v1.10.2
-	github.com/mattn/go-sqlite3 v1.14.7
+	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/prometheus/client_golang v1.11.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	gopkg.in/yaml.v2 v2.4.0
 	maunium.net/go/mauflag v1.0.0
 	maunium.net/go/maulogger/v2 v2.3.0
-	maunium.net/go/mautrix v0.9.19
+	maunium.net/go/mautrix v0.9.20
 )
 
-replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.5.9
+replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.5.10

go.sum

@@ -80,8 +80,8 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
-github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
+github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -138,8 +138,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
 github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/sjson v1.1.5 h1:wsUceI/XDyZk3J1FUvuuYlK62zJv2HO2Pzb8A5EWdUE=
 github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE=
-github.com/tulir/go-whatsapp v0.5.9 h1:IGM2O877eQx8uZLi2kt/5ZtkwrZ9dh9dSVkyO0S7wCM=
-github.com/tulir/go-whatsapp v0.5.9/go.mod h1:7J3IIL3bEQiBJGtiZst1N4PgXHlWIartdVQLe6lcx9A=
+github.com/tulir/go-whatsapp v0.5.10 h1:mcN9GuSZ4kLg9RyNfPOhBasapkPFA8nA4wFLEwA4AfE=
+github.com/tulir/go-whatsapp v0.5.10/go.mod h1:7J3IIL3bEQiBJGtiZst1N4PgXHlWIartdVQLe6lcx9A=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -217,5 +217,5 @@ maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfk
 maunium.net/go/maulogger/v2 v2.2.4/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
 maunium.net/go/maulogger/v2 v2.3.0 h1:TMCcO65fLk6+pJXo7sl38tzjzW0KBFgc6JWJMBJp4GE=
 maunium.net/go/maulogger/v2 v2.3.0/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
-maunium.net/go/mautrix v0.9.19 h1:8ZoDuijJOKxgEOMDoBN2B6at0Ba7EJpsqWA/5jV7ELw=
-maunium.net/go/mautrix v0.9.19/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8=
+maunium.net/go/mautrix v0.9.20 h1:MUKnH3etwsoi1K/1jhUg1NG8qcupzrj3w56rDnDF7mc=
+maunium.net/go/mautrix v0.9.20/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8=

portal.go

@@ -353,7 +353,6 @@ func (portal *Portal) markHandled(source *User, message *waProto.WebMessageInfo,
 			msg.Sender = message.GetParticipant()
 		}
 	}
-	msg.Content = message.Message
 	msg.Sent = isSent
 	msg.Insert()
 
@@ -2081,6 +2080,43 @@ func parseGeoURI(uri string) (lat, long float64, err error) {
 	return
 }
 
+func fallbackQuoteContent() *waProto.Message {
+	blankString := ""
+	return &waProto.Message{
+		Conversation: &blankString,
+	}
+}
+
+func loadQuoteContent(sender *User, msg *database.Message) *waProto.Message {
+	resp, err := sender.Conn.LoadMessagesBefore(msg.Chat.JID, msg.JID, msg.Sender == sender.JID, 1)
+	if err != nil {
+		return fallbackQuoteContent()
+	}
+	msgList, ok := resp.Content.([]interface{})
+	if !ok || len(msgList) == 0 {
+		return fallbackQuoteContent()
+	}
+	wmi, ok := msgList[0].(*waProto.WebMessageInfo)
+	if !ok {
+		return fallbackQuoteContent()
+	}
+	sender.log.Debugln("Loaded message before %s: %s", msg.JID, wmi.GetKey().GetId())
+	resp, err = sender.Conn.LoadMessagesAfter(msg.Chat.JID, wmi.GetKey().GetId(), wmi.GetKey().GetFromMe(), 1)
+	if err != nil {
+		return fallbackQuoteContent()
+	}
+	msgList, ok = resp.Content.([]interface{})
+	if !ok || len(msgList) == 0 {
+		return fallbackQuoteContent()
+	}
+	wmi, ok = msgList[0].(*waProto.WebMessageInfo)
+	if !ok {
+		return fallbackQuoteContent()
+	}
+	sender.log.Debugln("Loaded message %s", wmi.GetKey().GetId())
+	return wmi.GetMessage()
+}
+
 func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waProto.WebMessageInfo, *User) {
 	content, ok := evt.Content.Parsed.(*event.MessageEventContent)
 	if !ok {
@@ -2107,10 +2143,10 @@ func (portal *Portal) convertMatrixMessage(sender *User, evt *event.Event) (*waP
 	if len(replyToID) > 0 {
 		content.RemoveReplyFallback()
 		msg := portal.bridge.DB.Message.GetByMXID(replyToID)
-		if msg != nil && msg.Content != nil {
+		if msg != nil {
 			ctxInfo.StanzaId = &msg.JID
 			ctxInfo.Participant = &msg.Sender
-			ctxInfo.QuotedMessage = msg.Content
+			ctxInfo.QuotedMessage = loadQuoteContent(sender, msg)
 		}
 	}
 	relaybotFormatted := false