historysync.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593
  1. // mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
  2. // Copyright (C) 2021 Tulir Asokan
  3. //
  4. // This program is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU Affero General Public License as published by
  6. // the Free Software Foundation, either version 3 of the License, or
  7. // (at your option) any later version.
  8. //
  9. // This program is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU Affero General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU Affero General Public License
  15. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  16. package main
  17. import (
  18. "fmt"
  19. "time"
  20. waProto "go.mau.fi/whatsmeow/binary/proto"
  21. "go.mau.fi/whatsmeow/types"
  22. "maunium.net/go/mautrix"
  23. "maunium.net/go/mautrix/appservice"
  24. "maunium.net/go/mautrix/event"
  25. "maunium.net/go/mautrix/id"
  26. "maunium.net/go/mautrix-whatsapp/database"
  27. )
  28. // region User history sync handling
  29. type wrappedInfo struct {
  30. *types.MessageInfo
  31. Type database.MessageType
  32. Error database.MessageErrorType
  33. ExpirationStart uint64
  34. ExpiresIn uint32
  35. }
  36. func (user *User) handleHistorySyncsLoop() {
  37. reCheckQueue := make(chan bool, 1)
  38. if user.bridge.Config.Bridge.HistorySync.Backfill {
  39. // Start the backfill queue.
  40. user.BackfillQueue = &BackfillQueue{
  41. BackfillQuery: user.bridge.DB.BackfillQuery,
  42. ImmediateBackfillRequests: make(chan *database.Backfill, 1),
  43. DeferredBackfillRequests: make(chan *database.Backfill, 1),
  44. ReCheckQueue: make(chan bool, 1),
  45. log: user.log.Sub("BackfillQueue"),
  46. }
  47. reCheckQueue = user.BackfillQueue.ReCheckQueue
  48. // Immediate backfills can be done in parallel
  49. for i := 0; i < user.bridge.Config.Bridge.HistorySync.Immediate.WorkerCount; i++ {
  50. go user.handleBackfillRequestsLoop(user.BackfillQueue.ImmediateBackfillRequests)
  51. }
  52. // Deferred backfills should be handled synchronously so as not to
  53. // overload the homeserver. Users can configure their backfill stages
  54. // to be more or less aggressive with backfilling at this stage.
  55. go user.handleBackfillRequestsLoop(user.BackfillQueue.DeferredBackfillRequests)
  56. go user.BackfillQueue.RunLoops(user)
  57. }
  58. // Always save the history syncs for the user. If they want to enable
  59. // backfilling in the future, we will have it in the database.
  60. for evt := range user.historySyncs {
  61. user.handleHistorySync(reCheckQueue, evt.Data)
  62. }
  63. }
  64. func (user *User) handleBackfillRequestsLoop(backfillRequests chan *database.Backfill) {
  65. for req := range backfillRequests {
  66. user.log.Infof("Backfill request: %v", req)
  67. conv := user.bridge.DB.HistorySyncQuery.GetConversation(user.MXID, req.Portal)
  68. if conv == nil {
  69. user.log.Errorf("Could not find conversation for %s in %s", user.MXID, req.Portal.String())
  70. continue
  71. }
  72. // Update the client store with basic chat settings.
  73. if conv.MuteEndTime.After(time.Now()) {
  74. user.Client.Store.ChatSettings.PutMutedUntil(conv.PortalKey.JID, conv.MuteEndTime)
  75. }
  76. if conv.Archived {
  77. user.Client.Store.ChatSettings.PutArchived(conv.PortalKey.JID, true)
  78. }
  79. if conv.Pinned > 0 {
  80. user.Client.Store.ChatSettings.PutPinned(conv.PortalKey.JID, true)
  81. }
  82. portal := user.GetPortalByJID(conv.PortalKey.JID)
  83. if conv.EphemeralExpiration != nil && portal.ExpirationTime != *conv.EphemeralExpiration {
  84. portal.ExpirationTime = *conv.EphemeralExpiration
  85. portal.Update()
  86. }
  87. if !user.shouldCreatePortalForHistorySync(conv, portal) {
  88. continue
  89. }
  90. if len(portal.MXID) == 0 {
  91. user.log.Debugln("Creating portal for", portal.Key.JID, "as part of history sync handling")
  92. err := portal.CreateMatrixRoom(user, nil, true, false)
  93. if err != nil {
  94. user.log.Warnfln("Failed to create room for %s during backfill: %v", portal.Key.JID, err)
  95. continue
  96. }
  97. } else {
  98. portal.UpdateMatrixRoom(user, nil)
  99. }
  100. allMsgs := user.bridge.DB.HistorySyncQuery.GetMessagesBetween(user.MXID, conv.ConversationID, req.TimeStart, req.TimeEnd, req.MaxTotalEvents)
  101. if len(allMsgs) > 0 {
  102. user.log.Debugf("Backfilling %d messages in %s, %d messages at a time", len(allMsgs), portal.Key.JID, req.MaxBatchEvents)
  103. toBackfill := allMsgs[0:]
  104. insertionEventIds := []id.EventID{}
  105. for {
  106. if len(toBackfill) == 0 {
  107. break
  108. }
  109. var msgs []*waProto.WebMessageInfo
  110. if len(toBackfill) <= req.MaxBatchEvents {
  111. msgs = toBackfill
  112. toBackfill = toBackfill[0:0]
  113. } else {
  114. msgs = toBackfill[len(toBackfill)-req.MaxBatchEvents:]
  115. toBackfill = toBackfill[:len(toBackfill)-req.MaxBatchEvents]
  116. }
  117. if len(msgs) > 0 {
  118. time.Sleep(time.Duration(req.BatchDelay) * time.Second)
  119. user.log.Debugf("Backfilling %d messages in %s", len(msgs), portal.Key.JID)
  120. insertionEventIds = append(insertionEventIds, portal.backfill(user, msgs)...)
  121. }
  122. }
  123. user.log.Debugf("Finished backfilling %d messages in %s", len(allMsgs), portal.Key.JID)
  124. if len(insertionEventIds) > 0 {
  125. portal.sendPostBackfillDummy(
  126. time.Unix(int64(allMsgs[len(allMsgs)-1].GetMessageTimestamp()), 0),
  127. insertionEventIds[0])
  128. }
  129. } else {
  130. user.log.Debugfln("Not backfilling %s: no bridgeable messages found", portal.Key.JID)
  131. }
  132. if !conv.MarkedAsUnread && conv.UnreadCount == 0 {
  133. user.markSelfReadFull(portal)
  134. }
  135. }
  136. }
  137. func (user *User) shouldCreatePortalForHistorySync(conv *database.HistorySyncConversation, portal *Portal) bool {
  138. if len(portal.MXID) > 0 {
  139. user.log.Debugfln("Portal for %s already exists, ensuring user is invited", portal.Key.JID)
  140. portal.ensureUserInvited(user)
  141. // Portal exists, let backfill continue
  142. return true
  143. } else if !user.bridge.Config.Bridge.HistorySync.CreatePortals {
  144. user.log.Debugfln("Not creating portal for %s: creating rooms from history sync is disabled", portal.Key.JID)
  145. } else {
  146. // Portal doesn't exist, but should be created
  147. return true
  148. }
  149. // Portal shouldn't be created, reason logged above
  150. return false
  151. }
  152. func (user *User) handleHistorySync(reCheckQueue chan bool, evt *waProto.HistorySync) {
  153. if evt == nil || evt.SyncType == nil || evt.GetSyncType() == waProto.HistorySync_INITIAL_STATUS_V3 || evt.GetSyncType() == waProto.HistorySync_PUSH_NAME {
  154. return
  155. }
  156. description := fmt.Sprintf("type %s, %d conversations, chunk order %d", evt.GetSyncType(), len(evt.GetConversations()), evt.GetChunkOrder())
  157. user.log.Infoln("Storing history sync with", description)
  158. for _, conv := range evt.GetConversations() {
  159. jid, err := types.ParseJID(conv.GetId())
  160. if err != nil {
  161. user.log.Warnfln("Failed to parse chat JID '%s' in history sync: %v", conv.GetId(), err)
  162. continue
  163. }
  164. portal := user.GetPortalByJID(jid)
  165. historySyncConversation := user.bridge.DB.HistorySyncQuery.NewConversationWithValues(
  166. user.MXID,
  167. conv.GetId(),
  168. &portal.Key,
  169. getConversationTimestamp(conv),
  170. conv.GetMuteEndTime(),
  171. conv.GetArchived(),
  172. conv.GetPinned(),
  173. conv.GetDisappearingMode().GetInitiator(),
  174. conv.GetEndOfHistoryTransferType(),
  175. conv.EphemeralExpiration,
  176. conv.GetMarkedAsUnread(),
  177. conv.GetUnreadCount())
  178. historySyncConversation.Upsert()
  179. for _, msg := range conv.GetMessages() {
  180. // Don't store messages that will just be skipped.
  181. wmi := msg.GetMessage()
  182. msgType := getMessageType(wmi.GetMessage())
  183. if msgType == "unknown" || msgType == "ignore" || msgType == "unknown_protocol" {
  184. continue
  185. }
  186. // Don't store unsupported messages.
  187. if !containsSupportedMessage(msg.GetMessage().GetMessage()) {
  188. continue
  189. }
  190. message, err := user.bridge.DB.HistorySyncQuery.NewMessageWithValues(user.MXID, conv.GetId(), msg)
  191. if err != nil {
  192. user.log.Warnf("Failed to save message %s in %s. Error: %+v", msg.Message.Key.Id, conv.GetId(), err)
  193. continue
  194. }
  195. message.Insert()
  196. }
  197. }
  198. // If this was the initial bootstrap, enqueue immediate backfills for the
  199. // most recent portals. If it's the last history sync event, start
  200. // backfilling the rest of the history of the portals.
  201. if user.bridge.Config.Bridge.HistorySync.Backfill && evt.GetSyncType() == waProto.HistorySync_FULL {
  202. nMostRecent := user.bridge.DB.HistorySyncQuery.GetNMostRecentConversations(user.MXID, user.bridge.Config.Bridge.HistorySync.MaxInitialConversations)
  203. for i, conv := range nMostRecent {
  204. jid, err := types.ParseJID(conv.ConversationID)
  205. if err != nil {
  206. user.log.Warnfln("Failed to parse chat JID '%s' in history sync: %v", conv.ConversationID, err)
  207. continue
  208. }
  209. portal := user.GetPortalByJID(jid)
  210. // Enqueue immediate backfills for the most recent messages first.
  211. user.EnqueueImmedateBackfill(portal, i)
  212. // Enqueue deferred backfills as configured.
  213. user.EnqueueDeferredBackfills(portal, len(nMostRecent), i)
  214. }
  215. // Tell the queue to check for new backfill requests.
  216. reCheckQueue <- true
  217. }
  218. }
  219. func getConversationTimestamp(conv *waProto.Conversation) uint64 {
  220. convTs := conv.GetConversationTimestamp()
  221. if convTs == 0 && len(conv.GetMessages()) > 0 {
  222. convTs = conv.Messages[0].GetMessage().GetMessageTimestamp()
  223. }
  224. return convTs
  225. }
  226. func (user *User) EnqueueImmedateBackfill(portal *Portal, priority int) {
  227. maxMessages := user.bridge.Config.Bridge.HistorySync.Immediate.MaxEvents
  228. initialBackfill := user.bridge.DB.BackfillQuery.NewWithValues(user.MXID, database.BackfillImmediate, priority, &portal.Key, nil, nil, maxMessages, maxMessages, 0)
  229. initialBackfill.Insert()
  230. }
  231. func (user *User) EnqueueDeferredBackfills(portal *Portal, numConversations, priority int) {
  232. for j, backfillStage := range user.bridge.Config.Bridge.HistorySync.Deferred {
  233. var startDate *time.Time = nil
  234. if backfillStage.StartDaysAgo > 0 {
  235. startDaysAgo := time.Now().AddDate(0, 0, -backfillStage.StartDaysAgo)
  236. startDate = &startDaysAgo
  237. }
  238. backfill := user.bridge.DB.BackfillQuery.NewWithValues(
  239. user.MXID, database.BackfillDeferred, j*numConversations+priority, &portal.Key, startDate, nil, backfillStage.MaxBatchEvents, -1, backfillStage.BatchDelay)
  240. backfill.Insert()
  241. }
  242. }
  243. // endregion
  244. // region Portal backfilling
  245. var (
  246. PortalCreationDummyEvent = event.Type{Type: "fi.mau.dummy.portal_created", Class: event.MessageEventType}
  247. PreBackfillDummyEvent = event.Type{Type: "fi.mau.dummy.pre_backfill", Class: event.MessageEventType}
  248. // Marker events for when a backfill finishes
  249. BackfillEndDummyEvent = event.Type{Type: "fi.mau.dummy.backfill_end", Class: event.MessageEventType}
  250. RoomMarker = event.Type{Type: "m.room.marker", Class: event.MessageEventType}
  251. MSC2716Marker = event.Type{Type: "org.matrix.msc2716.marker", Class: event.MessageEventType}
  252. )
  253. func (portal *Portal) backfill(source *User, messages []*waProto.WebMessageInfo) []id.EventID {
  254. portal.backfillLock.Lock()
  255. defer portal.backfillLock.Unlock()
  256. var historyBatch, newBatch mautrix.ReqBatchSend
  257. var historyBatchInfos, newBatchInfos []*wrappedInfo
  258. firstMsgTimestamp := time.Unix(int64(messages[len(messages)-1].GetMessageTimestamp()), 0)
  259. historyBatch.StateEventsAtStart = make([]*event.Event, 0)
  260. newBatch.StateEventsAtStart = make([]*event.Event, 0)
  261. addedMembers := make(map[id.UserID]*event.MemberEventContent)
  262. addMember := func(puppet *Puppet) {
  263. if _, alreadyAdded := addedMembers[puppet.MXID]; alreadyAdded {
  264. return
  265. }
  266. mxid := puppet.MXID.String()
  267. content := event.MemberEventContent{
  268. Membership: event.MembershipJoin,
  269. Displayname: puppet.Displayname,
  270. AvatarURL: puppet.AvatarURL.CUString(),
  271. }
  272. inviteContent := content
  273. inviteContent.Membership = event.MembershipInvite
  274. historyBatch.StateEventsAtStart = append(historyBatch.StateEventsAtStart, &event.Event{
  275. Type: event.StateMember,
  276. Sender: portal.MainIntent().UserID,
  277. StateKey: &mxid,
  278. Timestamp: firstMsgTimestamp.UnixMilli(),
  279. Content: event.Content{Parsed: &inviteContent},
  280. }, &event.Event{
  281. Type: event.StateMember,
  282. Sender: puppet.MXID,
  283. StateKey: &mxid,
  284. Timestamp: firstMsgTimestamp.UnixMilli(),
  285. Content: event.Content{Parsed: &content},
  286. })
  287. addedMembers[puppet.MXID] = &content
  288. }
  289. firstMessage := portal.bridge.DB.Message.GetFirstInChat(portal.Key)
  290. lastMessage := portal.bridge.DB.Message.GetLastInChat(portal.Key)
  291. var historyMaxTs, newMinTs time.Time
  292. if portal.FirstEventID != "" || portal.NextBatchID != "" {
  293. historyBatch.PrevEventID = portal.FirstEventID
  294. historyBatch.BatchID = portal.NextBatchID
  295. if firstMessage == nil && lastMessage == nil {
  296. historyMaxTs = time.Now()
  297. } else {
  298. historyMaxTs = firstMessage.Timestamp
  299. }
  300. }
  301. if lastMessage != nil {
  302. newBatch.PrevEventID = lastMessage.MXID
  303. newMinTs = lastMessage.Timestamp
  304. }
  305. portal.log.Infofln("Processing history sync with %d messages", len(messages))
  306. // The messages are ordered newest to oldest, so iterate them in reverse order.
  307. for i := len(messages) - 1; i >= 0; i-- {
  308. webMsg := messages[i]
  309. msgType := getMessageType(webMsg.GetMessage())
  310. if msgType == "unknown" || msgType == "ignore" || msgType == "unknown_protocol" {
  311. if msgType != "ignore" {
  312. portal.log.Debugfln("Skipping message %s with unknown type in backfill", webMsg.GetKey().GetId())
  313. }
  314. continue
  315. }
  316. info := portal.parseWebMessageInfo(source, webMsg)
  317. if info == nil {
  318. continue
  319. }
  320. var batch *mautrix.ReqBatchSend
  321. var infos *[]*wrappedInfo
  322. if !historyMaxTs.IsZero() && info.Timestamp.Before(historyMaxTs) {
  323. batch, infos = &historyBatch, &historyBatchInfos
  324. } else if !newMinTs.IsZero() && info.Timestamp.After(newMinTs) {
  325. batch, infos = &newBatch, &newBatchInfos
  326. } else {
  327. continue
  328. }
  329. if webMsg.GetPushName() != "" && webMsg.GetPushName() != "-" {
  330. existingContact, _ := source.Client.Store.Contacts.GetContact(info.Sender)
  331. if !existingContact.Found || existingContact.PushName == "" {
  332. changed, _, err := source.Client.Store.Contacts.PutPushName(info.Sender, webMsg.GetPushName())
  333. if err != nil {
  334. source.log.Errorfln("Failed to save push name of %s from historical message in device store: %v", info.Sender, err)
  335. } else if changed {
  336. source.log.Debugfln("Got push name %s for %s from historical message", webMsg.GetPushName(), info.Sender)
  337. }
  338. }
  339. }
  340. puppet := portal.getMessagePuppet(source, info)
  341. intent := puppet.IntentFor(portal)
  342. if intent.IsCustomPuppet && !portal.bridge.Config.CanDoublePuppetBackfill(puppet.CustomMXID) {
  343. intent = puppet.DefaultIntent()
  344. }
  345. converted := portal.convertMessage(intent, source, info, webMsg.GetMessage())
  346. if converted == nil {
  347. portal.log.Debugfln("Skipping unsupported message %s in backfill", info.ID)
  348. continue
  349. }
  350. if !intent.IsCustomPuppet && !portal.bridge.StateStore.IsInRoom(portal.MXID, puppet.MXID) {
  351. addMember(puppet)
  352. }
  353. // TODO this won't work for history
  354. if len(converted.ReplyTo) > 0 {
  355. portal.SetReply(converted.Content, converted.ReplyTo)
  356. }
  357. err := portal.appendBatchEvents(converted, info, webMsg.GetEphemeralStartTimestamp(), &batch.Events, infos)
  358. if err != nil {
  359. portal.log.Errorfln("Error handling message %s during backfill: %v", info.ID, err)
  360. }
  361. }
  362. if (len(historyBatch.Events) > 0 && len(historyBatch.BatchID) == 0) || len(newBatch.Events) > 0 {
  363. portal.log.Debugln("Sending a dummy event to avoid forward extremity errors with backfill")
  364. _, err := portal.MainIntent().SendMessageEvent(portal.MXID, PreBackfillDummyEvent, struct{}{})
  365. if err != nil {
  366. portal.log.Warnln("Error sending pre-backfill dummy event:", err)
  367. }
  368. }
  369. insertionEventIds := []id.EventID{}
  370. if len(historyBatch.Events) > 0 && len(historyBatch.PrevEventID) > 0 {
  371. portal.log.Infofln("Sending %d historical messages...", len(historyBatch.Events))
  372. historyResp, err := portal.MainIntent().BatchSend(portal.MXID, &historyBatch)
  373. insertionEventIds = append(insertionEventIds, historyResp.BaseInsertionEventID)
  374. if err != nil {
  375. portal.log.Errorln("Error sending batch of historical messages:", err)
  376. } else {
  377. portal.finishBatch(historyResp.EventIDs, historyBatchInfos)
  378. portal.NextBatchID = historyResp.NextBatchID
  379. portal.Update()
  380. }
  381. }
  382. if len(newBatch.Events) > 0 && len(newBatch.PrevEventID) > 0 {
  383. portal.log.Infofln("Sending %d new messages...", len(newBatch.Events))
  384. newResp, err := portal.MainIntent().BatchSend(portal.MXID, &newBatch)
  385. insertionEventIds = append(insertionEventIds, newResp.BaseInsertionEventID)
  386. if err != nil {
  387. portal.log.Errorln("Error sending batch of new messages:", err)
  388. } else {
  389. portal.finishBatch(newResp.EventIDs, newBatchInfos)
  390. }
  391. }
  392. return insertionEventIds
  393. }
  394. func (portal *Portal) parseWebMessageInfo(source *User, webMsg *waProto.WebMessageInfo) *types.MessageInfo {
  395. info := types.MessageInfo{
  396. MessageSource: types.MessageSource{
  397. Chat: portal.Key.JID,
  398. IsFromMe: webMsg.GetKey().GetFromMe(),
  399. IsGroup: portal.Key.JID.Server == types.GroupServer,
  400. },
  401. ID: webMsg.GetKey().GetId(),
  402. PushName: webMsg.GetPushName(),
  403. Timestamp: time.Unix(int64(webMsg.GetMessageTimestamp()), 0),
  404. }
  405. var err error
  406. if info.IsFromMe {
  407. info.Sender = source.JID.ToNonAD()
  408. } else if portal.IsPrivateChat() {
  409. info.Sender = portal.Key.JID
  410. } else if webMsg.GetParticipant() != "" {
  411. info.Sender, err = types.ParseJID(webMsg.GetParticipant())
  412. } else if webMsg.GetKey().GetParticipant() != "" {
  413. info.Sender, err = types.ParseJID(webMsg.GetKey().GetParticipant())
  414. }
  415. if info.Sender.IsEmpty() {
  416. portal.log.Warnfln("Failed to get sender of message %s (parse error: %v)", info.ID, err)
  417. return nil
  418. }
  419. return &info
  420. }
  421. func (portal *Portal) appendBatchEvents(converted *ConvertedMessage, info *types.MessageInfo, expirationStart uint64, eventsArray *[]*event.Event, infoArray *[]*wrappedInfo) error {
  422. mainEvt, err := portal.wrapBatchEvent(info, converted.Intent, converted.Type, converted.Content, converted.Extra)
  423. if err != nil {
  424. return err
  425. }
  426. if converted.Caption != nil {
  427. captionEvt, err := portal.wrapBatchEvent(info, converted.Intent, converted.Type, converted.Caption, nil)
  428. if err != nil {
  429. return err
  430. }
  431. *eventsArray = append(*eventsArray, mainEvt, captionEvt)
  432. *infoArray = append(*infoArray, &wrappedInfo{info, database.MsgNormal, converted.Error, expirationStart, converted.ExpiresIn}, nil)
  433. } else {
  434. *eventsArray = append(*eventsArray, mainEvt)
  435. *infoArray = append(*infoArray, &wrappedInfo{info, database.MsgNormal, converted.Error, expirationStart, converted.ExpiresIn})
  436. }
  437. if converted.MultiEvent != nil {
  438. for _, subEvtContent := range converted.MultiEvent {
  439. subEvt, err := portal.wrapBatchEvent(info, converted.Intent, converted.Type, subEvtContent, nil)
  440. if err != nil {
  441. return err
  442. }
  443. *eventsArray = append(*eventsArray, subEvt)
  444. *infoArray = append(*infoArray, nil)
  445. }
  446. }
  447. return nil
  448. }
  449. const backfillIDField = "fi.mau.whatsapp.backfill_msg_id"
  450. func (portal *Portal) wrapBatchEvent(info *types.MessageInfo, intent *appservice.IntentAPI, eventType event.Type, content *event.MessageEventContent, extraContent map[string]interface{}) (*event.Event, error) {
  451. if extraContent == nil {
  452. extraContent = map[string]interface{}{}
  453. }
  454. extraContent[backfillIDField] = info.ID
  455. if intent.IsCustomPuppet {
  456. extraContent[doublePuppetKey] = doublePuppetValue
  457. }
  458. wrappedContent := event.Content{
  459. Parsed: content,
  460. Raw: extraContent,
  461. }
  462. newEventType, err := portal.encrypt(&wrappedContent, eventType)
  463. if err != nil {
  464. return nil, err
  465. }
  466. return &event.Event{
  467. Sender: intent.UserID,
  468. Type: newEventType,
  469. Timestamp: info.Timestamp.UnixMilli(),
  470. Content: wrappedContent,
  471. }, nil
  472. }
  473. func (portal *Portal) finishBatch(eventIDs []id.EventID, infos []*wrappedInfo) {
  474. if len(eventIDs) != len(infos) {
  475. portal.log.Errorfln("Length of event IDs (%d) and message infos (%d) doesn't match! Using slow path for mapping event IDs", len(eventIDs), len(infos))
  476. infoMap := make(map[types.MessageID]*wrappedInfo, len(infos))
  477. for _, info := range infos {
  478. infoMap[info.ID] = info
  479. }
  480. for _, eventID := range eventIDs {
  481. if evt, err := portal.MainIntent().GetEvent(portal.MXID, eventID); err != nil {
  482. portal.log.Warnfln("Failed to get event %s to register it in the database: %v", eventID, err)
  483. } else if msgID, ok := evt.Content.Raw[backfillIDField].(string); !ok {
  484. portal.log.Warnfln("Event %s doesn't include the WhatsApp message ID", eventID)
  485. } else if info, ok := infoMap[types.MessageID(msgID)]; !ok {
  486. portal.log.Warnfln("Didn't find info of message %s (event %s) to register it in the database", msgID, eventID)
  487. } else {
  488. portal.finishBatchEvt(info, eventID)
  489. }
  490. }
  491. } else {
  492. for i := 0; i < len(infos); i++ {
  493. portal.finishBatchEvt(infos[i], eventIDs[i])
  494. }
  495. portal.log.Infofln("Successfully sent %d events", len(eventIDs))
  496. }
  497. }
  498. func (portal *Portal) finishBatchEvt(info *wrappedInfo, eventID id.EventID) {
  499. if info == nil {
  500. return
  501. }
  502. portal.markHandled(nil, info.MessageInfo, eventID, true, false, info.Type, info.Error)
  503. if info.ExpiresIn > 0 {
  504. if info.ExpirationStart > 0 {
  505. remainingSeconds := time.Unix(int64(info.ExpirationStart), 0).Add(time.Duration(info.ExpiresIn) * time.Second).Sub(time.Now()).Seconds()
  506. portal.log.Debugfln("Disappearing history sync message: expires in %d, started at %d, remaining %d", info.ExpiresIn, info.ExpirationStart, int(remainingSeconds))
  507. portal.MarkDisappearing(eventID, uint32(remainingSeconds), true)
  508. } else {
  509. portal.log.Debugfln("Disappearing history sync message: expires in %d (not started)", info.ExpiresIn)
  510. portal.MarkDisappearing(eventID, info.ExpiresIn, false)
  511. }
  512. }
  513. }
  514. func (portal *Portal) sendPostBackfillDummy(lastTimestamp time.Time, insertionEventId id.EventID) {
  515. for _, evtType := range []event.Type{BackfillEndDummyEvent, RoomMarker, MSC2716Marker} {
  516. resp, err := portal.MainIntent().SendMessageEvent(portal.MXID, evtType, map[string]interface{}{
  517. "org.matrix.msc2716.marker.insertion": insertionEventId,
  518. "m.marker.insertion": insertionEventId,
  519. })
  520. if err != nil {
  521. portal.log.Errorln("Error sending post-backfill dummy event:", err)
  522. return
  523. }
  524. msg := portal.bridge.DB.Message.New()
  525. msg.Chat = portal.Key
  526. msg.MXID = resp.EventID
  527. msg.JID = types.MessageID(resp.EventID)
  528. msg.Timestamp = lastTimestamp.Add(1 * time.Second)
  529. msg.Sent = true
  530. msg.Insert()
  531. }
  532. }
  533. // endregion