Merge branch 'database-refactor' into emoji-reacts

Session.xcodeproj/project.pbxproj

@@ -3133,8 +3133,6 @@
 			children = (
 				C3C2A5B0255385C700C340D1 /* Meta */,
 				FD17D79D27F40CAA00122BE0 /* Database */,
-				FD17D7DF27F67BC400122BE0 /* Models */,
-				FD17D7D027F5795300122BE0 /* Types */,
 				FDC438AF27BB158500C60D73 /* Models */,
 				C3C2A5CD255385F300C340D1 /* Utilities */,
 				C3C2A5B9255385ED00C340D1 /* Configuration.swift */,
@@ -3624,20 +3622,6 @@
 			path = Models;
 			sourceTree = "<group>";
 		};
-		FD17D7D027F5795300122BE0 /* Types */ = {
-			isa = PBXGroup;
-			children = (
-			);
-			path = Types;
-			sourceTree = "<group>";
-		};
-		FD17D7DF27F67BC400122BE0 /* Models */ = {
-			isa = PBXGroup;
-			children = (
-			);
-			path = Models;
-			sourceTree = "<group>";
-		};
 		FD17D7E827F6A1B800122BE0 /* LegacyDatabase */ = {
 			isa = PBXGroup;
 			children = (
@@ -6909,7 +6893,7 @@
 				CODE_SIGN_ENTITLEMENTS = Session/Meta/Signal.entitlements;
 				CODE_SIGN_IDENTITY = "iPhone Developer";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer";
-				CURRENT_PROJECT_VERSION = 357;
+				CURRENT_PROJECT_VERSION = 360;
 				DEVELOPMENT_TEAM = SUQ8J2PCT7;
 				FRAMEWORK_SEARCH_PATHS = (
 					"$(inherited)",
@@ -6981,7 +6965,7 @@
 				CODE_SIGN_ENTITLEMENTS = Session/Meta/Signal.entitlements;
 				CODE_SIGN_IDENTITY = "iPhone Developer";
 				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer";
-				CURRENT_PROJECT_VERSION = 357;
+				CURRENT_PROJECT_VERSION = 360;
 				DEVELOPMENT_TEAM = SUQ8J2PCT7;
 				FRAMEWORK_SEARCH_PATHS = (
 					"$(inherited)",

Session/Calls/Call Management/SessionCall.swift

@@ -167,7 +167,10 @@ public final class SessionCall: CurrentCallProtocol, WebRTCSessionDelegate {
     }
     
     func reportIncomingCallIfNeeded(completion: @escaping (Error?) -> Void) {
-        guard case .answer = mode else { return }
+        guard case .answer = mode else {
+            SessionCallManager.reportFakeCall(info: "Call not in answer mode")
+            return
+        }
         
         setupTimeoutTimer()
         AppEnvironment.shared.callManager.reportIncomingCall(self, callerName: contactName) { error in

Session/Calls/Call Management/SessionCallManager.swift

@@ -72,6 +72,16 @@ public final class SessionCallManager: NSObject, CallManagerProtocol {
     
     // MARK: - Report calls
     
+    public static func reportFakeCall(info: String) {
+        SessionCallManager.sharedProvider(useSystemCallLog: false)
+            .reportNewIncomingCall(
+                with: UUID(),
+                update: CXCallUpdate()
+            ) { _ in
+                SNLog("[Calls] Reported fake incoming call to CallKit due to: \(info)")
+            }
+    }
+    
     public func reportOutgoingCall(_ call: SessionCall) {
         AssertIsOnMainThread()
         UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
@@ -109,7 +119,9 @@ public final class SessionCallManager: NSObject, CallManagerProtocol {
                 UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
                 completion(nil)
             }
-        } else {
+        }
+        else {
+            SessionCallManager.reportFakeCall(info: "No CXProvider instance")
             UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
             completion(nil)
         }

Session/Conversations/ConversationVC.swift

@@ -454,7 +454,7 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
         recoverInputView()
     }
     
@@ -464,7 +464,7 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes
         dataChangeObservable = Storage.shared.start(
             viewModel.observableThreadData,
@@ -510,6 +510,13 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
                     self?.viewModel.onInteractionChange = { [weak self] updatedInteractionData in
                         self?.handleInteractionUpdates(updatedInteractionData)
                     }
+                    
+                    // Note: When returning from the background we could have received notifications but the
+                    // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+                    // data to ensure everything is up to date
+                    if didReturnFromBackground {
+                        self?.viewModel.pagedDataObserver?.reload()
+                    }
                 }
             }
         )

Session/DMs/NewDMVC.swift

@@ -134,9 +134,13 @@ final class NewDMVC : BaseVC, UIPageViewControllerDataSource, UIPageViewControll
     }
     
     fileprivate func startNewDMIfPossible(with onsNameOrPublicKey: String) {
-        if ECKeyPair.isValidHexEncodedPublicKey(candidate: onsNameOrPublicKey) {
+        let maybeSessionId: SessionId? = SessionId(from: onsNameOrPublicKey)
+        
+        if ECKeyPair.isValidHexEncodedPublicKey(candidate: onsNameOrPublicKey) && maybeSessionId?.prefix == .standard {
             startNewDM(with: onsNameOrPublicKey)
-        } else {
+            return
+        }
+        
         // This could be an ONS name
         ModalActivityIndicatorViewController.present(fromViewController: navigationController!, canCancel: false) { [weak self] modalActivityIndicator in
             SnodeAPI.getSessionID(for: onsNameOrPublicKey).done { sessionID in
@@ -153,15 +157,23 @@ final class NewDMVC : BaseVC, UIPageViewControllerDataSource, UIPageViewControll
                             default: break
                         }
                     }
-                        let message = messageOrNil ?? "Please check the Session ID or ONS name and try again"
+                    let message: String = {
+                        if let messageOrNil: String = messageOrNil {
+                            return messageOrNil
+                        }
+                        
+                        return (maybeSessionId?.prefix == .blinded ?
+                            "You can only send messages to Blinded IDs from within an Open Group" :
+                            "Please check the Session ID or ONS name and try again"
+                        )
+                    }()
                     let alert = UIAlertController(title: "Error", message: message, preferredStyle: .alert)
-                        alert.addAction(UIAlertAction(title: NSLocalizedString("BUTTON_OK", comment: ""), style: .default, handler: nil))
+                    alert.addAction(UIAlertAction(title: "BUTTON_OK".localized(), style: .default, handler: nil))
                     self?.presentAlert(alert)
                 }
             }
         }
     }
-    }
 
     private func startNewDM(with sessionId: String) {
         let maybeThread: SessionThread? = Storage.shared.write { db in

Session/Home/HomeVC.swift

@@ -239,7 +239,7 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -248,7 +248,7 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes
         dataChangeObservable = Storage.shared.start(
             viewModel.observableState,
@@ -269,6 +269,13 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
         self.viewModel.onThreadChange = { [weak self] updatedThreadData in
             self?.handleThreadUpdates(updatedThreadData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func stopObservingChanges() {

Session/Home/Message Requests/MessageRequestsViewController.swift

@@ -147,7 +147,7 @@ class MessageRequestsViewController: BaseVC, UITableViewDelegate, UITableViewDat
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -186,10 +186,17 @@ class MessageRequestsViewController: BaseVC, UITableViewDelegate, UITableViewDat
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         self.viewModel.onThreadChange = { [weak self] updatedThreadData in
             self?.handleThreadUpdates(updatedThreadData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func handleThreadUpdates(_ updatedData: [MessageRequestsViewModel.SectionModel], initialLoad: Bool = false) {

Session/Media Viewing & Editing/MediaTileViewController.swift

@@ -171,7 +171,7 @@ public class MediaTileViewController: UIViewController, UICollectionViewDataSour
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -243,11 +243,18 @@ public class MediaTileViewController: UIViewController, UICollectionViewDataSour
         }
     }
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes (will callback on the main thread)
         self.viewModel.onGalleryChange = { [weak self] updatedGalleryData in
             self?.handleUpdates(updatedGalleryData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func stopObservingChanges() {

Session/Meta/AppDelegate.swift

@@ -114,6 +114,16 @@ class AppDelegate: UIResponder, UIApplicationDelegate, UNUserNotificationCenterD
         return true
     }
     
+    func applicationWillEnterForeground(_ application: UIApplication) {
+        /// **Note:** We _shouldn't_ need to call this here but for some reason the OS doesn't seems to
+        /// be calling the `userNotificationCenter(_:,didReceive:withCompletionHandler:)`
+        /// method when the device is locked while the app is in the foreground (or if the user returns to the
+        /// springboard without swapping to another app) - adding this here in addition to the one in
+        /// `appDidFinishLaunching` seems to fix this odd behaviour (even though it doesn't match
+        /// Apple's documentation on the matter)
+        UNUserNotificationCenter.current().delegate = self
+    }
+    
     func applicationDidEnterBackground(_ application: UIApplication) {
         DDLog.flushLog()
         
@@ -155,7 +165,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate, UNUserNotificationCenterD
         }
 
         // On every activation, clear old temp directories.
-        ClearOldTemporaryDirectories();
+        ClearOldTemporaryDirectories()
     }
     
     func applicationWillResignActive(_ application: UIApplication) {

Session/Notifications/PushRegistrationManager.swift

@@ -242,8 +242,16 @@ public enum PushRegistrationError: Error {
         owsAssertDebug(type == .voIP)
         let payload = payload.dictionaryPayload
         
-        if let uuid = payload["uuid"] as? String, let caller = payload["caller"] as? String, let timestampMs = payload["timestamp"] as? Int64 {
+        guard
-            let call: SessionCall? = Storage.shared.write { db in
+            let uuid: String = payload["uuid"] as? String,
+            let caller: String = payload["caller"] as? String,
+            let timestampMs: Int64 = payload["timestamp"] as? Int64
+        else {
+            SessionCallManager.reportFakeCall(info: "Missing payload data")
+            return
+        }
+        
+        let maybeCall: SessionCall? = Storage.shared.write { db in
             let messageInfo: CallMessage.MessageInfo = CallMessage.MessageInfo(
                 state: (caller == getUserHexEncodedPublicKey(db) ?
                     .outgoing :
@@ -269,17 +277,21 @@ public enum PushRegistrationError: Error {
             return call
         }
         
+        guard let call: SessionCall = maybeCall else {
+            SessionCallManager.reportFakeCall(info: "Could not retrieve call from database")
+            return
+        }
+        
         // NOTE: Just start 1-1 poller so that it won't wait for polling group messages
         (UIApplication.shared.delegate as? AppDelegate)?.startPollersIfNeeded(shouldStartGroupPollers: false)
         
-            call?.reportIncomingCallIfNeeded { error in
+        call.reportIncomingCallIfNeeded { error in
             if let error = error {
                 SNLog("[Calls] Failed to report incoming call to CallKit due to error: \(error)")
             }
         }
     }
 }
-}
 
 // We transmit pushToken data as hex encoded string to the server
 fileprivate extension Data {

Session/Settings/NukeDataModal.swift

@@ -15,7 +15,7 @@ final class NukeDataModal: Modal {
         let result = UILabel()
         result.textColor = Colors.text
         result.font = .boldSystemFont(ofSize: Values.mediumFontSize)
-        result.text = NSLocalizedString("modal_clear_all_data_title", comment: "")
+        result.text = "modal_clear_all_data_title".localized()
         result.numberOfLines = 0
         result.lineBreakMode = .byWordWrapping
         result.textAlignment = .center
@@ -27,7 +27,7 @@ final class NukeDataModal: Modal {
         let result = UILabel()
         result.textColor = Colors.text.withAlphaComponent(Values.mediumOpacity)
         result.font = .systemFont(ofSize: Values.smallFontSize)
-        result.text = NSLocalizedString("modal_clear_all_data_explanation", comment: "")
+        result.text = "modal_clear_all_data_explanation".localized()
         result.numberOfLines = 0
         result.textAlignment = .center
         result.lineBreakMode = .byWordWrapping
@@ -44,7 +44,7 @@ final class NukeDataModal: Modal {
         }
         result.titleLabel!.font = .systemFont(ofSize: Values.smallFontSize)
         result.setTitleColor(isLightMode ? Colors.destructive : Colors.text, for: UIControl.State.normal)
-        result.setTitle(NSLocalizedString("TXT_DELETE_TITLE", comment: ""), for: UIControl.State.normal)
+        result.setTitle("TXT_DELETE_TITLE".localized(), for: UIControl.State.normal)
         result.addTarget(self, action: #selector(clearAllData), for: UIControl.Event.touchUpInside)
         
         return result
@@ -66,7 +66,7 @@ final class NukeDataModal: Modal {
         result.backgroundColor = Colors.buttonBackground
         result.titleLabel!.font = .systemFont(ofSize: Values.smallFontSize)
         result.setTitleColor(Colors.text, for: UIControl.State.normal)
-        result.setTitle(NSLocalizedString("modal_clear_all_data_device_only_button_title", comment: ""), for: UIControl.State.normal)
+        result.setTitle("modal_clear_all_data_device_only_button_title".localized(), for: UIControl.State.normal)
         result.addTarget(self, action: #selector(clearDeviceOnly), for: UIControl.Event.touchUpInside)
         
         return result
@@ -81,7 +81,7 @@ final class NukeDataModal: Modal {
         }
         result.titleLabel!.font = .systemFont(ofSize: Values.smallFontSize)
         result.setTitleColor(isLightMode ? Colors.destructive : Colors.text, for: UIControl.State.normal)
-        result.setTitle(NSLocalizedString("modal_clear_all_data_entire_account_button_title", comment: ""), for: UIControl.State.normal)
+        result.setTitle("modal_clear_all_data_entire_account_button_title".localized(), for: UIControl.State.normal)
         result.addTarget(self, action: #selector(clearEntireAccount), for: UIControl.Event.touchUpInside)
         
         return result
@@ -211,6 +211,10 @@ final class NukeDataModal: Modal {
             PushNotificationAPI.unregister(data).retainUntilComplete()
         }
         
+        // Clear the app badge and notifications
+        AppEnvironment.shared.notificationPresenter.clearAllNotifications()
+        CurrentAppContext().setMainAppBadgeNumber(0)
+        
         // Clear out the user defaults
         UserDefaults.removeAll()
         

SessionMessagingKit/Database/Migrations/_001_InitialSetupMigration.swift

@@ -140,6 +140,9 @@ enum _001_InitialSetupMigration: Migration {
             t.column(.sequenceNumber, .integer).notNull()
             t.column(.inboxLatestMessageId, .integer).notNull()
             t.column(.outboxLatestMessageId, .integer).notNull()
+            t.column(.pollFailureCount, .integer)
+                .notNull()
+                .defaults(to: 0)
         }
         
         /// Create a full-text search table synchronized with the OpenGroup table

SessionMessagingKit/Database/Models/Attachment.swift

@@ -468,6 +468,7 @@ extension Attachment {
         public let attachmentId: String
         public let interactionId: Int64
         public let state: Attachment.State
+        public let downloadUrl: String?
     }
     
     public static func stateInfo(authorId: String, state: State? = nil) -> SQLRequest<Attachment.StateInfo> {
@@ -484,7 +485,8 @@ extension Attachment {
             SELECT DISTINCT
                 \(attachment[.id]) AS attachmentId,
                 \(interaction[.id]) AS interactionId,
-                \(attachment[.state]) AS state
+                \(attachment[.state]) AS state,
+                \(attachment[.downloadUrl]) AS downloadUrl
         
             FROM \(Attachment.self)
             
@@ -529,7 +531,8 @@ extension Attachment {
             SELECT DISTINCT
                 \(attachment[.id]) AS attachmentId,
                 \(interaction[.id]) AS interactionId,
-                \(attachment[.state]) AS state
+                \(attachment[.state]) AS state,
+                \(attachment[.downloadUrl]) AS downloadUrl
         
             FROM \(Attachment.self)
             
@@ -913,6 +916,16 @@ extension Attachment {
 
         return true
     }
+    
+    public static func fileId(for downloadUrl: String?) -> String? {
+        return downloadUrl
+            .map { urlString -> String? in
+                urlString
+                    .split(separator: "/")
+                    .last
+                    .map { String($0) }
+            }
+    }
 }
 
 // MARK: - Upload
@@ -923,14 +936,14 @@ extension Attachment {
         queue: DispatchQueue,
         using upload: (Database, Data) -> Promise<String>,
         encrypt: Bool,
-        success: (() -> Void)?,
+        success: ((String?) -> Void)?,
         failure: ((Error) -> Void)?
     ) {
         // This can occur if an AttachmnetUploadJob was explicitly created for a message
         // dependant on the attachment being uploaded (in this case the attachment has
         // already been uploaded so just succeed)
         guard state != .uploaded else {
-            success?()
+            success?(Attachment.fileId(for: self.downloadUrl))
             return
         }
         
@@ -982,7 +995,7 @@ extension Attachment {
                 return
             }
                 
-            success?()
+            success?(Attachment.fileId(for: self.downloadUrl))
             return
         }
         
@@ -1073,7 +1086,7 @@ extension Attachment {
                     return
                 }
                     
-                success?()
+                success?(fileId)
             }
             .catch(on: queue) { error in
                 Storage.shared.write { db in

SessionMessagingKit/Database/Models/OpenGroup.swift

@@ -26,6 +26,7 @@ public struct OpenGroup: Codable, Identifiable, FetchableRecord, PersistableReco
         case sequenceNumber
         case inboxLatestMessageId
         case outboxLatestMessageId
+        case pollFailureCount
     }
     
     public var id: String { threadId }  // Identifiable
@@ -86,6 +87,9 @@ public struct OpenGroup: Codable, Identifiable, FetchableRecord, PersistableReco
     /// updated whenever this value changes)
     public let outboxLatestMessageId: Int64
     
+    /// The number of times this room has failed to poll since the last successful poll
+    public let pollFailureCount: Int64
+    
     // MARK: - Relationships
     
     public var thread: QueryInterfaceRequest<SessionThread> {
@@ -117,7 +121,8 @@ public struct OpenGroup: Codable, Identifiable, FetchableRecord, PersistableReco
         infoUpdates: Int64,
         sequenceNumber: Int64 = 0,
         inboxLatestMessageId: Int64 = 0,
-        outboxLatestMessageId: Int64 = 0
+        outboxLatestMessageId: Int64 = 0,
+        pollFailureCount: Int64 = 0
     ) {
         self.threadId = OpenGroup.idFor(roomToken: roomToken, server: server)
         self.server = server.lowercased()
@@ -133,6 +138,7 @@ public struct OpenGroup: Codable, Identifiable, FetchableRecord, PersistableReco
         self.sequenceNumber = sequenceNumber
         self.inboxLatestMessageId = inboxLatestMessageId
         self.outboxLatestMessageId = outboxLatestMessageId
+        self.pollFailureCount = pollFailureCount
     }
 }
 
@@ -156,10 +162,11 @@ public extension OpenGroup {
                 imageId: nil,
                 imageData: nil,
                 userCount: 0,
-                infoUpdates: -1,
+                infoUpdates: 0,
                 sequenceNumber: 0,
                 inboxLatestMessageId: 0,
-                outboxLatestMessageId: 0
+                outboxLatestMessageId: 0,
+                pollFailureCount: 0
             )
         }
         
@@ -192,7 +199,8 @@ extension OpenGroup: CustomStringConvertible, CustomDebugStringConvertible {
             "infoUpdates: \(infoUpdates)",
             "sequenceNumber: \(sequenceNumber)",
             "inboxLatestMessageId: \(inboxLatestMessageId)",
-            "outboxLatestMessageId: \(outboxLatestMessageId))"
+            "outboxLatestMessageId: \(outboxLatestMessageId)",
+            "pollFailureCount: \(pollFailureCount))"
         ].joined(separator: ", ")
     }
 }

SessionMessagingKit/Jobs/Types/AttachmentDownloadJob.swift

@@ -87,10 +87,7 @@ public enum AttachmentDownloadJob: JobExecutor {
         let downloadPromise: Promise<Data> = {
             guard
                 let downloadUrl: String = attachment.downloadUrl,
-                let fileId: String = downloadUrl
+                let fileId: String = Attachment.fileId(for: downloadUrl)
-                    .split(separator: "/")
-                    .last
-                    .map({ String($0) })
             else {
                 return Promise(error: AttachmentDownloadError.invalidUrl)
             }

SessionMessagingKit/Jobs/Types/AttachmentUploadJob.swift

@@ -34,6 +34,15 @@ public enum AttachmentUploadJob: JobExecutor {
             return
         }
         
+        // If the original interaction no longer exists then don't bother uploading the attachment (ie. the
+        // message was deleted before it even got sent)
+        if let interactionId: Int64 = job.interactionId {
+            guard Storage.shared.read({ db in try Interaction.exists(db, id: interactionId) }) == true else {
+                failure(job, StorageError.objectNotFound, true)
+                return
+            }
+        }
+        
         // Note: In the AttachmentUploadJob we intentionally don't provide our own db instance to prevent reentrancy
         // issues when the success/failure closures get called before the upload as the JobRunner will attempt to
         // update the state of the job immediately
@@ -55,7 +64,7 @@ public enum AttachmentUploadJob: JobExecutor {
                     .map { response -> String in response.id }
             },
             encrypt: (openGroup == nil),
-            success: { success(job, false) },
+            success: { _ in success(job, false) },
             failure: { error in failure(job, error, false) }
         )
     }

SessionMessagingKit/Jobs/Types/MessageSendJob.swift

@@ -27,6 +27,10 @@ public enum MessageSendJob: JobExecutor {
             return
         }
         
+        // We need to include 'fileIds' when sending messages with attachments to Open Groups
+        // so extract them from any associated attachments
+        var messageFileIds: [String] = []
+        
         if details.message is VisibleMessage {
             guard
                 let jobId: Int64 = job.id,
@@ -36,20 +40,30 @@ public enum MessageSendJob: JobExecutor {
                 return
             }
             
+            // If the original interaction no longer exists then don't bother sending the message (ie. the
+            // message was deleted before it even got sent)
+            guard Storage.shared.read({ db in try Interaction.exists(db, id: interactionId) }) == true else {
+                failure(job, StorageError.objectNotFound, true)
+                return
+            }
+            
             // Check if there are any attachments associated to this message, and if so
             // upload them now
             //
             // Note: Normal attachments should be sent in a non-durable way but any
             // attachments for LinkPreviews and Quotes will be processed through this mechanism
-            let attachmentState: (shouldFail: Bool, shouldDefer: Bool)? = Storage.shared.write { db in
+            let attachmentState: (shouldFail: Bool, shouldDefer: Bool, fileIds: [String])? = Storage.shared.write { db in
                 let allAttachmentStateInfo: [Attachment.StateInfo] = try Attachment
                     .stateInfo(interactionId: interactionId)
                     .fetchAll(db)
+                let maybeFileIds: [String?] = allAttachmentStateInfo
+                    .map { Attachment.fileId(for: $0.downloadUrl) }
+                let fileIds: [String] = maybeFileIds.compactMap { $0 }
                 
                 // If there were failed attachments then this job should fail (can't send a
                 // message which has associated attachments if the attachments fail to upload)
                 guard !allAttachmentStateInfo.contains(where: { $0.state == .failedDownload }) else {
-                    return (true, false)
+                    return (true, false, fileIds)
                 }
                 
                 // Create jobs for any pending (or failed) attachment jobs and insert them into the
@@ -102,9 +116,13 @@ public enum MessageSendJob: JobExecutor {
                 // If there were pending or uploading attachments then stop here (we want to
                 // upload them first and then re-run this send job - the 'JobRunner.insert'
                 // method will take care of this)
+                let isMissingFileIds: Bool = (maybeFileIds.count != fileIds.count)
+                let hasPendingUploads: Bool = allAttachmentStateInfo.contains(where: { $0.state != .uploaded })
+                
                 return (
-                    false,
+                    (isMissingFileIds && !hasPendingUploads),
-                    allAttachmentStateInfo.contains(where: { $0.state != .uploaded })
+                    hasPendingUploads,
+                    fileIds
                 )
             }
             
@@ -122,6 +140,9 @@ public enum MessageSendJob: JobExecutor {
                 deferred(job)
                 return
             }
+            
+            // Store the fileIds so they can be sent with the open group message content
+            messageFileIds = (attachmentState?.fileIds ?? [])
         }
         
         // Store the sentTimestamp from the message in case it fails due to a clockOutOfSync error
@@ -135,7 +156,8 @@ public enum MessageSendJob: JobExecutor {
             try MessageSender.sendImmediate(
                 db,
                 message: details.message,
-                to: details.destination,
+                to: details.destination
+                    .with(fileIds: messageFileIds),
                 interactionId: job.interactionId
             )
         }

SessionMessagingKit/Jobs/Types/UpdateProfilePictureJob.swift

@@ -52,7 +52,14 @@ public enum UpdateProfilePictureJob: JobExecutor {
             image: nil,
             imageFilePath: profileFilePath,
             requiredSync: true,
-            success: { _, _ in success(job, false) },
+            success: { _, _ in
+                // Need to call the 'success' closure asynchronously on the queue to prevent a reentrancy
+                // issue as it will write to the database and this closure is already called within
+                // another database write
+                queue.async {
+                    success(job, false)
+                }
+            },
             failure: { error in failure(job, error, false) }
         )
     }

SessionMessagingKit/Messages/Message+Destination.swift

@@ -49,5 +49,21 @@ public extension Message {
                     return .openGroup(roomToken: openGroup.roomToken, server: openGroup.server, fileIds: fileIds)
             }
         }
+        
+        func with(fileIds: [String]) -> Message.Destination {
+            // Only Open Group messages support receiving the 'fileIds'
+            switch self {
+                case .openGroup(let roomToken, let server, let whisperTo, let whisperMods, _):
+                    return .openGroup(
+                        roomToken: roomToken,
+                        server: server,
+                        whisperTo: whisperTo,
+                        whisperMods: whisperMods,
+                        fileIds: fileIds
+                    )
+                    
+                default: return self
+            }
+        }
     }
 }

SessionMessagingKit/Open Groups/OpenGroupAPI.swift

@@ -382,6 +382,72 @@ public enum OpenGroupAPI {
             }
     }
     
+    /// This is a convenience method which constructs a `/sequence` of the `capabilities` and `rooms`  requests, refer to those
+    /// methods for the documented behaviour of each method
+    public static func capabilitiesAndRooms(
+        _ db: Database,
+        on server: String,
+        authenticated: Bool = true,
+        using dependencies: SMKDependencies = SMKDependencies()
+    ) -> Promise<(capabilities: (info: OnionRequestResponseInfoType, data: Capabilities), rooms: (info: OnionRequestResponseInfoType, data: [Room]))> {
+        let requestResponseType: [BatchRequestInfoType] = [
+            // Get the latest capabilities for the server (in case it's a new server or the cached ones are stale)
+            BatchRequestInfo(
+                request: Request<NoBody, Endpoint>(
+                    server: server,
+                    endpoint: .capabilities
+                ),
+                responseType: Capabilities.self
+            ),
+            
+            // And the room info
+            BatchRequestInfo(
+                request: Request<NoBody, Endpoint>(
+                    server: server,
+                    endpoint: .rooms
+                ),
+                responseType: [Room].self
+            )
+        ]
+        
+        return OpenGroupAPI
+            .sequence(
+                db,
+                server: server,
+                requests: requestResponseType,
+                authenticated: authenticated,
+                using: dependencies
+            )
+            .map { (response: [Endpoint: (OnionRequestResponseInfoType, Codable?)]) -> (capabilities: (OnionRequestResponseInfoType, Capabilities), rooms: (OnionRequestResponseInfoType, [Room])) in
+                let maybeCapabilities: (info: OnionRequestResponseInfoType, data: Capabilities?)? = response[.capabilities]
+                    .map { info, data in (info, (data as? BatchSubResponse<Capabilities>)?.body) }
+                let maybeRoomResponse: (OnionRequestResponseInfoType, Codable?)? = response
+                    .first(where: { key, _ in
+                        switch key {
+                            case .rooms: return true
+                            default: return false
+                        }
+                    })
+                    .map { _, value in value }
+                let maybeRooms: (info: OnionRequestResponseInfoType, data: [Room]?)? = maybeRoomResponse
+                    .map { info, data in (info, (data as? BatchSubResponse<[Room]>)?.body) }
+                
+                guard
+                    let capabilitiesInfo: OnionRequestResponseInfoType = maybeCapabilities?.info,
+                    let capabilities: Capabilities = maybeCapabilities?.data,
+                    let roomsInfo: OnionRequestResponseInfoType = maybeRooms?.info,
+                    let rooms: [Room] = maybeRooms?.data
+                else {
+                    throw HTTP.Error.parsingFailed
+                }
+                
+                return (
+                    (capabilitiesInfo, capabilities),
+                    (roomsInfo, rooms)
+                )
+            }
+    }
+    
     // MARK: - Messages
     
     /// Posts a new message to a room

SessionMessagingKit/Open Groups/OpenGroupManager.swift

@@ -779,13 +779,25 @@ public final class OpenGroupManager: NSObject {
         // Try to retrieve the default rooms 8 times
         attempt(maxRetryCount: 8, recoveringOn: OpenGroupAPI.workQueue) {
             dependencies.storage.read { db in
-                OpenGroupAPI.rooms(db, server: OpenGroupAPI.defaultServer, using: dependencies)
+                OpenGroupAPI.capabilitiesAndRooms(
+                    db,
+                    on: OpenGroupAPI.defaultServer,
+                    authenticated: false,
+                    using: dependencies
+                )
             }
-            .map { _, data in data }
         }
-        .done(on: OpenGroupAPI.workQueue) { items in
+        .done(on: OpenGroupAPI.workQueue) { response in
             dependencies.storage.writeAsync { db in
-                items
+                // Store the capabilities first
+                OpenGroupManager.handleCapabilities(
+                    db,
+                    capabilities: response.capabilities.data,
+                    on: OpenGroupAPI.defaultServer
+                )
+                    
+                // Then the rooms
+                response.rooms.data
                     .compactMap { room -> (String, String)? in
                         // Try to insert an inactive version of the OpenGroup (use 'insert' rather than 'save'
                         // as we want it to fail if the room already exists)
@@ -825,7 +837,7 @@ public final class OpenGroupManager: NSObject {
                     }
             }
             
-            seal.fulfill(items)
+            seal.fulfill(response.rooms.data)
         }
         .catch(on: OpenGroupAPI.workQueue) { error in
             dependencies.mutableCache.mutate { cache in

SessionMessagingKit/Sending & Receiving/MessageSender+Convenience.swift

@@ -100,7 +100,7 @@ extension MessageSender {
     }
     
     public static func sendNonDurably(_ db: Database, message: Message, interactionId: Int64?, to destination: Message.Destination) -> Promise<Void> {
-        var attachmentUploadPromises: [Promise<Void>] = [Promise.value(())]
+        var attachmentUploadPromises: [Promise<String?>] = [Promise.value(nil)]
         
         // If we have an interactionId then check if it has any attachments and process them first
         if let interactionId: Int64 = interactionId {
@@ -124,8 +124,8 @@ extension MessageSender {
                 .filter(ids: attachmentStateInfo.map { $0.attachmentId })
                 .fetchAll(db))
                 .defaulting(to: [])
-                .map { attachment -> Promise<Void> in
+                .map { attachment -> Promise<String?> in
-                    let (promise, seal) = Promise<Void>.pending()
+                    let (promise, seal) = Promise<String?>.pending()
     
                     attachment.upload(
                         db,
@@ -146,7 +146,7 @@ extension MessageSender {
                                 .map { response -> String in response.id }
                         },
                         encrypt: (openGroup == nil),
-                        success: { seal.fulfill(()) },
+                        success: { fileId in seal.fulfill(fileId) },
                         failure: { seal.reject($0) }
                     )
     
@@ -167,10 +167,18 @@ extension MessageSender {
                 if let error: Error = errors.first { return Promise(error: error) }
                 
                 return Storage.shared.writeAsync { db in
-                    try MessageSender.sendImmediate(
+                    let fileIds: [String] = results
+                        .compactMap { result -> String? in
+                            if case .fulfilled(let value) = result { return value }
+                            
+                            return nil
+                        }
+                    
+                    return try MessageSender.sendImmediate(
                         db,
                         message: message,
-                        to: destination,
+                        to: destination
+                            .with(fileIds: fileIds),
                         interactionId: interactionId
                     )
                 }

SessionMessagingKit/Sending & Receiving/Pollers/OpenGroupPoller.swift

@@ -15,7 +15,8 @@ extension OpenGroupAPI {
 
         // MARK: - Settings
         
-        private static let pollInterval: TimeInterval = 4
+        private static let minPollInterval: TimeInterval = 3
+        private static let maxPollInterval: Double = (60 * 60)
         internal static let maxInactivityPeriod: Double = (14 * 24 * 60 * 60)
         
         // MARK: - Lifecycle
@@ -28,10 +29,7 @@ extension OpenGroupAPI {
             guard !hasStarted else { return }
             
             hasStarted = true
-            timer = Timer.scheduledTimerOnMainThread(withTimeInterval: Poller.pollInterval, repeats: true) { _ in
+            pollRecursively(using: dependencies)
-                self.poll(using: dependencies).retainUntilComplete()
-            }
-            poll(using: dependencies).retainUntilComplete()
         }
 
         @objc public func stop() {
@@ -41,6 +39,30 @@ extension OpenGroupAPI {
 
         // MARK: - Polling
         
+        private func pollRecursively(using dependencies: OpenGroupManager.OGMDependencies = OpenGroupManager.OGMDependencies()) {
+            guard hasStarted else { return }
+            
+            let minPollFailureCount: TimeInterval = Storage.shared
+                .read { db in
+                    try OpenGroup
+                        .filter(OpenGroup.Columns.server == server)
+                        .select(min(OpenGroup.Columns.pollFailureCount))
+                        .asRequest(of: TimeInterval.self)
+                        .fetchOne(db)
+                }
+                .defaulting(to: 0)
+            let nextPollInterval: TimeInterval = getInterval(for: minPollFailureCount, minInterval: Poller.minPollInterval, maxInterval: Poller.maxPollInterval)
+            
+            poll(using: dependencies).retainUntilComplete()
+            timer = Timer.scheduledTimerOnMainThread(withTimeInterval: nextPollInterval, repeats: false) { [weak self] timer in
+                timer.invalidate()
+                
+                Threading.pollerQueue.async {
+                    self?.pollRecursively(using: dependencies)
+                }
+            }
+        }
+        
         @discardableResult
         public func poll(using dependencies: OpenGroupManager.OGMDependencies = OpenGroupManager.OGMDependencies()) -> Promise<Void> {
             return poll(isBackgroundPoll: false, isPostCapabilitiesRetry: false, using: dependencies)
@@ -83,6 +105,14 @@ extension OpenGroupAPI {
                             cache.timeSinceLastPoll[server] = Date().timeIntervalSince1970
                             UserDefaults.standard[.lastOpen] = Date()
                         }
+                        
+                        // Reset the failure count
+                        Storage.shared.writeAsync { db in
+                            try OpenGroup
+                                .filter(OpenGroup.Columns.server == server)
+                                .updateAll(db, OpenGroup.Columns.pollFailureCount.set(to: 0))
+                        }
+                        
                         SNLog("Open group polling finished for \(server).")
                         seal.fulfill(())
                     }
@@ -97,7 +127,24 @@ extension OpenGroupAPI {
                         )
                         .done(on: OpenGroupAPI.workQueue) { [weak self] didHandleError in
                             if !didHandleError {
-                                SNLog("Open group polling failed due to error: \(error).")
+                                // Increase the failure count
+                                let pollFailureCount: Int64 = Storage.shared
+                                    .read { db in
+                                        try OpenGroup
+                                            .filter(OpenGroup.Columns.server == server)
+                                            .select(max(OpenGroup.Columns.pollFailureCount))
+                                            .asRequest(of: Int64.self)
+                                            .fetchOne(db)
+                                    }
+                                    .defaulting(to: 0)
+                                
+                                Storage.shared.writeAsync { db in
+                                    try OpenGroup
+                                        .filter(OpenGroup.Columns.server == server)
+                                        .updateAll(db, OpenGroup.Columns.pollFailureCount.set(to: (pollFailureCount + 1)))
+                                }
+                                
+                                SNLog("Open group polling failed due to error: \(error). Setting failure count to \(pollFailureCount).")
                             }
                             
                             self?.isPolling = false
@@ -182,7 +229,7 @@ extension OpenGroupAPI {
                     switch endpoint {
                         case .capabilities:
                             guard let responseData: BatchSubResponse<Capabilities> = endpointResponse.data as? BatchSubResponse<Capabilities>, let responseBody: Capabilities = responseData.body else {
-                                SNLog("Open group polling failed due to invalid data.")
+                                SNLog("Open group polling failed due to invalid capability data.")
                                 return
                             }
                             
@@ -194,7 +241,10 @@ extension OpenGroupAPI {
                             
                         case .roomPollInfo(let roomToken, _):
                             guard let responseData: BatchSubResponse<RoomPollInfo> = endpointResponse.data as? BatchSubResponse<RoomPollInfo>, let responseBody: RoomPollInfo = responseData.body else {
-                                SNLog("Open group polling failed due to invalid data.")
+                                switch (endpointResponse.data as? BatchSubResponse<RoomPollInfo>)?.code {
+                                    case 404: SNLog("Open group polling failed to retrieve info for unknown room '\(roomToken)'.")
+                                    default: SNLog("Open group polling failed due to invalid room info data.")
+                                }
                                 return
                             }
                             
@@ -209,7 +259,10 @@ extension OpenGroupAPI {
                             
                         case .roomMessagesRecent(let roomToken), .roomMessagesBefore(let roomToken, _), .roomMessagesSince(let roomToken, _):
                             guard let responseData: BatchSubResponse<[Failable<Message>]> = endpointResponse.data as? BatchSubResponse<[Failable<Message>]>, let responseBody: [Failable<Message>] = responseData.body else {
-                                SNLog("Open group polling failed due to invalid data.")
+                                switch (endpointResponse.data as? BatchSubResponse<[Failable<Message>]>)?.code {
+                                    case 404: SNLog("Open group polling failed to retrieve messages for unknown room '\(roomToken)'.")
+                                    default: SNLog("Open group polling failed due to invalid messages data.")
+                                }
                                 return
                             }
                             let successfulMessages: [Message] = responseBody.compactMap { $0.value }
@@ -231,7 +284,7 @@ extension OpenGroupAPI {
                             
                         case .inbox, .inboxSince, .outbox, .outboxSince:
                             guard let responseData: BatchSubResponse<[DirectMessage]?> = endpointResponse.data as? BatchSubResponse<[DirectMessage]?>, !responseData.failedToParseBody else {
-                                SNLog("Open group polling failed due to invalid data.")
+                                SNLog("Open group polling failed due to invalid inbox/outbox data.")
                                 return
                             }
                             
@@ -259,4 +312,11 @@ extension OpenGroupAPI {
             }
         }
     }
+    
+    // MARK: - Convenience
+
+    fileprivate static func getInterval(for failureCount: TimeInterval, minInterval: TimeInterval, maxInterval: TimeInterval) -> TimeInterval {
+        // Arbitrary backoff factor...
+        return min(maxInterval, minInterval + pow(2, failureCount))
+    }
 }

SessionMessagingKit/Utilities/ProfileManager.swift

@@ -150,10 +150,7 @@ public struct ProfileManager {
             return
         }
         guard
-            let fileId: String = profileUrlStringAtStart
+            let fileId: String = Attachment.fileId(for: profileUrlStringAtStart),
-                .split(separator: "/")
-                .last
-                .map({ String($0) }),
             let profileKeyAtStart: OWSAES256Key = profile.profileEncryptionKey,
             profileKeyAtStart.keyData.count > 0
         else {

SessionUtilitiesKit/Database/Storage.swift

@@ -193,7 +193,21 @@ public final class Storage {
             if !jobTableInfo.contains(where: { $0["name"] == "shouldSkipLaunchBecomeActive" }) {
                 finalError = StorageError.devRemigrationRequired
             }
+            // Forcibly change any 'infoUpdates' on open groups from '-1' to '0' (-1 is invalid)
+            try? db.execute(literal: """
+                UPDATE openGroup
+                SET infoUpdates = 0
+                WHERE openGroup.infoUpdates = -1
+            """)
             // TODO: Remove this once everyone has updated
+            let openGroupTableInfo: [Row] = (try? Row.fetchAll(db, sql: "PRAGMA table_info(openGroup)"))
+                .defaulting(to: [])
+            if !openGroupTableInfo.contains(where: { $0["name"] == "pollFailureCount" }) {
+                try? db.execute(literal: """
+                    ALTER TABLE openGroup
+                    ADD pollFailureCount INTEGER NOT NULL DEFAULT 0
+                """)
+            }
             
             onComplete(finalError, needsConfigSync)
         }

SessionUtilitiesKit/Database/Types/PagedDatabaseObserver.swift

@@ -283,8 +283,10 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
             let indexesAreSequential: Bool = (indexes.map { $0 - 1 }.dropFirst() == indexes.dropLast())
             let hasOneValidIndex: Bool = indexInfo.contains(where: { info -> Bool in
                 info.rowIndex >= updatedPageInfo.pageOffset && (
-                    info.rowIndex < updatedPageInfo.currentCount ||
+                    info.rowIndex < updatedPageInfo.currentCount || (
-                    updatedPageInfo.currentCount == 0
+                        updatedPageInfo.currentCount < updatedPageInfo.pageSize &&
+                        info.rowIndex <= (updatedPageInfo.pageOffset + updatedPageInfo.pageSize)
+                    )
                 )
             })
             
@@ -293,8 +295,10 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
                 indexInfo
                     .filter { info -> Bool in
                         info.rowIndex >= updatedPageInfo.pageOffset && (
-                            info.rowIndex < updatedPageInfo.currentCount ||
+                            info.rowIndex < updatedPageInfo.currentCount || (
-                            updatedPageInfo.currentCount == 0
+                                updatedPageInfo.currentCount < updatedPageInfo.pageSize &&
+                                info.rowIndex <= (updatedPageInfo.pageOffset + updatedPageInfo.pageSize)
+                            )
                         )
                     }
                     .map { info -> Int64 in info.rowId }
@@ -477,6 +481,13 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
                             cacheCurrentEndIndex,
                             currentPageInfo.pageOffset
                         )
+                        
+                    case .reloadCurrent:
+                        return (
+                            currentPageInfo.currentCount,
+                            currentPageInfo.pageOffset,
+                            currentPageInfo.pageOffset
+                        )
                 }
             }()
             
@@ -570,6 +581,10 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
         
         triggerUpdates()
     }
+    
+    public func reload() {
+        self.load(.reloadCurrent)
+    }
 }
 
 // MARK: - Convenience
@@ -720,6 +735,7 @@ public enum PagedData {
             case pageBefore
             case pageAfter
             case untilInclusive(id: SQLExpression, padding: Int)
+            case reloadCurrent
         }
         
         public enum Target<ID: SQLExpressible> {
@@ -1092,8 +1108,10 @@ public class AssociatedRecord<T, PagedType>: ErasedAssociatedRecord where T: Fet
         /// commit - this will mean in some cases we cache data which is actually unrelated to the filtered paged data
         let hasOneValidIndex: Bool = pagedItemIndexes.contains(where: { info -> Bool in
             info.rowIndex >= pageInfo.pageOffset && (
-                info.rowIndex < pageInfo.currentCount ||
+                info.rowIndex < pageInfo.currentCount || (
-                pageInfo.currentCount == 0
+                    pageInfo.currentCount < pageInfo.pageSize &&
+                    info.rowIndex <= (pageInfo.pageOffset + pageInfo.pageSize)
+                )
             )
         })
         

SessionUtilitiesKit/Networking/HTTP.swift

@@ -86,6 +86,7 @@ public enum HTTP {
         case invalidResponse
         case maxFileSizeExceeded
         case httpRequestFailed(statusCode: UInt, data: Data?)
+        case timeout
         
         public var errorDescription: String? {
             switch self {
@@ -95,6 +96,7 @@ public enum HTTP {
                 case .parsingFailed, .invalidResponse: return "Invalid response."
                 case .maxFileSizeExceeded: return "Maximum file size exceeded."
                 case .httpRequestFailed(let statusCode, _): return "HTTP request failed with status code: \(statusCode)."
+                case .timeout: return "The request timed out."
             }
         }
     }
@@ -138,8 +140,13 @@ public enum HTTP {
                 } else {
                     SNLog("\(verb.rawValue) request to \(url) failed.")
                 }
+                
                 // Override the actual error so that we can correctly catch failed requests in sendOnionRequest(invoking:on:with:)
-                return seal.reject(Error.httpRequestFailed(statusCode: 0, data: nil))
+                switch (error as? NSError)?.code {
+                    case NSURLErrorTimedOut: return seal.reject(Error.timeout)
+                    default: return seal.reject(Error.httpRequestFailed(statusCode: 0, data: nil))
+                }
+                
             }
             if let error = error {
                 SNLog("\(verb.rawValue) request to \(url) failed due to error: \(error).")