Merge branch 'database-refactor' into quote-standardise

Session/Calls/Call Management/SessionCall.swift

@@ -167,7 +167,10 @@ public final class SessionCall: CurrentCallProtocol, WebRTCSessionDelegate {
     }
     
     func reportIncomingCallIfNeeded(completion: @escaping (Error?) -> Void) {
-        guard case .answer = mode else { return }
+        guard case .answer = mode else {
+            SessionCallManager.reportFakeCall(info: "Call not in answer mode")
+            return
+        }
         
         setupTimeoutTimer()
         AppEnvironment.shared.callManager.reportIncomingCall(self, callerName: contactName) { error in

Session/Calls/Call Management/SessionCallManager.swift

@@ -72,6 +72,16 @@ public final class SessionCallManager: NSObject, CallManagerProtocol {
     
     // MARK: - Report calls
     
+    public static func reportFakeCall(info: String) {
+        SessionCallManager.sharedProvider(useSystemCallLog: false)
+            .reportNewIncomingCall(
+                with: UUID(),
+                update: CXCallUpdate()
+            ) { _ in
+                SNLog("[Calls] Reported fake incoming call to CallKit due to: \(info)")
+            }
+    }
+    
     public func reportOutgoingCall(_ call: SessionCall) {
         AssertIsOnMainThread()
         UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
@@ -109,7 +119,9 @@ public final class SessionCallManager: NSObject, CallManagerProtocol {
                 UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
                 completion(nil)
             }
-        } else {
+        }
+        else {
+            SessionCallManager.reportFakeCall(info: "No CXProvider instance")
             UserDefaults.sharedLokiProject?.set(true, forKey: "isCallOngoing")
             completion(nil)
         }

Session/Conversations/ConversationVC.swift

@@ -450,7 +450,7 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
         recoverInputView()
     }
     
@@ -460,7 +460,7 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes
         dataChangeObservable = Storage.shared.start(
             viewModel.observableThreadData,
@@ -506,6 +506,13 @@ final class ConversationVC: BaseVC, OWSConversationSettingsViewDelegate, Convers
                     self?.viewModel.onInteractionChange = { [weak self] updatedInteractionData in
                         self?.handleInteractionUpdates(updatedInteractionData)
                     }
+                    
+                    // Note: When returning from the background we could have received notifications but the
+                    // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+                    // data to ensure everything is up to date
+                    if didReturnFromBackground {
+                        self?.viewModel.pagedDataObserver?.reload()
+                    }
                 }
             }
         )

Session/Home/HomeVC.swift

@@ -239,7 +239,7 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -248,7 +248,7 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes
         dataChangeObservable = Storage.shared.start(
             viewModel.observableState,
@@ -269,6 +269,13 @@ final class HomeVC: BaseVC, UITableViewDataSource, UITableViewDelegate, NewConve
         self.viewModel.onThreadChange = { [weak self] updatedThreadData in
             self?.handleThreadUpdates(updatedThreadData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func stopObservingChanges() {

Session/Home/Message Requests/MessageRequestsViewController.swift

@@ -147,7 +147,7 @@ class MessageRequestsViewController: BaseVC, UITableViewDelegate, UITableViewDat
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -186,10 +186,17 @@ class MessageRequestsViewController: BaseVC, UITableViewDelegate, UITableViewDat
     
     // MARK: - Updating
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         self.viewModel.onThreadChange = { [weak self] updatedThreadData in
             self?.handleThreadUpdates(updatedThreadData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func handleThreadUpdates(_ updatedData: [MessageRequestsViewModel.SectionModel], initialLoad: Bool = false) {

Session/Media Viewing & Editing/MediaTileViewController.swift

@@ -171,7 +171,7 @@ public class MediaTileViewController: UIViewController, UICollectionViewDataSour
     }
     
     @objc func applicationDidBecomeActive(_ notification: Notification) {
-        startObservingChanges()
+        startObservingChanges(didReturnFromBackground: true)
     }
     
     @objc func applicationDidResignActive(_ notification: Notification) {
@@ -243,11 +243,18 @@ public class MediaTileViewController: UIViewController, UICollectionViewDataSour
         }
     }
     
-    private func startObservingChanges() {
+    private func startObservingChanges(didReturnFromBackground: Bool = false) {
         // Start observing for data changes (will callback on the main thread)
         self.viewModel.onGalleryChange = { [weak self] updatedGalleryData in
             self?.handleUpdates(updatedGalleryData)
         }
+        
+        // Note: When returning from the background we could have received notifications but the
+        // PagedDatabaseObserver won't have them so we need to force a re-fetch of the current
+        // data to ensure everything is up to date
+        if didReturnFromBackground {
+            self.viewModel.pagedDataObserver?.reload()
+        }
     }
     
     private func stopObservingChanges() {

Session/Meta/AppDelegate.swift

@@ -149,13 +149,13 @@ class AppDelegate: UIResponder, UIApplicationDelegate, UNUserNotificationCenterD
             /// no longer always called before `applicationDidBecomeActive` we need to trigger the "clear notifications" logic
             /// within the `runNowOrWhenAppDidBecomeReady` callback and dispatch to the next run loop to ensure it runs after
             /// the notification has actually been handled
-            DispatchQueue.main.async { [weak self] in
+            DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(100)) { [weak self] in
                 self?.clearAllNotificationsAndRestoreBadgeCount()
             }
         }
 
         // On every activation, clear old temp directories.
-        ClearOldTemporaryDirectories();
+        ClearOldTemporaryDirectories()
     }
     
     func applicationWillResignActive(_ application: UIApplication) {

Session/Notifications/PushRegistrationManager.swift

@@ -242,8 +242,16 @@ public enum PushRegistrationError: Error {
         owsAssertDebug(type == .voIP)
         let payload = payload.dictionaryPayload
         
-        if let uuid = payload["uuid"] as? String, let caller = payload["caller"] as? String, let timestampMs = payload["timestamp"] as? Int64 {
+        guard
-            let call: SessionCall? = Storage.shared.write { db in
+            let uuid: String = payload["uuid"] as? String,
+            let caller: String = payload["caller"] as? String,
+            let timestampMs: Int64 = payload["timestamp"] as? Int64
+        else {
+            SessionCallManager.reportFakeCall(info: "Missing payload data")
+            return
+        }
+        
+        let maybeCall: SessionCall? = Storage.shared.write { db in
             let messageInfo: CallMessage.MessageInfo = CallMessage.MessageInfo(
                 state: (caller == getUserHexEncodedPublicKey(db) ?
                     .outgoing :
@@ -269,16 +277,20 @@ public enum PushRegistrationError: Error {
             return call
         }
         
+        guard let call: SessionCall = maybeCall else {
+            SessionCallManager.reportFakeCall(info: "Could not retrieve call from database")
+            return
+        }
+        
         // NOTE: Just start 1-1 poller so that it won't wait for polling group messages
         (UIApplication.shared.delegate as? AppDelegate)?.startPollersIfNeeded(shouldStartGroupPollers: false)
         
-            call?.reportIncomingCallIfNeeded { error in
+        call.reportIncomingCallIfNeeded { error in
             if let error = error {
                 SNLog("[Calls] Failed to report incoming call to CallKit due to error: \(error)")
             }
         }
     }
-    }
 }
 
 // We transmit pushToken data as hex encoded string to the server

SessionMessagingKit/Database/Models/Attachment.swift

@@ -468,6 +468,7 @@ extension Attachment {
         public let attachmentId: String
         public let interactionId: Int64
         public let state: Attachment.State
+        public let downloadUrl: String?
     }
     
     public static func stateInfo(authorId: String, state: State? = nil) -> SQLRequest<Attachment.StateInfo> {
@@ -484,7 +485,8 @@ extension Attachment {
             SELECT DISTINCT
                 \(attachment[.id]) AS attachmentId,
                 \(interaction[.id]) AS interactionId,
-                \(attachment[.state]) AS state
+                \(attachment[.state]) AS state,
+                \(attachment[.downloadUrl]) AS downloadUrl
         
             FROM \(Attachment.self)
             
@@ -529,7 +531,8 @@ extension Attachment {
             SELECT DISTINCT
                 \(attachment[.id]) AS attachmentId,
                 \(interaction[.id]) AS interactionId,
-                \(attachment[.state]) AS state
+                \(attachment[.state]) AS state,
+                \(attachment[.downloadUrl]) AS downloadUrl
         
             FROM \(Attachment.self)
             
@@ -913,6 +916,16 @@ extension Attachment {
 
         return true
     }
+    
+    public static func fileId(for downloadUrl: String?) -> String? {
+        return downloadUrl
+            .map { urlString -> String? in
+                urlString
+                    .split(separator: "/")
+                    .last
+                    .map { String($0) }
+            }
+    }
 }
 
 // MARK: - Upload
@@ -923,14 +936,14 @@ extension Attachment {
         queue: DispatchQueue,
         using upload: (Database, Data) -> Promise<String>,
         encrypt: Bool,
-        success: (() -> Void)?,
+        success: ((String?) -> Void)?,
         failure: ((Error) -> Void)?
     ) {
         // This can occur if an AttachmnetUploadJob was explicitly created for a message
         // dependant on the attachment being uploaded (in this case the attachment has
         // already been uploaded so just succeed)
         guard state != .uploaded else {
-            success?()
+            success?(Attachment.fileId(for: self.downloadUrl))
             return
         }
         
@@ -982,7 +995,7 @@ extension Attachment {
                 return
             }
                 
-            success?()
+            success?(Attachment.fileId(for: self.downloadUrl))
             return
         }
         
@@ -1073,7 +1086,7 @@ extension Attachment {
                     return
                 }
                     
-                success?()
+                success?(fileId)
             }
             .catch(on: queue) { error in
                 Storage.shared.write { db in

SessionMessagingKit/Jobs/Types/AttachmentDownloadJob.swift

@@ -87,10 +87,7 @@ public enum AttachmentDownloadJob: JobExecutor {
         let downloadPromise: Promise<Data> = {
             guard
                 let downloadUrl: String = attachment.downloadUrl,
-                let fileId: String = downloadUrl
+                let fileId: String = Attachment.fileId(for: downloadUrl)
-                    .split(separator: "/")
-                    .last
-                    .map({ String($0) })
             else {
                 return Promise(error: AttachmentDownloadError.invalidUrl)
             }

SessionMessagingKit/Jobs/Types/AttachmentUploadJob.swift

@@ -55,7 +55,7 @@ public enum AttachmentUploadJob: JobExecutor {
                     .map { response -> String in response.id }
             },
             encrypt: (openGroup == nil),
-            success: { success(job, false) },
+            success: { _ in success(job, false) },
             failure: { error in failure(job, error, false) }
         )
     }

SessionMessagingKit/Jobs/Types/MessageSendJob.swift

@@ -27,6 +27,10 @@ public enum MessageSendJob: JobExecutor {
             return
         }
         
+        // We need to include 'fileIds' when sending messages with attachments to Open Groups
+        // so extract them from any associated attachments
+        var messageFileIds: [String] = []
+        
         if details.message is VisibleMessage {
             guard
                 let jobId: Int64 = job.id,
@@ -36,20 +40,30 @@ public enum MessageSendJob: JobExecutor {
                 return
             }
             
+            // If the original interaction no longer exists then don't bother sending the message (ie. the
+            // message was deleted before it even got sent)
+            guard Storage.shared.read({ db in try Interaction.exists(db, id: interactionId) }) == true else {
+                failure(job, StorageError.objectNotFound, true)
+                return
+            }
+            
             // Check if there are any attachments associated to this message, and if so
             // upload them now
             //
             // Note: Normal attachments should be sent in a non-durable way but any
             // attachments for LinkPreviews and Quotes will be processed through this mechanism
-            let attachmentState: (shouldFail: Bool, shouldDefer: Bool)? = Storage.shared.write { db in
+            let attachmentState: (shouldFail: Bool, shouldDefer: Bool, fileIds: [String])? = Storage.shared.write { db in
                 let allAttachmentStateInfo: [Attachment.StateInfo] = try Attachment
                     .stateInfo(interactionId: interactionId)
                     .fetchAll(db)
+                let maybeFileIds: [String?] = allAttachmentStateInfo
+                    .map { Attachment.fileId(for: $0.downloadUrl) }
+                let fileIds: [String] = maybeFileIds.compactMap { $0 }
                 
                 // If there were failed attachments then this job should fail (can't send a
                 // message which has associated attachments if the attachments fail to upload)
                 guard !allAttachmentStateInfo.contains(where: { $0.state == .failedDownload }) else {
-                    return (true, false)
+                    return (true, false, fileIds)
                 }
                 
                 // Create jobs for any pending (or failed) attachment jobs and insert them into the
@@ -102,9 +116,13 @@ public enum MessageSendJob: JobExecutor {
                 // If there were pending or uploading attachments then stop here (we want to
                 // upload them first and then re-run this send job - the 'JobRunner.insert'
                 // method will take care of this)
+                let isMissingFileIds: Bool = (maybeFileIds.count != fileIds.count)
+                let hasPendingUploads: Bool = allAttachmentStateInfo.contains(where: { $0.state != .uploaded })
+                
                 return (
-                    false,
+                    (isMissingFileIds && !hasPendingUploads),
-                    allAttachmentStateInfo.contains(where: { $0.state != .uploaded })
+                    hasPendingUploads,
+                    fileIds
                 )
             }
             
@@ -122,6 +140,9 @@ public enum MessageSendJob: JobExecutor {
                 deferred(job)
                 return
             }
+            
+            // Store the fileIds so they can be sent with the open group message content
+            messageFileIds = (attachmentState?.fileIds ?? [])
         }
         
         // Store the sentTimestamp from the message in case it fails due to a clockOutOfSync error
@@ -135,7 +156,8 @@ public enum MessageSendJob: JobExecutor {
             try MessageSender.sendImmediate(
                 db,
                 message: details.message,
-                to: details.destination,
+                to: details.destination
+                    .with(fileIds: messageFileIds),
                 interactionId: job.interactionId
             )
         }

SessionMessagingKit/Messages/Message+Destination.swift

@@ -49,5 +49,21 @@ public extension Message {
                     return .openGroup(roomToken: openGroup.roomToken, server: openGroup.server, fileIds: fileIds)
             }
         }
+        
+        func with(fileIds: [String]) -> Message.Destination {
+            // Only Open Group messages support receiving the 'fileIds'
+            switch self {
+                case .openGroup(let roomToken, let server, let whisperTo, let whisperMods, _):
+                    return .openGroup(
+                        roomToken: roomToken,
+                        server: server,
+                        whisperTo: whisperTo,
+                        whisperMods: whisperMods,
+                        fileIds: fileIds
+                    )
+                    
+                default: return self
+            }
+        }
     }
 }

SessionMessagingKit/Sending & Receiving/MessageSender+Convenience.swift

@@ -100,7 +100,7 @@ extension MessageSender {
     }
     
     public static func sendNonDurably(_ db: Database, message: Message, interactionId: Int64?, to destination: Message.Destination) -> Promise<Void> {
-        var attachmentUploadPromises: [Promise<Void>] = [Promise.value(())]
+        var attachmentUploadPromises: [Promise<String?>] = [Promise.value(nil)]
         
         // If we have an interactionId then check if it has any attachments and process them first
         if let interactionId: Int64 = interactionId {
@@ -124,8 +124,8 @@ extension MessageSender {
                 .filter(ids: attachmentStateInfo.map { $0.attachmentId })
                 .fetchAll(db))
                 .defaulting(to: [])
-                .map { attachment -> Promise<Void> in
+                .map { attachment -> Promise<String?> in
-                    let (promise, seal) = Promise<Void>.pending()
+                    let (promise, seal) = Promise<String?>.pending()
     
                     attachment.upload(
                         db,
@@ -146,7 +146,7 @@ extension MessageSender {
                                 .map { response -> String in response.id }
                         },
                         encrypt: (openGroup == nil),
-                        success: { seal.fulfill(()) },
+                        success: { fileId in seal.fulfill(fileId) },
                         failure: { seal.reject($0) }
                     )
     
@@ -167,10 +167,18 @@ extension MessageSender {
                 if let error: Error = errors.first { return Promise(error: error) }
                 
                 return Storage.shared.writeAsync { db in
-                    try MessageSender.sendImmediate(
+                    let fileIds: [String] = results
+                        .compactMap { result -> String? in
+                            if case .fulfilled(let value) = result { return value }
+                            
+                            return nil
+                        }
+                    
+                    return try MessageSender.sendImmediate(
                         db,
                         message: message,
-                        to: destination,
+                        to: destination
+                            .with(fileIds: fileIds),
                         interactionId: interactionId
                     )
                 }

SessionMessagingKit/Utilities/ProfileManager.swift

@@ -150,10 +150,7 @@ public struct ProfileManager {
             return
         }
         guard
-            let fileId: String = profileUrlStringAtStart
+            let fileId: String = Attachment.fileId(for: profileUrlStringAtStart),
-                .split(separator: "/")
-                .last
-                .map({ String($0) }),
             let profileKeyAtStart: OWSAES256Key = profile.profileEncryptionKey,
             profileKeyAtStart.keyData.count > 0
         else {

SessionUtilitiesKit/Database/Types/PagedDatabaseObserver.swift

@@ -477,6 +477,13 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
                             cacheCurrentEndIndex,
                             currentPageInfo.pageOffset
                         )
+                        
+                    case .reloadCurrent:
+                        return (
+                            currentPageInfo.currentCount,
+                            currentPageInfo.pageOffset,
+                            currentPageInfo.pageOffset
+                        )
                 }
             }()
             
@@ -570,6 +577,10 @@ public class PagedDatabaseObserver<ObservedTable, T>: TransactionObserver where
         
         triggerUpdates()
     }
+    
+    public func reload() {
+        self.load(.reloadCurrent)
+    }
 }
 
 // MARK: - Convenience
@@ -718,6 +729,7 @@ public enum PagedData {
             case pageBefore
             case pageAfter
             case untilInclusive(id: SQLExpression, padding: Int)
+            case reloadCurrent
         }
         
         public enum Target<ID: SQLExpressible> {