Merge branch 'closed-groups' into multi-device

SessionMessagingKit/Jobs/JobQueue.swift

@@ -76,15 +76,14 @@ public final class JobQueue : NSObject, JobDelegate {
 
     private func getRetryInterval(for job: Job) -> TimeInterval {
         // Arbitrary backoff factor...
-        // try  1 delay:  0.00s
+        // try  1 delay: 0.5s
-        // try  2 delay:  0.19s
+        // try  2 delay: 1s
         // ...
-        // try  5 delay:  1.30s
+        // try  5 delay: 16s
         // ...
-        // try 11 delay: 61.31s
+        // try 11 delay: 512s
-        let backoffFactor = 1.9
+        let maxBackoff: Double = 10 * 60 // 10 minutes
-        let maxBackoff: Double = 60 * 60 * 1000
+        return 0.25 * min(maxBackoff, pow(2, Double(job.failureCount)))
-        return 0.1 * min(maxBackoff, pow(backoffFactor, Double(job.failureCount)))
     }
 
     @objc private func retry(_ timer: Timer) {

SessionMessagingKit/Jobs/MessageReceiveJob.swift

@@ -57,15 +57,17 @@ public final class MessageReceiveJob : NSObject, Job, NSCoding { // NSObject/NSC
         let (promise, seal) = Promise<Void>.pending()
         SNMessagingKitConfiguration.shared.storage.write(with: { transaction in // Intentionally capture self
             do {
-                let (message, proto) = try MessageReceiver.parse(self.data, openGroupMessageServerID: self.openGroupMessageServerID, using: transaction)
+                let isRetry = (self.failureCount != 0)
+                let (message, proto) = try MessageReceiver.parse(self.data, openGroupMessageServerID: self.openGroupMessageServerID, isRetry: isRetry, using: transaction)
                 try MessageReceiver.handle(message, associatedWithProto: proto, openGroupID: self.openGroupID, isBackgroundPoll: self.isBackgroundPoll, using: transaction)
                 self.handleSuccess()
                 seal.fulfill(())
             } catch {
-                SNLog("Couldn't receive message due to error: \(error).")
                 if let error = error as? MessageReceiver.Error, !error.isRetryable {
+                    SNLog("Message receive job permanently failed due to error: \(error).")
                     self.handlePermanentFailure(error: error)
                 } else {
+                    SNLog("Couldn't receive message due to error: \(error).")
                     self.handleFailure(error: error)
                 }
                 seal.fulfill(()) // The promise is just used to keep track of when we're done

SessionMessagingKit/Messages/Control Messages/ClosedGroupControlMessage.swift

@@ -18,11 +18,15 @@ public final class ClosedGroupControlMessage : ControlMessage {
         case new(publicKey: Data, name: String, encryptionKeyPair: ECKeyPair, members: [Data], admins: [Data])
         /// - Note: Deprecated in favor of more explicit group updates.
         case update(name: String, members: [Data])
-        case encryptionKeyPair([KeyPairWrapper]) // The new encryption key pair encrypted for each member individually
+        /// An encryption key pair encrypted for each member individually.
+        ///
+        /// - Note: `publicKey` is only set when an encryption key pair is sent in a one-to-one context (i.e. not in a group).
+        case encryptionKeyPair(publicKey: Data?, wrappers: [KeyPairWrapper])
         case nameChange(name: String)
         case membersAdded(members: [Data])
         case membersRemoved(members: [Data])
         case memberLeft
+        case encryptionKeyPairRequest
 
         public var description: String {
             switch self {
@@ -33,6 +37,7 @@ public final class ClosedGroupControlMessage : ControlMessage {
             case .membersAdded: return "membersAdded"
             case .membersRemoved: return "membersRemoved"
             case .memberLeft: return "memberLeft"
+            case .encryptionKeyPairRequest: return "encryptionKeyPairRequest"
             }
         }
     }
@@ -98,6 +103,7 @@ public final class ClosedGroupControlMessage : ControlMessage {
         case .membersAdded(let members): return !members.isEmpty
         case .membersRemoved(let members): return !members.isEmpty
         case .memberLeft: return true
+        case .encryptionKeyPairRequest: return true
         }
     }
 
@@ -118,8 +124,9 @@ public final class ClosedGroupControlMessage : ControlMessage {
                 let members = coder.decodeObject(forKey: "members") as? [Data] else { return nil }
             self.kind = .update(name: name, members: members)
         case "encryptionKeyPair":
+            let publicKey = coder.decodeObject(forKey: "publicKey") as? Data
             guard let wrappers = coder.decodeObject(forKey: "wrappers") as? [KeyPairWrapper] else { return nil }
-            self.kind = .encryptionKeyPair(wrappers)
+            self.kind = .encryptionKeyPair(publicKey: publicKey, wrappers: wrappers)
         case "nameChange":
             guard let name = coder.decodeObject(forKey: "name") as? String else { return nil }
             self.kind = .nameChange(name: name)
@@ -131,6 +138,8 @@ public final class ClosedGroupControlMessage : ControlMessage {
             self.kind = .membersRemoved(members: members)
         case "memberLeft":
             self.kind = .memberLeft
+        case "encryptionKeyPairRequest":
+            self.kind = .encryptionKeyPairRequest
         default: return nil
         }
     }
@@ -150,8 +159,9 @@ public final class ClosedGroupControlMessage : ControlMessage {
             coder.encode("update", forKey: "kind")
             coder.encode(name, forKey: "name")
             coder.encode(members, forKey: "members")
-        case .encryptionKeyPair(let wrappers):
+        case .encryptionKeyPair(let publicKey, let wrappers):
             coder.encode("encryptionKeyPair", forKey: "kind")
+            coder.encode(publicKey, forKey: "publicKey")
             coder.encode(wrappers, forKey: "wrappers")
         case .nameChange(let name):
             coder.encode("nameChange", forKey: "kind")
@@ -164,6 +174,8 @@ public final class ClosedGroupControlMessage : ControlMessage {
             coder.encode(members, forKey: "members")
         case .memberLeft:
             coder.encode("memberLeft", forKey: "kind")
+        case .encryptionKeyPairRequest:
+            coder.encode("encryptionKeyPairRequest", forKey: "kind")
         }
     }
 
@@ -187,8 +199,9 @@ public final class ClosedGroupControlMessage : ControlMessage {
             guard let name = closedGroupControlMessageProto.name else { return nil }
             kind = .update(name: name, members: closedGroupControlMessageProto.members)
         case .encryptionKeyPair:
+            let publicKey = closedGroupControlMessageProto.publicKey
             let wrappers = closedGroupControlMessageProto.wrappers.compactMap { KeyPairWrapper.fromProto($0) }
-            kind = .encryptionKeyPair(wrappers)
+            kind = .encryptionKeyPair(publicKey: publicKey, wrappers: wrappers)
         case .nameChange:
             guard let name = closedGroupControlMessageProto.name else { return nil }
             kind = .nameChange(name: name)
@@ -198,6 +211,8 @@ public final class ClosedGroupControlMessage : ControlMessage {
             kind = .membersRemoved(members: closedGroupControlMessageProto.members)
         case .memberLeft:
             kind = .memberLeft
+        case .encryptionKeyPairRequest:
+            kind = .encryptionKeyPairRequest
         }
         return ClosedGroupControlMessage(kind: kind)
     }
@@ -227,8 +242,11 @@ public final class ClosedGroupControlMessage : ControlMessage {
                 closedGroupControlMessage = SNProtoDataMessageClosedGroupControlMessage.builder(type: .update)
                 closedGroupControlMessage.setName(name)
                 closedGroupControlMessage.setMembers(members)
-            case .encryptionKeyPair(let wrappers):
+            case .encryptionKeyPair(let publicKey, let wrappers):
                 closedGroupControlMessage = SNProtoDataMessageClosedGroupControlMessage.builder(type: .encryptionKeyPair)
+                if let publicKey = publicKey {
+                    closedGroupControlMessage.setPublicKey(publicKey)
+                }
                 closedGroupControlMessage.setWrappers(wrappers.compactMap { $0.toProto() })
             case .nameChange(let name):
                 closedGroupControlMessage = SNProtoDataMessageClosedGroupControlMessage.builder(type: .nameChange)
@@ -241,6 +259,8 @@ public final class ClosedGroupControlMessage : ControlMessage {
                 closedGroupControlMessage.setMembers(members)
             case .memberLeft:
                 closedGroupControlMessage = SNProtoDataMessageClosedGroupControlMessage.builder(type: .memberLeft)
+            case .encryptionKeyPairRequest:
+                closedGroupControlMessage = SNProtoDataMessageClosedGroupControlMessage.builder(type: .encryptionKeyPairRequest)
             }
             let contentProto = SNProtoContent.builder()
             let dataMessageProto = SNProtoDataMessage.builder()

SessionMessagingKit/Protos/Generated/SNProto.swift

@@ -2410,6 +2410,7 @@ extension SNProtoDataMessageClosedGroupControlMessageKeyPairWrapper.SNProtoDataM
         case membersAdded = 5
         case membersRemoved = 6
         case memberLeft = 7
+        case encryptionKeyPairRequest = 8
     }
 
     private class func SNProtoDataMessageClosedGroupControlMessageTypeWrap(_ value: SessionProtos_DataMessage.ClosedGroupControlMessage.TypeEnum) -> SNProtoDataMessageClosedGroupControlMessageType {
@@ -2421,6 +2422,7 @@ extension SNProtoDataMessageClosedGroupControlMessageKeyPairWrapper.SNProtoDataM
         case .membersAdded: return .membersAdded
         case .membersRemoved: return .membersRemoved
         case .memberLeft: return .memberLeft
+        case .encryptionKeyPairRequest: return .encryptionKeyPairRequest
         }
     }
 
@@ -2433,6 +2435,7 @@ extension SNProtoDataMessageClosedGroupControlMessageKeyPairWrapper.SNProtoDataM
         case .membersAdded: return .membersAdded
         case .membersRemoved: return .membersRemoved
         case .memberLeft: return .memberLeft
+        case .encryptionKeyPairRequest: return .encryptionKeyPairRequest
         }
     }
 

SessionMessagingKit/Protos/Generated/SessionProtos.pb.swift

@@ -1152,6 +1152,9 @@ struct SessionProtos_DataMessage {
       case membersRemoved // = 6
       case memberLeft // = 7
 
+      /// wrappers
+      case encryptionKeyPairRequest // = 8
+
       init() {
         self = .new
       }
@@ -1165,6 +1168,7 @@ struct SessionProtos_DataMessage {
         case 5: self = .membersAdded
         case 6: self = .membersRemoved
         case 7: self = .memberLeft
+        case 8: self = .encryptionKeyPairRequest
         default: return nil
         }
       }
@@ -1178,6 +1182,7 @@ struct SessionProtos_DataMessage {
         case .membersAdded: return 5
         case .membersRemoved: return 6
         case .memberLeft: return 7
+        case .encryptionKeyPairRequest: return 8
         }
       }
 
@@ -3131,6 +3136,7 @@ extension SessionProtos_DataMessage.ClosedGroupControlMessage.TypeEnum: SwiftPro
     5: .same(proto: "MEMBERS_ADDED"),
     6: .same(proto: "MEMBERS_REMOVED"),
     7: .same(proto: "MEMBER_LEFT"),
+    8: .same(proto: "ENCRYPTION_KEY_PAIR_REQUEST"),
   ]
 }
 

SessionMessagingKit/Protos/SessionProtos.proto

@@ -171,11 +171,12 @@ message DataMessage {
     enum Type {
       NEW                         = 1; // publicKey, name, encryptionKeyPair, members, admins
       UPDATE                      = 2; // name, members
-      ENCRYPTION_KEY_PAIR = 3; // wrappers
+      ENCRYPTION_KEY_PAIR         = 3; // publicKey, wrappers
       NAME_CHANGE                 = 4; // name
       MEMBERS_ADDED               = 5; // members
       MEMBERS_REMOVED             = 6; // members
       MEMBER_LEFT                 = 7;
+      ENCRYPTION_KEY_PAIR_REQUEST = 8;
     }
 
     message KeyPairWrapper {

SessionMessagingKit/Sending & Receiving/MessageReceiver+Handling.swift

@@ -254,6 +254,7 @@ extension MessageReceiver {
         case .membersAdded: handleClosedGroupMembersAdded(message, using: transaction)
         case .membersRemoved: handleClosedGroupMembersRemoved(message, using: transaction)
         case .memberLeft: handleClosedGroupMemberLeft(message, using: transaction)
+        case .encryptionKeyPairRequest: handleClosedGroupEncryptionKeyPairRequest(message, using: transaction)
         }
     }
     
@@ -295,7 +296,8 @@ extension MessageReceiver {
 
     private static func handleClosedGroupEncryptionKeyPair(_ message: ClosedGroupControlMessage, using transaction: Any) {
         // Prepare
-        guard case let .encryptionKeyPair(wrappers) = message.kind, let groupPublicKey = message.groupPublicKey else { return }
+        guard case let .encryptionKeyPair(explicitGroupPublicKey, wrappers) = message.kind,
+            let groupPublicKey = explicitGroupPublicKey?.toHexString() ?? message.groupPublicKey else { return }
         let transaction = transaction as! YapDatabaseReadWriteTransaction
         let userPublicKey = getUserHexEncodedPublicKey()
         guard let userKeyPair = SNMessagingKitConfiguration.shared.storage.getUserKeyPair() else {
@@ -306,8 +308,8 @@ extension MessageReceiver {
         guard let thread = TSGroupThread.fetch(uniqueId: threadID, transaction: transaction) else {
             return SNLog("Ignoring closed group encryption key pair for nonexistent group.")
         }
-        guard thread.groupModel.groupAdminIds.contains(message.sender!) else {
+        guard thread.groupModel.groupMemberIds.contains(message.sender!) else {
-            return SNLog("Ignoring closed group encryption key pair from non-admin.")
+            return SNLog("Ignoring closed group encryption key pair from non-member.")
         }
         // Find our wrapper and decrypt it if possible
         guard let wrapper = wrappers.first(where: { $0.publicKey == userPublicKey }), let encryptedKeyPair = wrapper.encryptedKeyPair else { return }
@@ -330,7 +332,11 @@ extension MessageReceiver {
         } catch {
             return SNLog("Couldn't parse closed group encryption key pair.")
         }
-        // Store it
+        // Store it if needed
+        let closedGroupEncryptionKeyPairs = Storage.shared.getClosedGroupEncryptionKeyPairs(for: groupPublicKey)
+        guard !closedGroupEncryptionKeyPairs.contains(keyPair) else {
+            return SNLog("Ignoring duplicate closed group encryption key pair.")
+        }
         Storage.shared.addClosedGroupEncryptionKeyPair(keyPair, for: groupPublicKey, using: transaction)
         SNLog("Received a new closed group encryption key pair.")
     }
@@ -447,6 +453,30 @@ extension MessageReceiver {
         }
     }
     
+    private static func handleClosedGroupEncryptionKeyPairRequest(_ message: ClosedGroupControlMessage, using transaction: Any) {
+        guard case .encryptionKeyPairRequest = message.kind else { return }
+        let transaction = transaction as! YapDatabaseReadWriteTransaction
+        guard let groupPublicKey = message.groupPublicKey else { return }
+        performIfValid(for: message, using: transaction) { groupID, _, group in
+            let publicKey = message.sender!
+            // Guard against self-sends
+            guard publicKey != getUserHexEncodedPublicKey() else {
+                return SNLog("Ignoring invalid closed group update.")
+            }
+            // Get the latest encryption key pair
+            guard let encryptionKeyPair = Storage.shared.getLatestClosedGroupEncryptionKeyPair(for: groupPublicKey) else { return }
+            // Send it
+            guard let proto = try? SNProtoKeyPair.builder(publicKey: encryptionKeyPair.publicKey,
+                privateKey: encryptionKeyPair.privateKey).build(), let plaintext = try? proto.serializedData() else { return }
+            let thread = TSContactThread.getOrCreateThread(withContactId: publicKey, transaction: transaction)
+            guard let ciphertext = try? MessageSender.encryptWithSessionProtocol(plaintext, for: publicKey) else { return }
+            SNLog("Responding to closed group encryption key pair request from: \(publicKey).")
+            let wrapper = ClosedGroupControlMessage.KeyPairWrapper(publicKey: publicKey, encryptedKeyPair: ciphertext)
+            let closedGroupControlMessage = ClosedGroupControlMessage(kind: .encryptionKeyPair(publicKey: Data(hex: groupPublicKey), wrappers: [ wrapper ]))
+            MessageSender.send(closedGroupControlMessage, in: thread, using: transaction)
+        }
+    }
+    
     private static func performIfValid(for message: ClosedGroupControlMessage, using transaction: Any, _ update: (Data, TSGroupThread, TSGroupModel) -> Void) {
         // Prepare
         let transaction = transaction as! YapDatabaseReadWriteTransaction

SessionMessagingKit/Sending & Receiving/MessageReceiver.swift

@@ -1,6 +1,7 @@
 import SessionUtilitiesKit
 
 public enum MessageReceiver {
+    private static var lastEncryptionKeyPairRequest: [String:Date] = [:] 
 
     public enum Error : LocalizedError {
         case duplicateMessage
@@ -18,11 +19,10 @@ public enum MessageReceiver {
         // Shared sender keys
         case invalidGroupPublicKey
         case noGroupKeyPair
-        case sharedSecretGenerationFailed
 
         public var isRetryable: Bool {
             switch self {
-            case .duplicateMessage, .invalidMessage, .unknownMessage, .unknownEnvelopeType, .invalidSignature, .noData, .senderBlocked, .selfSend, .decryptionFailed: return false
+            case .duplicateMessage, .invalidMessage, .unknownMessage, .unknownEnvelopeType, .invalidSignature, .noData, .senderBlocked, .selfSend: return false
             default: return true
             }
         }
@@ -44,18 +44,20 @@ public enum MessageReceiver {
             // Shared sender keys
             case .invalidGroupPublicKey: return "Invalid group public key."
             case .noGroupKeyPair: return "Missing group key pair."
-            case .sharedSecretGenerationFailed: return "Couldn't generate a shared secret."
             }
         }
     }
 
-    public static func parse(_ data: Data, openGroupMessageServerID: UInt64?, using transaction: Any) throws -> (Message, SNProtoContent) {
+    public static func parse(_ data: Data, openGroupMessageServerID: UInt64?, isRetry: Bool = false, using transaction: Any) throws -> (Message, SNProtoContent) {
         let userPublicKey = SNMessagingKitConfiguration.shared.storage.getUserPublicKey()
         let isOpenGroupMessage = (openGroupMessageServerID != nil)
         // Parse the envelope
         let envelope = try SNProtoEnvelope.parseData(data)
         let storage = SNMessagingKitConfiguration.shared.storage
-        guard !Set(storage.getReceivedMessageTimestamps(using: transaction)).contains(envelope.timestamp) else { throw Error.duplicateMessage }
+        // If the message failed to process the first time around we retry it later (if the error is retryable). In this case the timestamp
+        // will already be in the database but we don't want to treat the message as a duplicate. The isRetry flag is a simple workaround
+        // for this issue.
+        guard !Set(storage.getReceivedMessageTimestamps(using: transaction)).contains(envelope.timestamp) || isRetry else { throw Error.duplicateMessage }
         storage.addReceivedMessageTimestamp(envelope.timestamp, using: transaction)
         // Decrypt the contents
         guard let ciphertext = envelope.content else { throw Error.noData }
@@ -88,8 +90,21 @@ public enum MessageReceiver {
                         }
                     }
                 }
-                try decrypt()
                 groupPublicKey = envelope.source
+                do {
+                    try decrypt()
+                } catch {
+                    do {
+                        let now = Date()
+                        // Don't spam encryption key pair requests
+                        let shouldRequestEncryptionKeyPair = given(lastEncryptionKeyPairRequest[groupPublicKey!]) { now.timeIntervalSince($0) > 30 } ?? true
+                        if shouldRequestEncryptionKeyPair {
+                            try MessageSender.requestEncryptionKeyPair(for: groupPublicKey!, using: transaction as! YapDatabaseReadWriteTransaction)
+                            lastEncryptionKeyPairRequest[groupPublicKey!] = now
+                        }
+                    }
+                    throw error // Throw the * decryption * error and not the error generated by requestEncryptionKeyPair (if it generated one)
+                }
             default: throw Error.unknownEnvelopeType
             }
         }

SessionMessagingKit/Sending & Receiving/MessageSender+ClosedGroups.swift

@@ -68,7 +68,7 @@ extension MessageSender {
             let ciphertext = try MessageSender.encryptWithSessionProtocol(plaintext, for: publicKey)
             return ClosedGroupControlMessage.KeyPairWrapper(publicKey: publicKey, encryptedKeyPair: ciphertext)
         }
-        let closedGroupControlMessage = ClosedGroupControlMessage(kind: .encryptionKeyPair(wrappers))
+        let closedGroupControlMessage = ClosedGroupControlMessage(kind: .encryptionKeyPair(publicKey: nil, wrappers: wrappers))
         let _ = MessageSender.sendNonDurably(closedGroupControlMessage, in: thread, using: transaction).done { // FIXME: It'd be great if we could make this a durable operation
             // Store it * after * having sent out the message to the group
             SNMessagingKitConfiguration.shared.storage.write { transaction in
@@ -233,6 +233,21 @@ extension MessageSender {
         infoMessage.save(with: transaction)
     }
     
+    public static func requestEncryptionKeyPair(for groupPublicKey: String, using transaction: YapDatabaseReadWriteTransaction) throws {
+        // Get the group, check preconditions & prepare
+        let groupID = LKGroupUtilities.getEncodedClosedGroupIDAsData(groupPublicKey)
+        let threadID = TSGroupThread.threadId(fromGroupId: groupID)
+        guard let thread = TSGroupThread.fetch(uniqueId: threadID, transaction: transaction) else {
+            SNLog("Can't request encryption key pair for nonexistent closed group.")
+            throw Error.noThread
+        }
+        let group = thread.groupModel
+        guard group.groupMemberIds.contains(getUserHexEncodedPublicKey()) else { return }
+        // Send the request to the group
+        let closedGroupControlMessage = ClosedGroupControlMessage(kind: .encryptionKeyPairRequest)
+        MessageSender.send(closedGroupControlMessage, in: thread, using: transaction)
+    }
+    
     
     
     // MARK: - Deprecated