Batch backup exports.

Signal/src/util/Backup/OWSBackupAPI.swift

@@ -57,6 +57,16 @@ import PromiseKit
                                recordType: signalBackupRecordType)
     }
 
+    // "Ephemeral" files are specific to this backup export and will always need to
+    // be saved.  For example, a complete image of the database is exported each time.
+    // We wouldn't want to overwrite previous images until the entire backup export is
+    // complete.
+    @objc
+    public class func recordNameForEphemeralFile(recipientId: String,
+                                                 label: String) -> String {
+        return "\(recordNamePrefix(forRecipientId: recipientId))ephemeral-\(label)-\(NSUUID().uuidString)"
+    }
+
     // "Ephemeral" files are specific to this backup export and will always need to
     // be saved.  For example, a complete image of the database is exported each time.
     // We wouldn't want to overwrite previous images until the entire backup export is
@@ -73,7 +83,7 @@ import PromiseKit
     public class func saveEphemeralFileToCloud(recipientId: String,
                                                label: String,
                                                fileUrl: URL) -> Promise<String> {
-        let recordName = "\(recordNamePrefix(forRecipientId: recipientId))ephemeral-\(label)-\(NSUUID().uuidString)"
+        let recordName = recordNameForEphemeralFile(recipientId: recipientId, label: label)
         return saveFileToCloud(fileUrl: fileUrl,
                                recordName: recordName,
                                recordType: signalBackupRecordType)
@@ -181,11 +191,10 @@ import PromiseKit
 
     @objc
     public class func saveFileToCloudObjc(fileUrl: URL,
-                                          recordName: String,
+                                          recordName: String) -> AnyPromise {
-                                          recordType: String) -> AnyPromise {
         return AnyPromise(saveFileToCloud(fileUrl: fileUrl,
                                           recordName: recordName,
-                                          recordType: recordType))
+                                          recordType: signalBackupRecordType))
     }
 
     public class func saveFileToCloud(fileUrl: URL,
@@ -216,7 +225,7 @@ import PromiseKit
 
         return Promise { resolver in
             let saveOperation = CKModifyRecordsOperation(recordsToSave: [record ], recordIDsToDelete: nil)
-            saveOperation.modifyRecordsCompletionBlock = { (records, recordIds, error) in
+            saveOperation.modifyRecordsCompletionBlock = { (_, _, error) in
 
                 let outcome = outcomeForCloudKitError(error: error,
                                                       remainingRetries: remainingRetries,
@@ -264,6 +273,99 @@ import PromiseKit
         }
     }
 
+    @objc
+    public class func record(forFileUrl fileUrl: URL,
+                             recordName: String) -> CKRecord {
+        let recordType = signalBackupRecordType
+        let recordID = CKRecordID(recordName: recordName)
+        let record = CKRecord(recordType: recordType, recordID: recordID)
+        let asset = CKAsset(fileURL: fileUrl)
+        record[payloadKey] = asset
+
+        return record
+    }
+
+    @objc
+    public class func saveRecordsToCloudObjc(records: [CKRecord]) -> AnyPromise {
+        return AnyPromise(saveRecordsToCloud(records: records))
+    }
+
+    public class func saveRecordsToCloud(records: [CKRecord]) -> Promise<Void> {
+        return saveRecordsToCloud(records: records,
+                                  remainingRetries: maxRetries)
+    }
+
+    private class func saveRecordsToCloud(records: [CKRecord],
+                                          remainingRetries: Int) -> Promise<Void> {
+
+        let recordNames = records.map { (record) in
+            return record.recordID.recordName
+        }
+        Logger.verbose("recordNames \(recordNames)")
+
+        return Promise { resolver in
+            let saveOperation = CKModifyRecordsOperation(recordsToSave: records, recordIDsToDelete: nil)
+            saveOperation.modifyRecordsCompletionBlock = { (savedRecords: [CKRecord]?, _, error) in
+
+                let retry = {
+                    // Only retry records which didn't already succeed.
+                    var savedRecordNames = [String]()
+                    if let savedRecords = savedRecords {
+                        savedRecordNames = savedRecords.map { (record) in
+                            return record.recordID.recordName
+                        }
+                    }
+                    let retryRecords = records.filter({ (record) in
+                        return !savedRecordNames.contains(record.recordID.recordName)
+                    })
+
+                    saveRecordsToCloud(records: retryRecords,
+                                       remainingRetries: remainingRetries - 1)
+                        .done { _ in
+                            resolver.fulfill(())
+                        }.catch { (error) in
+                            resolver.reject(error)
+                        }.retainUntilComplete()
+                }
+
+                let outcome = outcomeForCloudKitError(error: error,
+                                                      remainingRetries: remainingRetries,
+                                                      label: "Save Record")
+                switch outcome {
+                case .success:
+                    resolver.fulfill(())
+                case .failureDoNotRetry(let outcomeError):
+                    resolver.reject(outcomeError)
+                case .failureRetryAfterDelay(let retryDelay):
+                    DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
+                        retry()
+                    })
+                case .failureRetryWithoutDelay:
+                    DispatchQueue.global().async {
+                        retry()
+                    }
+                case .unknownItem:
+                    owsFailDebug("unexpected CloudKit response.")
+                    resolver.reject(invalidServiceResponseError())
+                }
+            }
+            saveOperation.isAtomic = false
+            saveOperation.savePolicy = .allKeys
+
+            // TODO: use perRecordProgressBlock and perRecordCompletionBlock.
+//            open var perRecordProgressBlock: ((CKRecord, Double) -> Void)?
+//            open var perRecordCompletionBlock: ((CKRecord, Error?) -> Void)?
+
+            // These APIs are only available in iOS 9.3 and later.
+            if #available(iOS 9.3, *) {
+                saveOperation.isLongLived = true
+                saveOperation.qualityOfService = .background
+            }
+
+            database().add(saveOperation)
+        }
+    }
+
     // Compare:
     // * An "upsert" creates a new record if none exists and
     //   or updates if there is an existing record.

Signal/src/util/Backup/OWSBackupExportJob.m

@@ -18,6 +18,8 @@
 #import <SignalServiceKit/TSMessage.h>
 #import <SignalServiceKit/TSThread.h>
 
+@import CloudKit;
+
 NS_ASSUME_NONNULL_BEGIN
 
 @class OWSAttachmentExport;
@@ -738,33 +740,39 @@ NS_ASSUME_NONNULL_BEGIN
 // This method returns YES IFF "work was done and there might be more work to do".
 - (AnyPromise *)saveDatabaseFilesToCloud
 {
-    AnyPromise *promise = [AnyPromise promiseWithValue:@(1)];
+    //    AnyPromise *promise = [AnyPromise promiseWithValue:@(1)];
 
-    // We need to preserve ordering of database shards.
+    if (self.isComplete) {
-    for (OWSBackupExportItem *item in self.unsavedDatabaseItems) {
+        return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")];
+    }
+
+    NSArray<OWSBackupExportItem *> *items = [self.unsavedDatabaseItems copy];
+    NSMutableArray<CKRecord *> *records = [NSMutableArray new];
+    for (OWSBackupExportItem *item in items) {
         OWSAssertDebug(item.encryptedItem.filePath.length > 0);
 
-        promise
+        NSString *recordName =
-            = promise
+            [OWSBackupAPI recordNameForEphemeralFileWithRecipientId:self.recipientId label:@"database"];
-                  .thenInBackground(^{
+        CKRecord *record =
-                      if (self.isComplete) {
+            [OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:item.encryptedItem.filePath] recordName:recordName];
-                          return [AnyPromise
+        [records addObject:record];
-                              promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")];
     }
 
-                      return [OWSBackupAPI
+    // TODO: Expose progress.
-                          saveEphemeralFileToCloudObjcWithRecipientId:self.recipientId
+    return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records].thenInBackground(^(NSString *recordName) {
-                                                                label:@"database"
+        OWSAssertDebug(items.count == records.count);
-                                                              fileUrl:[NSURL
+        NSUInteger count = MIN(items.count, records.count);
-                                                                          fileURLWithPath:item.encryptedItem.filePath]];
+        for (NSUInteger i = 0; i < count; i++) {
-                  })
+            OWSBackupExportItem *item = items[i];
-                  .thenInBackground(^(NSString *recordName) {
+            CKRecord *record = records[i];
-                      item.recordName = recordName;
+
-                      [self.savedDatabaseItems addObject:item];
+            OWSAssertDebug(record.recordID.recordName.length > 0);
-                  });
+            item.recordName = record.recordID.recordName;
         }
-    [self.unsavedDatabaseItems removeAllObjects];
+
-    return promise;
+        [self.savedDatabaseItems addObjectsFromArray:items];
+        [self.unsavedDatabaseItems removeObjectsInArray:items];
+    });
 }
 
 // This method returns YES IFF "work was done and there might be more work to do".
@@ -907,10 +915,11 @@ NS_ASSUME_NONNULL_BEGIN
     OWSBackupExportItem *exportItem = [OWSBackupExportItem new];
     exportItem.encryptedItem = encryptedItem;
 
-    return [OWSBackupAPI saveEphemeralFileToCloudObjcWithRecipientId:self.recipientId
+    NSString *recordName =
-                                                               label:@"local-profile-avatar"
+        [OWSBackupAPI recordNameForEphemeralFileWithRecipientId:self.recipientId label:@"local-profile-avatar"];
-                                                             fileUrl:[NSURL fileURLWithPath:encryptedItem.filePath]]
+    CKRecord *record =
-        .thenInBackground(^(NSString *recordName) {
+        [OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:encryptedItem.filePath] recordName:recordName];
+    return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]].thenInBackground(^{
         exportItem.recordName = recordName;
         self.localProfileAvatarItem = exportItem;