From c1ac5c1872ea3069878d1298d346bac44b41dbe6 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Fri, 30 Nov 2018 16:07:23 -0500 Subject: [PATCH] Batch backup exports. --- Signal/src/util/Backup/OWSBackupAPI.swift | 112 +++++++++++++++++++- Signal/src/util/Backup/OWSBackupExportJob.m | 71 +++++++------ 2 files changed, 147 insertions(+), 36 deletions(-) diff --git a/Signal/src/util/Backup/OWSBackupAPI.swift b/Signal/src/util/Backup/OWSBackupAPI.swift index 6981bcc36..a4a8489c4 100644 --- a/Signal/src/util/Backup/OWSBackupAPI.swift +++ b/Signal/src/util/Backup/OWSBackupAPI.swift @@ -57,6 +57,16 @@ import PromiseKit recordType: signalBackupRecordType) } + // "Ephemeral" files are specific to this backup export and will always need to + // be saved. For example, a complete image of the database is exported each time. + // We wouldn't want to overwrite previous images until the entire backup export is + // complete. + @objc + public class func recordNameForEphemeralFile(recipientId: String, + label: String) -> String { + return "\(recordNamePrefix(forRecipientId: recipientId))ephemeral-\(label)-\(NSUUID().uuidString)" + } + // "Ephemeral" files are specific to this backup export and will always need to // be saved. For example, a complete image of the database is exported each time. // We wouldn't want to overwrite previous images until the entire backup export is @@ -73,7 +83,7 @@ import PromiseKit public class func saveEphemeralFileToCloud(recipientId: String, label: String, fileUrl: URL) -> Promise { - let recordName = "\(recordNamePrefix(forRecipientId: recipientId))ephemeral-\(label)-\(NSUUID().uuidString)" + let recordName = recordNameForEphemeralFile(recipientId: recipientId, label: label) return saveFileToCloud(fileUrl: fileUrl, recordName: recordName, recordType: signalBackupRecordType) @@ -181,11 +191,10 @@ import PromiseKit @objc public class func saveFileToCloudObjc(fileUrl: URL, - recordName: String, - recordType: String) -> AnyPromise { + recordName: String) -> AnyPromise { return AnyPromise(saveFileToCloud(fileUrl: fileUrl, recordName: recordName, - recordType: recordType)) + recordType: signalBackupRecordType)) } public class func saveFileToCloud(fileUrl: URL, @@ -216,7 +225,7 @@ import PromiseKit return Promise { resolver in let saveOperation = CKModifyRecordsOperation(recordsToSave: [record ], recordIDsToDelete: nil) - saveOperation.modifyRecordsCompletionBlock = { (records, recordIds, error) in + saveOperation.modifyRecordsCompletionBlock = { (_, _, error) in let outcome = outcomeForCloudKitError(error: error, remainingRetries: remainingRetries, @@ -264,6 +273,99 @@ import PromiseKit } } + @objc + public class func record(forFileUrl fileUrl: URL, + recordName: String) -> CKRecord { + let recordType = signalBackupRecordType + let recordID = CKRecordID(recordName: recordName) + let record = CKRecord(recordType: recordType, recordID: recordID) + let asset = CKAsset(fileURL: fileUrl) + record[payloadKey] = asset + + return record + } + + @objc + public class func saveRecordsToCloudObjc(records: [CKRecord]) -> AnyPromise { + return AnyPromise(saveRecordsToCloud(records: records)) + } + + public class func saveRecordsToCloud(records: [CKRecord]) -> Promise { + return saveRecordsToCloud(records: records, + remainingRetries: maxRetries) + } + + private class func saveRecordsToCloud(records: [CKRecord], + remainingRetries: Int) -> Promise { + + let recordNames = records.map { (record) in + return record.recordID.recordName + } + Logger.verbose("recordNames \(recordNames)") + + return Promise { resolver in + let saveOperation = CKModifyRecordsOperation(recordsToSave: records, recordIDsToDelete: nil) + saveOperation.modifyRecordsCompletionBlock = { (savedRecords: [CKRecord]?, _, error) in + + let retry = { + // Only retry records which didn't already succeed. + var savedRecordNames = [String]() + if let savedRecords = savedRecords { + savedRecordNames = savedRecords.map { (record) in + return record.recordID.recordName + } + } + let retryRecords = records.filter({ (record) in + return !savedRecordNames.contains(record.recordID.recordName) + }) + + saveRecordsToCloud(records: retryRecords, + remainingRetries: remainingRetries - 1) + .done { _ in + resolver.fulfill(()) + }.catch { (error) in + resolver.reject(error) + }.retainUntilComplete() + } + + let outcome = outcomeForCloudKitError(error: error, + remainingRetries: remainingRetries, + label: "Save Record") + switch outcome { + case .success: + resolver.fulfill(()) + case .failureDoNotRetry(let outcomeError): + resolver.reject(outcomeError) + case .failureRetryAfterDelay(let retryDelay): + DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { + retry() + }) + case .failureRetryWithoutDelay: + DispatchQueue.global().async { + retry() + } + case .unknownItem: + owsFailDebug("unexpected CloudKit response.") + resolver.reject(invalidServiceResponseError()) + } + } + saveOperation.isAtomic = false + saveOperation.savePolicy = .allKeys + + // TODO: use perRecordProgressBlock and perRecordCompletionBlock. +// open var perRecordProgressBlock: ((CKRecord, Double) -> Void)? +// open var perRecordCompletionBlock: ((CKRecord, Error?) -> Void)? + + // These APIs are only available in iOS 9.3 and later. + if #available(iOS 9.3, *) { + saveOperation.isLongLived = true + saveOperation.qualityOfService = .background + } + + database().add(saveOperation) + } + } + // Compare: // * An "upsert" creates a new record if none exists and // or updates if there is an existing record. diff --git a/Signal/src/util/Backup/OWSBackupExportJob.m b/Signal/src/util/Backup/OWSBackupExportJob.m index 845885aa5..5a6dafcf1 100644 --- a/Signal/src/util/Backup/OWSBackupExportJob.m +++ b/Signal/src/util/Backup/OWSBackupExportJob.m @@ -18,6 +18,8 @@ #import #import +@import CloudKit; + NS_ASSUME_NONNULL_BEGIN @class OWSAttachmentExport; @@ -738,33 +740,39 @@ NS_ASSUME_NONNULL_BEGIN // This method returns YES IFF "work was done and there might be more work to do". - (AnyPromise *)saveDatabaseFilesToCloud { - AnyPromise *promise = [AnyPromise promiseWithValue:@(1)]; + // AnyPromise *promise = [AnyPromise promiseWithValue:@(1)]; - // We need to preserve ordering of database shards. - for (OWSBackupExportItem *item in self.unsavedDatabaseItems) { + if (self.isComplete) { + return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")]; + } + + NSArray *items = [self.unsavedDatabaseItems copy]; + NSMutableArray *records = [NSMutableArray new]; + for (OWSBackupExportItem *item in items) { OWSAssertDebug(item.encryptedItem.filePath.length > 0); - promise - = promise - .thenInBackground(^{ - if (self.isComplete) { - return [AnyPromise - promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")]; - } - - return [OWSBackupAPI - saveEphemeralFileToCloudObjcWithRecipientId:self.recipientId - label:@"database" - fileUrl:[NSURL - fileURLWithPath:item.encryptedItem.filePath]]; - }) - .thenInBackground(^(NSString *recordName) { - item.recordName = recordName; - [self.savedDatabaseItems addObject:item]; - }); + NSString *recordName = + [OWSBackupAPI recordNameForEphemeralFileWithRecipientId:self.recipientId label:@"database"]; + CKRecord *record = + [OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:item.encryptedItem.filePath] recordName:recordName]; + [records addObject:record]; } - [self.unsavedDatabaseItems removeAllObjects]; - return promise; + + // TODO: Expose progress. + return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records].thenInBackground(^(NSString *recordName) { + OWSAssertDebug(items.count == records.count); + NSUInteger count = MIN(items.count, records.count); + for (NSUInteger i = 0; i < count; i++) { + OWSBackupExportItem *item = items[i]; + CKRecord *record = records[i]; + + OWSAssertDebug(record.recordID.recordName.length > 0); + item.recordName = record.recordID.recordName; + } + + [self.savedDatabaseItems addObjectsFromArray:items]; + [self.unsavedDatabaseItems removeObjectsInArray:items]; + }); } // This method returns YES IFF "work was done and there might be more work to do". @@ -907,15 +915,16 @@ NS_ASSUME_NONNULL_BEGIN OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; exportItem.encryptedItem = encryptedItem; - return [OWSBackupAPI saveEphemeralFileToCloudObjcWithRecipientId:self.recipientId - label:@"local-profile-avatar" - fileUrl:[NSURL fileURLWithPath:encryptedItem.filePath]] - .thenInBackground(^(NSString *recordName) { - exportItem.recordName = recordName; - self.localProfileAvatarItem = exportItem; + NSString *recordName = + [OWSBackupAPI recordNameForEphemeralFileWithRecipientId:self.recipientId label:@"local-profile-avatar"]; + CKRecord *record = + [OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:encryptedItem.filePath] recordName:recordName]; + return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]].thenInBackground(^{ + exportItem.recordName = recordName; + self.localProfileAvatarItem = exportItem; - return [AnyPromise promiseWithValue:@(1)]; - }); + return [AnyPromise promiseWithValue:@(1)]; + }); } - (AnyPromise *)saveManifestFileToCloud