diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index 8f46dcbaf..71ea58eda 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -44,6 +44,10 @@ NS_ASSUME_NONNULL_BEGIN actionBlock:^{ [DebugUIBackup logDatabaseSizeStats]; }]]; + [items addObject:[OWSTableItem itemWithTitle:@"Clear All CloudKit Records" + actionBlock:^{ + [DebugUIBackup clearAllCloudKitRecords]; + }]]; return [OWSTableSection sectionWithTitle:self.name items:items]; } @@ -150,6 +154,13 @@ NS_ASSUME_NONNULL_BEGIN } } ++ (void)clearAllCloudKitRecords +{ + DDLogInfo(@"%@ clearAllCloudKitRecords.", self.logTag); + + [OWSBackup.sharedManager clearAllCloudKitRecords]; +} + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackup.h b/Signal/src/util/OWSBackup.h index 5327f78d8..bcae8148f 100644 --- a/Signal/src/util/OWSBackup.h +++ b/Signal/src/util/OWSBackup.h @@ -69,6 +69,7 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) { - (void)cancelImportBackup; - (void)logBackupRecords; +- (void)clearAllCloudKitRecords; @end diff --git a/Signal/src/util/OWSBackup.m b/Signal/src/util/OWSBackup.m index 504e1abd3..420cfcaf6 100644 --- a/Signal/src/util/OWSBackup.m +++ b/Signal/src/util/OWSBackup.m @@ -459,6 +459,30 @@ NS_ASSUME_NONNULL_BEGIN }]; } +- (void)clearAllCloudKitRecords +{ + OWSAssertIsOnMainThread(); + + DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + [OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray *recordNames) { + if (recordNames.count < 1) { + DDLogInfo(@"%@ No CloudKit records found to clear.", self.logTag); + return; + } + [OWSBackupAPI deleteRecordsFromCloudWithRecordNames:recordNames + success:^{ + DDLogInfo(@"%@ Clear all CloudKit records succeeded.", self.logTag); + } + failure:^(NSError *error) { + DDLogError(@"%@ Clear all CloudKit records failed: %@.", self.logTag, error); + }]; + } + failure:^(NSError *error) { + DDLogError(@"%@ Failed to retrieve CloudKit records: %@", self.logTag, error); + }]; +} + #pragma mark - Notifications - (void)postDidChangeNotification diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index be943ff95..ebead95ab 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -59,6 +59,14 @@ import CloudKit failure: failure) } + // "Persistent" files may be shared between backup export; they should only be saved + // once. For example, attachment files should only be uploaded once. Subsequent + // backups can reuse the same record. + @objc + public class func recordNameForPersistentFile(fileId: String) -> String { + return "persistentFile-\(fileId)" + } + // "Persistent" files may be shared between backup export; they should only be saved // once. For example, attachment files should only be uploaded once. Subsequent // backups can reuse the same record. @@ -67,7 +75,7 @@ import CloudKit fileUrlBlock: @escaping (()) -> URL?, success: @escaping (String) -> Void, failure: @escaping (Error) -> Void) { - saveFileOnceToCloud(recordName: "persistentFile-\(fileId)", + saveFileOnceToCloud(recordName: recordNameForPersistentFile(fileId: fileId), recordType: signalBackupRecordType, fileUrlBlock: fileUrlBlock, success: success, diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index f96e5585d..3905ca788 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -294,6 +294,9 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, nullable) OWSBackupExportItem *manifestItem; +// If we are replacing an existing backup, we use some of its contents for continuity. +@property (nonatomic, nullable) NSDictionary *lastManifestItemMap; + @end #pragma mark - @@ -338,29 +341,41 @@ NS_ASSUME_NONNULL_BEGIN if (self.isComplete) { return; } - [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_EXPORT", - @"Indicates that the backup export data is being exported.") - progress:nil]; - if (![self exportDatabase]) { - [self failWithErrorDescription: - NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", - @"Error indicating the a backup export could not export the user's data.")]; - return; - } - if (self.isComplete) { - return; - } - [self saveToCloudWithCompletion:^(NSError *_Nullable saveError) { - if (saveError) { - [weakSelf failWithError:saveError]; + [self tryToFetchManifestWithCompletion:^(BOOL tryToFetchManifestSuccess) { + if (!tryToFetchManifestSuccess) { + [self failWithErrorDescription: + NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", + @"Error indicating the a backup export could not export the user's data.")]; return; } - [self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) { - if (cleanUpError) { - [weakSelf failWithError:cleanUpError]; + + if (self.isComplete) { + return; + } + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_EXPORT", + @"Indicates that the backup export data is being exported.") + progress:nil]; + if (![self exportDatabase]) { + [self failWithErrorDescription: + NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", + @"Error indicating the a backup export could not export the user's data.")]; + return; + } + if (self.isComplete) { + return; + } + [self saveToCloudWithCompletion:^(NSError *_Nullable saveError) { + if (saveError) { + [weakSelf failWithError:saveError]; return; } - [weakSelf succeed]; + [self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) { + if (cleanUpError) { + [weakSelf failWithError:cleanUpError]; + return; + } + [weakSelf succeed]; + }]; }]; }]; }]; @@ -402,6 +417,80 @@ NS_ASSUME_NONNULL_BEGIN }]; } +- (void)tryToFetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion +{ + OWSAssert(completion); + + if (self.isComplete) { + return; + } + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_CHECK_BACKUP", + @"Indicates that the backup import is checking for an existing backup.") + progress:nil]; + + __weak OWSBackupExportJob *weakSelf = self; + + [OWSBackupAPI checkForManifestInCloudWithSuccess:^(BOOL value) { + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + if (value) { + [weakSelf fetchManifestWithCompletion:completion]; + } else { + // There is no existing manifest; continue. + completion(YES); + } + }); + } + failure:^(NSError *error) { + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + completion(NO); + }); + }]; +} + +- (void)fetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion +{ + OWSAssert(completion); + + if (self.isComplete) { + return; + } + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + __weak OWSBackupExportJob *weakSelf = self; + [weakSelf downloadAndProcessManifestWithSuccess:^(OWSBackupManifestContents *manifest) { + OWSBackupExportJob *strongSelf = weakSelf; + if (!strongSelf) { + return; + } + if (self.isComplete) { + return; + } + OWSCAssert(manifest.databaseItems.count > 0); + OWSCAssert(manifest.attachmentsItems); + [strongSelf processLastManifest:manifest]; + completion(YES); + } + failure:^(NSError *manifestError) { + completion(NO); + } + backupIO:self.backupIO]; +} + +- (void)processLastManifest:(OWSBackupManifestContents *)manifest +{ + OWSAssert(manifest); + + NSMutableDictionary *lastManifestItemMap = [NSMutableDictionary new]; + for (OWSBackupManifestItem *manifestItem in manifest.attachmentsItems) { + lastManifestItemMap[manifestItem.recordName] = manifestItem; + } + self.lastManifestItemMap = [lastManifestItemMap copy]; +} + - (BOOL)exportDatabase { OWSAssert(self.backupIO); @@ -678,6 +767,39 @@ NS_ASSUME_NONNULL_BEGIN OWSAttachmentExport *attachmentExport = self.unsavedAttachmentExports.lastObject; [self.unsavedAttachmentExports removeLastObject]; + if (self.lastManifestItemMap) { + // Wherever possible, we do incremental backups and re-use fragments of the last backup. + // Recycling fragments doesn't just reduce redundant network activity, + // it allows us to skip the local export work, i.e. encryption. + // To do so, we must preserve the metadata for these fragments. + + NSString *lastRecordName = [OWSBackupAPI recordNameForPersistentFileWithFileId:attachmentExport.attachmentId]; + OWSBackupManifestItem *_Nullable lastManifestItem = self.lastManifestItemMap[lastRecordName]; + if (lastManifestItem) { + OWSAssert(lastManifestItem.encryptionKey.length > 0); + OWSAssert(lastManifestItem.relativeFilePath.length > 0); + + // Recycle the metadata from the last backup's manifest. + OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new]; + encryptedItem.encryptionKey = lastManifestItem.encryptionKey; + attachmentExport.encryptedItem = encryptedItem; + attachmentExport.relativeFilePath = lastManifestItem.relativeFilePath; + + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = attachmentExport.encryptedItem; + exportItem.recordName = lastRecordName; + exportItem.attachmentExport = attachmentExport; + [self.savedAttachmentItems addObject:exportItem]; + + DDLogVerbose(@"%@ recycled attachment: %@ as %@", + self.logTag, + attachmentExport.attachmentFilePath, + attachmentExport.relativeFilePath); + [self saveNextFileToCloudWithCompletion:completion]; + return YES; + } + } + @autoreleasepool { // OWSAttachmentExport is used to lazily write an encrypted copy of the // attachment to disk. @@ -815,7 +937,6 @@ NS_ASSUME_NONNULL_BEGIN OWSAssert(item.recordName.length > 0); itemJson[kOWSBackup_ManifestKey_RecordName] = item.recordName; - OWSAssert(item.encryptedItem.filePath.length > 0); OWSAssert(item.encryptedItem.encryptionKey.length > 0); itemJson[kOWSBackup_ManifestKey_EncryptionKey] = item.encryptedItem.encryptionKey.base64EncodedString; if (item.attachmentExport) { diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index 9cb109365..965a2021a 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -76,7 +76,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe progress:nil]; __weak OWSBackupImportJob *weakSelf = self; - [weakSelf downloadAndProcessManifestWithSuccess:^(OWSBackupManifestContents *_Nullable manifest) { + [weakSelf downloadAndProcessManifestWithSuccess:^(OWSBackupManifestContents *manifest) { OWSBackupImportJob *strongSelf = weakSelf; if (!strongSelf) { return; @@ -84,12 +84,6 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe if (self.isComplete) { return; } - if (!manifest) { - [strongSelf failWithErrorDescription:NSLocalizedString(@"BACKUP_IMPORT_ERROR_COULD_NOT_IMPORT", - @"Error indicating the a backup import " - @"could not import the user's data.")]; - return; - } OWSCAssert(manifest.databaseItems.count > 0); OWSCAssert(manifest.attachmentsItems); strongSelf.databaseItems = manifest.databaseItems; @@ -97,10 +91,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe [strongSelf downloadAndProcessImport]; } failure:^(NSError *manifestError) { - if (manifestError) { - [weakSelf failWithError:manifestError]; - return; - } + [weakSelf failWithError:manifestError]; } backupIO:self.backupIO]; } diff --git a/Signal/src/util/OWSBackupJob.h b/Signal/src/util/OWSBackupJob.h index 31dc17c66..7c72511a7 100644 --- a/Signal/src/util/OWSBackupJob.h +++ b/Signal/src/util/OWSBackupJob.h @@ -17,7 +17,7 @@ extern NSString *const kOWSBackup_ManifestKey_DataSize; typedef void (^OWSBackupJobBoolCompletion)(BOOL success); typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); -typedef void (^OWSBackupJobManifestSuccess)(OWSBackupManifestContents *_Nullable manifest); +typedef void (^OWSBackupJobManifestSuccess)(OWSBackupManifestContents *manifest); typedef void (^OWSBackupJobManifestFailure)(NSError *error); @interface OWSBackupManifestItem : NSObject @@ -26,10 +26,13 @@ typedef void (^OWSBackupJobManifestFailure)(NSError *error); @property (nonatomic) NSData *encryptionKey; +// This property is only set for certain types of manifest item; @property (nonatomic, nullable) NSString *relativeFilePath; +// This property is only set if the manifest item is downloaded. @property (nonatomic, nullable) NSString *downloadFilePath; +// This property is only set if the manifest item is compressed. @property (nonatomic, nullable) NSNumber *uncompressedDataLength; @end