diff --git a/Pods b/Pods index 1d47ca77e..10be6cb36 160000 --- a/Pods +++ b/Pods @@ -1 +1 @@ -Subproject commit 1d47ca77ea929a2fd76b2b3410487b61f18f5b54 +Subproject commit 10be6cb3689bd1815ffcecc2f9500b2b55f60962 diff --git a/Signal.xcodeproj/project.pbxproj b/Signal.xcodeproj/project.pbxproj index c35a62037..6ee946dd6 100644 --- a/Signal.xcodeproj/project.pbxproj +++ b/Signal.xcodeproj/project.pbxproj @@ -205,6 +205,7 @@ 34D1F0C01F8EC1760066283D /* MessageRecipientStatusUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D1F0BF1F8EC1760066283D /* MessageRecipientStatusUtils.swift */; }; 34D2CCD4206294B900CB1A14 /* OWSScreenLock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D2CCD3206294B900CB1A14 /* OWSScreenLock.swift */; }; 34D2CCDA2062E7D000CB1A14 /* OWSScreenLockUI.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D2CCD92062E7D000CB1A14 /* OWSScreenLockUI.m */; }; + 34D2CCD220618B3000CB1A14 /* OWSBackupLazyRestoreJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D2CCD120618B2F00CB1A14 /* OWSBackupLazyRestoreJob.swift */; }; 34D5CCA91EAE3D30005515DB /* AvatarViewHelper.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D5CCA81EAE3D30005515DB /* AvatarViewHelper.m */; }; 34D8C0271ED3673300188D7C /* DebugUIMessages.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C0241ED3673300188D7C /* DebugUIMessages.m */; }; 34D8C0281ED3673300188D7C /* DebugUITableViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C0261ED3673300188D7C /* DebugUITableViewController.m */; }; @@ -812,6 +813,7 @@ 34D2CCD3206294B900CB1A14 /* OWSScreenLock.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OWSScreenLock.swift; sourceTree = ""; }; 34D2CCD82062E7D000CB1A14 /* OWSScreenLockUI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSScreenLockUI.h; sourceTree = ""; }; 34D2CCD92062E7D000CB1A14 /* OWSScreenLockUI.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSScreenLockUI.m; sourceTree = ""; }; + 34D2CCD120618B2F00CB1A14 /* OWSBackupLazyRestoreJob.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OWSBackupLazyRestoreJob.swift; sourceTree = ""; }; 34D5CCA71EAE3D30005515DB /* AvatarViewHelper.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AvatarViewHelper.h; sourceTree = ""; }; 34D5CCA81EAE3D30005515DB /* AvatarViewHelper.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AvatarViewHelper.m; sourceTree = ""; }; 34D8C0231ED3673300188D7C /* DebugUIMessages.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DebugUIMessages.h; sourceTree = ""; }; @@ -1974,6 +1976,7 @@ 34D2CCD3206294B900CB1A14 /* OWSScreenLock.swift */, 34D2CCD82062E7D000CB1A14 /* OWSScreenLockUI.h */, 34D2CCD92062E7D000CB1A14 /* OWSScreenLockUI.m */, + 34D2CCD120618B2F00CB1A14 /* OWSBackupLazyRestoreJob.swift */, 4579431C1E7C8CE9008ED0C0 /* Pastelog.h */, 4579431D1E7C8CE9008ED0C0 /* Pastelog.m */, 450DF2041E0D74AC003D14BE /* Platform.swift */, @@ -3143,6 +3146,7 @@ 458DE9D91DEE7B360071BB03 /* OWSWebRTCDataProtos.pb.m in Sources */, 76EB063C18170B33006006FC /* NumberUtil.m in Sources */, 451166C01FD86B98000739BA /* AccountManager.swift in Sources */, + 34D2CCD220618B3000CB1A14 /* OWSBackupLazyRestoreJob.swift in Sources */, 3430FE181F7751D4000EC51B /* GiphyAPI.swift in Sources */, 34A55F3720485465002CC6DE /* OWS2FARegistrationViewController.m in Sources */, 340FC8AD204DAC8D007AEB0F /* OWSLinkedDevicesTableViewController.m in Sources */, diff --git a/Signal/src/AppDelegate.m b/Signal/src/AppDelegate.m index 6d2b0d1cd..eae87d402 100644 --- a/Signal/src/AppDelegate.m +++ b/Signal/src/AppDelegate.m @@ -418,8 +418,8 @@ static NSString *const kURLHostVerifyPrefix = @"verify"; // Every time we change or add a database view in such a way that // might cause a delay on launch, we need to bump this constant. // - // We upgraded YapDatabase in v2.20.0 and need to regenerate all database views. - NSString *kLastVersionWithDatabaseViewChange = @"2.20.0"; + // We added a database view in v2.23.0. + NSString *kLastVersionWithDatabaseViewChange = @"2.23.0"; BOOL mayNeedUpgrade = ([TSAccountManager isRegistered] && lastLaunchedAppVersion && (!lastCompletedLaunchAppVersion || [VersionMigrations isVersion:lastCompletedLaunchAppVersion @@ -1134,6 +1134,9 @@ static NSString *const kURLHostVerifyPrefix = @"verify"; [self ensureRootViewController]; [OWSBackup.sharedManager setup]; + + // Resume lazy restore. + [OWSBackupLazyRestoreJob runAsync]; } - (void)registrationStateDidChange diff --git a/Signal/src/Signal-Bridging-Header.h b/Signal/src/Signal-Bridging-Header.h index 0c93f10df..a950f6f37 100644 --- a/Signal/src/Signal-Bridging-Header.h +++ b/Signal/src/Signal-Bridging-Header.h @@ -19,6 +19,8 @@ #import "NotificationsManager.h" #import "OWSAnyTouchGestureRecognizer.h" #import "OWSAudioPlayer.h" +#import "OWSBackup.h" +#import "OWSBackupIO.h" #import "OWSBezierPathView.h" #import "OWSCallNotificationsAdaptee.h" #import "OWSDatabaseMigration.h" diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index 71ea58eda..027a350f9 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -48,6 +48,10 @@ NS_ASSUME_NONNULL_BEGIN actionBlock:^{ [DebugUIBackup clearAllCloudKitRecords]; }]]; + [items addObject:[OWSTableItem itemWithTitle:@"Clear Backup Metadata Cache" + actionBlock:^{ + [DebugUIBackup clearBackupMetadataCache]; + }]]; return [OWSTableSection sectionWithTitle:self.name items:items]; } @@ -161,6 +165,16 @@ NS_ASSUME_NONNULL_BEGIN [OWSBackup.sharedManager clearAllCloudKitRecords]; } ++ (void)clearBackupMetadataCache +{ + DDLogInfo(@"%@ ClearBackupMetadataCache.", self.logTag); + + [OWSPrimaryStorage.sharedManager.newDatabaseConnection + readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + [transaction removeAllObjectsInCollection:[OWSBackupFragment collection]]; + }]; +} + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackup.h b/Signal/src/util/OWSBackup.h index bcae8148f..ddcdb2552 100644 --- a/Signal/src/util/OWSBackup.h +++ b/Signal/src/util/OWSBackup.h @@ -20,13 +20,15 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) { OWSBackupState_Succeeded, }; +@class OWSBackupIO; +@class TSAttachmentStream; @class TSThread; @interface OWSBackup : NSObject - (instancetype)init NS_UNAVAILABLE; -+ (instancetype)sharedManager; ++ (instancetype)sharedManager NS_SWIFT_NAME(shared()); - (void)setup; @@ -71,6 +73,16 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) { - (void)logBackupRecords; - (void)clearAllCloudKitRecords; +#pragma mark - Lazy Restore + +- (NSArray *)attachmentRecordNamesForLazyRestore; + +- (NSArray *)attachmentIdsForLazyRestore; + +- (void)lazyRestoreAttachment:(TSAttachmentStream *)attachment + backupIO:(OWSBackupIO *)backupIO + completion:(OWSBackupBoolBlock)completion; + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/util/OWSBackup.m b/Signal/src/util/OWSBackup.m index 420cfcaf6..1bc178db6 100644 --- a/Signal/src/util/OWSBackup.m +++ b/Signal/src/util/OWSBackup.m @@ -4,6 +4,7 @@ #import "OWSBackup.h" #import "OWSBackupExportJob.h" +#import "OWSBackupIO.h" #import "OWSBackupImportJob.h" #import "Signal-Swift.h" #import @@ -483,6 +484,164 @@ NS_ASSUME_NONNULL_BEGIN }]; } +#pragma mark - Lazy Restore + +- (NSArray *)attachmentRecordNamesForLazyRestore +{ + NSMutableArray *recordNames = [NSMutableArray new]; + [self.dbConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) { + id ext = [transaction ext:TSLazyRestoreAttachmentsDatabaseViewExtensionName]; + if (!ext) { + OWSProdLogAndFail(@"%@ Could not load database view.", self.logTag); + return; + } + + [ext enumerateKeysAndObjectsInGroup:TSLazyRestoreAttachmentsGroup + usingBlock:^( + NSString *collection, NSString *key, id object, NSUInteger index, BOOL *stop) { + if (![object isKindOfClass:[TSAttachmentStream class]]) { + OWSProdLogAndFail(@"%@ Unexpected object: %@ in collection:%@", + self.logTag, + [object class], + collection); + return; + } + TSAttachmentStream *attachmentStream = object; + if (!attachmentStream.lazyRestoreFragment) { + OWSProdLogAndFail(@"%@ Invalid object: %@ in collection:%@", + self.logTag, + [object class], + collection); + return; + } + [recordNames addObject:attachmentStream.lazyRestoreFragment.recordName]; + }]; + }]; + return recordNames; +} + +- (NSArray *)attachmentIdsForLazyRestore +{ + NSMutableArray *attachmentIds = [NSMutableArray new]; + [self.dbConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) { + id ext = [transaction ext:TSLazyRestoreAttachmentsDatabaseViewExtensionName]; + if (!ext) { + OWSProdLogAndFail(@"%@ Could not load database view.", self.logTag); + return; + } + + [ext enumerateKeysInGroup:TSLazyRestoreAttachmentsGroup + usingBlock:^(NSString *collection, NSString *key, NSUInteger index, BOOL *stop) { + [attachmentIds addObject:key]; + }]; + }]; + return attachmentIds; +} + +- (void)lazyRestoreAttachment:(TSAttachmentStream *)attachment + backupIO:(OWSBackupIO *)backupIO + completion:(OWSBackupBoolBlock)completion +{ + OWSAssert(attachment); + OWSAssert(backupIO); + OWSAssert(completion); + + NSString *_Nullable attachmentFilePath = [attachment filePath]; + if (attachmentFilePath.length < 1) { + DDLogError(@"%@ Attachment has invalid file path.", self.logTag); + return completion(NO); + } + if ([NSFileManager.defaultManager fileExistsAtPath:attachmentFilePath]) { + DDLogError(@"%@ Attachment already has file.", self.logTag); + return completion(NO); + } + + OWSBackupFragment *_Nullable lazyRestoreFragment = attachment.lazyRestoreFragment; + if (!lazyRestoreFragment) { + DDLogWarn(@"%@ Attachment missing lazy restore metadata.", self.logTag); + return completion(NO); + } + if (lazyRestoreFragment.recordName.length < 1 || lazyRestoreFragment.encryptionKey.length < 1) { + DDLogError(@"%@ Incomplete lazy restore metadata.", self.logTag); + return completion(NO); + } + + // Use a predictable file path so that multiple "import backup" attempts + // will leverage successful file downloads from previous attempts. + // + // TODO: This will also require imports using a predictable jobTempDirPath. + NSString *tempFilePath = [backupIO generateTempFilePath]; + + [OWSBackupAPI downloadFileFromCloudWithRecordName:lazyRestoreFragment.recordName + toFileUrl:[NSURL fileURLWithPath:tempFilePath] + success:^{ + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + [self lazyRestoreAttachment:attachment + backupIO:backupIO + encryptedFilePath:tempFilePath + encryptionKey:lazyRestoreFragment.encryptionKey + completion:completion]; + }); + } + failure:^(NSError *error) { + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + completion(NO); + }); + }]; +} + +- (void)lazyRestoreAttachment:(TSAttachmentStream *)attachment + backupIO:(OWSBackupIO *)backupIO + encryptedFilePath:(NSString *)encryptedFilePath + encryptionKey:(NSData *)encryptionKey + completion:(OWSBackupBoolBlock)completion +{ + OWSAssert(attachment); + OWSAssert(backupIO); + OWSAssert(encryptedFilePath.length > 0); + OWSAssert(encryptionKey.length > 0); + OWSAssert(completion); + + NSData *_Nullable data = [NSData dataWithContentsOfFile:encryptedFilePath]; + if (!data) { + DDLogError(@"%@ Could not load encrypted file.", self.logTag); + return completion(NO); + } + + NSString *decryptedFilePath = [backupIO generateTempFilePath]; + + @autoreleasepool { + if (![backupIO decryptFileAsFile:encryptedFilePath dstFilePath:decryptedFilePath encryptionKey:encryptionKey]) { + DDLogError(@"%@ Could not load decrypt file.", self.logTag); + return completion(NO); + } + } + + NSString *_Nullable attachmentFilePath = [attachment filePath]; + if (attachmentFilePath.length < 1) { + DDLogError(@"%@ Attachment has invalid file path.", self.logTag); + return completion(NO); + } + + NSString *attachmentDirPath = [attachmentFilePath stringByDeletingLastPathComponent]; + if (![OWSFileSystem ensureDirectoryExists:attachmentDirPath]) { + DDLogError(@"%@ Couldn't create directory for attachment file.", self.logTag); + return completion(NO); + } + + NSError *error; + BOOL success = + [NSFileManager.defaultManager moveItemAtPath:decryptedFilePath toPath:attachmentFilePath error:&error]; + if (!success || error) { + DDLogError(@"%@ Attachment file could not be restored: %@.", self.logTag, error); + return completion(NO); + } + + [attachment updateWithLazyRestoreComplete]; + + completion(YES); +} + #pragma mark - Notifications - (void)postDidChangeNotification diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index a3a53e0f0..4dce41255 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -6,6 +6,12 @@ import Foundation import SignalServiceKit import CloudKit +// We don't worry about atomic writes. Each backup export +// will diff against last successful backup. +// +// Note that all of our CloudKit records are immutable. +// "Persistent" records are only uploaded once. +// "Ephemeral" records are always uploaded to a new record name. @objc public class OWSBackupAPI: NSObject { // If we change the record types, we need to ensure indices diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 28d82b1f0..89a686ec3 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -295,8 +295,7 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, nullable) OWSBackupExportItem *manifestItem; // If we are replacing an existing backup, we use some of its contents for continuity. -@property (nonatomic, nullable) NSDictionary *lastManifestItemMap; -@property (nonatomic, nullable) NSSet *lastRecordNames; +@property (nonatomic, nullable) NSSet *lastValidRecordNames; @end @@ -346,7 +345,7 @@ NS_ASSUME_NONNULL_BEGIN if (self.isComplete) { return; } - [self tryToFetchManifestWithCompletion:^(BOOL tryToFetchManifestSuccess) { + [self fetchAllRecordsWithCompletion:^(BOOL tryToFetchManifestSuccess) { if (!tryToFetchManifestSuccess) { [self failWithErrorDescription: NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT", @@ -374,7 +373,7 @@ NS_ASSUME_NONNULL_BEGIN [weakSelf failWithError:saveError]; return; } - [self cleanUpCloudWithCompletion:^(NSError *_Nullable cleanUpError) { + [self cleanUpWithCompletion:^(NSError *_Nullable cleanUpError) { if (cleanUpError) { [weakSelf failWithError:cleanUpError]; return; @@ -422,69 +421,6 @@ NS_ASSUME_NONNULL_BEGIN }]; } -- (void)tryToFetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion -{ - OWSAssert(completion); - - if (self.isComplete) { - return; - } - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - - [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_CHECK_BACKUP", - @"Indicates that the backup import is checking for an existing backup.") - progress:nil]; - - __weak OWSBackupExportJob *weakSelf = self; - - [OWSBackupAPI checkForManifestInCloudWithSuccess:^(BOOL value) { - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - if (value) { - [weakSelf fetchManifestWithCompletion:completion]; - } else { - // There is no existing manifest; continue. - completion(YES); - } - }); - } - failure:^(NSError *error) { - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - completion(NO); - }); - }]; -} - -- (void)fetchManifestWithCompletion:(OWSBackupJobBoolCompletion)completion -{ - OWSAssert(completion); - - if (self.isComplete) { - return; - } - - DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - - __weak OWSBackupExportJob *weakSelf = self; - [weakSelf downloadAndProcessManifestWithSuccess:^(OWSBackupManifestContents *manifest) { - OWSBackupExportJob *strongSelf = weakSelf; - if (!strongSelf) { - return; - } - if (strongSelf.isComplete) { - return; - } - OWSCAssert(manifest.databaseItems.count > 0); - OWSCAssert(manifest.attachmentsItems); - [strongSelf processLastManifest:manifest]; - [strongSelf fetchAllRecordsWithCompletion:completion]; - } - failure:^(NSError *manifestError) { - completion(NO); - } - backupIO:self.backupIO]; -} - - (void)fetchAllRecordsWithCompletion:(OWSBackupJobBoolCompletion)completion { OWSAssert(completion); @@ -505,7 +441,7 @@ NS_ASSUME_NONNULL_BEGIN if (strongSelf.isComplete) { return; } - strongSelf.lastRecordNames = [NSSet setWithArray:recordNames]; + strongSelf.lastValidRecordNames = [NSSet setWithArray:recordNames]; completion(YES); }); } @@ -516,17 +452,6 @@ NS_ASSUME_NONNULL_BEGIN }]; } -- (void)processLastManifest:(OWSBackupManifestContents *)manifest -{ - OWSAssert(manifest); - - NSMutableDictionary *lastManifestItemMap = [NSMutableDictionary new]; - for (OWSBackupManifestItem *manifestItem in manifest.attachmentsItems) { - lastManifestItemMap[manifestItem.recordName] = manifestItem; - } - self.lastManifestItemMap = [lastManifestItemMap copy]; -} - - (BOOL)exportDatabase { OWSAssert(self.backupIO); @@ -806,28 +731,29 @@ NS_ASSUME_NONNULL_BEGIN OWSAttachmentExport *attachmentExport = self.unsavedAttachmentExports.lastObject; [self.unsavedAttachmentExports removeLastObject]; - if (self.lastManifestItemMap && self.lastRecordNames) { - // Wherever possible, we do incremental backups and re-use fragments of the last backup. + if (self.lastValidRecordNames) { + // Wherever possible, we do incremental backups and re-use fragments of the last + // backup and/or restore. // Recycling fragments doesn't just reduce redundant network activity, // it allows us to skip the local export work, i.e. encryption. // To do so, we must preserve the metadata for these fragments. // // We check two things: // - // * That the "last known backup manifest" contains an item from which we can recover - // this record's metadata. + // * That we already know the metadata for this fragment (from a previous backup + // or restore). // * That this record does in fact exist in our CloudKit database. NSString *lastRecordName = [OWSBackupAPI recordNameForPersistentFileWithFileId:attachmentExport.attachmentId]; - OWSBackupManifestItem *_Nullable lastManifestItem = self.lastManifestItemMap[lastRecordName]; - if (lastManifestItem && [self.lastRecordNames containsObject:lastRecordName]) { - OWSAssert(lastManifestItem.encryptionKey.length > 0); - OWSAssert(lastManifestItem.relativeFilePath.length > 0); + OWSBackupFragment *_Nullable lastBackupFragment = [OWSBackupFragment fetchObjectWithUniqueID:lastRecordName]; + if (lastBackupFragment && [self.lastValidRecordNames containsObject:lastRecordName]) { + OWSAssert(lastBackupFragment.encryptionKey.length > 0); + OWSAssert(lastBackupFragment.relativeFilePath.length > 0); // Recycle the metadata from the last backup's manifest. OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new]; - encryptedItem.encryptionKey = lastManifestItem.encryptionKey; + encryptedItem.encryptionKey = lastBackupFragment.encryptionKey; attachmentExport.encryptedItem = encryptedItem; - attachmentExport.relativeFilePath = lastManifestItem.relativeFilePath; + attachmentExport.relativeFilePath = lastBackupFragment.relativeFilePath; OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; exportItem.encryptedItem = attachmentExport.encryptedItem; @@ -887,6 +813,15 @@ NS_ASSUME_NONNULL_BEGIN exportItem.attachmentExport = attachmentExport; [strongSelf.savedAttachmentItems addObject:exportItem]; + // Immediately save the record metadata to facilitate export resume. + OWSBackupFragment *backupFragment = [OWSBackupFragment new]; + backupFragment.recordName = recordName; + backupFragment.encryptionKey = exportItem.encryptedItem.encryptionKey; + backupFragment.relativeFilePath = attachmentExport.relativeFilePath; + backupFragment.attachmentId = attachmentExport.attachmentId; + backupFragment.uncompressedDataLength = exportItem.uncompressedDataLength; + [backupFragment save]; + DDLogVerbose(@"%@ saved attachment: %@ as %@", self.logTag, attachmentExport.attachmentFilePath, @@ -990,6 +925,10 @@ NS_ASSUME_NONNULL_BEGIN OWSAssert(item.attachmentExport.relativeFilePath.length > 0); itemJson[kOWSBackup_ManifestKey_RelativeFilePath] = item.attachmentExport.relativeFilePath; } + if (item.attachmentExport.attachmentId) { + OWSAssert(item.attachmentExport.attachmentId.length > 0); + itemJson[kOWSBackup_ManifestKey_AttachmentId] = item.attachmentExport.attachmentId; + } if (item.uncompressedDataLength) { itemJson[kOWSBackup_ManifestKey_DataSize] = item.uncompressedDataLength; } @@ -999,10 +938,15 @@ NS_ASSUME_NONNULL_BEGIN return result; } -- (void)cleanUpCloudWithCompletion:(OWSBackupJobCompletion)completion +- (void)cleanUpWithCompletion:(OWSBackupJobCompletion)completion { OWSAssert(completion); + if (self.isComplete) { + // Job was aborted. + return completion(nil); + } + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_CLEAN_UP", @@ -1029,9 +973,47 @@ NS_ASSUME_NONNULL_BEGIN OWSAssert(![activeRecordNames containsObject:self.manifestItem.recordName]); [activeRecordNames addObject:self.manifestItem.recordName]; - // TODO: If we implement "lazy restores" where attachments (etc.) are - // restored lazily, we need to include the record names for all + // Because we do "lazy attachment restores", we need to include the record names for all // records that haven't been restored yet. + NSArray *restoringRecordNames = [OWSBackup.sharedManager attachmentRecordNamesForLazyRestore]; + [activeRecordNames addObjectsFromArray:restoringRecordNames]; + + [self cleanUpMetadataCacheWithActiveRecordNames:activeRecordNames]; + + [self cleanUpCloudWithActiveRecordNames:activeRecordNames completion:completion]; +} + +- (void)cleanUpMetadataCacheWithActiveRecordNames:(NSSet *)activeRecordNames +{ + OWSAssert(activeRecordNames.count > 0); + + if (self.isComplete) { + // Job was aborted. + return; + } + + // After every successful backup export, we can (and should) cull metadata + // for any backup fragment (i.e. CloudKit record) that wasn't involved in + // the latest backup export. + [self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + NSMutableSet *obsoleteRecordNames = [NSMutableSet new]; + [obsoleteRecordNames addObjectsFromArray:[transaction allKeysInCollection:[OWSBackupFragment collection]]]; + [obsoleteRecordNames minusSet:activeRecordNames]; + + [transaction removeObjectsForKeys:obsoleteRecordNames.allObjects inCollection:[OWSBackupFragment collection]]; + }]; +} + +- (void)cleanUpCloudWithActiveRecordNames:(NSSet *)activeRecordNames + completion:(OWSBackupJobCompletion)completion +{ + OWSAssert(activeRecordNames.count > 0); + OWSAssert(completion); + + if (self.isComplete) { + // Job was aborted. + return completion(nil); + } __weak OWSBackupExportJob *weakSelf = self; [OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray *recordNames) { diff --git a/Signal/src/util/OWSBackupIO.h b/Signal/src/util/OWSBackupIO.h index fabb0088d..ec9ef0575 100644 --- a/Signal/src/util/OWSBackupIO.h +++ b/Signal/src/util/OWSBackupIO.h @@ -20,6 +20,10 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)initWithJobTempDirPath:(NSString *)jobTempDirPath; +- (NSString *)generateTempFilePath; + +- (nullable NSString *)createTempFile; + #pragma mark - Encrypt - (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath; diff --git a/Signal/src/util/OWSBackupIO.m b/Signal/src/util/OWSBackupIO.m index 02b50d9fb..7044c284c 100644 --- a/Signal/src/util/OWSBackupIO.m +++ b/Signal/src/util/OWSBackupIO.m @@ -44,6 +44,21 @@ static const compression_algorithm SignalCompressionAlgorithm = COMPRESSION_LZMA return self; } +- (NSString *)generateTempFilePath +{ + return [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; +} + +- (nullable NSString *)createTempFile +{ + NSString *filePath = [self generateTempFilePath]; + if (![OWSFileSystem ensureFileExists:filePath]) { + OWSProdLogAndFail(@"%@ could not create temp file.", self.logTag); + return nil; + } + return filePath; +} + #pragma mark - Encrypt - (nullable OWSBackupEncryptedItem *)encryptFileAsTempFile:(NSString *)srcFilePath @@ -92,7 +107,10 @@ static const compression_algorithm SignalCompressionAlgorithm = COMPRESSION_LZMA // TODO: Encrypt the data using key; NSData *encryptedData = unencryptedData; - NSString *dstFilePath = [self.jobTempDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; + NSString *_Nullable dstFilePath = [self createTempFile]; + if (!dstFilePath) { + return nil; + } NSError *error; BOOL success = [encryptedData writeToFile:dstFilePath options:NSDataWritingAtomic error:&error]; if (!success || error) { diff --git a/Signal/src/util/OWSBackupImportJob.m b/Signal/src/util/OWSBackupImportJob.m index 7b75ee502..752d2bd8a 100644 --- a/Signal/src/util/OWSBackupImportJob.m +++ b/Signal/src/util/OWSBackupImportJob.m @@ -26,8 +26,8 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe @property (nonatomic) OWSBackupIO *backupIO; -@property (nonatomic) NSArray *databaseItems; -@property (nonatomic) NSArray *attachmentsItems; +@property (nonatomic) NSArray *databaseItems; +@property (nonatomic) NSArray *attachmentsItems; @end @@ -105,10 +105,17 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe OWSAssert(self.databaseItems); OWSAssert(self.attachmentsItems); - NSMutableArray *allItems = [NSMutableArray new]; + NSMutableArray *allItems = [NSMutableArray new]; [allItems addObjectsFromArray:self.databaseItems]; [allItems addObjectsFromArray:self.attachmentsItems]; + // Record metadata for all items, so that we can re-use them in incremental backups after the restore. + [self.primaryStorage.newDatabaseConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + for (OWSBackupFragment *item in allItems) { + [item saveWithTransaction:transaction]; + } + }]; + __weak OWSBackupImportJob *weakSelf = self; [weakSelf downloadFilesFromCloud:allItems @@ -122,12 +129,6 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } - [weakSelf restoreAttachmentFiles]; - - if (weakSelf.isComplete) { - return; - } - [weakSelf restoreDatabaseWithCompletion:^(BOOL restoreDatabaseSuccess) { if (!restoreDatabaseSuccess) { [weakSelf @@ -154,6 +155,15 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return; } + [weakSelf restoreAttachmentFiles]; + + if (weakSelf.isComplete) { + return; + } + + // Kick off lazy restore. + [OWSBackupLazyRestoreJob runAsync]; + [weakSelf succeed]; }]; }]; @@ -174,7 +184,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe return YES; } -- (void)downloadFilesFromCloud:(NSMutableArray *)items +- (void)downloadFilesFromCloud:(NSMutableArray *)items completion:(OWSBackupJobCompletion)completion { OWSAssert(items.count > 0); @@ -185,7 +195,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe [self downloadNextItemFromCloud:items recordCount:items.count completion:completion]; } -- (void)downloadNextItemFromCloud:(NSMutableArray *)items +- (void)downloadNextItemFromCloud:(NSMutableArray *)items recordCount:(NSUInteger)recordCount completion:(OWSBackupJobCompletion)completion { @@ -201,7 +211,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe // All downloads are complete; exit. return completion(nil); } - OWSBackupManifestItem *item = items.lastObject; + OWSBackupFragment *item = items.lastObject; [items removeLastObject]; CGFloat progress = (recordCount > 0 ? ((recordCount - items.count) / (CGFloat)recordCount) : 0.f); @@ -209,7 +219,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe @"Indicates that the backup import data is being downloaded.") progress:@(progress)]; - // Use a predictable file path so that multiple "import backup" attempts + // TODO: Use a predictable file path so that multiple "import backup" attempts // will leverage successful file downloads from previous attempts. // // TODO: This will also require imports using a predictable jobTempDirPath. @@ -248,53 +258,44 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe { DDLogVerbose(@"%@ %s: %zd", self.logTag, __PRETTY_FUNCTION__, self.attachmentsItems.count); - NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; - - NSUInteger count = 0; - for (OWSBackupManifestItem *item in self.attachmentsItems) { - if (self.isComplete) { - return; - } - if (item.recordName.length < 1) { - DDLogError(@"%@ attachment was not downloaded.", self.logTag); - // Attachment-related errors are recoverable and can be ignored. - continue; - } - if (item.relativeFilePath.length < 1) { - DDLogError(@"%@ attachment missing relative file path.", self.logTag); - // Attachment-related errors are recoverable and can be ignored. - continue; - } - - count++; - [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_FILES", - @"Indicates that the backup import data is being restored.") - progress:@(count / (CGFloat)self.attachmentsItems.count)]; - - NSString *dstFilePath = [attachmentsDirPath stringByAppendingPathComponent:item.relativeFilePath]; - if ([NSFileManager.defaultManager fileExistsAtPath:dstFilePath]) { - DDLogError(@"%@ skipping redundant file restore: %@.", self.logTag, dstFilePath); - continue; - } - NSString *dstDirPath = [dstFilePath stringByDeletingLastPathComponent]; - if (![NSFileManager.defaultManager fileExistsAtPath:dstDirPath]) { - if (![OWSFileSystem ensureDirectoryExists:dstDirPath]) { - DDLogError(@"%@ couldn't create directory for file restore: %@.", self.logTag, dstFilePath); + __block NSUInteger count = 0; + YapDatabaseConnection *dbConnection = self.primaryStorage.newDatabaseConnection; + [dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + for (OWSBackupFragment *item in self.attachmentsItems) { + if (self.isComplete) { + return; + } + if (item.recordName.length < 1) { + DDLogError(@"%@ attachment was not downloaded.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. continue; } - } - @autoreleasepool { - if (![self.backupIO decryptFileAsFile:item.downloadFilePath - dstFilePath:dstFilePath - encryptionKey:item.encryptionKey]) { - DDLogError(@"%@ attachment could not be restored.", self.logTag); + if (item.attachmentId.length < 1) { + DDLogError(@"%@ attachment missing attachment id.", self.logTag); // Attachment-related errors are recoverable and can be ignored. continue; } + if (item.relativeFilePath.length < 1) { + DDLogError(@"%@ attachment missing relative file path.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } + TSAttachmentStream *_Nullable attachment = + [TSAttachmentStream fetchObjectWithUniqueID:item.attachmentId transaction:transaction]; + if (!attachment) { + DDLogError(@"%@ attachment to restore could not be found.", self.logTag); + // Attachment-related errors are recoverable and can be ignored. + continue; + } + [attachment markForLazyRestoreWithFragment:item transaction:transaction]; + count++; + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_IMPORT_PHASE_RESTORING_FILES", + @"Indicates that the backup import data is being restored.") + progress:@(count / (CGFloat)self.attachmentsItems.count)]; } + }]; - DDLogError(@"%@ restored file: %@.", self.logTag, item.relativeFilePath); - } + DDLogError(@"%@ enqueued lazy restore of %zd files.", self.logTag, count); } - (void)restoreDatabaseWithCompletion:(OWSBackupJobBoolCompletion)completion @@ -350,7 +351,7 @@ NSString *const kOWSBackup_ImportDatabaseKeySpec = @"kOWSBackup_ImportDatabaseKe } NSUInteger count = 0; - for (OWSBackupManifestItem *item in self.databaseItems) { + for (OWSBackupFragment *item in self.databaseItems) { if (self.isComplete) { return; } diff --git a/Signal/src/util/OWSBackupJob.h b/Signal/src/util/OWSBackupJob.h index 95a1fb7c9..663ed9e5a 100644 --- a/Signal/src/util/OWSBackupJob.h +++ b/Signal/src/util/OWSBackupJob.h @@ -2,6 +2,9 @@ // Copyright (c) 2018 Open Whisper Systems. All rights reserved. // +#import "TSYapDatabaseObject.h" +#import + NS_ASSUME_NONNULL_BEGIN extern NSString *const kOWSBackup_ManifestKey_DatabaseFiles; @@ -9,6 +12,7 @@ extern NSString *const kOWSBackup_ManifestKey_AttachmentFiles; extern NSString *const kOWSBackup_ManifestKey_RecordName; extern NSString *const kOWSBackup_ManifestKey_EncryptionKey; extern NSString *const kOWSBackup_ManifestKey_RelativeFilePath; +extern NSString *const kOWSBackup_ManifestKey_AttachmentId; extern NSString *const kOWSBackup_ManifestKey_DataSize; @class OWSBackupIO; @@ -20,31 +24,10 @@ typedef void (^OWSBackupJobCompletion)(NSError *_Nullable error); typedef void (^OWSBackupJobManifestSuccess)(OWSBackupManifestContents *manifest); typedef void (^OWSBackupJobManifestFailure)(NSError *error); -@interface OWSBackupManifestItem : NSObject - -@property (nonatomic) NSString *recordName; - -@property (nonatomic) NSData *encryptionKey; - -// This property is only set for certain types of manifest item, -// namely attachments where we need to know where the attachment's -// file should reside relative to the attachments folder. -@property (nonatomic, nullable) NSString *relativeFilePath; - -// This property is only set if the manifest item is downloaded. -@property (nonatomic, nullable) NSString *downloadFilePath; - -// This property is only set if the manifest item is compressed. -@property (nonatomic, nullable) NSNumber *uncompressedDataLength; - -@end - -#pragma mark - - @interface OWSBackupManifestContents : NSObject -@property (nonatomic) NSArray *databaseItems; -@property (nonatomic) NSArray *attachmentsItems; +@property (nonatomic) NSArray *databaseItems; +@property (nonatomic) NSArray *attachmentsItems; @end diff --git a/Signal/src/util/OWSBackupJob.m b/Signal/src/util/OWSBackupJob.m index 35a0c91a0..990cf0e30 100644 --- a/Signal/src/util/OWSBackupJob.m +++ b/Signal/src/util/OWSBackupJob.m @@ -16,16 +16,11 @@ NSString *const kOWSBackup_ManifestKey_AttachmentFiles = @"attachment_files"; NSString *const kOWSBackup_ManifestKey_RecordName = @"record_name"; NSString *const kOWSBackup_ManifestKey_EncryptionKey = @"encryption_key"; NSString *const kOWSBackup_ManifestKey_RelativeFilePath = @"relative_file_path"; +NSString *const kOWSBackup_ManifestKey_AttachmentId = @"attachment_id"; NSString *const kOWSBackup_ManifestKey_DataSize = @"data_size"; NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; -@implementation OWSBackupManifestItem - -@end - -#pragma mark - - @implementation OWSBackupManifestContents @end @@ -86,16 +81,12 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; // might want to use a predictable directory so that repeated // import attempts can reuse downloads from previous attempts. NSString *temporaryDirectory = NSTemporaryDirectory(); - self.jobTempDirPath = [temporaryDirectory stringByAppendingString:[NSUUID UUID].UUIDString]; + self.jobTempDirPath = [temporaryDirectory stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; if (![OWSFileSystem ensureDirectoryExists:self.jobTempDirPath]) { OWSProdLogAndFail(@"%@ Could not create jobTempDirPath.", self.logTag); return NO; } - if (![OWSFileSystem protectFileOrFolderAtPath:self.jobTempDirPath]) { - OWSProdLogAndFail(@"%@ Could not protect jobTempDirPath.", self.logTag); - return NO; - } return YES; } @@ -226,12 +217,12 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; DDLogVerbose(@"%@ json: %@", self.logTag, json); - NSArray *_Nullable databaseItems = + NSArray *_Nullable databaseItems = [self parseItems:json key:kOWSBackup_ManifestKey_DatabaseFiles]; if (!databaseItems) { return failure(); } - NSArray *_Nullable attachmentsItems = + NSArray *_Nullable attachmentsItems = [self parseItems:json key:kOWSBackup_ManifestKey_AttachmentFiles]; if (!attachmentsItems) { return failure(); @@ -244,7 +235,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; return success(contents); } -- (nullable NSArray *)parseItems:(id)json key:(NSString *)key +- (nullable NSArray *)parseItems:(id)json key:(NSString *)key { OWSAssert(json); OWSAssert(key.length); @@ -258,7 +249,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; OWSProdLogAndFail(@"%@ manifest has invalid data: %@.", self.logTag, key); return nil; } - NSMutableArray *items = [NSMutableArray new]; + NSMutableArray *items = [NSMutableArray new]; for (NSDictionary *itemMap in itemMaps) { if (![itemMap isKindOfClass:[NSDictionary class]]) { OWSProdLogAndFail(@"%@ manifest has invalid item: %@.", self.logTag, key); @@ -267,6 +258,7 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; NSString *_Nullable recordName = itemMap[kOWSBackup_ManifestKey_RecordName]; NSString *_Nullable encryptionKeyString = itemMap[kOWSBackup_ManifestKey_EncryptionKey]; NSString *_Nullable relativeFilePath = itemMap[kOWSBackup_ManifestKey_RelativeFilePath]; + NSString *_Nullable attachmentId = itemMap[kOWSBackup_ManifestKey_AttachmentId]; NSNumber *_Nullable uncompressedDataLength = itemMap[kOWSBackup_ManifestKey_DataSize]; if (![recordName isKindOfClass:[NSString class]]) { OWSProdLogAndFail(@"%@ manifest has invalid recordName: %@.", self.logTag, key); @@ -281,6 +273,11 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; OWSProdLogAndFail(@"%@ manifest has invalid relativeFilePath: %@.", self.logTag, key); return nil; } + // attachmentId is an optional field. + if (attachmentId && ![attachmentId isKindOfClass:[NSString class]]) { + OWSProdLogAndFail(@"%@ manifest has invalid attachmentId: %@.", self.logTag, key); + return nil; + } NSData *_Nullable encryptionKey = [NSData dataFromBase64String:encryptionKeyString]; if (!encryptionKey) { OWSProdLogAndFail(@"%@ manifest has corrupt encryptionKey: %@.", self.logTag, key); @@ -292,10 +289,11 @@ NSString *const kOWSBackup_KeychainService = @"kOWSBackup_KeychainService"; return nil; } - OWSBackupManifestItem *item = [OWSBackupManifestItem new]; + OWSBackupFragment *item = [OWSBackupFragment new]; item.recordName = recordName; item.encryptionKey = encryptionKey; item.relativeFilePath = relativeFilePath; + item.attachmentId = attachmentId; item.uncompressedDataLength = uncompressedDataLength; [items addObject:item]; } diff --git a/Signal/src/util/OWSBackupLazyRestoreJob.swift b/Signal/src/util/OWSBackupLazyRestoreJob.swift new file mode 100644 index 000000000..456302050 --- /dev/null +++ b/Signal/src/util/OWSBackupLazyRestoreJob.swift @@ -0,0 +1,92 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +import Foundation +import PromiseKit +import SignalServiceKit + +@objc +public class OWSBackupLazyRestoreJob: NSObject { + + let primaryStorage: OWSPrimaryStorage + + private var jobTempDirPath: String? + + deinit { + if let jobTempDirPath = self.jobTempDirPath { + DispatchQueue.global().async { + OWSFileSystem.deleteFile(jobTempDirPath) + } + } + } + + @objc + public class func runAsync() { + OWSBackupLazyRestoreJob().runAsync() + } + + public override init() { + self.primaryStorage = OWSPrimaryStorage.shared() + } + + private func runAsync() { + AssertIsOnMainThread() + + DispatchQueue.global().async { + self.restoreAttachments() + } + } + + private func restoreAttachments() { + let temporaryDirectory = NSTemporaryDirectory() + let jobTempDirPath = (temporaryDirectory as NSString).appendingPathComponent(NSUUID().uuidString) + + guard OWSFileSystem.ensureDirectoryExists(jobTempDirPath) else { + Logger.error("\(logTag) could not create temp directory.") + return + } + + self.jobTempDirPath = jobTempDirPath + + let backupIO = OWSBackupIO(jobTempDirPath: jobTempDirPath) + + let attachmentIds = OWSBackup.shared().attachmentIdsForLazyRestore() + guard attachmentIds.count > 0 else { + Logger.info("\(logTag) No attachments need lazy restore.") + return + } + Logger.info("\(logTag) Lazy restoring \(attachmentIds.count) attachments.") + self.tryToRestoreNextAttachment(attachmentIds: attachmentIds, backupIO: backupIO) + } + + private func tryToRestoreNextAttachment(attachmentIds: [String], backupIO: OWSBackupIO) { + var attachmentIdsCopy = attachmentIds + guard let attachmentId = attachmentIdsCopy.last else { + // This job is done. + Logger.verbose("\(logTag) job is done.") + return + } + attachmentIdsCopy.removeLast() + guard let attachment = TSAttachmentStream.fetch(uniqueId: attachmentId) else { + Logger.warn("\(logTag) could not load attachment.") + // Not necessarily an error. + // The attachment might have been deleted since the job began. + // Continue trying to restore the other attachments. + tryToRestoreNextAttachment(attachmentIds: attachmentIds, backupIO: backupIO) + return + } + OWSBackup.shared().lazyRestoreAttachment(attachment, + backupIO: backupIO, + completion: { (success) in + if success { + Logger.info("\(self.logTag) restored attachment.") + } else { + Logger.warn("\(self.logTag) could not restore attachment.") + } + // Continue trying to restore the other attachments. + self.tryToRestoreNextAttachment(attachmentIds: attachmentIdsCopy, backupIO: backupIO) + }) + + } +} diff --git a/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.h b/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.h index 1349a34e0..97c73e47c 100644 --- a/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.h +++ b/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.h @@ -3,6 +3,7 @@ // #import "DataSource.h" +#import "OWSBackupFragment.h" #import "TSAttachment.h" #if TARGET_OS_IPHONE @@ -62,6 +63,17 @@ NS_ASSUME_NONNULL_BEGIN + (nullable NSError *)migrateToSharedData; +// Non-nil for attachments which need "lazy backup restore." +- (nullable OWSBackupFragment *)lazyRestoreFragment; + +#pragma mark - Update With... Methods + +// Marks attachment as needing "lazy backup restore." +- (void)markForLazyRestoreWithFragment:(OWSBackupFragment *)lazyRestoreFragment + transaction:(YapDatabaseReadWriteTransaction *)transaction; +// Marks attachment as having completed "lazy backup restore." +- (void)updateWithLazyRestoreComplete; + @end NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.m b/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.m index 1bef2f1da..15ca2fbd9 100644 --- a/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.m +++ b/SignalServiceKit/src/Messages/Attachments/TSAttachmentStream.m @@ -26,6 +26,9 @@ NS_ASSUME_NONNULL_BEGIN // This property should only be accessed on the main thread. @property (nullable, nonatomic) NSNumber *cachedAudioDurationSeconds; +// Optional property. Only set for attachments which need "lazy backup restore." +@property (nonatomic, nullable) NSString *lazyRestoreFragmentId; + @end #pragma mark - @@ -359,7 +362,9 @@ NS_ASSUME_NONNULL_BEGIN } if (![[NSFileManager defaultManager] fileExistsAtPath:self.mediaURL.path]) { - OWSFail(@"%@ while generating thumbnail, source file doesn't exist: %@", self.logTag, self.mediaURL); + DDLogError(@"%@ while generating thumbnail, source file doesn't exist: %@", self.logTag, self.mediaURL); + // If we're not lazy-restoring this message, the attachment should exist on disk. + OWSAssert(self.lazyRestoreFragmentId); return; } @@ -610,6 +615,44 @@ NS_ASSUME_NONNULL_BEGIN return audioDurationSeconds; } +- (nullable OWSBackupFragment *)lazyRestoreFragment +{ + if (!self.lazyRestoreFragmentId) { + return nil; + } + return [OWSBackupFragment fetchObjectWithUniqueID:self.lazyRestoreFragmentId]; +} + +#pragma mark - Update With... Methods + +- (void)markForLazyRestoreWithFragment:(OWSBackupFragment *)lazyRestoreFragment + transaction:(YapDatabaseReadWriteTransaction *)transaction +{ + OWSAssert(lazyRestoreFragment); + OWSAssert(transaction); + + if (!lazyRestoreFragment.uniqueId) { + // If metadata hasn't been saved yet, save now. + [lazyRestoreFragment saveWithTransaction:transaction]; + + OWSAssert(lazyRestoreFragment.uniqueId); + } + [self applyChangeToSelfAndLatestCopy:transaction + changeBlock:^(TSAttachmentStream *attachment) { + [attachment setLazyRestoreFragmentId:lazyRestoreFragment.uniqueId]; + }]; +} + +- (void)updateWithLazyRestoreComplete +{ + [self.dbReadWriteConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + [self applyChangeToSelfAndLatestCopy:transaction + changeBlock:^(TSAttachmentStream *attachment) { + [attachment setLazyRestoreFragmentId:nil]; + }]; + }]; +} + @end NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m index e05947567..ddff2a59a 100644 --- a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m +++ b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m @@ -57,6 +57,7 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage) [OWSFailedMessagesJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage]; [OWSFailedAttachmentDownloadsJob asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage]; [OWSMediaGalleryFinder asyncRegisterDatabaseExtensionsWithPrimaryStorage:storage]; + [TSDatabaseView asyncRegisterLazyRestoreAttachmentsDatabaseView:storage]; } #pragma mark - diff --git a/SignalServiceKit/src/Storage/TSDatabaseView.h b/SignalServiceKit/src/Storage/TSDatabaseView.h index 59357856f..3e83d174b 100644 --- a/SignalServiceKit/src/Storage/TSDatabaseView.h +++ b/SignalServiceKit/src/Storage/TSDatabaseView.h @@ -17,6 +17,9 @@ extern NSString *const TSUnreadDatabaseViewExtensionName; extern NSString *const TSSecondaryDevicesDatabaseViewExtensionName; +extern NSString *const TSLazyRestoreAttachmentsGroup; +extern NSString *const TSLazyRestoreAttachmentsDatabaseViewExtensionName; + @interface TSDatabaseView : NSObject - (instancetype)init NS_UNAVAILABLE; @@ -55,4 +58,6 @@ extern NSString *const TSSecondaryDevicesDatabaseViewExtensionName; + (void)asyncRegisterSecondaryDevicesDatabaseView:(OWSStorage *)storage; ++ (void)asyncRegisterLazyRestoreAttachmentsDatabaseView:(OWSStorage *)storage; + @end diff --git a/SignalServiceKit/src/Storage/TSDatabaseView.m b/SignalServiceKit/src/Storage/TSDatabaseView.m index 77d4a651f..25f967607 100644 --- a/SignalServiceKit/src/Storage/TSDatabaseView.m +++ b/SignalServiceKit/src/Storage/TSDatabaseView.m @@ -5,6 +5,8 @@ #import "TSDatabaseView.h" #import "OWSDevice.h" #import "OWSReadTracking.h" +#import "TSAttachment.h" +#import "TSAttachmentStream.h" #import "TSIncomingMessage.h" #import "TSInvalidIdentityKeyErrorMessage.h" #import "TSOutgoingMessage.h" @@ -28,6 +30,9 @@ NSString *const TSUnreadDatabaseViewExtensionName = @"TSUnreadDatabaseViewExtens NSString *const TSUnseenDatabaseViewExtensionName = @"TSUnseenDatabaseViewExtensionName"; NSString *const TSThreadSpecialMessagesDatabaseViewExtensionName = @"TSThreadSpecialMessagesDatabaseViewExtensionName"; NSString *const TSSecondaryDevicesDatabaseViewExtensionName = @"TSSecondaryDevicesDatabaseViewExtensionName"; +NSString *const TSLazyRestoreAttachmentsDatabaseViewExtensionName + = @"TSLazyRestoreAttachmentsDatabaseViewExtensionName"; +NSString *const TSLazyRestoreAttachmentsGroup = @"TSLazyRestoreAttachmentsGroup"; @interface OWSStorage (TSDatabaseView) @@ -295,30 +300,26 @@ NSString *const TSSecondaryDevicesDatabaseViewExtensionName = @"TSSecondaryDevic + (void)asyncRegisterSecondaryDevicesDatabaseView:(OWSStorage *)storage { - YapDatabaseViewGrouping *viewGrouping = - [YapDatabaseViewGrouping withObjectBlock:^NSString *_Nullable(YapDatabaseReadTransaction *_Nonnull transaction, - NSString *_Nonnull collection, - NSString *_Nonnull key, - id _Nonnull object) { - if ([object isKindOfClass:[OWSDevice class]]) { - OWSDevice *device = (OWSDevice *)object; - if (![device isPrimaryDevice]) { - return TSSecondaryDevicesGroup; - } + YapDatabaseViewGrouping *viewGrouping = [YapDatabaseViewGrouping withObjectBlock:^NSString *_Nullable( + YapDatabaseReadTransaction *transaction, NSString *collection, NSString *key, id object) { + if ([object isKindOfClass:[OWSDevice class]]) { + OWSDevice *device = (OWSDevice *)object; + if (![device isPrimaryDevice]) { + return TSSecondaryDevicesGroup; } - return nil; - }]; + } + return nil; + }]; YapDatabaseViewSorting *viewSorting = - [YapDatabaseViewSorting withObjectBlock:^NSComparisonResult(YapDatabaseReadTransaction *_Nonnull transaction, - NSString *_Nonnull group, - NSString *_Nonnull collection1, - NSString *_Nonnull key1, - id _Nonnull object1, - NSString *_Nonnull collection2, - NSString *_Nonnull key2, - id _Nonnull object2) { - + [YapDatabaseViewSorting withObjectBlock:^NSComparisonResult(YapDatabaseReadTransaction *transaction, + NSString *group, + NSString *collection1, + NSString *key1, + id object1, + NSString *collection2, + NSString *key2, + id object2) { if ([object1 isKindOfClass:[OWSDevice class]] && [object2 isKindOfClass:[OWSDevice class]]) { OWSDevice *device1 = (OWSDevice *)object1; OWSDevice *device2 = (OWSDevice *)object2; @@ -341,6 +342,58 @@ NSString *const TSSecondaryDevicesDatabaseViewExtensionName = @"TSSecondaryDevic [storage asyncRegisterExtension:view withName:TSSecondaryDevicesDatabaseViewExtensionName]; } ++ (void)asyncRegisterLazyRestoreAttachmentsDatabaseView:(OWSStorage *)storage +{ + YapDatabaseViewGrouping *viewGrouping = [YapDatabaseViewGrouping withObjectBlock:^NSString *_Nullable( + YapDatabaseReadTransaction *transaction, NSString *collection, NSString *key, id object) { + if (![object isKindOfClass:[TSAttachment class]]) { + OWSProdLogAndFail(@"%@ Unexpected entity %@ in collection: %@", self.logTag, [object class], collection); + return nil; + } + if (![object isKindOfClass:[TSAttachmentStream class]]) { + return nil; + } + TSAttachmentStream *attachmentStream = (TSAttachmentStream *)object; + if (attachmentStream.lazyRestoreFragment) { + return TSLazyRestoreAttachmentsGroup; + } else { + return nil; + } + }]; + + YapDatabaseViewSorting *viewSorting = [YapDatabaseViewSorting withObjectBlock:^NSComparisonResult( + YapDatabaseReadTransaction *transaction, + NSString *group, + NSString *collection1, + NSString *key1, + id object1, + NSString *collection2, + NSString *key2, + id object2) { + if (![object1 isKindOfClass:[TSAttachment class]]) { + OWSProdLogAndFail(@"%@ Unexpected entity %@ in collection: %@", self.logTag, [object1 class], collection1); + return NSOrderedSame; + } + if (![object2 isKindOfClass:[TSAttachment class]]) { + OWSProdLogAndFail(@"%@ Unexpected entity %@ in collection: %@", self.logTag, [object2 class], collection2); + return NSOrderedSame; + } + + // Specific ordering doesn't matter; we just need a stable ordering. + TSAttachmentStream *attachmentStream1 = (TSAttachmentStream *)object1; + TSAttachmentStream *attachmentStream2 = (TSAttachmentStream *)object2; + return [attachmentStream2.creationTimestamp compare:attachmentStream1.creationTimestamp]; + }]; + + YapDatabaseViewOptions *options = [YapDatabaseViewOptions new]; + options.isPersistent = YES; + options.allowedCollections = + [[YapWhitelistBlacklist alloc] initWithWhitelist:[NSSet setWithObject:[TSAttachment collection]]]; + YapDatabaseView *view = + [[YapDatabaseAutoView alloc] initWithGrouping:viewGrouping sorting:viewSorting versionTag:@"2" options:options]; + [storage asyncRegisterExtension:view withName:TSLazyRestoreAttachmentsDatabaseViewExtensionName]; +} + + (id)unseenDatabaseViewExtension:(YapDatabaseReadTransaction *)transaction { OWSAssert(transaction); diff --git a/SignalServiceKit/src/Util/OWSBackupFragment.h b/SignalServiceKit/src/Util/OWSBackupFragment.h new file mode 100644 index 000000000..6db141c00 --- /dev/null +++ b/SignalServiceKit/src/Util/OWSBackupFragment.h @@ -0,0 +1,42 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +#import "TSYapDatabaseObject.h" + +NS_ASSUME_NONNULL_BEGIN + +// We store metadata for known backup fragments (i.e. CloudKit record) in +// the database. We might learn about them from: +// +// * Past backup exports. +// * An import downloading and parsing the manifest of the last complete backup. +// +// Storing this data in the database provides continuity. +// +// * Backup exports can reuse fragments from previous Backup exports even if they +// don't complete (i.e. backup export resume). +// * Backup exports can reuse fragments from the backup import, if any. +@interface OWSBackupFragment : TSYapDatabaseObject + +@property (nonatomic) NSString *recordName; + +@property (nonatomic) NSData *encryptionKey; + +// This property is only set for certain types of manifest item, +// namely attachments where we need to know where the attachment's +// file should reside relative to the attachments folder. +@property (nonatomic, nullable) NSString *relativeFilePath; + +// This property is only set for attachments. +@property (nonatomic, nullable) NSString *attachmentId; + +// This property is only set if the manifest item is downloaded. +@property (nonatomic, nullable) NSString *downloadFilePath; + +// This property is only set if the manifest item is compressed. +@property (nonatomic, nullable) NSNumber *uncompressedDataLength; + +@end + +NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Util/OWSBackupFragment.m b/SignalServiceKit/src/Util/OWSBackupFragment.m new file mode 100644 index 000000000..4561707e7 --- /dev/null +++ b/SignalServiceKit/src/Util/OWSBackupFragment.m @@ -0,0 +1,23 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +#import "OWSBackupFragment.h" + +NS_ASSUME_NONNULL_BEGIN + +@implementation OWSBackupFragment + +- (void)saveWithTransaction:(YapDatabaseReadWriteTransaction *)transaction +{ + OWSAssert(self.recordName.length > 0); + + if (!self.uniqueId) { + self.uniqueId = self.recordName; + } + [super saveWithTransaction:transaction]; +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/SignalServiceKit/src/Util/OWSFileSystem.h b/SignalServiceKit/src/Util/OWSFileSystem.h index e9a180289..b56e9f49d 100644 --- a/SignalServiceKit/src/Util/OWSFileSystem.h +++ b/SignalServiceKit/src/Util/OWSFileSystem.h @@ -28,6 +28,8 @@ NS_ASSUME_NONNULL_BEGIN // Returns NO IFF the directory does not exist and could not be created. + (BOOL)ensureDirectoryExists:(NSString *)dirPath; ++ (BOOL)ensureFileExists:(NSString *)filePath; + + (BOOL)deleteFile:(NSString *)filePath; + (BOOL)deleteFileIfExists:(NSString *)filePath; diff --git a/SignalServiceKit/src/Util/OWSFileSystem.m b/SignalServiceKit/src/Util/OWSFileSystem.m index fa83b41a3..88c53f091 100644 --- a/SignalServiceKit/src/Util/OWSFileSystem.m +++ b/SignalServiceKit/src/Util/OWSFileSystem.m @@ -227,6 +227,21 @@ NS_ASSUME_NONNULL_BEGIN } } ++ (BOOL)ensureFileExists:(NSString *)filePath +{ + BOOL exists = [[NSFileManager defaultManager] fileExistsAtPath:filePath]; + if (exists) { + return [self protectFileOrFolderAtPath:filePath]; + } else { + BOOL success = [[NSFileManager defaultManager] createFileAtPath:filePath contents:nil attributes:nil]; + if (!success) { + OWSFail(@"%@ Failed to create file.", self.logTag); + return NO; + } + return [self protectFileOrFolderAtPath:filePath]; + } +} + + (BOOL)deleteFile:(NSString *)filePath { NSError *error;