diff --git a/js/background.js b/js/background.js index d72edc427..7fc323e8e 100644 --- a/js/background.js +++ b/js/background.js @@ -11,12 +11,13 @@ /* global Whisper: false */ /* global wrapDeferred: false */ -;(function() { +;(async function() { 'use strict'; const { IdleDetector, MessageDataMigrator } = Signal.Workflow; const { Errors, Message } = window.Signal.Types; const { upgradeMessageSchema } = window.Signal.Migrations; + const { Migrations0DatabaseWithAttachmentData } = window.Signal.Database; const { Views } = window.Signal; // Implicitly used in `indexeddb-backbonejs-adapter`: @@ -75,13 +76,17 @@ return accountManager; }; - const cancelInitializationMessage = Views.Initialization.setMessage(); - console.log('Start IndexedDB migrations'); - storage.fetch(); - - /* eslint-enable */ /* jshint ignore:start */ + const cancelInitializationMessage = Views.Initialization.setMessage(); + console.log('Start IndexedDB migrations'); + + console.log('Migrate database with attachments'); + await Migrations0DatabaseWithAttachmentData.run({ Backbone }); + + console.log('Migrate database without attachments'); + storage.fetch(); + const NUM_MESSAGE_UPGRADES_PER_IDLE = 2; const idleDetector = new IdleDetector(); idleDetector.on('idle', async () => { diff --git a/js/database.js b/js/database.js index a7335abd3..db0688d35 100644 --- a/js/database.js +++ b/js/database.js @@ -2,13 +2,11 @@ /* global Backbone: false */ /* global _: false */ -/* eslint-disable more/no-then */ - // eslint-disable-next-line func-names (function () { 'use strict'; - const { Migrations } = window.Signal; + const { Migrations1DatabaseWithoutAttachmentData } = window.Signal.Database; window.Whisper = window.Whisper || {}; window.Whisper.Database = window.Whisper.Database || {}; @@ -125,132 +123,5 @@ request.onsuccess = resolve; })); - Whisper.Database.migrations = [ - { - version: '12.0', - migrate(transaction, next) { - console.log('migration 12.0'); - console.log('creating object stores'); - const messages = transaction.db.createObjectStore('messages'); - messages.createIndex('conversation', ['conversationId', 'received_at'], { - unique: false, - }); - messages.createIndex('receipt', 'sent_at', { unique: false }); - messages.createIndex('unread', ['conversationId', 'unread'], { unique: false }); - messages.createIndex('expires_at', 'expires_at', { unique: false }); - - const conversations = transaction.db.createObjectStore('conversations'); - conversations.createIndex('inbox', 'active_at', { unique: false }); - conversations.createIndex('group', 'members', { - unique: false, - multiEntry: true, - }); - conversations.createIndex('type', 'type', { - unique: false, - }); - conversations.createIndex('search', 'tokens', { - unique: false, - multiEntry: true, - }); - - transaction.db.createObjectStore('groups'); - - transaction.db.createObjectStore('sessions'); - transaction.db.createObjectStore('identityKeys'); - transaction.db.createObjectStore('preKeys'); - transaction.db.createObjectStore('signedPreKeys'); - transaction.db.createObjectStore('items'); - - console.log('creating debug log'); - transaction.db.createObjectStore('debug'); - - next(); - }, - }, - { - version: '13.0', - migrate(transaction, next) { - console.log('migration 13.0'); - console.log('Adding fields to identity keys'); - const identityKeys = transaction.objectStore('identityKeys'); - const request = identityKeys.openCursor(); - const promises = []; - request.onsuccess = (event) => { - const cursor = event.target.result; - if (cursor) { - const attributes = cursor.value; - attributes.timestamp = 0; - attributes.firstUse = false; - attributes.nonblockingApproval = false; - attributes.verified = 0; - promises.push(new Promise(((resolve, reject) => { - const putRequest = identityKeys.put(attributes, attributes.id); - putRequest.onsuccess = resolve; - putRequest.onerror = (e) => { - console.log(e); - reject(e); - }; - }))); - cursor.continue(); - } else { - // no more results - Promise.all(promises).then(() => { - next(); - }); - } - }; - request.onerror = (event) => { - console.log(event); - }; - }, - }, - { - version: '14.0', - migrate(transaction, next) { - console.log('migration 14.0'); - console.log('Adding unprocessed message store'); - const unprocessed = transaction.db.createObjectStore('unprocessed'); - unprocessed.createIndex('received', 'timestamp', { unique: false }); - next(); - }, - }, - { - version: '15.0', - migrate(transaction, next) { - console.log('migration 15.0'); - console.log('Adding messages index for de-duplication'); - const messages = transaction.objectStore('messages'); - messages.createIndex('unique', ['source', 'sourceDevice', 'sent_at'], { - unique: true, - }); - next(); - }, - }, - { - version: '16.0', - migrate(transaction, next) { - console.log('migration 16.0'); - console.log('Dropping log table, since we now log to disk'); - transaction.db.deleteObjectStore('debug'); - next(); - }, - }, - { - version: 17, - async migrate(transaction, next) { - console.log('migration 17'); - console.log('Start migration to database version 17'); - - const start = Date.now(); - await Migrations.V17.run(transaction); - const duration = Date.now() - start; - - console.log( - 'Complete migration to database version 17.', - `Duration: ${duration}ms` - ); - next(); - }, - }, - ]; + Whisper.Database.migrations = Migrations1DatabaseWithoutAttachmentData.migrations; }()); diff --git a/js/modules/migrations/17/index.js b/js/modules/migrations/17/index.js deleted file mode 100644 index feffbb058..000000000 --- a/js/modules/migrations/17/index.js +++ /dev/null @@ -1,55 +0,0 @@ -const Message = require('../../types/message'); - - -exports.run = async (transaction) => { - const messagesStore = transaction.objectStore('messages'); - - console.log('Initialize messages schema version'); - const numUpgradedMessages = await _initializeMessageSchemaVersion(messagesStore); - console.log('Complete messages schema version initialization', { numUpgradedMessages }); - - console.log('Create index from attachment schema version to attachment'); - messagesStore.createIndex('schemaVersion', 'schemaVersion', { unique: false }); -}; - -const _initializeMessageSchemaVersion = messagesStore => - new Promise((resolve, reject) => { - const messagePutOperations = []; - - const cursorRequest = messagesStore.openCursor(); - cursorRequest.onsuccess = async (event) => { - const cursor = event.target.result; - const hasMoreData = Boolean(cursor); - if (!hasMoreData) { - await Promise.all(messagePutOperations); - return resolve(messagePutOperations.length); - } - - const message = cursor.value; - const messageWithSchemaVersion = Message.initializeSchemaVersion(message); - messagePutOperations.push(putItem( - messagesStore, - messageWithSchemaVersion, - messageWithSchemaVersion.id - )); - - return cursor.continue(); - }; - - cursorRequest.onerror = event => - reject(event.target.error); - }); - -// putItem :: IDBObjectStore -> Item -> Key -> Promise Item -const putItem = (store, item, key) => - new Promise((resolve, reject) => { - try { - const request = store.put(item, key); - request.onsuccess = event => - resolve(event.target.result); - request.onerror = event => - reject(event.target.error); - } catch (error) { - reject(error); - } - }); diff --git a/js/modules/migrations/migrations_0_database_with_attachment_data.js b/js/modules/migrations/migrations_0_database_with_attachment_data.js new file mode 100644 index 000000000..45fc11505 --- /dev/null +++ b/js/modules/migrations/migrations_0_database_with_attachment_data.js @@ -0,0 +1,166 @@ +const isFunction = require('lodash/isFunction'); +const isObject = require('lodash/isObject'); + + +// IMPORTANT: The migrations below are run on a database that may be very large +// due to attachments being directly stored inside the database. Please avoid +// any expensive operations, e.g. modifying all messages / attachments, etc., as +// it may cause out-of-memory errors for users with long histories: +// https://github.com/signalapp/Signal-Desktop/issues/2163 +const migrations = [ + { + version: '12.0', + migrate(transaction, next) { + console.log('Migration 12'); + console.log('creating object stores'); + const messages = transaction.db.createObjectStore('messages'); + messages.createIndex('conversation', ['conversationId', 'received_at'], { + unique: false, + }); + messages.createIndex('receipt', 'sent_at', { unique: false }); + messages.createIndex('unread', ['conversationId', 'unread'], { unique: false }); + messages.createIndex('expires_at', 'expires_at', { unique: false }); + + const conversations = transaction.db.createObjectStore('conversations'); + conversations.createIndex('inbox', 'active_at', { unique: false }); + conversations.createIndex('group', 'members', { + unique: false, + multiEntry: true, + }); + conversations.createIndex('type', 'type', { + unique: false, + }); + conversations.createIndex('search', 'tokens', { + unique: false, + multiEntry: true, + }); + + transaction.db.createObjectStore('groups'); + + transaction.db.createObjectStore('sessions'); + transaction.db.createObjectStore('identityKeys'); + transaction.db.createObjectStore('preKeys'); + transaction.db.createObjectStore('signedPreKeys'); + transaction.db.createObjectStore('items'); + + console.log('creating debug log'); + transaction.db.createObjectStore('debug'); + + next(); + }, + }, + { + version: '13.0', + migrate(transaction, next) { + console.log('Migration 13'); + console.log('Adding fields to identity keys'); + const identityKeys = transaction.objectStore('identityKeys'); + const request = identityKeys.openCursor(); + const promises = []; + request.onsuccess = (event) => { + const cursor = event.target.result; + if (cursor) { + const attributes = cursor.value; + attributes.timestamp = 0; + attributes.firstUse = false; + attributes.nonblockingApproval = false; + attributes.verified = 0; + promises.push(new Promise(((resolve, reject) => { + const putRequest = identityKeys.put(attributes, attributes.id); + putRequest.onsuccess = resolve; + putRequest.onerror = (e) => { + console.log(e); + reject(e); + }; + }))); + cursor.continue(); + } else { + // no more results + // eslint-disable-next-line more/no-then + Promise.all(promises).then(() => { + next(); + }); + } + }; + request.onerror = (event) => { + console.log(event); + }; + }, + }, + { + version: '14.0', + migrate(transaction, next) { + console.log('Migration 14'); + console.log('Adding unprocessed message store'); + const unprocessed = transaction.db.createObjectStore('unprocessed'); + unprocessed.createIndex('received', 'timestamp', { unique: false }); + next(); + }, + }, + { + version: '15.0', + migrate(transaction, next) { + console.log('Migration 15'); + console.log('Adding messages index for de-duplication'); + const messages = transaction.objectStore('messages'); + messages.createIndex('unique', ['source', 'sourceDevice', 'sent_at'], { + unique: true, + }); + next(); + }, + }, + { + version: '16.0', + migrate(transaction, next) { + console.log('Migration 16'); + console.log('Dropping log table, since we now log to disk'); + transaction.db.deleteObjectStore('debug'); + next(); + }, + }, + { + version: 17, + async migrate(transaction, next) { + console.log('Migration 17'); + console.log('Start migration to database version 17'); + + const start = Date.now(); + + const messagesStore = transaction.objectStore('messages'); + console.log('Create index from attachment schema version to attachment'); + messagesStore.createIndex('schemaVersion', 'schemaVersion', { unique: false }); + + const duration = Date.now() - start; + + console.log( + 'Complete migration to database version 17.', + `Duration: ${duration}ms` + ); + next(); + }, + }, +]; + +const database = { + id: 'signal', + nolog: true, + migrations, +}; + +exports.run = ({ Backbone } = {}) => { + if (!isObject(Backbone) || !isObject(Backbone.Collection) || + !isFunction(Backbone.Collection.extend)) { + throw new TypeError('"Backbone" is required'); + } + + const migrationCollection = new (Backbone.Collection.extend({ + database, + storeName: 'conversations', + }))(); + + return new Promise((resolve) => { + // NOTE: This `then` refers to a jQuery `Deferred`: + // eslint-disable-next-line more/no-then + migrationCollection.fetch().then(() => resolve()); + }); +}; diff --git a/js/modules/migrations/migrations_1_database_without_attachment_data.js b/js/modules/migrations/migrations_1_database_without_attachment_data.js new file mode 100644 index 000000000..5cb5446c8 --- /dev/null +++ b/js/modules/migrations/migrations_1_database_without_attachment_data.js @@ -0,0 +1,9 @@ +exports.migrations = [ + { + version: 18, + async migrate(transaction, next) { + console.log('Migration 18'); + next(); + }, + }, +]; diff --git a/preload.js b/preload.js index c10ed563a..a2feb6736 100644 --- a/preload.js +++ b/preload.js @@ -127,12 +127,16 @@ window.Signal = {}; window.Signal.Backup = require('./js/modules/backup'); window.Signal.Crypto = require('./js/modules/crypto'); + window.Signal.Database = {}; + window.Signal.Database.Migrations0DatabaseWithAttachmentData = + require('./js/modules/migrations/migrations_0_database_with_attachment_data'); + window.Signal.Database.Migrations1DatabaseWithoutAttachmentData = + require('./js/modules/migrations/migrations_1_database_without_attachment_data'); window.Signal.Logs = require('./js/modules/logs'); window.Signal.Migrations = {}; window.Signal.Migrations.loadAttachmentData = Attachment.loadData(readAttachmentData); window.Signal.Migrations.deleteAttachmentData = Attachment.deleteData(deleteAttachmentData); window.Signal.Migrations.upgradeMessageSchema = upgradeMessageSchema; - window.Signal.Migrations.V17 = require('./js/modules/migrations/17'); window.Signal.OS = require('./js/modules/os'); window.Signal.Types = {}; window.Signal.Types.Attachment = Attachment;