Merge branch 'dev' of https://github.com/oxen-io/session-android into client-side-nickname

app/build.gradle

@@ -158,7 +158,7 @@ dependencies {
     testImplementation 'org.robolectric:shadows-multidex:4.2'
 }
 
-def canonicalVersionCode = 159
+def canonicalVersionCode = 161
 def canonicalVersionName = "1.10.2"
 
 def postFixSize = 10

app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java

@@ -327,7 +327,6 @@ public class ApplicationContext extends MultiDexApplication implements Dependenc
                 .setJobStorage(new FastJobStorage(DatabaseFactory.getJobDatabase(this)))
                 .setDependencyInjector(this)
                 .build());
-        JobQueue.getShared().resumePendingJobs();
     }
 
     private void initializeDependencyInjection() {
@@ -455,7 +454,6 @@ public class ApplicationContext extends MultiDexApplication implements Dependenc
             poller.setUserPublicKey(userPublicKey);
             return;
         }
-        LokiAPIDatabase apiDB = DatabaseFactory.getLokiAPIDatabase(this);
         poller = new Poller();
         closedGroupPoller = new ClosedGroupPoller();
     }

app/src/main/java/org/thoughtcrime/securesms/database/Storage.kt

@@ -184,8 +184,8 @@ class Storage(context: Context, helper: SQLCipherOpenHelper) : Database(context,
         DatabaseFactory.getSessionJobDatabase(context).markJobAsSucceeded(jobId)
     }
 
-    override fun markJobAsFailed(jobId: String) {
+    override fun markJobAsFailedPermanently(jobId: String) {
-        DatabaseFactory.getSessionJobDatabase(context).markJobAsFailed(jobId)
+        DatabaseFactory.getSessionJobDatabase(context).markJobAsFailedPermanently(jobId)
     }
 
     override fun getAllPendingJobs(type: String): Map<String, Job?> {
@@ -576,7 +576,7 @@ class Storage(context: Context, helper: SQLCipherOpenHelper) : Database(context,
         val database = DatabaseFactory.getThreadDatabase(context)
         if (!openGroupID.isNullOrEmpty()) {
             val recipient = Recipient.from(context, Address.fromSerialized(GroupUtil.getEncodedOpenGroupID(openGroupID.toByteArray())), false)
-            return database.getOrCreateThreadIdFor(recipient)
+            return database.getThreadIdIfExistsFor(recipient)
         } else if (!groupPublicKey.isNullOrEmpty()) {
             val recipient = Recipient.from(context, Address.fromSerialized(GroupUtil.doubleEncodeGroupID(groupPublicKey)), false)
             return database.getOrCreateThreadIdFor(recipient)

app/src/main/java/org/thoughtcrime/securesms/loki/api/BackgroundPollWorker.kt

@@ -8,7 +8,6 @@ import nl.komponents.kovenant.Promise
 import nl.komponents.kovenant.all
 import nl.komponents.kovenant.functional.map
 import org.session.libsession.messaging.jobs.MessageReceiveJob
-import org.session.libsession.messaging.open_groups.OpenGroup
 import org.session.libsession.messaging.open_groups.OpenGroupV2
 import org.session.libsession.messaging.sending_receiving.pollers.ClosedGroupPoller
 import org.session.libsession.messaging.sending_receiving.pollers.OpenGroupPoller
@@ -17,7 +16,6 @@ import org.session.libsession.snode.SnodeAPI
 import org.session.libsession.utilities.TextSecurePreferences
 import org.session.libsignal.utilities.logging.Log
 import org.thoughtcrime.securesms.database.DatabaseFactory
-import java.io.IOException
 import java.util.concurrent.TimeUnit
 
 class BackgroundPollWorker(val context: Context, params: WorkerParameters) : Worker(context, params) {
@@ -25,45 +23,23 @@ class BackgroundPollWorker(val context: Context, params: WorkerParameters) : Wor
     companion object {
         const val TAG = "BackgroundPollWorker"
 
-        private const val RETRY_ATTEMPTS = 3
-
-        @JvmStatic
-        fun scheduleInstant(context: Context) {
-            val workRequest = OneTimeWorkRequestBuilder<BackgroundPollWorker>()
-                    .setConstraints(Constraints.Builder()
-                            .setRequiredNetworkType(NetworkType.CONNECTED)
-                            .build()
-                    )
-                    .build()
-
-            WorkManager
-                    .getInstance(context)
-                    .enqueue(workRequest)
-        }
-
         @JvmStatic
         fun schedulePeriodic(context: Context) {
             Log.v(TAG, "Scheduling periodic work.")
-            val workRequest = PeriodicWorkRequestBuilder<BackgroundPollWorker>(15, TimeUnit.MINUTES)
+            val builder = PeriodicWorkRequestBuilder<BackgroundPollWorker>(5, TimeUnit.MINUTES)
-                    .setConstraints(Constraints.Builder()
+            builder.setConstraints(Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build())
-                            .setRequiredNetworkType(NetworkType.CONNECTED)
+            val workRequest = builder.build()
-                            .build()
+            WorkManager.getInstance(context).enqueueUniquePeriodicWork(
-                    )
+                TAG,
-                    .build()
+                ExistingPeriodicWorkPolicy.REPLACE,
-
+                workRequest
-            WorkManager
+            )
-                    .getInstance(context)
-                    .enqueueUniquePeriodicWork(
-                            TAG,
-                            ExistingPeriodicWorkPolicy.KEEP,
-                            workRequest
-                    )
         }
     }
 
     override fun doWork(): Result {
         if (TextSecurePreferences.getLocalNumber(context) == null) {
-            Log.v(TAG, "Background poll is canceled due to the Session user is not set up yet.")
+            Log.v(TAG, "User not registered yet.")
             return Result.failure()
         }
 
@@ -71,44 +47,41 @@ class BackgroundPollWorker(val context: Context, params: WorkerParameters) : Wor
             Log.v(TAG, "Performing background poll.")
             val promises = mutableListOf<Promise<Unit, Exception>>()
 
-            // Private chats
+            // DMs
             val userPublicKey = TextSecurePreferences.getLocalNumber(context)!!
-            val privateChatsPromise = SnodeAPI.getMessages(userPublicKey).map { envelopes ->
+            val dmsPromise = SnodeAPI.getMessages(userPublicKey).map { envelopes ->
                 envelopes.map { envelope ->
                     // FIXME: Using a job here seems like a bad idea...
                     MessageReceiveJob(envelope.toByteArray(), false).executeAsync()
                 }
             }
-            promises.addAll(privateChatsPromise.get())
+            promises.addAll(dmsPromise.get())
 
             // Closed groups
             promises.addAll(ClosedGroupPoller().pollOnce())
 
             // Open Groups
-            val openGroups = DatabaseFactory.getLokiThreadDatabase(context).getAllPublicChats().map { (_,chat)->
+            val openGroups = DatabaseFactory.getLokiThreadDatabase(context).getAllPublicChats().values
-                OpenGroup(chat.channel, chat.server, chat.displayName, chat.isDeletable)
-            }
             for (openGroup in openGroups) {
                 val poller = OpenGroupPoller(openGroup)
                 promises.add(poller.pollForNewMessages())
             }
 
-            val openGroupsV2 = DatabaseFactory.getLokiThreadDatabase(context).getAllV2OpenGroups().values.groupBy(OpenGroupV2::server)
+            val v2OpenGroups = DatabaseFactory.getLokiThreadDatabase(context).getAllV2OpenGroups().values.groupBy(OpenGroupV2::server)
 
-            openGroupsV2.values.map { groups ->
+            v2OpenGroups.values.map { groups ->
                 OpenGroupV2Poller(groups)
             }.forEach { poller ->
-                promises.add(poller.compactPoll(true).map{ /*Unit*/ })
+                promises.add(poller.compactPoll(true).map { })
             }
 
-            // Wait till all the promises get resolved
+            // Wait until all the promises are resolved
             all(promises).get()
 
             return Result.success()
         } catch (exception: Exception) {
-            Log.v(TAG, "Background poll failed due to error: ${exception.message}.", exception)
+            Log.e(TAG, "Background poll failed due to error: ${exception.message}.", exception)
-
+            return Result.retry()
-            return if (runAttemptCount < RETRY_ATTEMPTS) Result.retry() else Result.failure()
         }
     }
 
@@ -117,8 +90,7 @@ class BackgroundPollWorker(val context: Context, params: WorkerParameters) : Wor
         override fun onReceive(context: Context, intent: Intent) {
             if (intent.action == Intent.ACTION_BOOT_COMPLETED) {
                 Log.v(TAG, "Boot broadcast caught.")
-                BackgroundPollWorker.scheduleInstant(context)
+                schedulePeriodic(context)
-                BackgroundPollWorker.schedulePeriodic(context)
             }
         }
     }

app/src/main/java/org/thoughtcrime/securesms/loki/api/PublicChatManager.kt

@@ -31,7 +31,7 @@ class PublicChatManager(private val context: Context) {
     refreshChatsAndPollers()
     for ((threadID, _) in chats) {
       val poller = pollers[threadID]
-      areAllCaughtUp = if (poller != null) areAllCaughtUp && poller.isCaughtUp else true
+      areAllCaughtUp = if (poller != null) areAllCaughtUp && poller.isCaughtUp else areAllCaughtUp
     }
     return areAllCaughtUp
   }
@@ -42,6 +42,9 @@ class PublicChatManager(private val context: Context) {
       val poller = pollers[threadID] ?: OpenGroupPoller(chat, executorService)
       poller.isCaughtUp = false
     }
+    for ((_,poller) in v2Pollers) {
+      poller.isCaughtUp = false
+    }
   }
 
   public fun startPollersIfNeeded() {

app/src/main/java/org/thoughtcrime/securesms/loki/api/SessionProtocolImpl.kt

@@ -23,7 +23,6 @@ class SessionProtocolImpl(private val context: Context) : SessionProtocol {
     override fun decrypt(ciphertext: ByteArray, x25519KeyPair: ECKeyPair): Pair<ByteArray, String> {
         val recipientX25519PrivateKey = x25519KeyPair.privateKey.serialize()
         val recipientX25519PublicKey = Hex.fromStringCondensed(x25519KeyPair.hexEncodedPublicKey.removing05PrefixIfNeeded())
-        Log.d("Test", "recipientX25519PublicKey: $recipientX25519PublicKey")
         val signatureSize = Sign.BYTES
         val ed25519PublicKeySize = Sign.PUBLICKEYBYTES
 

app/src/main/java/org/thoughtcrime/securesms/loki/database/SessionJobDatabase.kt

@@ -25,19 +25,19 @@ class SessionJobDatabase(context: Context, helper: SQLCipherOpenHelper) : Databa
     fun persistJob(job: Job) {
         val database = databaseHelper.writableDatabase
         val contentValues = ContentValues(4)
-        contentValues.put(jobID, job.id)
+        contentValues.put(jobID, job.id!!)
         contentValues.put(jobType, job.getFactoryKey())
         contentValues.put(failureCount, job.failureCount)
         contentValues.put(serializedData, SessionJobHelper.dataSerializer.serialize(job.serialize()))
-        database.insertOrUpdate(sessionJobTable, contentValues, "$jobID = ?", arrayOf(jobID))
+        database.insertOrUpdate(sessionJobTable, contentValues, "$jobID = ?", arrayOf( job.id!! ))
     }
 
     fun markJobAsSucceeded(jobID: String) {
-        databaseHelper.writableDatabase.delete(sessionJobTable, "$jobID = ?", arrayOf( jobID ))
+        databaseHelper.writableDatabase.delete(sessionJobTable, "${Companion.jobID} = ?", arrayOf( jobID ))
     }
 
-    fun markJobAsFailed(jobID: String) {
+    fun markJobAsFailedPermanently(jobID: String) {
-        databaseHelper.writableDatabase.delete(sessionJobTable, "$jobID = ?", arrayOf( jobID ))
+        databaseHelper.writableDatabase.delete(sessionJobTable, "${Companion.jobID} = ?", arrayOf( jobID ))
     }
 
     fun getAllPendingJobs(type: String): Map<String, Job?> {
@@ -74,8 +74,8 @@ class SessionJobDatabase(context: Context, helper: SQLCipherOpenHelper) : Databa
         val database = databaseHelper.readableDatabase
         var cursor: android.database.Cursor? = null
         try {
-            cursor = database.rawQuery("SELECT * FROM $sessionJobTable WHERE $jobID = ?", arrayOf( job.id ))
+            cursor = database.rawQuery("SELECT * FROM $sessionJobTable WHERE $jobID = ?", arrayOf( job.id!! ))
-            return cursor != null && cursor.moveToFirst()
+            return cursor == null || !cursor.moveToFirst()
         } catch (e: Exception) {
             // Do nothing
         }  finally {

libsession/src/main/java/org/session/libsession/messaging/StorageProtocol.kt

@@ -45,7 +45,7 @@ interface StorageProtocol {
     // Jobs
     fun persistJob(job: Job)
     fun markJobAsSucceeded(jobId: String)
-    fun markJobAsFailed(jobId: String)
+    fun markJobAsFailedPermanently(jobId: String)
     fun getAllPendingJobs(type: String): Map<String,Job?>
     fun getAttachmentUploadJob(attachmentID: Long): AttachmentUploadJob?
     fun getMessageSendJob(messageSendJobID: String): MessageSendJob?

libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentUploadJob.kt

@@ -103,7 +103,7 @@ class AttachmentUploadJob(val attachmentID: Long, val threadID: String, val mess
         val messageSendJob = storage.getMessageSendJob(messageSendJobID)
         MessageSender.handleFailedMessageSend(this.message, e)
         if (messageSendJob != null) {
-            storage.markJobAsFailed(messageSendJobID)
+            storage.markJobAsFailedPermanently(messageSendJobID)
         }
     }
 
@@ -131,6 +131,7 @@ class AttachmentUploadJob(val attachmentID: Long, val threadID: String, val mess
         override fun create(data: Data): AttachmentUploadJob {
             val serializedMessage = data.getByteArray(MESSAGE_KEY)
             val kryo = Kryo()
+            kryo.isRegistrationRequired = false
             val input = Input(serializedMessage)
             val message: Message = kryo.readObject(input, Message::class.java)
             input.close()

libsession/src/main/java/org/session/libsession/messaging/jobs/Job.kt

@@ -12,6 +12,7 @@ interface Job {
         // Keys used for database storage
         private val ID_KEY = "id"
         private val FAILURE_COUNT_KEY = "failure_count"
+        internal const val MAX_BUFFER_SIZE = 1_000_000 // bytes
     }
 
     fun execute()

libsession/src/main/java/org/session/libsession/messaging/jobs/JobQueue.kt

@@ -5,6 +5,7 @@ import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED
 import org.session.libsession.messaging.MessagingModuleConfiguration
 import org.session.libsignal.utilities.logging.Log
+import java.lang.IllegalStateException
 import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.Executors
@@ -17,29 +18,49 @@ import kotlin.math.roundToLong
 class JobQueue : JobDelegate {
     private var hasResumedPendingJobs = false // Just for debugging
     private val jobTimestampMap = ConcurrentHashMap<Long, AtomicInteger>()
-    private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
+    private val rxDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
-    private val multiDispatcher = Executors.newFixedThreadPool(2).asCoroutineDispatcher()
+    private val txDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
+    private val attachmentDispatcher = Executors.newFixedThreadPool(2).asCoroutineDispatcher()
     private val scope = GlobalScope + SupervisorJob()
     private val queue = Channel<Job>(UNLIMITED)
 
     val timer = Timer()
 
+    private fun CoroutineScope.processWithDispatcher(channel: Channel<Job>, dispatcher: CoroutineDispatcher) = launch(dispatcher) {
+        for (job in channel) {
+            if (!isActive) break
+            job.delegate = this@JobQueue
+            job.execute()
+        }
+    }
+
     init {
         // Process jobs
-        scope.launch(dispatcher) {
+        scope.launch {
+            val rxQueue = Channel<Job>(capacity = 1024)
+            val txQueue = Channel<Job>(capacity = 1024)
+            val attachmentQueue = Channel<Job>(capacity = 1024)
+
+            val receiveJob = processWithDispatcher(rxQueue, rxDispatcher)
+            val txJob = processWithDispatcher(txQueue, txDispatcher)
+            val attachmentJob = processWithDispatcher(attachmentQueue, attachmentDispatcher)
+
             while (isActive) {
-                queue.receive().let { job ->
+                for (job in queue) {
-                    if (job.canExecuteParallel()) {
+                    when (job) {
-                        launch(multiDispatcher) {
+                        is NotifyPNServerJob, is AttachmentUploadJob, is MessageSendJob -> txQueue.send(job)
-                            job.delegate = this@JobQueue
+                        is AttachmentDownloadJob -> attachmentQueue.send(job)
-                            job.execute()
+                        is MessageReceiveJob -> rxQueue.send(job)
-                        }
+                        else -> throw IllegalStateException("Unexpected job type.")
-                    } else {
-                        job.delegate = this@JobQueue
-                        job.execute()
                     }
                 }
             }
+
+            // The job has been cancelled
+            receiveJob.cancel()
+            txJob.cancel()
+            attachmentJob.cancel()
+
         }
     }
 
@@ -49,13 +70,6 @@ class JobQueue : JobDelegate {
         val shared: JobQueue by lazy { JobQueue() }
     }
 
-    private fun Job.canExecuteParallel(): Boolean {
-        return this.javaClass in arrayOf(
-            AttachmentUploadJob::class.java,
-            AttachmentDownloadJob::class.java
-        )
-    }
-
     fun add(job: Job) {
         addWithoutExecuting(job)
         queue.offer(job) // offer always called on unlimited capacity
@@ -109,10 +123,24 @@ class JobQueue : JobDelegate {
     }
 
     override fun handleJobFailed(job: Job, error: Exception) {
-        job.failureCount += 1
+        // Canceled
         val storage = MessagingModuleConfiguration.shared.storage
-        if (storage.isJobCanceled(job)) { return Log.i("Loki", "${job::class.simpleName} canceled.")}
+        if (storage.isJobCanceled(job)) {
-        if (job.failureCount == job.maxFailureCount) {
+            return Log.i("Loki", "${job::class.simpleName} canceled.")
+        }
+        // Message send jobs waiting for the attachment to upload
+        if (job is MessageSendJob && error is MessageSendJob.AwaitingAttachmentUploadException) {
+            val retryInterval: Long = 1000 * 4
+            Log.i("Loki", "Message send job waiting for attachment upload to finish.")
+            timer.schedule(delay = retryInterval) {
+                Log.i("Loki", "Retrying ${job::class.simpleName}.")
+                queue.offer(job)
+            }
+            return
+        }
+        // Regular job failure
+        job.failureCount += 1
+        if (job.failureCount >= job.maxFailureCount) {
             handleJobFailedPermanently(job, error)
         } else {
             storage.persistJob(job)
@@ -132,7 +160,7 @@ class JobQueue : JobDelegate {
 
     private fun handleJobFailedPermanently(jobId: String) {
         val storage = MessagingModuleConfiguration.shared.storage
-        storage.markJobAsFailed(jobId)
+        storage.markJobAsFailedPermanently(jobId)
     }
 
     private fun getRetryInterval(job: Job): Long {

libsession/src/main/java/org/session/libsession/messaging/jobs/MessageSendJob.kt

@@ -4,6 +4,7 @@ import com.esotericsoftware.kryo.Kryo
 import com.esotericsoftware.kryo.io.Input
 import com.esotericsoftware.kryo.io.Output
 import org.session.libsession.messaging.MessagingModuleConfiguration
+import org.session.libsession.messaging.jobs.Job.Companion.MAX_BUFFER_SIZE
 import org.session.libsession.messaging.messages.Destination
 import org.session.libsession.messaging.messages.Message
 import org.session.libsession.messaging.messages.visible.VisibleMessage
@@ -11,6 +12,9 @@ import org.session.libsession.messaging.sending_receiving.MessageSender
 import org.session.libsignal.utilities.logging.Log
 
 class MessageSendJob(val message: Message, val destination: Destination) : Job {
+
+    object AwaitingAttachmentUploadException : Exception("Awaiting attachment upload.")
+
     override var delegate: JobDelegate? = null
     override var id: String? = null
     override var failureCount: Int = 0
@@ -45,7 +49,10 @@ class MessageSendJob(val message: Message, val destination: Destination) : Job {
                     JobQueue.shared.add(job)
                 }
             }
-            if (attachmentsToUpload.isNotEmpty()) return // Wait for all attachments to upload before continuing
+            if (attachmentsToUpload.isNotEmpty()) {
+                this.handleFailure(AwaitingAttachmentUploadException)
+                return
+            } // Wait for all attachments to upload before continuing
         }
         MessageSender.send(this.message, this.destination).success {
             this.handleSuccess()
@@ -80,7 +87,7 @@ class MessageSendJob(val message: Message, val destination: Destination) : Job {
     override fun serialize(): Data {
         val kryo = Kryo()
         kryo.isRegistrationRequired = false
-        val output = Output(ByteArray(4096), 10_000_000)
+        val output = Output(ByteArray(4096), MAX_BUFFER_SIZE)
         // Message
         kryo.writeClassAndObject(output, message)
         output.close()

libsession/src/main/java/org/session/libsession/messaging/jobs/NotifyPNServerJob.kt

@@ -78,6 +78,7 @@ class NotifyPNServerJob(val message: SnodeMessage) : Job {
         override fun create(data: Data): NotifyPNServerJob {
             val serializedMessage = data.getByteArray(MESSAGE_KEY)
             val kryo = Kryo()
+            kryo.isRegistrationRequired = false
             val input = Input(serializedMessage)
             val message: SnodeMessage = kryo.readObject(input, SnodeMessage::class.java)
             input.close()

libsession/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageHandler.kt

@@ -1,7 +1,6 @@
 package org.session.libsession.messaging.sending_receiving
 
 import android.text.TextUtils
-import okhttp3.HttpUrl
 import org.session.libsession.messaging.MessagingModuleConfiguration
 import org.session.libsession.messaging.jobs.AttachmentDownloadJob
 import org.session.libsession.messaging.jobs.JobQueue
@@ -156,7 +155,12 @@ fun MessageReceiver.handleVisibleMessage(message: VisibleMessage, proto: SignalS
 
     // Get or create thread
     val threadID = storage.getOrCreateThreadIdFor(message.syncTarget
-            ?: message.sender!!, message.groupPublicKey, openGroupID)
+        ?: message.sender!!, message.groupPublicKey, openGroupID)
+
+    if (threadID < 0) {
+        // thread doesn't exist, should only be reached in a case where we are processing open group messages for no longer existent thread
+        throw MessageReceiver.Error.NoThread
+    }
 
     val openGroup = threadID.let {
         storage.getOpenGroup(it.toString())
@@ -236,7 +240,7 @@ fun MessageReceiver.handleVisibleMessage(message: VisibleMessage, proto: SignalS
     }
     val openGroupServerID = message.openGroupServerMessageID
     if (openGroupServerID != null) {
-        storage.setOpenGroupServerMessageID(messageID, openGroupServerID, threadID, !(message.isMediaMessage() || attachments.isNotEmpty()))
+        storage.setOpenGroupServerMessageID(messageID, openGroupServerID, threadID, !message.isMediaMessage())
     }
     // Cancel any typing indicators if needed
     cancelTypingIndicatorsIfNeeded(message.sender!!)

libsession/src/main/java/org/session/libsession/snode/utilities/Random.kt

@@ -6,6 +6,7 @@ import java.security.SecureRandom
  * Uses `SecureRandom` to pick an element from this collection.
  */
 fun <T> Collection<T>.getRandomElementOrNull(): T? {
+    if (isEmpty()) return null
     val index = SecureRandom().nextInt(size) // SecureRandom() should be cryptographically secure
     return elementAtOrNull(index)
 }

libsignal/src/main/java/org/session/libsignal/service/loki/Random.kt

@@ -6,6 +6,7 @@ import java.security.SecureRandom
  * Uses `SecureRandom` to pick an element from this collection.
  */
 fun <T> Collection<T>.getRandomElementOrNull(): T? {
+    if (isEmpty()) return null
     val index = SecureRandom().nextInt(size) // SecureRandom() should be cryptographically secure
     return elementAtOrNull(index)
 }