keejef
/
session-android
mirror of https://github.com/oxen-io/session-android
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

Video management logic update

Browse Source 
Rounded corners for floating inset
Proper handling of video scaling based on video proportions
Proper handling of mirroring logic for floating/fullscreen videos depending on whether they are the user or the remote video and whether the camera is front facing or not
pr/1127
ThomasSession 10 months ago
parent 95dc1d9f54
commit 1bc35723fa
7 changed files with 150 additions and 94 deletions
Show Stats Download Patch File Download Diff File

82
app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt
Unescape Escape View File

@ -5,6 +5,7 @@ import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.graphics.Outline
import android.hardware.Sensor
import android.hardware.SensorEvent
import android.hardware.SensorEventListener
@ -14,7 +15,11 @@ import android.os.Build
import android.os.Bundle
import android.provider.Settings
import android.view.MenuItem
import android.view.View
import android.view.ViewGroup
import android.view.ViewOutlineProvider
import android.view.WindowManager
import android.widget.FrameLayout
import androidx.activity.viewModels
import androidx.core.content.ContextCompat
import androidx.core.view.isVisible
@ -49,6 +54,7 @@ import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_RINGING
import org.thoughtcrime.securesms.webrtc.Orientation
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.EARPIECE
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.SPEAKER_PHONE
import org.webrtc.RendererCommon
import kotlin.math.asin
@AndroidEntryPoint
@ -137,9 +143,7 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
}
binding.floatingRendererContainer.setOnClickListener {
val swapVideoViewIntent =
WebRtcCallService.swapVideoViews(this, viewModel.toggleVideoSwap())
startService(swapVideoViewIntent)
viewModel.swapVideos()
}
binding.microphoneButton.setOnClickListener {
@ -180,7 +184,7 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
Permissions.with(this)
.request(Manifest.permission.CAMERA)
.onAllGranted {
val intent = WebRtcCallService.cameraEnabled(this, !viewModel.videoEnabled)
val intent = WebRtcCallService.cameraEnabled(this, !viewModel.videoState.value.userVideoEnabled)
startService(intent)
}
.execute()
@ -197,6 +201,26 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
onBackPressed()
}
clipFloatingInsets()
}
/**
* Makes sure the floating video inset has clipped rounded corners, included with the video stream itself
*/
private fun clipFloatingInsets() {
// clip the video inset with rounded corners
val videoInsetProvider = object : ViewOutlineProvider() {
override fun getOutline(view: View, outline: Outline) {
// all corners
outline.setRoundRect(
0, 0, view.width, view.height,
resources.getDimensionPixelSize(R.dimen.video_inset_radius).toFloat()
)
}
}
binding.floatingRendererContainer.outlineProvider = videoInsetProvider
binding.floatingRendererContainer.clipToOutline = true
}
//Function to check if Android System Auto-rotate is on or off
@ -216,7 +240,11 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
override fun onPause() {
super.onPause()
sensorManager.unregisterListener(this)
try {
sensorManager.unregisterListener(this)
} catch (e: Exception) {
// the unregister can throw if the activity dies too quickly and the sensorManager is not initialised yet
}
}
override fun onSensorChanged(event: SensorEvent) {
@ -394,33 +422,37 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
}
}
// handle video windows
launch {
viewModel.localVideoEnabledState.collect { isEnabled ->
viewModel.videoState.collect { state ->
binding.floatingRenderer.removeAllViews()
if (isEnabled) {
viewModel.floatingRenderer?.let { surfaceView ->
surfaceView.setZOrderOnTop(true)
binding.floatingRenderer.addView(surfaceView)
}
}
binding.floatingRenderer.isVisible = isEnabled
binding.enableCameraButton.isSelected = isEnabled
//binding.swapViewIcon.bringToFront()
}
}
launch {
viewModel.remoteVideoEnabledState.collect { isEnabled ->
binding.fullscreenRenderer.removeAllViews()
if (isEnabled) {
// handle fullscreen video window
if(state.showFullscreenVideo()){
viewModel.fullscreenRenderer?.let { surfaceView ->
binding.fullscreenRenderer.addView(surfaceView)
binding.fullscreenRenderer.isVisible = true
binding.remoteRecipient.isVisible = false
}
} else {
binding.fullscreenRenderer.isVisible = false
binding.remoteRecipient.isVisible = true
}
// handle floating video window
if(state.showFloatingVideo()){
viewModel.floatingRenderer?.let { surfaceView ->
binding.floatingRenderer.addView(surfaceView)
binding.floatingRenderer.isVisible = true
binding.swapViewIcon.bringToFront()
}
} else {
binding.floatingRenderer.isVisible = false
}
binding.fullscreenRenderer.isVisible = isEnabled
binding.remoteRecipient.isVisible = !isEnabled
//binding.swapViewIcon.bringToFront()
// handle buttons
binding.enableCameraButton.isSelected = state.userVideoEnabled
}
}
}

12
app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt
Unescape Escape View File

@ -62,7 +62,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
const val ACTION_LOCAL_HANGUP = "LOCAL_HANGUP"
const val ACTION_SET_MUTE_AUDIO = "SET_MUTE_AUDIO"
const val ACTION_SET_MUTE_VIDEO = "SET_MUTE_VIDEO"
const val ACTION_SWAP_VIDEO_VIEW = "SWAP_VIDEO_VIEW"
const val ACTION_FLIP_CAMERA = "FLIP_CAMERA"
const val ACTION_UPDATE_AUDIO = "UPDATE_AUDIO"
const val ACTION_WIRED_HEADSET_CHANGE = "WIRED_HEADSET_CHANGE"
@ -110,11 +109,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
fun acceptCallIntent(context: Context) = Intent(context, WebRtcCallService::class.java)
.setAction(ACTION_ANSWER_CALL)
fun swapVideoViews(context: Context, swapped: Boolean) =
Intent(context, WebRtcCallService::class.java)
.setAction(ACTION_SWAP_VIDEO_VIEW)
.putExtra(EXTRA_SWAPPED, swapped)
fun microphoneIntent(context: Context, enabled: Boolean) =
Intent(context, WebRtcCallService::class.java)
.setAction(ACTION_SET_MUTE_AUDIO)
@ -299,7 +293,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
action == ACTION_DENY_CALL -> handleDenyCall(intent)
action == ACTION_LOCAL_HANGUP -> handleLocalHangup(intent)
action == ACTION_REMOTE_HANGUP -> handleRemoteHangup(intent)
action == ACTION_SWAP_VIDEO_VIEW ->handleSwapVideoView(intent)
action == ACTION_SET_MUTE_AUDIO -> handleSetMuteAudio(intent)
action == ACTION_SET_MUTE_VIDEO -> handleSetMuteVideo(intent)
action == ACTION_FLIP_CAMERA -> handleSetCameraFlip(intent)
@ -602,11 +595,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
onHangup()
}
private fun handleSwapVideoView(intent: Intent) {
val swapped = intent.getBooleanExtra(EXTRA_SWAPPED, false)
callManager.handleSwapVideoView(swapped)
}
private fun handleSetMuteAudio(intent: Intent) {
val muted = intent.getBooleanExtra(EXTRA_MUTE, false)
callManager.handleSetMuteAudio(muted)

74
app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt
Unescape Escape View File

@ -6,6 +6,7 @@ import android.telephony.TelephonyManager
import androidx.core.content.ContextCompat
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
@ -51,6 +52,7 @@ import org.webrtc.MediaStream
import org.webrtc.PeerConnection
import org.webrtc.PeerConnection.IceConnectionState
import org.webrtc.PeerConnectionFactory
import org.webrtc.RendererCommon
import org.webrtc.RtpReceiver
import org.webrtc.SessionDescription
import org.webrtc.SurfaceViewRenderer
@ -67,7 +69,6 @@ class CallManager(
SignalAudioManager.EventListener, CameraEventListener, DataChannel.Observer {
sealed class StateEvent {
data class VideoSwapped(val isSwapped: Boolean): StateEvent()
data class AudioEnabled(val isEnabled: Boolean): StateEvent()
data class VideoEnabled(val isEnabled: Boolean): StateEvent()
data class CallStateUpdate(val state: CallState): StateEvent()
@ -106,10 +107,15 @@ class CallManager(
private val _audioEvents = MutableStateFlow(AudioEnabled(false))
val audioEvents = _audioEvents.asSharedFlow()
private val _videoEvents = MutableStateFlow(VideoEnabled(false))
val videoEvents = _videoEvents.asSharedFlow()
private val _remoteVideoEvents = MutableStateFlow(VideoEnabled(false))
val remoteVideoEvents = _remoteVideoEvents.asSharedFlow()
private val _videoState: MutableStateFlow<VideoState> = MutableStateFlow(
VideoState(
swapped = false,
userVideoEnabled = false,
remoteVideoEnabled = false
)
)
val videoState = _videoState
private val stateProcessor = StateProcessor(CallState.Idle)
@ -221,8 +227,10 @@ class CallManager(
val base = EglBase.create()
eglBase = base
floatingRenderer = SurfaceViewRenderer(context)
floatingRenderer?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT)
fullscreenRenderer = SurfaceViewRenderer(context)
fullscreenRenderer?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT)
remoteRotationSink = RemoteRotationVideoProxySink()
@ -363,7 +371,8 @@ class CallManager(
val byteArray = ByteArray(buffer.data.remaining()) { buffer.data[it] }
val json = Json.parseToJsonElement(byteArray.decodeToString()) as JsonObject
if (json.containsKey("video")) {
_remoteVideoEvents.value = VideoEnabled((json["video"] as JsonPrimitive).boolean)
_videoState.value = _videoState.value.copy(remoteVideoEnabled = (json["video"] as JsonPrimitive).boolean)
handleMirroring()
} else if (json.containsKey("hangup")) {
peerConnectionObservers.forEach(WebRtcListener::onHangup)
}
@ -399,8 +408,11 @@ class CallManager(
pendingOffer = null
callStartTime = -1
_audioEvents.value = AudioEnabled(false)
_videoEvents.value = VideoEnabled(false)
_remoteVideoEvents.value = VideoEnabled(false)
_videoState.value = VideoState(
swapped = false,
userVideoEnabled = false,
remoteVideoEnabled = false
)
pendingOutgoingIceUpdates.clear()
pendingIncomingIceUpdates.clear()
}
@ -411,7 +423,7 @@ class CallManager(
// If the camera we've switched to is the front one then mirror it to match what someone
// would see when looking in the mirror rather than the left<-->right flipped version.
handleUserMirroring()
handleMirroring()
}
fun onPreOffer(callId: UUID, recipient: Recipient, onSuccess: () -> Unit) {
@ -609,10 +621,19 @@ class CallManager(
}
}
fun handleSwapVideoView(swapped: Boolean) {
videoSwapped = swapped
fun swapVideos() {
videoSwapped = !videoSwapped
// update the state
_videoState.value = _videoState.value.copy(swapped = videoSwapped)
handleMirroring()
if (!swapped) {
//todo TOM received rotated video shouldn't be full scale
//todo TOM make sure the swap icon is visible
//todo TOM Should we show the 'no video' inset straight away?
//todo TOM ios rotates the controls in landscape ( just the buttons though, not the whole ui??)
if (!videoSwapped) {
peerConnection?.rotationVideoSink?.apply {
setSink(floatingRenderer)
}
@ -633,18 +654,31 @@ class CallManager(
*/
private fun getUserRenderer() = if(videoSwapped) fullscreenRenderer else floatingRenderer
/**
* Returns the renderer currently showing the contact's video, not the user's
*/
private fun getRemoteRenderer() = if(videoSwapped) floatingRenderer else fullscreenRenderer
/**
* Makes sure the user's renderer applies mirroring if necessary
*/
private fun handleUserMirroring() = getUserRenderer()?.setMirror(isCameraFrontFacing())
private fun handleMirroring() {
val videoState = _videoState.value
fun handleSetMuteVideo(muted: Boolean, lockManager: LockManager) {
_videoEvents.value = VideoEnabled(!muted)
// if we have user video and the camera is front facing, make sure to mirror stream
if(videoState.userVideoEnabled) {
getUserRenderer()?.setMirror(isCameraFrontFacing())
}
// if we have video and the camera is not front facing, make sure to mirror stream
if(!muted){
handleUserMirroring()
// the remote video is never mirrored
if(videoState.remoteVideoEnabled){
getRemoteRenderer()?.setMirror(false)
}
}
fun handleSetMuteVideo(muted: Boolean, lockManager: LockManager) {
_videoState.value = _videoState.value.copy(userVideoEnabled = !muted)
handleMirroring()
val connection = peerConnection ?: return
connection.setVideoEnabled(!muted)
@ -761,7 +795,7 @@ class CallManager(
connection.setCommunicationMode()
setAudioEnabled(true)
dataChannel?.let { channel ->
val toSend = if (!_videoEvents.value.isEnabled) VIDEO_DISABLED_JSON else VIDEO_ENABLED_JSON
val toSend = if (!_videoState.value.userVideoEnabled) VIDEO_DISABLED_JSON else VIDEO_ENABLED_JSON
val buffer = DataChannel.Buffer(ByteBuffer.wrap(toSend.toString().encodeToByteArray()), false)
channel.send(buffer)
}
@ -790,7 +824,7 @@ class CallManager(
fun isInitiator(): Boolean = peerConnection?.isInitiator() == true
fun isCameraFrontFacing() = localCameraState.activeDirection == CameraState.Direction.FRONT
fun isCameraFrontFacing() = localCameraState.activeDirection != CameraState.Direction.BACK
interface WebRtcListener: PeerConnection.Observer {
fun onHangup()

30
app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt
Unescape Escape View File

@ -2,6 +2,8 @@ package org.thoughtcrime.securesms.webrtc
import androidx.lifecycle.ViewModel
import dagger.hilt.android.lifecycle.HiltViewModel
import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.onEach
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager
@ -35,15 +37,6 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V
val fullscreenRenderer: SurfaceViewRenderer?
get() = callManager.fullscreenRenderer
private var _videoEnabled: Boolean = false
val videoEnabled: Boolean
get() = _videoEnabled
private var _remoteVideoEnabled: Boolean = false
private var _videoViewSwapped: Boolean = false
private var _microphoneEnabled: Boolean = true
val microphoneEnabled: Boolean
@ -63,15 +56,8 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V
get() = callManager.audioEvents.map { it.isEnabled }
.onEach { _microphoneEnabled = it }
val localVideoEnabledState
get() = callManager.videoEvents
.map { it.isEnabled }
.onEach { _videoEnabled = it }
val remoteVideoEnabledState
get() = callManager.remoteVideoEvents
.map { it.isEnabled }
.onEach { _remoteVideoEnabled = it }
val videoState: StateFlow<VideoState>
get() = callManager.videoState
var deviceOrientation: Orientation = Orientation.UNKNOWN
set(value) {
@ -91,11 +77,7 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V
val callStartTime: Long
get() = callManager.callStartTime
/**
* Toggles the video swapped state, and return the value post toggle
*/
fun toggleVideoSwap(): Boolean {
_videoViewSwapped = !_videoViewSwapped
return _videoViewSwapped
fun swapVideos() {
callManager.swapVideos()
}
}

17
app/src/main/java/org/thoughtcrime/securesms/webrtc/VideoState.kt
Unescape Escape View File

@ -0,0 +1,17 @@
package org.thoughtcrime.securesms.webrtc
data class VideoState (
val swapped: Boolean,
val userVideoEnabled: Boolean,
val remoteVideoEnabled: Boolean
){
fun showFloatingVideo(): Boolean {
return userVideoEnabled && !swapped ||
remoteVideoEnabled && swapped
}
fun showFullscreenVideo(): Boolean {
return userVideoEnabled && swapped ||
remoteVideoEnabled && !swapped
}
}

28
app/src/main/res/layout/activity_webrtc.xml
Unescape Escape View File

@ -20,7 +20,7 @@
<FrameLayout
android:id="@+id/fullscreen_renderer"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_height="wrap_content"
android:layout_gravity="center"/>
</FrameLayout>
<ImageView
@ -132,7 +132,8 @@
android:elevation="8dp"
android:id="@+id/floating_renderer"
android:layout_width="match_parent"
android:layout_height="match_parent"/>
android:layout_height="wrap_content"
android:layout_gravity="center"/>
<com.github.ybq.android.spinkit.SpinKitView
android:id="@+id/local_loading_view"
style="@style/SpinKitView.Large.ThreeBounce"
@ -142,19 +143,20 @@
android:layout_gravity="center"
tools:visibility="visible"
android:visibility="gone" />
<ImageView
android:id="@+id/swap_view_icon"
android:src="@drawable/ic_baseline_screen_rotation_alt_24"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintTop_toTopOf="parent"
app:tint="?android:textColorPrimary"
android:layout_marginTop="@dimen/very_small_spacing"
android:layout_marginEnd="@dimen/very_small_spacing"
android:layout_gravity="end"
android:layout_width="14dp"
android:layout_height="14dp"/>
</FrameLayout>
<ImageView
android:id="@+id/swap_view_icon"
android:src="@drawable/ic_baseline_screen_rotation_alt_24"
app:layout_constraintTop_toTopOf="@id/floating_renderer_container"
app:layout_constraintEnd_toEndOf="@id/floating_renderer_container"
app:tint="?android:textColorPrimary"
android:layout_marginTop="@dimen/very_small_spacing"
android:layout_marginEnd="@dimen/very_small_spacing"
android:layout_width="14dp"
android:layout_height="14dp"/>
<ImageView
android:id="@+id/endCallButton"
android:background="@drawable/circle_tintable"

1
app/src/main/res/values/dimens.xml
Unescape Escape View File

@ -32,6 +32,7 @@
<dimen name="fake_chat_view_height">250dp</dimen>
<dimen name="setting_button_height">64dp</dimen>
<dimen name="dialog_corner_radius">8dp</dimen>
<dimen name="video_inset_radius">11dp</dimen>
<dimen name="dialog_button_corner_radius">4dp</dimen>
<dimen name="pn_option_corner_radius">8dp</dimen>
<dimen name="path_status_view_size">8dp</dimen>

Write Preview
Loading…
Cancel
Save