From 08b441a05e2594c1a32a08d2b491fbb58e5cb87d Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 20 Oct 2026 11:36:34 +0200 Subject: [PATCH 01/16] feat(call): share own screen from the device Add an entry to the more call actions dialog to start and stop sharing the device screen. The screen is captured via MediaProjection while the call foreground service additionally has the mediaProjection type. Without MCU a screen peer connection is created for each participant in the call; with MCU a screen publisher is created and the participants are requested to receive an offer for it. "unshareScreen" is sent when the sharing stops. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- app/src/main/AndroidManifest.xml | 3 +- .../nextcloud/talk/activities/CallActivity.kt | 278 +++++++++++++++++- .../talk/activities/CallViewModel.kt | 7 + .../json/signaling/NCSignalingMessageDto.kt | 6 +- .../talk/services/CallForegroundService.kt | 84 ++++++ .../talk/ui/dialog/MoreCallActionsDialog.kt | 29 ++ .../talk/utils/ParticipantPermissions.kt | 7 + .../talk/webrtc/PeerConnectionWrapper.java | 40 ++- .../main/res/drawable/ic_screen_share_24.xml | 16 + .../res/drawable/ic_stop_screen_share_24.xml | 16 + .../res/layout/dialog_more_call_actions.xml | 32 ++ app/src/main/res/values/strings.xml | 3 + 12 files changed, 516 insertions(+), 5 deletions(-) create mode 100644 app/src/main/res/drawable/ic_screen_share_24.xml create mode 100644 app/src/main/res/drawable/ic_stop_screen_share_24.xml diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 8884e456bb1..69fe41e6c15 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -44,6 +44,7 @@ + @@ -369,7 +370,7 @@ + android:foregroundServiceType="microphone|camera|mediaProjection" /> diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 2ad7065a292..af1b5782a2b 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -29,11 +29,14 @@ import android.media.AudioFormat import android.media.AudioRecord import android.media.MediaPlayer import android.media.MediaRecorder +import android.media.projection.MediaProjection +import android.media.projection.MediaProjectionManager import android.net.Uri import android.os.Build import android.os.Bundle import android.os.Handler import android.os.Looper +import android.os.ResultReceiver import android.provider.Settings import android.text.TextUtils import android.text.format.DateUtils @@ -45,6 +48,7 @@ import android.view.View.OnTouchListener import android.widget.Toast import androidx.activity.SystemBarStyle import androidx.activity.enableEdgeToEdge +import androidx.activity.result.ActivityResult import androidx.activity.result.contract.ActivityResultContracts import androidx.annotation.DrawableRes import androidx.annotation.StringRes @@ -108,6 +112,7 @@ import com.nextcloud.talk.models.json.generic.GenericOverall import com.nextcloud.talk.models.json.participants.ParticipantDto import com.nextcloud.talk.models.json.signaling.DataChannelMessageDto import com.nextcloud.talk.models.json.signaling.NCSignalingMessageDto +import com.nextcloud.talk.utils.ParticipantPermissions import com.nextcloud.talk.models.json.signaling.SignalingDto import com.nextcloud.talk.models.json.signaling.SignalingOverall import com.nextcloud.talk.models.json.signaling.settings.SignalingSettingsOverall @@ -196,6 +201,7 @@ import org.webrtc.PeerConnection import org.webrtc.PeerConnection.IceConnectionState import org.webrtc.PeerConnectionFactory import org.webrtc.RendererCommon +import org.webrtc.ScreenCapturerAndroid import org.webrtc.SoftwareVideoDecoderFactory import org.webrtc.SoftwareVideoEncoderFactory import org.webrtc.SurfaceTextureHelper @@ -209,6 +215,7 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject import kotlin.math.abs +import kotlin.math.max @AutoInjector(NextcloudTalkApplication::class) @Suppress("TooManyFunctions", "ReturnCount", "LargeClass") @@ -288,6 +295,20 @@ class CallActivity : CallBaseActivity() { private val callParticipantMessageListeners: MutableMap = HashMap() private val selfPeerConnectionObserver: PeerConnectionObserver = CallActivitySelfPeerConnectionObserver() + private var screenCapturer: ScreenCapturerAndroid? = null + private var screenSurfaceTextureHelper: SurfaceTextureHelper? = null + private var screenVideoSource: VideoSource? = null + + @Volatile + private var localScreenStream: MediaStream? = null + + // Without MCU there is one connection per remote participant, with MCU just the own publisher connection. + private val screenSharePeerConnectionWrapperList: MutableList = CopyOnWriteArrayList() + + @Volatile + private var isScreenSharePublisherConnected = false + private val screenSharePublisherObserver: PeerConnectionObserver = ScreenSharePublisherPeerConnectionObserver() + private val callParticipantListObserver: CallParticipantList.Observer = object : CallParticipantList.Observer { override fun onCallParticipantsChanged( joined: Collection, @@ -368,6 +389,12 @@ class CallActivity : CallBaseActivity() { private var moreCallActionsDialog: MoreCallActionsDialog? = null private var elapsedSeconds: Long = 0 + private val screenSharingPermissionLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult() + ) { result: ActivityResult -> + onScreenSharingPermissionResult(result) + } + private var requestPermissionLauncher = registerForActivityResult( ActivityResultContracts.RequestMultiplePermissions() ) { permissionMap: Map -> @@ -903,7 +930,7 @@ class CallActivity : CallBaseActivity() { } private fun initFeaturesVisibility() { - if (isAllowedToStartOrStopRecording || isAllowedToRaiseHand) { + if (isAllowedToStartOrStopRecording || isAllowedToRaiseHand || isAllowedToShareScreen) { binding!!.moreCallActions.visibility = View.VISIBLE } else { binding!!.moreCallActions.visibility = View.GONE @@ -2282,6 +2309,7 @@ class CallActivity : CallBaseActivity() { private fun hangup(shutDownView: Boolean, endCallForAll: Boolean) { Log.d(TAG, "hangup! shutDownView=$shutDownView, endCallForAll=$endCallForAll") + stopScreenSharing() joinRoomInitiated = false if (shutDownView) { setCallState(CallStatus.LEAVING) @@ -2599,6 +2627,7 @@ class CallActivity : CallBaseActivity() { Log.d(TAG, " session that will be removed is: $session") endPeerConnection(session, "video") endPeerConnection(session, "screen") + endScreenSharePeerConnection(session) removeCallParticipant(session) } } @@ -2892,6 +2921,8 @@ class CallActivity : CallBaseActivity() { // and send the local state exactly when the data channel becomes ready. localStateBroadcaster!!.handleCallParticipantAdded(callViewModel.getParticipant(sessionId)!!.uiState) + shareScreenWithJoinedParticipant(sessionId) + initPipMode() } @@ -3364,6 +3395,32 @@ class CallActivity : CallBaseActivity() { } } + private inner class ScreenSharePublisherPeerConnectionObserver : PeerConnectionObserver { + override fun onStreamAdded(mediaStream: MediaStream) { + // unused atm + } + + override fun onStreamRemoved(mediaStream: MediaStream) { + // unused atm + } + + override fun onIceConnectionStateChanged(iceConnectionState: IceConnectionState) { + runOnUiThread { + if (localScreenStream == null) { + return@runOnUiThread + } + if (iceConnectionState == IceConnectionState.CONNECTED && !isScreenSharePublisherConnected) { + isScreenSharePublisherConnected = true + getParticipantSessionKeys().forEach { requestScreenShareOffer(it) } + } else if (iceConnectionState == IceConnectionState.FAILED) { + Log.e(TAG, "Screen share publisher connection failed") + stopScreenSharing() + showScreenSharingError() + } + } + } + } + private inner class CallActivitySelfPeerConnectionObserver : PeerConnectionObserver { override fun onStreamAdded(mediaStream: MediaStream) { // unused atm @@ -3636,6 +3693,222 @@ class CallActivity : CallBaseActivity() { override fun onConfigurationChanged(newConfig: Configuration) { super.onConfigurationChanged(newConfig) eventBus.post(ConfigurationChangeEvent()) + screenCapturer?.let { + val (width, height) = getScreenCaptureSize() + it.changeCaptureFormat(width, height, SCREEN_SHARE_FRAME_RATE) + } + } + + val isAllowedToShareScreen: Boolean + get() { + val conversation = currentConversation ?: return false + return ParticipantPermissions( + conversationUser.capabilities?.spreedCapability, + ConversationModel.mapToConversationModel(conversation, conversationUser) + ).canPublishScreen() + } + + fun toggleScreenSharing() { + if (localScreenStream != null) { + stopScreenSharing() + return + } + if (!isAllowedToShareScreen || !isConnectionEstablished) { + Log.w(TAG, "Screen sharing is not allowed or the call is not established") + return + } + val mediaProjectionManager = getSystemService(MediaProjectionManager::class.java) + screenSharingPermissionLauncher.launch(mediaProjectionManager.createScreenCaptureIntent()) + } + + private fun onScreenSharingPermissionResult(result: ActivityResult) { + val permissionResultData = result.data + if (result.resultCode != RESULT_OK || permissionResultData == null) { + Log.d(TAG, "Screen sharing permission was not granted") + return + } + + // On Android 14+ the media projection can only be started once the foreground service has the media + // projection type, which in turn requires the permission to have been granted. + val resultReceiver = object : ResultReceiver(Handler(Looper.getMainLooper())) { + override fun onReceiveResult(resultCode: Int, resultData: Bundle?) { + if (resultCode == CallForegroundService.RESULT_SCREEN_SHARING_READY) { + startScreenSharing(permissionResultData) + } else { + showScreenSharingError() + } + } + } + CallForegroundService.startScreenSharing(applicationContext, resultReceiver) + } + + private fun startScreenSharing(permissionResultData: Intent) { + if (localScreenStream != null || !isConnectionEstablished || currentCallStatus === CallStatus.LEAVING) { + CallForegroundService.stopScreenSharing(applicationContext) + return + } + + val capturer = ScreenCapturerAndroid( + permissionResultData, + object : MediaProjection.Callback() { + override fun onStop() { + runOnUiThread { stopScreenSharing() } + } + } + ) + val surfaceTextureHelper = SurfaceTextureHelper.create("ScreenCaptureThread", rootEglBase!!.eglBaseContext) + val videoSource = peerConnectionFactory!!.createVideoSource(true) + screenCapturer = capturer + screenSurfaceTextureHelper = surfaceTextureHelper + screenVideoSource = videoSource + + capturer.initialize(surfaceTextureHelper, applicationContext, videoSource.capturerObserver) + try { + val (width, height) = getScreenCaptureSize() + capturer.startCapture(width, height, SCREEN_SHARE_FRAME_RATE) + } catch (e: SecurityException) { + Log.e(TAG, "Failed to start screen capture", e) + releaseScreenCapture() + CallForegroundService.stopScreenSharing(applicationContext) + showScreenSharingError() + return + } + + val stream = peerConnectionFactory!!.createLocalMediaStream("NCSS") + stream.addTrack(peerConnectionFactory!!.createVideoTrack("NCs0", videoSource)) + localScreenStream = stream + callViewModel.setLocalScreenSharing(true) + + if (hasMCU) { + addScreenSharePeerConnection(webSocketClient!!.sessionId) + } else { + getParticipantSessionKeys().forEach { addScreenSharePeerConnection(it) } + } + } + + fun stopScreenSharing() { + val stream = localScreenStream ?: return + localScreenStream = null + + getParticipantSessionKeys().forEach { sessionId -> + messageSender?.send( + NCSignalingMessageDto(type = "unshareScreen", roomType = VIDEO_STREAM_TYPE_SCREEN), + sessionId + ) + } + + for (peerConnectionWrapper in screenSharePeerConnectionWrapperList) { + peerConnectionWrapper.removeObserver(screenSharePublisherObserver) + peerConnectionWrapper.removePeerConnection() + } + screenSharePeerConnectionWrapperList.clear() + isScreenSharePublisherConnected = false + + releaseScreenCapture() + stream.dispose() + + CallForegroundService.stopScreenSharing(applicationContext) + callViewModel.setLocalScreenSharing(false) + } + + private fun releaseScreenCapture() { + screenCapturer?.let { + try { + it.stopCapture() + } catch (e: InterruptedException) { + Log.e(TAG, "Failed to stop screen capture", e) + } + it.dispose() + } + screenCapturer = null + screenVideoSource?.dispose() + screenVideoSource = null + screenSurfaceTextureHelper?.dispose() + screenSurfaceTextureHelper = null + } + + private fun showScreenSharingError() { + Snackbar.make(binding!!.root, R.string.nc_screen_sharing_failed, Snackbar.LENGTH_LONG).show() + } + + private fun getScreenCaptureSize(): Pair { + val displayMetrics = resources.displayMetrics + val scale = max( + 1f, + max(displayMetrics.widthPixels, displayMetrics.heightPixels).toFloat() / SCREEN_SHARE_MAX_DIMENSION + ) + // Video encoders expect even dimensions. + val width = (displayMetrics.widthPixels / scale).toInt() and 1.inv() + val height = (displayMetrics.heightPixels / scale).toInt() and 1.inv() + return Pair(width, height) + } + + private fun shareScreenWithJoinedParticipant(sessionId: String) { + if (localScreenStream == null) { + return + } + if (hasMCU) { + // Participants joining before the publisher is connected are requested once it is connected. + if (isScreenSharePublisherConnected) { + requestScreenShareOffer(sessionId) + } + } else { + addScreenSharePeerConnection(sessionId) + } + } + + /** + * With MCU the offer for the screen publisher is sent by the MCU to the remote participant on request. + */ + private fun requestScreenShareOffer(sessionId: String) { + messageSender?.send( + NCSignalingMessageDto(type = "sendoffer", roomType = VIDEO_STREAM_TYPE_SCREEN), + sessionId + ) + } + + private fun addScreenSharePeerConnection(sessionId: String?) { + val stream = localScreenStream ?: return + if (sessionId == null || screenSharePeerConnectionWrapperList.any { it.sessionId == sessionId }) { + return + } + + val (senderKeyRing, receiverKeyRing) = keyRingsFor(hasMCU, hasMCU, sessionId) + val peerConnectionWrapper = try { + PeerConnectionWrapper( + peerConnectionFactory, + iceServers, + sdpConstraintsForMCUPublisher, + sessionId, + if (webSocketClient != null) webSocketClient!!.sessionId else callSession, + stream, + hasMCU, + hasMCU, + VIDEO_STREAM_TYPE_SCREEN, + signalingMessageReceiver, + signalingMessageSender, + isCallEndToEndEncryptionEnabled, + senderKeyRing, + receiverKeyRing, + !hasMCU + ) + } catch (e: IllegalStateException) { + Log.e(TAG, "Not creating a screen share connection to $sessionId", e) + failCallEncryption(R.string.nc_call_e2ee_setup_failed) + return + } + if (hasMCU) { + peerConnectionWrapper.addObserver(screenSharePublisherObserver) + } + screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) + } + + private fun endScreenSharePeerConnection(sessionId: String?) { + val peerConnectionWrapper = screenSharePeerConnectionWrapperList.firstOrNull { it.sessionId == sessionId } + ?: return + peerConnectionWrapper.removeObserver(screenSharePublisherObserver) + peerConnectionWrapper.removePeerConnection() + screenSharePeerConnectionWrapperList.remove(peerConnectionWrapper) } val isAllowedToStartOrStopRecording: Boolean @@ -3669,6 +3942,7 @@ class CallActivity : CallBaseActivity() { var active = false const val VIDEO_STREAM_TYPE_VIDEO = "video" + const val VIDEO_STREAM_TYPE_SCREEN = "screen" private val TAG = CallActivity::class.java.simpleName private val PERMISSIONS_CAMERA = arrayOf( Manifest.permission.CAMERA @@ -3696,6 +3970,8 @@ class CallActivity : CallBaseActivity() { private const val MICROPHONE_VALUE_SLEEP: Long = 1000 private const val FRAME_RATE: Int = 30 + private const val SCREEN_SHARE_FRAME_RATE: Int = 15 + private const val SCREEN_SHARE_MAX_DIMENSION: Int = 1920 private const val WIDTH_16_TO_9_RATIO: Int = 1280 private const val HEIGHT_16_TO_9_RATIO: Int = 720 private const val WIDTH_4_TO_3_RATIO: Int = 640 diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt b/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt index ee4c0a2b2f0..9fce4075258 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt @@ -27,6 +27,9 @@ class CallViewModel @Inject constructor() : ViewModel() { private val _activeScreenShareSession = MutableStateFlow(null) val activeScreenShareSession: StateFlow = _activeScreenShareSession.asStateFlow() + private val _isLocalScreenSharing = MutableStateFlow(false) + val isLocalScreenSharing: StateFlow = _isLocalScreenSharing.asStateFlow() + fun getParticipant(sessionId: String?): ParticipantHandler? { if (sessionId == null) { Log.w(TAG, "Attempted to get participant with null sessionId.") @@ -88,6 +91,10 @@ class CallViewModel @Inject constructor() : ViewModel() { } } + fun setLocalScreenSharing(isSharing: Boolean) { + _isLocalScreenSharing.value = isSharing + } + public override fun onCleared() { participantHandlers.values.forEach { it.destroy() } participantHandlers.clear() diff --git a/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCSignalingMessageDto.kt b/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCSignalingMessageDto.kt index c8a9b5e980f..f3139258e4a 100644 --- a/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCSignalingMessageDto.kt +++ b/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCSignalingMessageDto.kt @@ -28,8 +28,10 @@ data class NCSignalingMessageDto( @JsonField(name = ["sid"]) var sid: String? = null, @JsonField(name = ["prefix"]) - var prefix: String? = null + var prefix: String? = null, + @JsonField(name = ["broadcaster"]) + var broadcaster: String? = null ) : Parcelable { // This constructor is added to work with the 'com.bluelinelabs.logansquare.annotation.JsonObject' - constructor() : this(null, null, null, null, null, null, null) + constructor() : this(null, null, null, null, null, null, null, null) } diff --git a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt index 35ec17783ea..2a3f7c00299 100644 --- a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt +++ b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt @@ -25,11 +25,13 @@ import android.os.Bundle import android.os.Handler import android.os.IBinder import android.os.Looper +import android.os.ResultReceiver import android.util.Log import androidx.annotation.RequiresApi import androidx.core.app.NotificationCompat import androidx.core.app.NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE import androidx.core.content.ContextCompat +import androidx.core.content.IntentCompat import com.nextcloud.talk.R import com.nextcloud.talk.activities.CallActivity import com.nextcloud.talk.application.NextcloudTalkApplication @@ -71,12 +73,18 @@ class CallForegroundService : Service() { private var currentCredentials: String? = null private var currentIsOneToOne = false private var conversationAvatarBitmap: Bitmap? = null + private var isScreenSharing = false override fun onBind(intent: Intent?): IBinder? = null @SuppressLint("ForegroundServiceType") override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { Log.d(TAG, "onStartCommand called") + + if (handleScreenSharingAction(intent)) { + return START_STICKY + } + handler.removeCallbacksAndMessages(null) val conversationName = intent?.getStringExtra(EXTRA_CONVERSATION_NAME) @@ -104,6 +112,53 @@ class CallForegroundService : Service() { return START_STICKY } + private fun handleScreenSharingAction(intent: Intent?): Boolean = + when (intent?.action) { + ACTION_START_SCREEN_SHARING -> { + val resultReceiver = + IntentCompat.getParcelableExtra(intent, EXTRA_RESULT_RECEIVER, ResultReceiver::class.java) + setScreenSharing(true, resultReceiver) + true + } + + ACTION_STOP_SCREEN_SHARING -> { + if (isScreenSharing) { + setScreenSharing(false, null) + } + true + } + + else -> false + } + + /** + * Updates the foreground service type to include or exclude media projection, which is required to capture the + * screen. The result receiver is notified once the type was updated, as only then the capture can be started. + */ + @SuppressLint("ForegroundServiceType") + private fun setScreenSharing(enabled: Boolean, resultReceiver: ResultReceiver?) { + isScreenSharing = enabled + + var resultCode = RESULT_SCREEN_SHARING_READY + try { + val notification = buildNotification(currentConversationName, currentCallExtras) + val foregroundServiceType = resolveForegroundServiceType(currentCallExtras) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && + foregroundServiceType != FOREGROUND_SERVICE_TYPE_ZERO + ) { + startForeground(NOTIFICATION_ID, notification, foregroundServiceType) + } else { + startForeground(NOTIFICATION_ID, notification) + } + } catch (e: SecurityException) { + Log.e(TAG, "Failed to update foreground service type for screen sharing", e) + isScreenSharing = false + resultCode = RESULT_SCREEN_SHARING_FAILED + } + + resultReceiver?.send(resultCode, null) + } + override fun onDestroy() { Log.d(TAG, "onDestroy called") handler.removeCallbacksAndMessages(null) @@ -316,6 +371,10 @@ class CallForegroundService : Service() { if (!isVoiceOnlyCall && canPublishVideo && isCameraPermissionGranted) { serviceType = serviceType or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA } + + if (isScreenSharing) { + serviceType = serviceType or ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION + } } return serviceType } @@ -331,6 +390,12 @@ class CallForegroundService : Service() { private const val EXTRA_CREDENTIALS = "extra_credentials" private const val EXTRA_IS_ONE_TO_ONE = "extra_is_one_to_one" private const val CALL_DURATION_UPDATE_INTERVAL = 1000L + private const val ACTION_START_SCREEN_SHARING = "action_start_screen_sharing" + private const val ACTION_STOP_SCREEN_SHARING = "action_stop_screen_sharing" + private const val EXTRA_RESULT_RECEIVER = "extra_result_receiver" + + const val RESULT_SCREEN_SHARING_READY = 0 + const val RESULT_SCREEN_SHARING_FAILED = 1 fun start(context: Context, conversationName: String?, callIntentExtras: Bundle?, avatarInfo: AvatarInfo) { val serviceIntent = Intent(context, CallForegroundService::class.java).apply { @@ -344,6 +409,25 @@ class CallForegroundService : Service() { ContextCompat.startForegroundService(context, serviceIntent) } + /** + * Must be called after the user granted the media projection and before the screen capture is started. The + * result receiver is notified with [RESULT_SCREEN_SHARING_READY] or [RESULT_SCREEN_SHARING_FAILED]. + */ + fun startScreenSharing(context: Context, resultReceiver: ResultReceiver) { + val serviceIntent = Intent(context, CallForegroundService::class.java).apply { + action = ACTION_START_SCREEN_SHARING + putExtra(EXTRA_RESULT_RECEIVER, resultReceiver) + } + context.startService(serviceIntent) + } + + fun stopScreenSharing(context: Context) { + val serviceIntent = Intent(context, CallForegroundService::class.java).apply { + action = ACTION_STOP_SCREEN_SHARING + } + context.startService(serviceIntent) + } + fun stop(context: Context) { context.stopService(Intent(context, CallForegroundService::class.java)) } diff --git a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt index 453212ffef2..935abdfe7b1 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt @@ -13,6 +13,7 @@ import android.view.ViewGroup import android.widget.LinearLayout import androidx.core.content.ContextCompat import androidx.emoji2.widget.EmojiTextView +import androidx.lifecycle.lifecycleScope import autodagger.AutoInjector import com.google.android.material.bottomsheet.BottomSheetBehavior import com.google.android.material.bottomsheet.BottomSheetDialog @@ -28,6 +29,7 @@ import com.nextcloud.talk.ui.theme.hostViewThemeUtils import com.nextcloud.talk.utils.CapabilitiesUtil import com.nextcloud.talk.utils.DisplayUtils import com.nextcloud.talk.viewmodels.CallRecordingViewModel +import kotlinx.coroutines.launch import javax.inject.Inject @AutoInjector(NextcloudTalkApplication::class) @@ -84,6 +86,12 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee if (!ImageSegmenterHelper.isAvailable()) { binding.backgroundBlur.visibility = View.GONE } + + if (callActivity.isAllowedToShareScreen) { + binding.shareScreen.visibility = View.VISIBLE + } else { + binding.shareScreen.visibility = View.GONE + } } private fun initClickListeners() { @@ -98,6 +106,11 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee binding.backgroundBlur.setOnClickListener { callActivity.toggleBackgroundBlur() } + + binding.shareScreen.setOnClickListener { + callActivity.toggleScreenSharing() + dismiss() + } } private fun initEmojiBar() { @@ -205,6 +218,22 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee } } } + + lifecycleScope.launch { + callActivity.callViewModel.isLocalScreenSharing.collect { isSharing -> + if (isSharing) { + binding.shareScreenText.text = context.getText(R.string.nc_stop_screen_sharing) + binding.shareScreenIcon.setImageDrawable( + ContextCompat.getDrawable(context, R.drawable.ic_stop_screen_share_24) + ) + } else { + binding.shareScreenText.text = context.getText(R.string.nc_start_screen_sharing) + binding.shareScreenIcon.setImageDrawable( + ContextCompat.getDrawable(context, R.drawable.ic_screen_share_24) + ) + } + } + } } companion object { diff --git a/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt b/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt index 2b376cf16f2..4b6eec553e5 100644 --- a/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt +++ b/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt @@ -63,6 +63,13 @@ class ParticipantPermissions( true } + fun canPublishScreen(): Boolean = + if (hasConversationPermissions()) { + canPublishScreen + } else { + true + } + fun hasChatPermission(): Boolean { if (CapabilitiesUtil.hasSpreedFeatureCapability(spreedCapabilities, SpreedFeatures.CHAT_PERMISSION)) { return hasChatPermission diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 10a755c8b7b..8481e34f35a 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -66,6 +66,8 @@ public final class PeerConnectionWrapper { private final SdpObserver sdpObserver; private final boolean isMCUPublisher; + private final boolean isScreenSharer; + private final String localSession; private final String videoStreamType; // Keys of the remote participant when the call is end-to-end encrypted, attached to every receiver @@ -132,6 +134,27 @@ public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, boolean encryptionRequired, @Nullable FrameKeyRing senderKeyRing, @Nullable FrameKeyRing receiverKeyRing) { + this(peerConnectionFactory, iceServerList, mediaConstraints, sessionId, localSession, localStream, + isMCUPublisher, hasMCU, videoStreamType, signalingMessageReceiver, signalingMessageSender, + encryptionRequired, senderKeyRing, receiverKeyRing, false); + } + + /** + * @param isScreenSharer true if the local screen is sent directly (without MCU) to the remote participant. The + * connection then always sends the offer and ignores offers from the remote participant, + * as those belong to the connection receiving the screen of the remote participant. + */ + public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, + List iceServerList, + MediaConstraints mediaConstraints, + String sessionId, String localSession, @Nullable MediaStream localStream, + boolean isMCUPublisher, boolean hasMCU, String videoStreamType, + SignalingMessageReceiver signalingMessageReceiver, + SignalingMessageSender signalingMessageSender, + boolean encryptionRequired, + @Nullable FrameKeyRing senderKeyRing, + @Nullable FrameKeyRing receiverKeyRing, + boolean isScreenSharer) { boolean missingSenderKeys = (isMCUPublisher || localStream != null) && senderKeyRing == null; boolean missingReceiverKeys = !isMCUPublisher && receiverKeyRing == null; if (encryptionRequired && (missingSenderKeys || missingReceiverKeys)) { @@ -140,6 +163,8 @@ public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, } this.videoStreamType = videoStreamType; + this.isScreenSharer = isScreenSharer; + this.localSession = localSession; this.receiverKeyRing = receiverKeyRing; this.preferredVideoCodec = senderKeyRing != null || receiverKeyRing != null ? "VP8" : "H264"; @@ -181,7 +206,9 @@ public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, } } - if (hasMCU || hasInitiated) { + if (isScreenSharer) { + peerConnection.createOffer(sdpObserver, mediaConstraints); + } else if (hasMCU || hasInitiated) { DataChannel.Init init = new DataChannel.Init(); init.negotiated = false; @@ -474,6 +501,9 @@ private NCSignalingMessageDto createBaseSignalingMessage(String type) { ncSignalingMessage.setTo(sessionId); ncSignalingMessage.setRoomType(videoStreamType); ncSignalingMessage.setType(type); + if (isScreenSharer) { + ncSignalingMessage.setBroadcaster(localSession); + } return ncSignalingMessage; } @@ -481,10 +511,18 @@ private NCSignalingMessageDto createBaseSignalingMessage(String type) { private class WebRtcMessageListener implements SignalingMessageReceiver.WebRtcMessageListener { public void onOffer(String sdp, String nick) { + if (isScreenSharer) { + return; + } onOfferOrAnswer("offer", sdp); } public void onAnswer(String sdp, String nick) { + // Answers for "screen" are meant for the connection sharing the local screen, not for the one receiving + // the screen of the remote participant. + if ("screen".equals(videoStreamType) && !isScreenSharer && !isMCUPublisher) { + return; + } onOfferOrAnswer("answer", sdp); } diff --git a/app/src/main/res/drawable/ic_screen_share_24.xml b/app/src/main/res/drawable/ic_screen_share_24.xml new file mode 100644 index 00000000000..909ebe7ceec --- /dev/null +++ b/app/src/main/res/drawable/ic_screen_share_24.xml @@ -0,0 +1,16 @@ + + + + diff --git a/app/src/main/res/drawable/ic_stop_screen_share_24.xml b/app/src/main/res/drawable/ic_stop_screen_share_24.xml new file mode 100644 index 00000000000..270c7074336 --- /dev/null +++ b/app/src/main/res/drawable/ic_stop_screen_share_24.xml @@ -0,0 +1,16 @@ + + + + diff --git a/app/src/main/res/layout/dialog_more_call_actions.xml b/app/src/main/res/layout/dialog_more_call_actions.xml index 385b1a6318d..86e79cb9544 100644 --- a/app/src/main/res/layout/dialog_more_call_actions.xml +++ b/app/src/main/res/layout/dialog_more_call_actions.xml @@ -150,5 +150,37 @@ + + + + + + + diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 2ef2c7618a5..7c011d08257 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1098,6 +1098,9 @@ How to translate with transifex: Nextcloud app suggestions in account chooser dialog Turn on background blur Turn off background blur + Share screen + Stop screen sharing + Screen sharing could not be started Account not found Password is secure From 881c410a364d892659bb2dfef6cd96aa94b9aeb7 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 20 Oct 2026 14:05:31 +0200 Subject: [PATCH 02/16] fix(call): harden sharing of own screen - Capture the whole display instead of the call window, which shrinks in picture in picture or split screen - Route answers and candidates of "screen" connections to the connection sharing the local screen unless the sender is the broadcaster, so they do not reach the connection receiving the remote screen - Do not open data channels on the MCU screen publisher and always send the offer from connections sharing the local screen - Guard screen sharing state against concurrent changes from the main and signaling threads - Stop screen sharing when the own participant is no longer in the call - Update the screen sharing permission when the conversation is loaded and when the participant permissions change during the call, stopping an active share if the permission is revoked - Inform the user when screen sharing can not be started yet Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 205 ++++++++++++------ .../json/participants/ParticipantDto.kt | 5 +- .../signaling/SignalingMessageReceiver.kt | 58 ++++- .../talk/signaling/WebRtcMessageNotifier.java | 4 + .../talk/webrtc/PeerConnectionWrapper.java | 19 +- app/src/main/res/values/strings.xml | 2 + ...ingMessageReceiverParticipantListTest.java | 2 + ...gnalingMessageReceiverScreenSharingTest.kt | 159 ++++++++++++++ 8 files changed, 371 insertions(+), 83 deletions(-) create mode 100644 app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverScreenSharingTest.kt diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index af1b5782a2b..1918867d466 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -40,6 +40,7 @@ import android.os.ResultReceiver import android.provider.Settings import android.text.TextUtils import android.text.format.DateUtils +import android.util.DisplayMetrics import android.util.Log import android.view.MotionEvent import android.view.OrientationEventListener @@ -299,11 +300,15 @@ class CallActivity : CallBaseActivity() { private var screenSurfaceTextureHelper: SurfaceTextureHelper? = null private var screenVideoSource: VideoSource? = null + // Guards localScreenStream and screenSharePeerConnectionWrapperList, which are changed both from the main thread + // and from the signaling thread. + private val screenShareLock = Any() + @Volatile private var localScreenStream: MediaStream? = null // Without MCU there is one connection per remote participant, with MCU just the own publisher connection. - private val screenSharePeerConnectionWrapperList: MutableList = CopyOnWriteArrayList() + private val screenSharePeerConnectionWrapperList: MutableList = ArrayList() @Volatile private var isScreenSharePublisherConnected = false @@ -460,6 +465,17 @@ class CallActivity : CallBaseActivity() { private var othersInCall = false private var isOneToOneConversation = false private var currentConversation: ConversationDto? = null + set(value) { + field = value + isAllowedToShareScreen = value != null && + ParticipantPermissions( + conversationUser.capabilities?.spreedCapability, + ConversationModel.mapToConversationModel(value, conversationUser) + ).canPublishScreen() + if (binding != null) { + initFeaturesVisibility() + } + } private lateinit var micInputAudioRecorder: AudioRecord private var micInputAudioRecordThread: Thread? = null @@ -2004,6 +2020,8 @@ class CallActivity : CallBaseActivity() { } callParticipantList = CallParticipantList(signalingMessageReceiver) callParticipantList!!.addObserver(callParticipantListObserver) + signalingMessageReceiver!!.removeListener(screenSharePermissionListener) + signalingMessageReceiver!!.addListener(screenSharePermissionListener) if (hasMCU) { localStateBroadcaster = LocalStateBroadcasterMcu(localCallParticipantModel, messageSender) @@ -2414,6 +2432,7 @@ class CallActivity : CallBaseActivity() { callParticipantList!!.removeObserver(callParticipantListObserver) callParticipantList!!.destroy() } + signalingMessageReceiver?.removeListener(screenSharePermissionListener) val endCall: Boolean? = if (endCallForAll) true else null // Fire DELETE best-effort; do not block the UI waiting for the server response. @@ -2600,6 +2619,7 @@ class CallActivity : CallBaseActivity() { if (!isSelfInCall) { Log.d(TAG, "Self not in call, disconnecting from all other sessions") + stopScreenSharing() removeSessions(participantsInCall) return } @@ -3699,22 +3719,53 @@ class CallActivity : CallBaseActivity() { } } - val isAllowedToShareScreen: Boolean - get() { - val conversation = currentConversation ?: return false - return ParticipantPermissions( - conversationUser.capabilities?.spreedCapability, - ConversationModel.mapToConversationModel(conversation, conversationUser) - ).canPublishScreen() + @Volatile + var isAllowedToShareScreen = false + private set + + private val screenSharePermissionListener = object : SignalingMessageReceiver.ParticipantListMessageListener { + override fun onUsersInRoom(participants: MutableList) { + updateScreenSharePermission(participants) + } + + override fun onParticipantsUpdate(participants: MutableList) { + updateScreenSharePermission(participants) + } + + override fun onAllParticipantsUpdate(inCall: Long) { + // unused atm } + } + + private fun updateScreenSharePermission(participants: List) { + val ownSessionId = webSocketClient?.sessionId ?: callSession + val permissions = participants.firstOrNull { it?.sessionId == ownSessionId }?.participantPermissions ?: return + val isAllowed = (permissions and ParticipantPermissions.PUBLISH_SCREEN) == ParticipantPermissions.PUBLISH_SCREEN + + runOnUiThread { + if (isAllowed == isAllowedToShareScreen) { + return@runOnUiThread + } + isAllowedToShareScreen = isAllowed + initFeaturesVisibility() + if (!isAllowed && localScreenStream != null) { + stopScreenSharing() + Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_allowed, Snackbar.LENGTH_LONG).show() + } + } + } fun toggleScreenSharing() { if (localScreenStream != null) { stopScreenSharing() return } - if (!isAllowedToShareScreen || !isConnectionEstablished) { - Log.w(TAG, "Screen sharing is not allowed or the call is not established") + if (!isAllowedToShareScreen) { + Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_allowed, Snackbar.LENGTH_LONG).show() + return + } + if (!isConnectionEstablished) { + Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_connected, Snackbar.LENGTH_LONG).show() return } val mediaProjectionManager = getSystemService(MediaProjectionManager::class.java) @@ -3743,7 +3794,11 @@ class CallActivity : CallBaseActivity() { } private fun startScreenSharing(permissionResultData: Intent) { - if (localScreenStream != null || !isConnectionEstablished || currentCallStatus === CallStatus.LEAVING) { + if (localScreenStream != null || + !isAllowedToShareScreen || + !isConnectionEstablished || + currentCallStatus === CallStatus.LEAVING + ) { CallForegroundService.stopScreenSharing(applicationContext) return } @@ -3776,7 +3831,9 @@ class CallActivity : CallBaseActivity() { val stream = peerConnectionFactory!!.createLocalMediaStream("NCSS") stream.addTrack(peerConnectionFactory!!.createVideoTrack("NCs0", videoSource)) - localScreenStream = stream + synchronized(screenShareLock) { + localScreenStream = stream + } callViewModel.setLocalScreenSharing(true) if (hasMCU) { @@ -3786,9 +3843,19 @@ class CallActivity : CallBaseActivity() { } } + /** + * Can be called from any thread; only the first call after starting the screen sharing tears it down. + */ fun stopScreenSharing() { - val stream = localScreenStream ?: return - localScreenStream = null + val stream: MediaStream + val peerConnectionWrappers: List + synchronized(screenShareLock) { + stream = localScreenStream ?: return + localScreenStream = null + isScreenSharePublisherConnected = false + peerConnectionWrappers = screenSharePeerConnectionWrapperList.toList() + screenSharePeerConnectionWrapperList.clear() + } getParticipantSessionKeys().forEach { sessionId -> messageSender?.send( @@ -3797,12 +3864,7 @@ class CallActivity : CallBaseActivity() { ) } - for (peerConnectionWrapper in screenSharePeerConnectionWrapperList) { - peerConnectionWrapper.removeObserver(screenSharePublisherObserver) - peerConnectionWrapper.removePeerConnection() - } - screenSharePeerConnectionWrapperList.clear() - isScreenSharePublisherConnected = false + peerConnectionWrappers.forEach { closeScreenSharePeerConnection(it) } releaseScreenCapture() stream.dispose() @@ -3831,15 +3893,29 @@ class CallActivity : CallBaseActivity() { Snackbar.make(binding!!.root, R.string.nc_screen_sharing_failed, Snackbar.LENGTH_LONG).show() } + /** + * Returns the size of the whole display, independently of the size of the call window (for example, in picture + * in picture or split screen), scaled down to [SCREEN_SHARE_MAX_DIMENSION]. + */ private fun getScreenCaptureSize(): Pair { - val displayMetrics = resources.displayMetrics - val scale = max( - 1f, - max(displayMetrics.widthPixels, displayMetrics.heightPixels).toFloat() / SCREEN_SHARE_MAX_DIMENSION - ) + val displayWidth: Int + val displayHeight: Int + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + val bounds = windowManager.maximumWindowMetrics.bounds + displayWidth = bounds.width() + displayHeight = bounds.height() + } else { + val displayMetrics = DisplayMetrics() + @Suppress("DEPRECATION") + windowManager.defaultDisplay.getRealMetrics(displayMetrics) + displayWidth = displayMetrics.widthPixels + displayHeight = displayMetrics.heightPixels + } + + val scale = max(1f, max(displayWidth, displayHeight).toFloat() / SCREEN_SHARE_MAX_DIMENSION) // Video encoders expect even dimensions. - val width = (displayMetrics.widthPixels / scale).toInt() and 1.inv() - val height = (displayMetrics.heightPixels / scale).toInt() and 1.inv() + val width = (displayWidth / scale).toInt() and 1.inv() + val height = (displayHeight / scale).toInt() and 1.inv() return Pair(width, height) } @@ -3868,47 +3944,54 @@ class CallActivity : CallBaseActivity() { } private fun addScreenSharePeerConnection(sessionId: String?) { - val stream = localScreenStream ?: return - if (sessionId == null || screenSharePeerConnectionWrapperList.any { it.sessionId == sessionId }) { - return - } + synchronized(screenShareLock) { + val stream = localScreenStream ?: return + if (sessionId == null || screenSharePeerConnectionWrapperList.any { it.sessionId == sessionId }) { + return + } - val (senderKeyRing, receiverKeyRing) = keyRingsFor(hasMCU, hasMCU, sessionId) - val peerConnectionWrapper = try { - PeerConnectionWrapper( - peerConnectionFactory, - iceServers, - sdpConstraintsForMCUPublisher, - sessionId, - if (webSocketClient != null) webSocketClient!!.sessionId else callSession, - stream, - hasMCU, - hasMCU, - VIDEO_STREAM_TYPE_SCREEN, - signalingMessageReceiver, - signalingMessageSender, - isCallEndToEndEncryptionEnabled, - senderKeyRing, - receiverKeyRing, - !hasMCU - ) - } catch (e: IllegalStateException) { - Log.e(TAG, "Not creating a screen share connection to $sessionId", e) - failCallEncryption(R.string.nc_call_e2ee_setup_failed) - return - } - if (hasMCU) { - peerConnectionWrapper.addObserver(screenSharePublisherObserver) + val (senderKeyRing, receiverKeyRing) = keyRingsFor(hasMCU, hasMCU, sessionId) + val peerConnectionWrapper = try { + PeerConnectionWrapper( + peerConnectionFactory, + iceServers, + sdpConstraintsForMCUPublisher, + sessionId, + if (webSocketClient != null) webSocketClient!!.sessionId else callSession, + stream, + hasMCU, + hasMCU, + VIDEO_STREAM_TYPE_SCREEN, + signalingMessageReceiver, + signalingMessageSender, + isCallEndToEndEncryptionEnabled, + senderKeyRing, + receiverKeyRing, + true + ) + } catch (e: IllegalStateException) { + Log.e(TAG, "Not creating a screen share connection to $sessionId", e) + failCallEncryption(R.string.nc_call_e2ee_setup_failed) + return + } + if (hasMCU) { + peerConnectionWrapper.addObserver(screenSharePublisherObserver) + } + screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) } - screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) } private fun endScreenSharePeerConnection(sessionId: String?) { - val peerConnectionWrapper = screenSharePeerConnectionWrapperList.firstOrNull { it.sessionId == sessionId } - ?: return + val peerConnectionWrapper = synchronized(screenShareLock) { + screenSharePeerConnectionWrapperList.firstOrNull { it.sessionId == sessionId } + ?.also { screenSharePeerConnectionWrapperList.remove(it) } + } ?: return + closeScreenSharePeerConnection(peerConnectionWrapper) + } + + private fun closeScreenSharePeerConnection(peerConnectionWrapper: PeerConnectionWrapper) { peerConnectionWrapper.removeObserver(screenSharePublisherObserver) peerConnectionWrapper.removePeerConnection() - screenSharePeerConnectionWrapperList.remove(peerConnectionWrapper) } val isAllowedToStartOrStopRecording: Boolean diff --git a/app/src/main/java/com/nextcloud/talk/models/json/participants/ParticipantDto.kt b/app/src/main/java/com/nextcloud/talk/models/json/participants/ParticipantDto.kt index 6abd48e2836..da95bf77cd1 100644 --- a/app/src/main/java/com/nextcloud/talk/models/json/participants/ParticipantDto.kt +++ b/app/src/main/java/com/nextcloud/talk/models/json/participants/ParticipantDto.kt @@ -75,7 +75,10 @@ data class ParticipantDto( @JsonField(name = ["invitedActorId"]) var invitedActorId: String? = null, - var selected: Boolean = false + var selected: Boolean = false, + + // Only set from signaling messages + var participantPermissions: Int? = null ) : Parcelable { // This constructor is added to work with the 'com.bluelinelabs.logansquare.annotation.JsonObject' constructor() : this( diff --git a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.kt b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.kt index 26b64ff6c0c..79857832072 100644 --- a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.kt +++ b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.kt @@ -79,12 +79,12 @@ abstract class SignalingMessageReceiver { * Only the following participant properties are set: * - inCall * - lastPing + * - participantPermissions (since Talk 13) * - sessionId * - userId (if the participant is not a guest) * - * "participantPermissions" is provided in the message (since Talk 13), but not currently set in the - * participant. "publishingPermissions" was provided instead in Talk 12, but it was not used anywhere, so it is - * ignored. + * "publishingPermissions" was provided instead of "participantPermissions" in Talk 12, but it was not used + * anywhere, so it is ignored. * * @param participants all the participants (users and guests) in the room */ @@ -104,6 +104,7 @@ abstract class SignalingMessageReceiver { * Only the following participant properties are set: * - inCall * - lastPing + * - participantPermissions (since Talk 13) * - sessionId * - type * - userId (if the participant is not a guest) @@ -111,9 +112,8 @@ abstract class SignalingMessageReceiver { * "nextcloudSessionId" is provided in the message (when the "inCall" property of any participant changed), but * not currently set in the participant. * - * "participantPermissions" is provided in the message (since Talk 13), but not currently set in the - * participant. "publishingPermissions" was provided instead in Talk 12, but it was not used anywhere, so it is - * ignored. + * "publishingPermissions" was provided instead of "participantPermissions" in Talk 12, but it was not used + * anywhere, so it is ignored. * * @param participants all the participants (users and guests) in the room */ @@ -629,6 +629,10 @@ abstract class SignalingMessageReceiver { participant.actorId = participantMap["actorId"].toString() } + if (participantMap["participantPermissions"] is Number) { + participant.participantPermissions = (participantMap["participantPermissions"] as Number).toInt() + } + // Only in external signaling messages if (participantMap["participantType"] != null) { val participantTypeInt = participantMap["participantType"].toString().toInt() @@ -883,6 +887,8 @@ abstract class SignalingMessageReceiver { // although extremely unlikely, that the WebRtcMessageListeners for the second offer are notified before the // WebRtcMessageListeners for the first offer. This should not be a problem, though, so for simplicity // the statements are not synchronized. + // Offers are always sent by the participant sharing its screen, so they never belong to a connection + // sharing the local screen. offerMessageNotifier.notifyOffer(sessionId, roomType, sdp, nick) webRtcMessageNotifier.notifyOffer(sessionId, roomType, sdp, nick) @@ -897,7 +903,7 @@ abstract class SignalingMessageReceiver { val sdp = payload.sdp val nick = payload.nick - webRtcMessageNotifier.notifyAnswer(sessionId, roomType, sdp, nick) + webRtcMessageNotifier.notifyAnswer(sessionId, getListenerRoomTypeForAnswer(sessionId, roomType), sdp, nick) return } @@ -952,7 +958,7 @@ abstract class SignalingMessageReceiver { webRtcMessageNotifier.notifyCandidate( sessionId, - roomType, + getListenerRoomType(sessionId, roomType, signalingMessage.broadcaster), ncIceCandidate.sdpMid, ncIceCandidate.sdpMLineIndex, ncIceCandidate.candidate @@ -962,9 +968,43 @@ abstract class SignalingMessageReceiver { } if ("endOfCandidates" == type) { - webRtcMessageNotifier.notifyEndOfCandidates(sessionId, roomType) + webRtcMessageNotifier.notifyEndOfCandidates( + sessionId, + getListenerRoomType(sessionId, roomType, signalingMessage.broadcaster) + ) return } } + + /** + * Without MCU both the connection sharing the local screen and the connection receiving the screen of a remote + * participant use the "screen" room type with the same remote session. Messages for the former are delivered to + * listeners added with [ROOM_TYPE_SCREEN_SHARING] instead. + * + * The participant sharing a screen sets itself as the "broadcaster" of its messages, so messages from a + * broadcaster belong to the receiving connection. Answers are only sent to the participant sharing a screen. + */ + private fun getListenerRoomType(sessionId: String?, roomType: String?, broadcaster: String?): String? { + if (ROOM_TYPE_SCREEN != roomType || (broadcaster != null && broadcaster == sessionId)) { + return roomType + } + return getListenerRoomTypeForAnswer(sessionId, roomType) + } + + private fun getListenerRoomTypeForAnswer(sessionId: String?, roomType: String?): String? = + if (ROOM_TYPE_SCREEN == roomType && webRtcMessageNotifier.hasListenerFor(sessionId, ROOM_TYPE_SCREEN_SHARING)) { + ROOM_TYPE_SCREEN_SHARING + } else { + roomType + } + + companion object { + private const val ROOM_TYPE_SCREEN = "screen" + + /** + * Room type to add a [WebRtcMessageListener] for a connection sharing the local screen. + */ + const val ROOM_TYPE_SCREEN_SHARING = "screen-sharing" + } } diff --git a/app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java b/app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java index 4bc6f3cb7e7..98a2f8819dd 100644 --- a/app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java +++ b/app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java @@ -68,6 +68,10 @@ public synchronized void removeListener(SignalingMessageReceiver.WebRtcMessageLi } } + public synchronized boolean hasListenerFor(String sessionId, String roomType) { + return !getListenersFor(sessionId, roomType).isEmpty(); + } + private List getListenersFor(String sessionId, String roomType) { List webRtcMessageListeners = new ArrayList<>(webRtcMessageListenersFrom.size()); diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 8481e34f35a..830a48e9bf0 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -140,9 +140,9 @@ public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, } /** - * @param isScreenSharer true if the local screen is sent directly (without MCU) to the remote participant. The - * connection then always sends the offer and ignores offers from the remote participant, - * as those belong to the connection receiving the screen of the remote participant. + * @param isScreenSharer true if the connection sends the local screen, either directly to the remote participant + * or, with MCU, to the own publisher. The connection then always sends the offer and does + * not open data channels. */ public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, List iceServerList, @@ -180,7 +180,10 @@ public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, peerConnection = peerConnectionFactory.createPeerConnection(configuration, new InitialPeerConnectionObserver()); this.signalingMessageReceiver = signalingMessageReceiver; - this.signalingMessageReceiver.addListener(webRtcMessageListener, sessionId, videoStreamType); + this.signalingMessageReceiver.addListener( + webRtcMessageListener, + sessionId, + isScreenSharer ? SignalingMessageReceiver.ROOM_TYPE_SCREEN_SHARING : videoStreamType); this.signalingMessageSender = signalingMessageSender; @@ -511,18 +514,10 @@ private NCSignalingMessageDto createBaseSignalingMessage(String type) { private class WebRtcMessageListener implements SignalingMessageReceiver.WebRtcMessageListener { public void onOffer(String sdp, String nick) { - if (isScreenSharer) { - return; - } onOfferOrAnswer("offer", sdp); } public void onAnswer(String sdp, String nick) { - // Answers for "screen" are meant for the connection sharing the local screen, not for the one receiving - // the screen of the remote participant. - if ("screen".equals(videoStreamType) && !isScreenSharer && !isMCUPublisher) { - return; - } onOfferOrAnswer("answer", sdp); } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 7c011d08257..f4fce449eeb 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1101,6 +1101,8 @@ How to translate with transifex: Share screen Stop screen sharing Screen sharing could not be started + You are not allowed to share your screen + You can share your screen once the call is connected Account not found Password is secure diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java index e1fdb3dc552..1037296e7e7 100644 --- a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java @@ -82,6 +82,7 @@ public void testInternalSignalingParticipantListMessageUsersInRoom() { expectedParticipant1.setUserId("theUserId"); expectedParticipant1.setActorType(ParticipantDto.ActorType.FEDERATED); expectedParticipant1.setActorId("theActorId"); + expectedParticipant1.setParticipantPermissions(42); expectedParticipantList.add(expectedParticipant1); ParticipantDto expectedParticipant2 = new ParticipantDto(); @@ -297,6 +298,7 @@ public void testExternalSignalingParticipantListMessageParticipantsUpdate() { expectedParticipant1.setUserId("theUserId"); expectedParticipant1.setActorType(ParticipantDto.ActorType.FEDERATED); expectedParticipant1.setActorId("theActorId"); + expectedParticipant1.setParticipantPermissions(42); expectedParticipantList.add(expectedParticipant1); ParticipantDto expectedParticipant2 = new ParticipantDto(); diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverScreenSharingTest.kt b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverScreenSharingTest.kt new file mode 100644 index 00000000000..3fee622a4bb --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverScreenSharingTest.kt @@ -0,0 +1,159 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ +package com.nextcloud.talk.signaling + +import com.nextcloud.talk.models.json.signaling.NCIceCandidateDto +import com.nextcloud.talk.models.json.signaling.NCMessagePayloadDto +import com.nextcloud.talk.models.json.signaling.NCSignalingMessageDto +import org.junit.Before +import org.junit.Test +import org.mockito.Mockito.mock +import org.mockito.Mockito.only +import org.mockito.Mockito.verify +import org.mockito.Mockito.verifyNoInteractions + +class SignalingMessageReceiverScreenSharingTest { + + private lateinit var signalingMessageReceiver: SignalingMessageReceiver + private lateinit var receivingListener: SignalingMessageReceiver.WebRtcMessageListener + private lateinit var sharingListener: SignalingMessageReceiver.WebRtcMessageListener + + @Before + fun setUp() { + signalingMessageReceiver = object : SignalingMessageReceiver() {} + receivingListener = mock(SignalingMessageReceiver.WebRtcMessageListener::class.java) + sharingListener = mock(SignalingMessageReceiver.WebRtcMessageListener::class.java) + } + + private fun addBothListeners() { + signalingMessageReceiver.addListener(receivingListener, REMOTE_SESSION_ID, "screen") + signalingMessageReceiver.addListener( + sharingListener, + REMOTE_SESSION_ID, + SignalingMessageReceiver.ROOM_TYPE_SCREEN_SHARING + ) + } + + private fun createMessage(type: String, broadcaster: String? = null): NCSignalingMessageDto { + val payload = NCMessagePayloadDto() + payload.type = type + if (type == "candidate") { + val iceCandidate = NCIceCandidateDto() + iceCandidate.sdpMid = "theSdpMid" + iceCandidate.sdpMLineIndex = 42 + iceCandidate.candidate = "theCandidate" + payload.iceCandidate = iceCandidate + } else { + payload.sdp = "theSdp" + } + + val message = NCSignalingMessageDto() + message.from = REMOTE_SESSION_ID + message.type = type + message.roomType = "screen" + message.broadcaster = broadcaster + message.payload = payload + return message + } + + @Test + fun testOfferIsDeliveredToReceivingConnection() { + addBothListeners() + + signalingMessageReceiver.processSignalingMessage(createMessage("offer", REMOTE_SESSION_ID)) + + verify(receivingListener, only()).onOffer("theSdp", null) + verifyNoInteractions(sharingListener) + } + + @Test + fun testAnswerIsDeliveredToSharingConnection() { + addBothListeners() + + signalingMessageReceiver.processSignalingMessage(createMessage("answer")) + + verify(sharingListener, only()).onAnswer("theSdp", null) + verifyNoInteractions(receivingListener) + } + + @Test + fun testAnswerWithoutSharingConnectionIsDeliveredToScreenListener() { + signalingMessageReceiver.addListener(receivingListener, REMOTE_SESSION_ID, "screen") + + signalingMessageReceiver.processSignalingMessage(createMessage("answer")) + + verify(receivingListener, only()).onAnswer("theSdp", null) + } + + @Test + fun testCandidateFromBroadcasterIsDeliveredToReceivingConnection() { + addBothListeners() + + signalingMessageReceiver.processSignalingMessage(createMessage("candidate", REMOTE_SESSION_ID)) + + verify(receivingListener, only()).onCandidate("theSdpMid", 42, "theCandidate") + verifyNoInteractions(sharingListener) + } + + @Test + fun testCandidateWithoutBroadcasterIsDeliveredToSharingConnection() { + addBothListeners() + + signalingMessageReceiver.processSignalingMessage(createMessage("candidate")) + + verify(sharingListener, only()).onCandidate("theSdpMid", 42, "theCandidate") + verifyNoInteractions(receivingListener) + } + + @Test + fun testCandidateWithLocalBroadcasterIsDeliveredToSharingConnection() { + addBothListeners() + + signalingMessageReceiver.processSignalingMessage(createMessage("candidate", "theLocalSessionId")) + + verify(sharingListener, only()).onCandidate("theSdpMid", 42, "theCandidate") + verifyNoInteractions(receivingListener) + } + + @Test + fun testCandidateWithoutSharingConnectionIsDeliveredToReceivingConnection() { + signalingMessageReceiver.addListener(receivingListener, REMOTE_SESSION_ID, "screen") + + signalingMessageReceiver.processSignalingMessage(createMessage("candidate")) + + verify(receivingListener, only()).onCandidate("theSdpMid", 42, "theCandidate") + } + + @Test + fun testEndOfCandidatesFromBroadcasterIsDeliveredToReceivingConnection() { + addBothListeners() + + signalingMessageReceiver.processSignalingMessage(createMessage("endOfCandidates", REMOTE_SESSION_ID)) + + verify(receivingListener, only()).onEndOfCandidates() + verifyNoInteractions(sharingListener) + } + + @Test + fun testVideoCandidateIsNotDeliveredToSharingConnection() { + val videoListener = mock(SignalingMessageReceiver.WebRtcMessageListener::class.java) + signalingMessageReceiver.addListener(videoListener, REMOTE_SESSION_ID, "video") + addBothListeners() + + val message = createMessage("candidate") + message.roomType = "video" + signalingMessageReceiver.processSignalingMessage(message) + + verify(videoListener, only()).onCandidate("theSdpMid", 42, "theCandidate") + verifyNoInteractions(sharingListener) + verifyNoInteractions(receivingListener) + } + + companion object { + private const val REMOTE_SESSION_ID = "theRemoteSessionId" + } +} From 3277ca295455f5f18ecdca63b92436b41b324c74 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 20 Oct 2026 14:29:15 +0200 Subject: [PATCH 03/16] fix(call): handle more edge cases when sharing own screen - Clean up when the screen capture fails to start due to an expired or revoked media projection - Ignore screen sharing actions in the call foreground service when no call is ongoing, so the service is not kept running after a call - Guard the screen capturer against concurrent use from the main and signaling threads - Prefer the screen sharing permission from signaling over the one from the conversation, and share the permission check between both - Follow display rotation while the call is in background - Refresh the items and observers of the more call actions dialog every time it is shown Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 164 +++++++++++++----- .../talk/services/CallForegroundService.kt | 44 +++-- .../talk/ui/dialog/MoreCallActionsDialog.kt | 6 +- .../talk/utils/ParticipantPermissions.kt | 29 ++-- .../talk/utils/ParticipantPermissionsTest.kt | 27 +++ 5 files changed, 194 insertions(+), 76 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 1918867d466..a1d380c3f79 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -24,6 +24,7 @@ import android.graphics.Color import android.graphics.Rect import android.graphics.Typeface import android.graphics.drawable.Icon +import android.hardware.display.DisplayManager import android.media.AudioAttributes import android.media.AudioFormat import android.media.AudioRecord @@ -42,6 +43,7 @@ import android.text.TextUtils import android.text.format.DateUtils import android.util.DisplayMetrics import android.util.Log +import android.view.Display import android.view.MotionEvent import android.view.OrientationEventListener import android.view.View @@ -299,9 +301,16 @@ class CallActivity : CallBaseActivity() { private var screenCapturer: ScreenCapturerAndroid? = null private var screenSurfaceTextureHelper: SurfaceTextureHelper? = null private var screenVideoSource: VideoSource? = null + private var screenCaptureSize: Pair? = null + private val screenCaptureDisplayListener = ScreenCaptureDisplayListener() - // Guards localScreenStream and screenSharePeerConnectionWrapperList, which are changed both from the main thread - // and from the signaling thread. + private var conversationScreenSharePermission = false + + // The permission from signaling is more recent than the one from the conversation, so it takes precedence. + private var signalingScreenSharePermission: Boolean? = null + + // Guards the screen sharing state (stream, peer connections and capturer), which is changed both from the main + // thread and from the signaling thread. private val screenShareLock = Any() @Volatile @@ -467,14 +476,12 @@ class CallActivity : CallBaseActivity() { private var currentConversation: ConversationDto? = null set(value) { field = value - isAllowedToShareScreen = value != null && + conversationScreenSharePermission = value != null && ParticipantPermissions( conversationUser.capabilities?.spreedCapability, ConversationModel.mapToConversationModel(value, conversationUser) ).canPublishScreen() - if (binding != null) { - initFeaturesVisibility() - } + applyScreenSharePermission() } private lateinit var micInputAudioRecorder: AudioRecord @@ -3415,6 +3422,22 @@ class CallActivity : CallBaseActivity() { } } + private inner class ScreenCaptureDisplayListener : DisplayManager.DisplayListener { + override fun onDisplayAdded(displayId: Int) { + // unused atm + } + + override fun onDisplayRemoved(displayId: Int) { + // unused atm + } + + override fun onDisplayChanged(displayId: Int) { + if (displayId == Display.DEFAULT_DISPLAY) { + updateScreenCaptureSize() + } + } + } + private inner class ScreenSharePublisherPeerConnectionObserver : PeerConnectionObserver { override fun onStreamAdded(mediaStream: MediaStream) { // unused atm @@ -3713,10 +3736,6 @@ class CallActivity : CallBaseActivity() { override fun onConfigurationChanged(newConfig: Configuration) { super.onConfigurationChanged(newConfig) eventBus.post(ConfigurationChangeEvent()) - screenCapturer?.let { - val (width, height) = getScreenCaptureSize() - it.changeCaptureFormat(width, height, SCREEN_SHARE_FRAME_RATE) - } } @Volatile @@ -3740,18 +3759,33 @@ class CallActivity : CallBaseActivity() { private fun updateScreenSharePermission(participants: List) { val ownSessionId = webSocketClient?.sessionId ?: callSession val permissions = participants.firstOrNull { it?.sessionId == ownSessionId }?.participantPermissions ?: return - val isAllowed = (permissions and ParticipantPermissions.PUBLISH_SCREEN) == ParticipantPermissions.PUBLISH_SCREEN + val isAllowed = ParticipantPermissions.canPublishScreen( + conversationUser.capabilities?.spreedCapability, + permissions + ) runOnUiThread { - if (isAllowed == isAllowedToShareScreen) { - return@runOnUiThread - } - isAllowedToShareScreen = isAllowed - initFeaturesVisibility() - if (!isAllowed && localScreenStream != null) { - stopScreenSharing() - Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_allowed, Snackbar.LENGTH_LONG).show() - } + signalingScreenSharePermission = isAllowed + applyScreenSharePermission() + } + } + + /** + * Must be called on the main thread. + */ + private fun applyScreenSharePermission() { + val isAllowed = signalingScreenSharePermission ?: conversationScreenSharePermission + if (isAllowed == isAllowedToShareScreen) { + return + } + isAllowedToShareScreen = isAllowed + if (binding == null) { + return + } + initFeaturesVisibility() + if (!isAllowed && localScreenStream != null) { + stopScreenSharing() + Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_allowed, Snackbar.LENGTH_LONG).show() } } @@ -3778,6 +3812,10 @@ class CallActivity : CallBaseActivity() { Log.d(TAG, "Screen sharing permission was not granted") return } + if (!isConnectionEstablished || currentCallStatus === CallStatus.LEAVING) { + Log.d(TAG, "Call ended while asking for the screen sharing permission") + return + } // On Android 14+ the media projection can only be started once the foreground service has the media // projection type, which in turn requires the permission to have been granted. @@ -3813,21 +3851,33 @@ class CallActivity : CallBaseActivity() { ) val surfaceTextureHelper = SurfaceTextureHelper.create("ScreenCaptureThread", rootEglBase!!.eglBaseContext) val videoSource = peerConnectionFactory!!.createVideoSource(true) - screenCapturer = capturer - screenSurfaceTextureHelper = surfaceTextureHelper - screenVideoSource = videoSource + val captureSize = getScreenCaptureSize() + synchronized(screenShareLock) { + screenCapturer = capturer + screenSurfaceTextureHelper = surfaceTextureHelper + screenVideoSource = videoSource + screenCaptureSize = captureSize + } capturer.initialize(surfaceTextureHelper, applicationContext, videoSource.capturerObserver) - try { - val (width, height) = getScreenCaptureSize() - capturer.startCapture(width, height, SCREEN_SHARE_FRAME_RATE) + // The media projection may fail to start, for example, if the permission expired or was revoked. + val captureError = try { + capturer.startCapture(captureSize.first, captureSize.second, SCREEN_SHARE_FRAME_RATE) + null } catch (e: SecurityException) { - Log.e(TAG, "Failed to start screen capture", e) + e + } catch (e: IllegalStateException) { + e + } + if (captureError != null) { + Log.e(TAG, "Failed to start screen capture", captureError) releaseScreenCapture() CallForegroundService.stopScreenSharing(applicationContext) showScreenSharingError() return } + getSystemService(DisplayManager::class.java) + .registerDisplayListener(screenCaptureDisplayListener, Handler(Looper.getMainLooper())) val stream = peerConnectionFactory!!.createLocalMediaStream("NCSS") stream.addTrack(peerConnectionFactory!!.createVideoTrack("NCs0", videoSource)) @@ -3874,7 +3924,22 @@ class CallActivity : CallBaseActivity() { } private fun releaseScreenCapture() { - screenCapturer?.let { + getSystemService(DisplayManager::class.java).unregisterDisplayListener(screenCaptureDisplayListener) + + val capturer: ScreenCapturerAndroid? + val videoSource: VideoSource? + val surfaceTextureHelper: SurfaceTextureHelper? + synchronized(screenShareLock) { + capturer = screenCapturer + videoSource = screenVideoSource + surfaceTextureHelper = screenSurfaceTextureHelper + screenCapturer = null + screenVideoSource = null + screenSurfaceTextureHelper = null + screenCaptureSize = null + } + + capturer?.let { try { it.stopCapture() } catch (e: InterruptedException) { @@ -3882,11 +3947,20 @@ class CallActivity : CallBaseActivity() { } it.dispose() } - screenCapturer = null - screenVideoSource?.dispose() - screenVideoSource = null - screenSurfaceTextureHelper?.dispose() - screenSurfaceTextureHelper = null + videoSource?.dispose() + surfaceTextureHelper?.dispose() + } + + private fun updateScreenCaptureSize() { + val captureSize = getScreenCaptureSize() + synchronized(screenShareLock) { + val capturer = screenCapturer ?: return + if (captureSize == screenCaptureSize) { + return + } + screenCaptureSize = captureSize + capturer.changeCaptureFormat(captureSize.first, captureSize.second, SCREEN_SHARE_FRAME_RATE) + } } private fun showScreenSharingError() { @@ -3894,23 +3968,17 @@ class CallActivity : CallBaseActivity() { } /** - * Returns the size of the whole display, independently of the size of the call window (for example, in picture - * in picture or split screen), scaled down to [SCREEN_SHARE_MAX_DIMENSION]. + * Returns the size of the whole display in its current rotation, independently of the size of the call window + * (for example, in picture in picture or split screen) and of whether the call is visible, scaled down to + * [SCREEN_SHARE_MAX_DIMENSION]. */ private fun getScreenCaptureSize(): Pair { - val displayWidth: Int - val displayHeight: Int - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { - val bounds = windowManager.maximumWindowMetrics.bounds - displayWidth = bounds.width() - displayHeight = bounds.height() - } else { - val displayMetrics = DisplayMetrics() - @Suppress("DEPRECATION") - windowManager.defaultDisplay.getRealMetrics(displayMetrics) - displayWidth = displayMetrics.widthPixels - displayHeight = displayMetrics.heightPixels - } + val displayMetrics = DisplayMetrics() + // The configuration of the activity is not updated while it is stopped, so the display is queried directly. + @Suppress("DEPRECATION") + getSystemService(DisplayManager::class.java).getDisplay(Display.DEFAULT_DISPLAY).getRealMetrics(displayMetrics) + val displayWidth = displayMetrics.widthPixels + val displayHeight = displayMetrics.heightPixels val scale = max(1f, max(displayWidth, displayHeight).toFloat() / SCREEN_SHARE_MAX_DIMENSION) // Video encoders expect even dimensions. diff --git a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt index 2a3f7c00299..8062aad01ac 100644 --- a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt +++ b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt @@ -74,6 +74,7 @@ class CallForegroundService : Service() { private var currentIsOneToOne = false private var conversationAvatarBitmap: Bitmap? = null private var isScreenSharing = false + private var isCallStarted = false override fun onBind(intent: Intent?): IBinder? = null @@ -81,10 +82,9 @@ class CallForegroundService : Service() { override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { Log.d(TAG, "onStartCommand called") - if (handleScreenSharingAction(intent)) { - return START_STICKY - } + handleScreenSharingAction(intent, startId)?.let { return it } + isCallStarted = true handler.removeCallbacksAndMessages(null) val conversationName = intent?.getStringExtra(EXTRA_CONVERSATION_NAME) @@ -112,24 +112,38 @@ class CallForegroundService : Service() { return START_STICKY } - private fun handleScreenSharingAction(intent: Intent?): Boolean = - when (intent?.action) { - ACTION_START_SCREEN_SHARING -> { - val resultReceiver = - IntentCompat.getParcelableExtra(intent, EXTRA_RESULT_RECEIVER, ResultReceiver::class.java) - setScreenSharing(true, resultReceiver) - true + /** + * Returns the value to return from onStartCommand if the intent was a screen sharing action, or null otherwise. + * + * Screen sharing actions are ignored if the service was started by them rather than by an ongoing call (for + * example, if the call ended in the meantime), as the service must not be kept running in that case. + */ + private fun handleScreenSharingAction(intent: Intent?, startId: Int): Int? { + val action = intent?.action + val resultReceiver = intent?.let { + IntentCompat.getParcelableExtra(it, EXTRA_RESULT_RECEIVER, ResultReceiver::class.java) + } + + return when { + action != ACTION_START_SCREEN_SHARING && action != ACTION_STOP_SCREEN_SHARING -> null + + !isCallStarted -> { + Log.w(TAG, "Ignoring screen sharing action without an ongoing call") + resultReceiver?.send(RESULT_SCREEN_SHARING_FAILED, null) + stopSelf(startId) + START_NOT_STICKY } - ACTION_STOP_SCREEN_SHARING -> { - if (isScreenSharing) { + else -> { + if (action == ACTION_START_SCREEN_SHARING) { + setScreenSharing(true, resultReceiver) + } else if (isScreenSharing) { setScreenSharing(false, null) } - true + START_STICKY } - - else -> false } + } /** * Updates the foreground service type to include or exclude media projection, which is required to capture the diff --git a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt index 935abdfe7b1..04ef00337f8 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt @@ -51,10 +51,8 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee viewThemeUtils.platform.themeDialogDark(binding.root) - initItemsVisibility() initEmojiBar() initClickListeners() - initObservers() } override fun onStart() { @@ -62,6 +60,10 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee val bottomSheet = findViewById(R.id.design_bottom_sheet) val behavior = BottomSheetBehavior.from(bottomSheet as View) behavior.state = BottomSheetBehavior.STATE_COLLAPSED + + // The dialog may be shown again after being dismissed, which destroys its lifecycle and thus the observers. + initItemsVisibility() + initObservers() } private fun initItemsVisibility() { diff --git a/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt b/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt index 4b6eec553e5..7140f6a00e1 100644 --- a/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt +++ b/app/src/main/java/com/nextcloud/talk/utils/ParticipantPermissions.kt @@ -28,11 +28,7 @@ class ParticipantPermissions( private val hasChatPermission = (conversation.permissions and CHAT) == CHAT private val hasReactPermission = (conversation.permissions and REACT) == REACT - private fun hasConversationPermissions(): Boolean = - CapabilitiesUtil.hasSpreedFeatureCapability( - spreedCapabilities, - SpreedFeatures.CONVERSATION_PERMISSION - ) + private fun hasConversationPermissions(): Boolean = hasConversationPermissions(spreedCapabilities) fun canIgnoreLobby(): Boolean { if (hasConversationPermissions()) { @@ -63,12 +59,7 @@ class ParticipantPermissions( true } - fun canPublishScreen(): Boolean = - if (hasConversationPermissions()) { - canPublishScreen - } else { - true - } + fun canPublishScreen(): Boolean = canPublishScreen(spreedCapabilities, conversation.permissions) fun hasChatPermission(): Boolean { if (CapabilitiesUtil.hasSpreedFeatureCapability(spreedCapabilities, SpreedFeatures.CHAT_PERMISSION)) { @@ -103,5 +94,21 @@ class ParticipantPermissions( const val DEFAULT_GROUP_PERMISSIONS = JOIN_CALL or PUBLISH_AUDIO or PUBLISH_VIDEO or PUBLISH_SCREEN or CHAT or REACT + + private fun hasConversationPermissions(spreedCapabilities: SpreedCapabilityDto?): Boolean = + CapabilitiesUtil.hasSpreedFeatureCapability( + spreedCapabilities, + SpreedFeatures.CONVERSATION_PERMISSION + ) + + /** + * @param permissions the permissions of the participant, either from the conversation or from signaling + */ + fun canPublishScreen(spreedCapabilities: SpreedCapabilityDto?, permissions: Int): Boolean = + if (hasConversationPermissions(spreedCapabilities)) { + (permissions and PUBLISH_SCREEN) == PUBLISH_SCREEN + } else { + true + } } } diff --git a/app/src/test/java/com/nextcloud/talk/utils/ParticipantPermissionsTest.kt b/app/src/test/java/com/nextcloud/talk/utils/ParticipantPermissionsTest.kt index 28cdf580c05..493b85fa2d2 100644 --- a/app/src/test/java/com/nextcloud/talk/utils/ParticipantPermissionsTest.kt +++ b/app/src/test/java/com/nextcloud/talk/utils/ParticipantPermissionsTest.kt @@ -18,6 +18,33 @@ import org.junit.Test class ParticipantPermissionsTest : TestCase() { + @Test + fun test_canPublishScreenWithConversationPermissionsCapability() { + val spreedCapability = SpreedCapabilityDto() + spreedCapability.features = listOf("conversation-permissions") + + assertTrue( + ParticipantPermissions.canPublishScreen( + spreedCapability, + ParticipantPermissions.CUSTOM or ParticipantPermissions.PUBLISH_SCREEN + ) + ) + assertFalse( + ParticipantPermissions.canPublishScreen( + spreedCapability, + ParticipantPermissions.CUSTOM or ParticipantPermissions.PUBLISH_VIDEO + ) + ) + } + + @Test + fun test_canPublishScreenWithoutConversationPermissionsCapability() { + val spreedCapability = SpreedCapabilityDto() + spreedCapability.features = emptyList() + + assertTrue(ParticipantPermissions.canPublishScreen(spreedCapability, ParticipantPermissions.CUSTOM)) + } + @Test fun test_areFlagsSet() { val spreedCapability = SpreedCapabilityDto() From f09b8864a4806d49ae2ebedde3b7bb57d23686b0 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 20 Oct 2026 14:45:13 +0200 Subject: [PATCH 04/16] feat(call): show screen sharing in status bar instead of picture in picture While the own screen is shared, leaving the call with the back or home buttons moves it to the background instead of showing picture in picture on top of the shared screen. The call notification shows a screen sharing icon in the status bar and states that the screen is shared. The screen sharing icons are now the same as in the web client. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 8 +++++ .../talk/activities/CallBaseActivity.java | 32 ++++++++++++++++--- .../talk/services/CallForegroundService.kt | 25 +++++++++++++-- .../main/res/drawable/ic_screen_share_24.xml | 4 +-- .../drawable/ic_screen_share_white_24dp.xml | 15 +++++++++ .../res/drawable/ic_stop_screen_share_24.xml | 4 +-- app/src/main/res/values/strings.xml | 1 + 7 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 app/src/main/res/drawable/ic_screen_share_white_24dp.xml diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index a1d380c3f79..fe9e6f438fb 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -3885,6 +3885,7 @@ class CallActivity : CallBaseActivity() { localScreenStream = stream } callViewModel.setLocalScreenSharing(true) + updatePipAutoEnter() if (hasMCU) { addScreenSharePeerConnection(webSocketClient!!.sessionId) @@ -3921,8 +3922,15 @@ class CallActivity : CallBaseActivity() { CallForegroundService.stopScreenSharing(applicationContext) callViewModel.setLocalScreenSharing(false) + runOnUiThread { updatePipAutoEnter() } } + /** + * Picture in picture would be shown on top of the shared screen, so the call is just moved to the background + * while sharing; the call notification shows that the screen is being shared. + */ + override fun isAutomaticPipModeSuppressed(): Boolean = localScreenStream != null + private fun releaseScreenCapture() { getSystemService(DisplayManager::class.java).unregisterDisplayListener(screenCaptureDisplayListener) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java index ec6ff7930f4..f807683f53f 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java @@ -33,7 +33,7 @@ public abstract class CallBaseActivity extends BaseActivity { private final OnBackPressedCallback onBackPressedCallback = new OnBackPressedCallback(true) { @Override public void handleOnBackPressed() { - if (isPipModePossible()) { + if (shouldEnterPipModeAutomatically()) { enterPipMode(); } else { moveTaskToBack(true); @@ -110,7 +110,7 @@ public void onTopResumedActivityChanged(boolean isTopResumedActivity) { Log.d(TAG, "onTopResumedActivityChanged: isTopResumedActivity=" + isTopResumedActivity + " isInPictureInPictureMode=" + isInPictureInPictureMode()); if (isTopResumedActivity || isInPictureInPictureMode() - || !isPipModePossible() + || !shouldEnterPipModeAutomatically() || isChangingConfigurations() || isFinishing()) { return; @@ -129,7 +129,7 @@ public void onPause() { // On API 29+, onTopResumedActivityChanged already handled this. if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q && !isInPictureInPictureMode() - && isPipModePossible() + && shouldEnterPipModeAutomatically() && !isChangingConfigurations() && !isFinishing()) { enterPipMode(); @@ -152,7 +152,7 @@ protected void onUserLeaveHint() { // moveTaskToBack here: onUserLeaveHint also fires when a transient overlay like the // permission dialog appears at call start, which would throw the call to the background. if (!isInPipMode - && isPipModePossible() + && shouldEnterPipModeAutomatically() && Build.VERSION.SDK_INT < Build.VERSION_CODES.S) { enterPipMode(); } @@ -173,6 +173,30 @@ void enterPipMode() { } } + /** + * Whether picture in picture should not be entered when the user leaves the activity (for example, with the back + * or home buttons). Picture in picture can still be entered explicitly. + *

+ * {@link #updatePipAutoEnter()} must be called whenever the returned value changes. + */ + protected boolean isAutomaticPipModeSuppressed() { + return false; + } + + private boolean shouldEnterPipModeAutomatically() { + return isPipModePossible() && !isAutomaticPipModeSuppressed(); + } + + /** + * Updates whether the system enters picture in picture automatically when leaving the activity. + */ + void updatePipAutoEnter() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && isPipModePossible()) { + mPictureInPictureParamsBuilder.setAutoEnterEnabled(!isAutomaticPipModeSuppressed()); + setPictureInPictureParams(mPictureInPictureParamsBuilder.build()); + } + } + boolean isPipModePossible() { boolean deviceHasPipFeature = getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE); diff --git a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt index 8062aad01ac..5b06c666390 100644 --- a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt +++ b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt @@ -209,10 +209,16 @@ class CallForegroundService : Service() { return buildCallStyleNotification(contentTitle, pendingIntent) } + val contentText = if (isScreenSharing) { + getString(R.string.nc_call_ongoing_notification_screen_sharing) + } else { + getString(R.string.nc_call_ongoing_notification_content) + } + return NotificationCompat.Builder(this, channelId) .setContentTitle(contentTitle) - .setContentText(getString(R.string.nc_call_ongoing_notification_content)) - .setSmallIcon(R.drawable.ic_call_white_24dp) + .setContentText(contentText) + .setSmallIcon(getSmallIcon()) .setOngoing(true) .setCategory(NotificationCompat.CATEGORY_CALL) .setPriority(NotificationCompat.PRIORITY_LOW) @@ -249,7 +255,7 @@ class CallForegroundService : Service() { return Notification.Builder(this, channelId) .setStyle(callStyle) - .setSmallIcon(R.drawable.ic_call_white_24dp) + .setSmallIcon(getSmallIcon()) .setContentIntent(pendingIntent) .setOngoing(true) .setCategory(Notification.CATEGORY_CALL) @@ -261,10 +267,23 @@ class CallForegroundService : Service() { builder.setWhen(callStartTime) builder.setShowWhen(true) } + if (isScreenSharing) { + builder.setContentText(getString(R.string.nc_call_ongoing_notification_screen_sharing)) + } } .build() } + /** + * The small icon is shown in the status bar, so it indicates when the screen is being shared. + */ + private fun getSmallIcon(): Int = + if (isScreenSharing) { + R.drawable.ic_screen_share_white_24dp + } else { + R.drawable.ic_call_white_24dp + } + /** * The CallStyle notification's Person icon defaults to a generic phone icon; this fetches the * conversation avatar in the background and re-posts the notification once it is available. diff --git a/app/src/main/res/drawable/ic_screen_share_24.xml b/app/src/main/res/drawable/ic_screen_share_24.xml index 909ebe7ceec..0392f9ef320 100644 --- a/app/src/main/res/drawable/ic_screen_share_24.xml +++ b/app/src/main/res/drawable/ic_screen_share_24.xml @@ -1,7 +1,7 @@ + android:pathData="M23 4V16C23 17.11 22.11 18 21 18H15V16H21V4H3V16H9V18H3C1.9 18 1 17.11 1 16V4C1 2.89 1.89 2 3 2H21C22.1 2 23 2.89 23 4M13 13H16L12 9L8 13H11V20H8V22H16V20H13V13Z"/> diff --git a/app/src/main/res/drawable/ic_screen_share_white_24dp.xml b/app/src/main/res/drawable/ic_screen_share_white_24dp.xml new file mode 100644 index 00000000000..ba300b929cf --- /dev/null +++ b/app/src/main/res/drawable/ic_screen_share_white_24dp.xml @@ -0,0 +1,15 @@ + + + + diff --git a/app/src/main/res/drawable/ic_stop_screen_share_24.xml b/app/src/main/res/drawable/ic_stop_screen_share_24.xml index 270c7074336..15daa8de532 100644 --- a/app/src/main/res/drawable/ic_stop_screen_share_24.xml +++ b/app/src/main/res/drawable/ic_stop_screen_share_24.xml @@ -1,7 +1,7 @@ + android:pathData="M14,18V20H16V22H8V20H10V18H3A2,2 0 0,1 1,16V4L0,3L1.41,1.58L22.16,22.34L20.75,23.75L15,18H14M3,16H13L3,6V16M21,2A2,2 0 0,1 23,4V16A2,2 0 0,1 21,18H20.66L18.66,16H21V4H6.66L4.66,2H21Z"/> diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index f4fce449eeb..f2362cd1f42 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -399,6 +399,7 @@ How to translate with transifex: You missed a call from %s Call in progress Tap to return to your call. + You are sharing your screen Return to call End call Open picture-in-picture mode From 67f4a497ace14417bd996ff79368416bbab7cd31 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 20 Oct 2026 15:24:30 +0200 Subject: [PATCH 05/16] fix(call): handle starting and stopping screen sharing more reliably - Do not enter picture in picture while asking for the screen sharing permission, as the permission dialog moves the call to background - Cancel screen sharing that is still being started when it is stopped from the signaling thread, for example when leaving the call - Use the signaling session to find the own participant permissions - Do not crash when updating picture in picture if it was not possible when the call started - Update the call notification even if the foreground service type can not be changed while in background - Handle devices without a screen capture permission dialog Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 105 +++++++++++++++--- .../talk/activities/CallBaseActivity.java | 3 +- .../talk/services/CallForegroundService.kt | 26 +++-- 3 files changed, 105 insertions(+), 29 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index fe9e6f438fb..aedb42862d5 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -14,6 +14,7 @@ import android.annotation.SuppressLint import android.app.ActivityManager import android.app.PendingIntent import android.app.RemoteAction +import android.content.ActivityNotFoundException import android.content.BroadcastReceiver import android.content.Context import android.content.DialogInterface @@ -316,6 +317,10 @@ class CallActivity : CallBaseActivity() { @Volatile private var localScreenStream: MediaStream? = null + // Set from asking for the permission until the screen sharing started, failed or was cancelled. + @Volatile + private var isScreenSharingPending = false + // Without MCU there is one connection per remote participant, with MCU just the own publisher connection. private val screenSharePeerConnectionWrapperList: MutableList = ArrayList() @@ -3757,7 +3762,7 @@ class CallActivity : CallBaseActivity() { } private fun updateScreenSharePermission(participants: List) { - val ownSessionId = webSocketClient?.sessionId ?: callSession + val ownSessionId = if (webSocketClient != null) webSocketClient!!.sessionId else callSession val permissions = participants.firstOrNull { it?.sessionId == ownSessionId }?.participantPermissions ?: return val isAllowed = ParticipantPermissions.canPublishScreen( conversationUser.capabilities?.spreedCapability, @@ -3794,6 +3799,9 @@ class CallActivity : CallBaseActivity() { stopScreenSharing() return } + if (isScreenSharingPending) { + return + } if (!isAllowedToShareScreen) { Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_allowed, Snackbar.LENGTH_LONG).show() return @@ -3802,18 +3810,36 @@ class CallActivity : CallBaseActivity() { Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_connected, Snackbar.LENGTH_LONG).show() return } + // The permission dialog moves the call to the background, which must not enter picture in picture. + isScreenSharingPending = true + updatePipAutoEnter() val mediaProjectionManager = getSystemService(MediaProjectionManager::class.java) - screenSharingPermissionLauncher.launch(mediaProjectionManager.createScreenCaptureIntent()) + try { + screenSharingPermissionLauncher.launch(mediaProjectionManager.createScreenCaptureIntent()) + } catch (e: ActivityNotFoundException) { + Log.e(TAG, "Screen capture is not supported on this device", e) + cancelPendingScreenSharing() + showScreenSharingError() + } + } + + private fun cancelPendingScreenSharing() { + synchronized(screenShareLock) { + isScreenSharingPending = false + } + runOnUiThread { updatePipAutoEnter() } } private fun onScreenSharingPermissionResult(result: ActivityResult) { val permissionResultData = result.data if (result.resultCode != RESULT_OK || permissionResultData == null) { Log.d(TAG, "Screen sharing permission was not granted") + cancelPendingScreenSharing() return } - if (!isConnectionEstablished || currentCallStatus === CallStatus.LEAVING) { - Log.d(TAG, "Call ended while asking for the screen sharing permission") + if (!isScreenSharingPending || !isCallActiveForScreenSharing()) { + Log.d(TAG, "Screen sharing was cancelled while asking for the permission") + cancelPendingScreenSharing() return } @@ -3824,6 +3850,7 @@ class CallActivity : CallBaseActivity() { if (resultCode == CallForegroundService.RESULT_SCREEN_SHARING_READY) { startScreenSharing(permissionResultData) } else { + cancelPendingScreenSharing() showScreenSharingError() } } @@ -3831,16 +3858,28 @@ class CallActivity : CallBaseActivity() { CallForegroundService.startScreenSharing(applicationContext, resultReceiver) } + private fun isCallActiveForScreenSharing(): Boolean = + isConnectionEstablished && currentCallStatus !== CallStatus.LEAVING + private fun startScreenSharing(permissionResultData: Intent) { if (localScreenStream != null || + !isScreenSharingPending || !isAllowedToShareScreen || - !isConnectionEstablished || - currentCallStatus === CallStatus.LEAVING + !isCallActiveForScreenSharing() ) { + cancelPendingScreenSharing() CallForegroundService.stopScreenSharing(applicationContext) return } + val videoSource = startScreenCapture(permissionResultData) ?: return + publishScreenShare(videoSource) + } + + /** + * Returns the video source of the started capture, or null if it could not be started. + */ + private fun startScreenCapture(permissionResultData: Intent): VideoSource? { val capturer = ScreenCapturerAndroid( permissionResultData, object : MediaProjection.Callback() { @@ -3872,17 +3911,34 @@ class CallActivity : CallBaseActivity() { if (captureError != null) { Log.e(TAG, "Failed to start screen capture", captureError) releaseScreenCapture() + cancelPendingScreenSharing() CallForegroundService.stopScreenSharing(applicationContext) showScreenSharingError() - return + return null } getSystemService(DisplayManager::class.java) .registerDisplayListener(screenCaptureDisplayListener, Handler(Looper.getMainLooper())) + return videoSource + } + private fun publishScreenShare(videoSource: VideoSource) { val stream = peerConnectionFactory!!.createLocalMediaStream("NCSS") stream.addTrack(peerConnectionFactory!!.createVideoTrack("NCs0", videoSource)) - synchronized(screenShareLock) { - localScreenStream = stream + val isCancelled = synchronized(screenShareLock) { + // stopScreenSharing() may have been called from the signaling thread while the capture was being started. + if (isScreenSharingPending) { + localScreenStream = stream + isScreenSharingPending = false + } + localScreenStream !== stream + } + if (isCancelled) { + Log.d(TAG, "Screen sharing was cancelled while starting it") + releaseScreenCapture() + stream.dispose() + CallForegroundService.stopScreenSharing(applicationContext) + updatePipAutoEnter() + return } callViewModel.setLocalScreenSharing(true) updatePipAutoEnter() @@ -3895,17 +3951,30 @@ class CallActivity : CallBaseActivity() { } /** - * Can be called from any thread; only the first call after starting the screen sharing tears it down. + * Can be called from any thread; only the first call after starting the screen sharing tears it down. If the + * screen sharing is still being started it is cancelled instead. */ fun stopScreenSharing() { - val stream: MediaStream - val peerConnectionWrappers: List + val stream: MediaStream? + var peerConnectionWrappers: List = emptyList() + var wasPending = false synchronized(screenShareLock) { - stream = localScreenStream ?: return - localScreenStream = null - isScreenSharePublisherConnected = false - peerConnectionWrappers = screenSharePeerConnectionWrapperList.toList() - screenSharePeerConnectionWrapperList.clear() + stream = localScreenStream + if (stream == null) { + wasPending = isScreenSharingPending + isScreenSharingPending = false + } else { + localScreenStream = null + isScreenSharePublisherConnected = false + peerConnectionWrappers = screenSharePeerConnectionWrapperList.toList() + screenSharePeerConnectionWrapperList.clear() + } + } + if (stream == null) { + if (wasPending) { + runOnUiThread { updatePipAutoEnter() } + } + return } getParticipantSessionKeys().forEach { sessionId -> @@ -3929,7 +3998,7 @@ class CallActivity : CallBaseActivity() { * Picture in picture would be shown on top of the shared screen, so the call is just moved to the background * while sharing; the call notification shows that the screen is being shared. */ - override fun isAutomaticPipModeSuppressed(): Boolean = localScreenStream != null + override fun isAutomaticPipModeSuppressed(): Boolean = localScreenStream != null || isScreenSharingPending private fun releaseScreenCapture() { getSystemService(DisplayManager::class.java).unregisterDisplayListener(screenCaptureDisplayListener) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java index f807683f53f..7762a9b2864 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java @@ -191,7 +191,8 @@ private boolean shouldEnterPipModeAutomatically() { * Updates whether the system enters picture in picture automatically when leaving the activity. */ void updatePipAutoEnter() { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && isPipModePossible()) { + // The builder is only created if picture in picture was possible when the activity was created. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && mPictureInPictureParamsBuilder != null) { mPictureInPictureParamsBuilder.setAutoEnterEnabled(!isAutomaticPipModeSuppressed()); setPictureInPictureParams(mPictureInPictureParamsBuilder.build()); } diff --git a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt index 5b06c666390..5d62ab2ab7a 100644 --- a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt +++ b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt @@ -168,6 +168,10 @@ class CallForegroundService : Service() { Log.e(TAG, "Failed to update foreground service type for screen sharing", e) isScreenSharing = false resultCode = RESULT_SCREEN_SHARING_FAILED + // The type may not be changed while in background, but the notification must not keep stating that the + // screen is shared. + getSystemService(NotificationManager::class.java) + .notify(NOTIFICATION_ID, buildNotification(currentConversationName, currentCallExtras)) } resultReceiver?.send(resultCode, null) @@ -209,15 +213,9 @@ class CallForegroundService : Service() { return buildCallStyleNotification(contentTitle, pendingIntent) } - val contentText = if (isScreenSharing) { - getString(R.string.nc_call_ongoing_notification_screen_sharing) - } else { - getString(R.string.nc_call_ongoing_notification_content) - } - return NotificationCompat.Builder(this, channelId) .setContentTitle(contentTitle) - .setContentText(contentText) + .setContentText(getScreenSharingText() ?: getString(R.string.nc_call_ongoing_notification_content)) .setSmallIcon(getSmallIcon()) .setOngoing(true) .setCategory(NotificationCompat.CATEGORY_CALL) @@ -267,13 +265,21 @@ class CallForegroundService : Service() { builder.setWhen(callStartTime) builder.setShowWhen(true) } - if (isScreenSharing) { - builder.setContentText(getString(R.string.nc_call_ongoing_notification_screen_sharing)) - } + getScreenSharingText()?.let { builder.setContentText(it) } } .build() } + /** + * Returns the text shown while the screen is shared, or null to use the default text of the notification. + */ + private fun getScreenSharingText(): String? = + if (isScreenSharing) { + getString(R.string.nc_call_ongoing_notification_screen_sharing) + } else { + null + } + /** * The small icon is shown in the status bar, so it indicates when the screen is being shared. */ From 4e4e7eb1f613f601b95e3213f35c3b50f5c2eb61 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 20 Oct 2026 15:49:11 +0200 Subject: [PATCH 06/16] fix(call): cover remaining screen sharing edge cases - Explain that notifications and microphone access are needed to share the screen when the call foreground service is not running - Ignore connection events from a previous screen share publisher - Do not update picture in picture after the activity was destroyed - Do not crash when the call is hung up while the screen capture starts - Update the screen sharing entry of the more call actions dialog while it is shown Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 60 ++++++++++++++----- .../talk/activities/CallBaseActivity.java | 4 ++ .../talk/activities/CallViewModel.kt | 7 +++ .../talk/ui/dialog/MoreCallActionsDialog.kt | 12 ++-- app/src/main/res/values/strings.xml | 1 + 5 files changed, 63 insertions(+), 21 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index aedb42862d5..a9de4b62d8c 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -326,7 +326,11 @@ class CallActivity : CallBaseActivity() { @Volatile private var isScreenSharePublisherConnected = false - private val screenSharePublisherObserver: PeerConnectionObserver = ScreenSharePublisherPeerConnectionObserver() + private val screenSharePublisherObservers: MutableMap = HashMap() + + // Screen sharing needs the foreground service, which is not started without microphone and notification permission. + @Volatile + private var isCallForegroundServiceStarted = false private val callParticipantListObserver: CallParticipantList.Observer = object : CallParticipantList.Observer { override fun onCallParticipantsChanged( @@ -1337,6 +1341,7 @@ class CallActivity : CallBaseActivity() { SEC_10 ).show() } else { + isCallForegroundServiceStarted = true CallForegroundService.start( applicationContext, conversationName, @@ -1713,6 +1718,7 @@ class CallActivity : CallBaseActivity() { if (currentCallStatus !== CallStatus.LEAVING) { hangup(true, false) } + isCallForegroundServiceStarted = false CallForegroundService.stop(applicationContext) powerManagerUtils!!.updatePhoneState(PowerManagerUtils.PhoneState.IDLE) @@ -2377,6 +2383,7 @@ class CallActivity : CallBaseActivity() { if (shutDownView) { Log.d(TAG, "Stopping foreground service from hangup()") + isCallForegroundServiceStarted = false CallForegroundService.stop(applicationContext) } @@ -3443,7 +3450,9 @@ class CallActivity : CallBaseActivity() { } } - private inner class ScreenSharePublisherPeerConnectionObserver : PeerConnectionObserver { + private inner class ScreenSharePublisherPeerConnectionObserver( + private val peerConnectionWrapper: PeerConnectionWrapper + ) : PeerConnectionObserver { override fun onStreamAdded(mediaStream: MediaStream) { // unused atm } @@ -3454,7 +3463,11 @@ class CallActivity : CallBaseActivity() { override fun onIceConnectionStateChanged(iceConnectionState: IceConnectionState) { runOnUiThread { - if (localScreenStream == null) { + // The event may have been queued before the publisher was closed, even if sharing started again. + val isCurrentPublisher = synchronized(screenShareLock) { + screenSharePeerConnectionWrapperList.contains(peerConnectionWrapper) + } + if (!isCurrentPublisher) { return@runOnUiThread } if (iceConnectionState == IceConnectionState.CONNECTED && !isScreenSharePublisherConnected) { @@ -3784,6 +3797,7 @@ class CallActivity : CallBaseActivity() { return } isAllowedToShareScreen = isAllowed + callViewModel.setScreenSharingAllowed(isAllowed) if (binding == null) { return } @@ -3810,6 +3824,10 @@ class CallActivity : CallBaseActivity() { Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_connected, Snackbar.LENGTH_LONG).show() return } + if (!isCallForegroundServiceStarted) { + Snackbar.make(binding!!.root, R.string.nc_screen_sharing_permissions_missing, Snackbar.LENGTH_LONG).show() + return + } // The permission dialog moves the call to the background, which must not enter picture in picture. isScreenSharingPending = true updatePipAutoEnter() @@ -3872,14 +3890,14 @@ class CallActivity : CallBaseActivity() { return } - val videoSource = startScreenCapture(permissionResultData) ?: return - publishScreenShare(videoSource) + val (factory, videoSource) = startScreenCapture(permissionResultData) ?: return + publishScreenShare(factory, videoSource) } /** - * Returns the video source of the started capture, or null if it could not be started. + * Returns the factory used and the video source of the started capture, or null if it could not be started. */ - private fun startScreenCapture(permissionResultData: Intent): VideoSource? { + private fun startScreenCapture(permissionResultData: Intent): Pair? { val capturer = ScreenCapturerAndroid( permissionResultData, object : MediaProjection.Callback() { @@ -3888,8 +3906,16 @@ class CallActivity : CallBaseActivity() { } } ) - val surfaceTextureHelper = SurfaceTextureHelper.create("ScreenCaptureThread", rootEglBase!!.eglBaseContext) - val videoSource = peerConnectionFactory!!.createVideoSource(true) + // Both are released when hanging up, which may happen in parallel from the signaling thread. + val eglBase = rootEglBase + val factory = peerConnectionFactory + if (eglBase == null || factory == null) { + cancelPendingScreenSharing() + CallForegroundService.stopScreenSharing(applicationContext) + return null + } + val surfaceTextureHelper = SurfaceTextureHelper.create("ScreenCaptureThread", eglBase.eglBaseContext) + val videoSource = factory.createVideoSource(true) val captureSize = getScreenCaptureSize() synchronized(screenShareLock) { screenCapturer = capturer @@ -3918,12 +3944,12 @@ class CallActivity : CallBaseActivity() { } getSystemService(DisplayManager::class.java) .registerDisplayListener(screenCaptureDisplayListener, Handler(Looper.getMainLooper())) - return videoSource + return Pair(factory, videoSource) } - private fun publishScreenShare(videoSource: VideoSource) { - val stream = peerConnectionFactory!!.createLocalMediaStream("NCSS") - stream.addTrack(peerConnectionFactory!!.createVideoTrack("NCs0", videoSource)) + private fun publishScreenShare(factory: PeerConnectionFactory, videoSource: VideoSource) { + val stream = factory.createLocalMediaStream("NCSS") + stream.addTrack(factory.createVideoTrack("NCs0", videoSource)) val isCancelled = synchronized(screenShareLock) { // stopScreenSharing() may have been called from the signaling thread while the capture was being started. if (isScreenSharingPending) { @@ -4120,7 +4146,9 @@ class CallActivity : CallBaseActivity() { return } if (hasMCU) { - peerConnectionWrapper.addObserver(screenSharePublisherObserver) + val observer = ScreenSharePublisherPeerConnectionObserver(peerConnectionWrapper) + screenSharePublisherObservers[peerConnectionWrapper] = observer + peerConnectionWrapper.addObserver(observer) } screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) } @@ -4135,7 +4163,9 @@ class CallActivity : CallBaseActivity() { } private fun closeScreenSharePeerConnection(peerConnectionWrapper: PeerConnectionWrapper) { - peerConnectionWrapper.removeObserver(screenSharePublisherObserver) + synchronized(screenShareLock) { + screenSharePublisherObservers.remove(peerConnectionWrapper) + }?.let { peerConnectionWrapper.removeObserver(it) } peerConnectionWrapper.removePeerConnection() } diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java index 7762a9b2864..a1827e62fd5 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallBaseActivity.java @@ -191,6 +191,10 @@ private boolean shouldEnterPipModeAutomatically() { * Updates whether the system enters picture in picture automatically when leaving the activity. */ void updatePipAutoEnter() { + // It may be called from a task posted before the activity was destroyed. + if (isFinishing() || isDestroyed()) { + return; + } // The builder is only created if picture in picture was possible when the activity was created. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && mPictureInPictureParamsBuilder != null) { mPictureInPictureParamsBuilder.setAutoEnterEnabled(!isAutomaticPipModeSuppressed()); diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt b/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt index 9fce4075258..ff0a6560115 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallViewModel.kt @@ -30,6 +30,9 @@ class CallViewModel @Inject constructor() : ViewModel() { private val _isLocalScreenSharing = MutableStateFlow(false) val isLocalScreenSharing: StateFlow = _isLocalScreenSharing.asStateFlow() + private val _isScreenSharingAllowed = MutableStateFlow(false) + val isScreenSharingAllowed: StateFlow = _isScreenSharingAllowed.asStateFlow() + fun getParticipant(sessionId: String?): ParticipantHandler? { if (sessionId == null) { Log.w(TAG, "Attempted to get participant with null sessionId.") @@ -95,6 +98,10 @@ class CallViewModel @Inject constructor() : ViewModel() { _isLocalScreenSharing.value = isSharing } + fun setScreenSharingAllowed(isAllowed: Boolean) { + _isScreenSharingAllowed.value = isAllowed + } + public override fun onCleared() { participantHandlers.values.forEach { it.destroy() } participantHandlers.clear() diff --git a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt index 04ef00337f8..5b0d719a953 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt @@ -88,12 +88,6 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee if (!ImageSegmenterHelper.isAvailable()) { binding.backgroundBlur.visibility = View.GONE } - - if (callActivity.isAllowedToShareScreen) { - binding.shareScreen.visibility = View.VISIBLE - } else { - binding.shareScreen.visibility = View.GONE - } } private fun initClickListeners() { @@ -221,6 +215,12 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee } } + lifecycleScope.launch { + callActivity.callViewModel.isScreenSharingAllowed.collect { isAllowed -> + binding.shareScreen.visibility = if (isAllowed) View.VISIBLE else View.GONE + } + } + lifecycleScope.launch { callActivity.callViewModel.isLocalScreenSharing.collect { isSharing -> if (isSharing) { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index f2362cd1f42..c2d9561b256 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1104,6 +1104,7 @@ How to translate with transifex: Screen sharing could not be started You are not allowed to share your screen You can share your screen once the call is connected + To share your screen, allow notifications and microphone access Account not found Password is secure From 82ba851ffbb18007ecc9fc8739369bc5d96af111 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 20 Oct 2026 17:29:38 +0200 Subject: [PATCH 07/16] feat(call): show indicator while sharing own screen Like for call recording, a red indicator is shown next to the conversation name while the own screen is shared. Clicking it asks whether to stop sharing the screen. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 26 +++++++++++++++++++ .../res/drawable/screen_share_indicator.xml | 15 +++++++++++ app/src/main/res/layout/call_activity.xml | 12 +++++++++ app/src/main/res/values/strings.xml | 1 + 4 files changed, 54 insertions(+) create mode 100644 app/src/main/res/drawable/screen_share_indicator.xml diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index a9de4b62d8c..323f22903e4 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -622,6 +622,7 @@ class CallActivity : CallBaseActivity() { initRaiseHandViewModel() initCallRecordingViewModel(extras.getInt(KEY_RECORDING_STATE, 0)) + initScreenSharingIndicator() initClickListeners(isModerator, isOneToOneConversation) binding!!.microphoneButton.setOnTouchListener(MicrophoneButtonTouchListener()) @@ -746,6 +747,31 @@ class CallActivity : CallBaseActivity() { } } + private fun initScreenSharingIndicator() { + lifecycleScope.launch { + callViewModel.isLocalScreenSharing.collect { isSharing -> + binding?.callScreenSharingIndicator?.visibility = if (isSharing) View.VISIBLE else View.GONE + } + } + binding!!.callScreenSharingIndicator.setOnClickListener { showStopScreenSharingDialog() } + } + + private fun showStopScreenSharingDialog() { + val dialogBuilder = MaterialAlertDialogBuilder(this) + .setTitle(R.string.nc_stop_screen_sharing) + .setMessage(R.string.nc_stop_screen_sharing_confirm_message) + .setPositiveButton(R.string.nc_stop_screen_sharing) { _: DialogInterface?, _: Int -> + stopScreenSharing() + } + .setNegativeButton(R.string.nc_common_dismiss, null) + viewThemeUtils.dialog.colorMaterialAlertDialogBackground(this, dialogBuilder) + val dialog = dialogBuilder.show() + viewThemeUtils.platform.colorTextButtons( + dialog.getButton(AlertDialog.BUTTON_POSITIVE), + dialog.getButton(AlertDialog.BUTTON_NEGATIVE) + ) + } + private fun initRaiseHandViewModel() { raiseHandViewModel = ViewModelProvider(this, viewModelFactory).get(RaiseHandViewModel::class.java) raiseHandViewModel!!.setData(conversationUser, roomToken!!, isBreakoutRoom) diff --git a/app/src/main/res/drawable/screen_share_indicator.xml b/app/src/main/res/drawable/screen_share_indicator.xml new file mode 100644 index 00000000000..85856de6ad6 --- /dev/null +++ b/app/src/main/res/drawable/screen_share_indicator.xml @@ -0,0 +1,15 @@ + + + + diff --git a/app/src/main/res/layout/call_activity.xml b/app/src/main/res/layout/call_activity.xml index d8ae0b660d3..fcbe1cde97a 100644 --- a/app/src/main/res/layout/call_activity.xml +++ b/app/src/main/res/layout/call_activity.xml @@ -81,6 +81,18 @@ tools:visibility="visible"> + + + You are not allowed to share your screen You can share your screen once the call is connected To share your screen, allow notifications and microphone access + Do you really want to stop sharing your screen? Account not found Password is secure From 6faa1e72e8e889d6e07c22a6be90ecabbfd9f5e6 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Mon, 26 Oct 2026 18:42:50 +0100 Subject: [PATCH 08/16] fix(call): do not hang up while holding the screen sharing lock - End the call after releasing the screen sharing lock when the screen share connection can not be encrypted, as hanging up also stops the screen sharing - Add tests for peer connections sharing the own screen Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 24 +- .../PeerConnectionWrapperScreenSharingTest.kt | 226 ++++++++++++++++++ 2 files changed, 241 insertions(+), 9 deletions(-) create mode 100644 app/src/test/java/com/nextcloud/talk/webrtc/PeerConnectionWrapperScreenSharingTest.kt diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 323f22903e4..ea588528c4b 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -116,7 +116,6 @@ import com.nextcloud.talk.models.json.generic.GenericOverall import com.nextcloud.talk.models.json.participants.ParticipantDto import com.nextcloud.talk.models.json.signaling.DataChannelMessageDto import com.nextcloud.talk.models.json.signaling.NCSignalingMessageDto -import com.nextcloud.talk.utils.ParticipantPermissions import com.nextcloud.talk.models.json.signaling.SignalingDto import com.nextcloud.talk.models.json.signaling.SignalingOverall import com.nextcloud.talk.models.json.signaling.settings.SignalingSettingsOverall @@ -138,6 +137,7 @@ import com.nextcloud.talk.utils.CapabilitiesUtil.hasSpreedFeatureCapability import com.nextcloud.talk.utils.CapabilitiesUtil.isCallRecordingAvailable import com.nextcloud.talk.utils.ConversationUtils.checkIfVoiceRoom import com.nextcloud.talk.utils.NotificationUtils.getCallRingtoneUri +import com.nextcloud.talk.utils.ParticipantPermissions import com.nextcloud.talk.utils.ReceiverFlag import com.nextcloud.talk.utils.SpreedFeatures import com.nextcloud.talk.utils.VibrationUtils.vibrateShort @@ -4141,7 +4141,7 @@ class CallActivity : CallBaseActivity() { } private fun addScreenSharePeerConnection(sessionId: String?) { - synchronized(screenShareLock) { + val isEncryptionFailed = synchronized(screenShareLock) { val stream = localScreenStream ?: return if (sessionId == null || screenSharePeerConnectionWrapperList.any { it.sessionId == sessionId }) { return @@ -4168,15 +4168,21 @@ class CallActivity : CallBaseActivity() { ) } catch (e: IllegalStateException) { Log.e(TAG, "Not creating a screen share connection to $sessionId", e) - failCallEncryption(R.string.nc_call_e2ee_setup_failed) - return + null } - if (hasMCU) { - val observer = ScreenSharePublisherPeerConnectionObserver(peerConnectionWrapper) - screenSharePublisherObservers[peerConnectionWrapper] = observer - peerConnectionWrapper.addObserver(observer) + if (peerConnectionWrapper != null) { + if (hasMCU) { + val observer = ScreenSharePublisherPeerConnectionObserver(peerConnectionWrapper) + screenSharePublisherObservers[peerConnectionWrapper] = observer + peerConnectionWrapper.addObserver(observer) + } + screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) } - screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) + peerConnectionWrapper == null + } + // Hanging up also stops the screen sharing, so it is not done while holding the lock. + if (isEncryptionFailed) { + failCallEncryption(R.string.nc_call_e2ee_setup_failed) } } diff --git a/app/src/test/java/com/nextcloud/talk/webrtc/PeerConnectionWrapperScreenSharingTest.kt b/app/src/test/java/com/nextcloud/talk/webrtc/PeerConnectionWrapperScreenSharingTest.kt new file mode 100644 index 00000000000..faa66956995 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/webrtc/PeerConnectionWrapperScreenSharingTest.kt @@ -0,0 +1,226 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ +package com.nextcloud.talk.webrtc + +import com.nextcloud.talk.call.e2ee.FrameKeyRing +import com.nextcloud.talk.models.json.signaling.NCSignalingMessageDto +import com.nextcloud.talk.signaling.SignalingMessageReceiver +import com.nextcloud.talk.signaling.SignalingMessageSender +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Before +import org.junit.Test +import org.mockito.ArgumentCaptor +import org.mockito.ArgumentMatchers.any +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq +import org.mockito.Mockito +import org.mockito.Mockito.never +import org.mockito.Mockito.verify +import org.webrtc.DataChannel +import org.webrtc.FrameDecryptor +import org.webrtc.FrameEncryptor +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.RtpReceiver +import org.webrtc.RtpSender +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.VideoTrack + +class PeerConnectionWrapperScreenSharingTest { + + private class FakeKeyRing : FrameKeyRing { + val encryptors = ArrayList() + + override fun setKey(key: ByteArray, keyIndex: Int): Boolean = true + + override fun attachEncryptor(sender: RtpSender): Boolean { + sender.setFrameEncryptor(FrameEncryptor { 0 }.also { encryptors.add(it) }) + return true + } + + override fun attachDecryptor(receiver: RtpReceiver): Boolean { + receiver.setFrameDecryptor(FrameDecryptor { 0 }) + return true + } + + override fun dispose() { + // unused atm + } + } + + private lateinit var mockedPeerConnection: PeerConnection + private lateinit var mockedPeerConnectionFactory: PeerConnectionFactory + private lateinit var mockedSignalingMessageReceiver: SignalingMessageReceiver + private lateinit var mockedSignalingMessageSender: SignalingMessageSender + + @Before + fun setUp() { + mockedPeerConnection = Mockito.mock(PeerConnection::class.java) + mockedPeerConnectionFactory = Mockito.mock(PeerConnectionFactory::class.java) + mockedSignalingMessageReceiver = Mockito.mock(SignalingMessageReceiver::class.java) + mockedSignalingMessageSender = Mockito.mock(SignalingMessageSender::class.java) + Mockito.`when`( + mockedPeerConnectionFactory.createPeerConnection( + any(PeerConnection.RTCConfiguration::class.java), + any(PeerConnection.Observer::class.java) + ) + ).thenReturn(mockedPeerConnection) + val mockedStatusDataChannel = Mockito.mock(DataChannel::class.java) + Mockito.`when`(mockedStatusDataChannel.label()).thenReturn("status") + Mockito.`when`(mockedPeerConnection.createDataChannel(eq("status"), any())).thenReturn(mockedStatusDataChannel) + } + + /** + * The local session is lower than the remote one, so the connection would not send the offer if it did not + * share the screen. + */ + private fun createWrapper( + isScreenSharer: Boolean, + hasMCU: Boolean = false, + localStream: MediaStream? = createScreenStream(), + videoStreamType: String = "screen", + encryptionRequired: Boolean = false, + senderKeyRing: FrameKeyRing? = null + ): PeerConnectionWrapper = + PeerConnectionWrapper( + mockedPeerConnectionFactory, + ArrayList(), + MediaConstraints(), + if (hasMCU) LOCAL_SESSION_ID else REMOTE_SESSION_ID, + LOCAL_SESSION_ID, + localStream, + hasMCU, + hasMCU, + videoStreamType, + mockedSignalingMessageReceiver, + mockedSignalingMessageSender, + encryptionRequired, + senderKeyRing, + null, + isScreenSharer + ) + + // The tracks are public fields initialized in the constructor, which a plain mock would skip. + private fun createScreenStream(): MediaStream { + val stream = Mockito.mock(MediaStream::class.java, Mockito.withSettings().useConstructor(0L)) + Mockito.`when`(stream.id).thenReturn("NCSS") + stream.videoTracks.add(Mockito.mock(VideoTrack::class.java)) + return stream + } + + private fun captureOfferObserver(): SdpObserver { + val captor = ArgumentCaptor.forClass(SdpObserver::class.java) + verify(mockedPeerConnection).createOffer(captor.capture(), any(MediaConstraints::class.java)) + return captor.value + } + + private fun captureSentMessage(): NCSignalingMessageDto { + val captor = ArgumentCaptor.forClass(NCSignalingMessageDto::class.java) + verify(mockedSignalingMessageSender).send(captor.capture()) + return captor.value + } + + @Test + fun testScreenSharerSendsOfferWithoutDataChannel() { + createWrapper(isScreenSharer = true) + + verify(mockedPeerConnection).addTrack(any(VideoTrack::class.java), eq(listOf("NCSS"))) + verify(mockedPeerConnection).createOffer(any(SdpObserver::class.java), any(MediaConstraints::class.java)) + verify(mockedPeerConnection, never()).createDataChannel(anyString(), any()) + } + + @Test + fun testNotOffererDoesNotSendOfferWithoutSharingScreen() { + createWrapper(isScreenSharer = false, localStream = null, videoStreamType = "video") + + verify(mockedPeerConnection, never()).createOffer( + any(SdpObserver::class.java), + any(MediaConstraints::class.java) + ) + } + + @Test + fun testScreenSharerListensWithScreenSharingRoomType() { + createWrapper(isScreenSharer = true) + createWrapper(isScreenSharer = false, localStream = null) + + verify(mockedSignalingMessageReceiver).addListener( + any(SignalingMessageReceiver.WebRtcMessageListener::class.java), + eq(REMOTE_SESSION_ID), + eq(SignalingMessageReceiver.ROOM_TYPE_SCREEN_SHARING) + ) + verify(mockedSignalingMessageReceiver).addListener( + any(SignalingMessageReceiver.WebRtcMessageListener::class.java), + eq(REMOTE_SESSION_ID), + eq("screen") + ) + } + + @Test + fun testScreenSharerSetsItselfAsBroadcaster() { + createWrapper(isScreenSharer = true) + + captureOfferObserver().onCreateSuccess(SessionDescription(SessionDescription.Type.OFFER, SDP)) + + val message = captureSentMessage() + assertEquals("offer", message.type) + assertEquals("screen", message.roomType) + assertEquals(REMOTE_SESSION_ID, message.to) + assertEquals(LOCAL_SESSION_ID, message.broadcaster) + } + + @Test + fun testOtherConnectionsDoNotSetBroadcaster() { + createWrapper(isScreenSharer = false, hasMCU = true, localStream = null, videoStreamType = "video") + + captureOfferObserver().onCreateSuccess(SessionDescription(SessionDescription.Type.OFFER, SDP)) + + assertNull(captureSentMessage().broadcaster) + } + + @Test + fun testEncryptedScreenShareIsEncryptedBeforeOffer() { + val sender = Mockito.mock(RtpSender::class.java) + Mockito.`when`(mockedPeerConnection.senders).thenReturn(listOf(sender)) + val keyRing = FakeKeyRing() + + createWrapper(isScreenSharer = true, hasMCU = true, encryptionRequired = true, senderKeyRing = keyRing) + + assertEquals(1, keyRing.encryptors.size) + val inOrder = Mockito.inOrder(sender, mockedPeerConnection) + inOrder.verify(sender).setFrameEncryptor(keyRing.encryptors[0]) + inOrder.verify(mockedPeerConnection).createOffer( + any(SdpObserver::class.java), + any(MediaConstraints::class.java) + ) + } + + @Test + fun testEncryptedScreenShareWithoutKeysIsRefused() { + for (hasMCU in listOf(true, false)) { + assertThrows(IllegalStateException::class.java) { + createWrapper(isScreenSharer = true, hasMCU = hasMCU, encryptionRequired = true) + } + } + + verify(mockedPeerConnectionFactory, never()).createPeerConnection( + any(PeerConnection.RTCConfiguration::class.java), + any(PeerConnection.Observer::class.java) + ) + } + + companion object { + private const val LOCAL_SESSION_ID = "a-local-session-id" + private const val REMOTE_SESSION_ID = "b-remote-session-id" + private const val SDP = "v=0\r\nm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=rtpmap:96 H264/90000\r\n" + } +} From 9d3c21c8298fb5305f87515681141a39129c52c6 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Mon, 26 Oct 2026 19:07:19 +0100 Subject: [PATCH 09/16] fix(call): keep sharing the screen when connections fail - Keep capturing the screen while the call reconnects and share it again once back in the call, instead of stopping the screen sharing - Reconnect when the publisher fails while the screen is shared, even if the call is in background - Create failed screen share connections again, also without MCU, before giving up Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 140 +++++++++++++----- 1 file changed, 106 insertions(+), 34 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index ea588528c4b..1a53ff206f9 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -326,7 +326,10 @@ class CallActivity : CallBaseActivity() { @Volatile private var isScreenSharePublisherConnected = false - private val screenSharePublisherObservers: MutableMap = HashMap() + private val screenShareObservers: MutableMap = HashMap() + + // Failed connections are created again a few times per remote session before giving up. + private val screenShareRetryCounts: MutableMap = HashMap() // Screen sharing needs the foreground service, which is not started without microphone and notification permission. @Volatile @@ -2371,7 +2374,7 @@ class CallActivity : CallBaseActivity() { private fun hangup(shutDownView: Boolean, endCallForAll: Boolean) { Log.d(TAG, "hangup! shutDownView=$shutDownView, endCallForAll=$endCallForAll") - stopScreenSharing() + stopOrSuspendScreenSharing(shutDownView) joinRoomInitiated = false if (shutDownView) { setCallState(CallStatus.LEAVING) @@ -2664,7 +2667,7 @@ class CallActivity : CallBaseActivity() { if (!isSelfInCall) { Log.d(TAG, "Self not in call, disconnecting from all other sessions") - stopScreenSharing() + stopOrSuspendScreenSharing(shutDownView = false) removeSessions(participantsInCall) return } @@ -2672,12 +2675,7 @@ class CallActivity : CallBaseActivity() { return } if (hasMCU) { - // Ensure that own publishing peer is set up. - getOrCreatePeerConnectionWrapperForSessionIdAndType( - webSocketClient!!.sessionId, - VIDEO_STREAM_TYPE_VIDEO, - true - ) + setUpOwnPublishers() } handleJoinedCallParticipantsChanged(selfParticipant, joined, currentSessionId) @@ -2687,6 +2685,15 @@ class CallActivity : CallBaseActivity() { removeSessions(left) } + /** + * Ensures that the own publishing peers are set up. The screen share publisher is closed while reconnecting, so it + * is set up again once back in the call. + */ + private fun setUpOwnPublishers() { + getOrCreatePeerConnectionWrapperForSessionIdAndType(webSocketClient!!.sessionId, VIDEO_STREAM_TYPE_VIDEO, true) + addScreenSharePeerConnection(webSocketClient!!.sessionId) + } + private fun removeSessions(sessions: Collection) { for ((_, _, _, _, _, _, _, _, _, _, session) in sessions) { Log.d(TAG, " session that will be removed is: $session") @@ -3476,9 +3483,8 @@ class CallActivity : CallBaseActivity() { } } - private inner class ScreenSharePublisherPeerConnectionObserver( - private val peerConnectionWrapper: PeerConnectionWrapper - ) : PeerConnectionObserver { + private inner class ScreenSharePeerConnectionObserver(private val peerConnectionWrapper: PeerConnectionWrapper) : + PeerConnectionObserver { override fun onStreamAdded(mediaStream: MediaStream) { // unused atm } @@ -3489,20 +3495,23 @@ class CallActivity : CallBaseActivity() { override fun onIceConnectionStateChanged(iceConnectionState: IceConnectionState) { runOnUiThread { - // The event may have been queued before the publisher was closed, even if sharing started again. - val isCurrentPublisher = synchronized(screenShareLock) { + // The event may have been queued before the connection was closed, even if sharing started again. + val isCurrentConnection = synchronized(screenShareLock) { screenSharePeerConnectionWrapperList.contains(peerConnectionWrapper) } - if (!isCurrentPublisher) { + if (!isCurrentConnection) { return@runOnUiThread } - if (iceConnectionState == IceConnectionState.CONNECTED && !isScreenSharePublisherConnected) { - isScreenSharePublisherConnected = true - getParticipantSessionKeys().forEach { requestScreenShareOffer(it) } + if (iceConnectionState == IceConnectionState.CONNECTED) { + synchronized(screenShareLock) { + screenShareRetryCounts.remove(peerConnectionWrapper.sessionId) + } + if (hasMCU && !isScreenSharePublisherConnected) { + isScreenSharePublisherConnected = true + getParticipantSessionKeys().forEach { requestScreenShareOffer(it) } + } } else if (iceConnectionState == IceConnectionState.FAILED) { - Log.e(TAG, "Screen share publisher connection failed") - stopScreenSharing() - showScreenSharingError() + retryScreenSharePeerConnection(peerConnectionWrapper) } } } @@ -3523,7 +3532,9 @@ class CallActivity : CallBaseActivity() { // Don't hang up if the activity is just backgrounded (e.g., task switching). // The ICE failure is likely transient due to the activity being stopped. // The connection will recover when the activity resumes. - if (!active && currentCallStatus === CallStatus.IN_CONVERSATION) { + // While sharing the screen the call is expected to be in background and may not be resumed for + // a long time, so it reconnects instead. + if (!active && localScreenStream == null && currentCallStatus === CallStatus.IN_CONVERSATION) { Log.d(TAG, "ICE FAILED while backgrounded, skipping hangup (will recover on resume)") return@runOnUiThread } @@ -4018,6 +4029,7 @@ class CallActivity : CallBaseActivity() { } else { localScreenStream = null isScreenSharePublisherConnected = false + screenShareRetryCounts.clear() peerConnectionWrappers = screenSharePeerConnectionWrapperList.toList() screenSharePeerConnectionWrapperList.clear() } @@ -4029,12 +4041,7 @@ class CallActivity : CallBaseActivity() { return } - getParticipantSessionKeys().forEach { sessionId -> - messageSender?.send( - NCSignalingMessageDto(type = "unshareScreen", roomType = VIDEO_STREAM_TYPE_SCREEN), - sessionId - ) - } + sendUnshareScreen(getParticipantSessionKeys()) peerConnectionWrappers.forEach { closeScreenSharePeerConnection(it) } @@ -4046,6 +4053,72 @@ class CallActivity : CallBaseActivity() { runOnUiThread { updatePipAutoEnter() } } + /** + * While reconnecting only the connections are closed, so the screen keeps being captured and is shared again + * once back in the call, without asking the user for the permission again. + */ + private fun stopOrSuspendScreenSharing(shutDownView: Boolean) { + val isReconnecting = currentCallStatus === CallStatus.RECONNECTING || + currentCallStatus === CallStatus.PUBLISHER_FAILED + if (shutDownView || !isReconnecting) { + stopScreenSharing() + return + } + + val peerConnectionWrappers = synchronized(screenShareLock) { + isScreenSharePublisherConnected = false + screenShareRetryCounts.clear() + screenSharePeerConnectionWrapperList.toList().also { screenSharePeerConnectionWrapperList.clear() } + } + peerConnectionWrappers.forEach { closeScreenSharePeerConnection(it) } + } + + /** + * Must be called on the main thread. + */ + private fun retryScreenSharePeerConnection(peerConnectionWrapper: PeerConnectionWrapper) { + val sessionId = peerConnectionWrapper.sessionId + val shouldRetry = synchronized(screenShareLock) { + if (!screenSharePeerConnectionWrapperList.remove(peerConnectionWrapper)) { + return + } + if (hasMCU) { + isScreenSharePublisherConnected = false + } + val retryCount = screenShareRetryCounts[sessionId] ?: 0 + screenShareRetryCounts[sessionId] = retryCount + 1 + retryCount < SCREEN_SHARE_MAX_RETRIES + } + closeScreenSharePeerConnection(peerConnectionWrapper) + + if (!isCallActiveForScreenSharing()) { + // The connections are set up again once reconnected. + return + } + if (shouldRetry) { + Log.w(TAG, "Screen share connection to $sessionId failed, creating it again") + // The remote participants close the connection receiving the screen, so the new offer is not handled by + // the failed connection. + sendUnshareScreen(if (hasMCU) getParticipantSessionKeys() else listOf(sessionId)) + addScreenSharePeerConnection(sessionId) + } else if (hasMCU) { + Log.e(TAG, "Screen share publisher connection failed") + stopScreenSharing() + showScreenSharingError() + } else { + Log.e(TAG, "Screen share connection to $sessionId failed, not sharing the screen with it anymore") + } + } + + private fun sendUnshareScreen(sessionIds: Collection) { + sessionIds.forEach { sessionId -> + messageSender?.send( + NCSignalingMessageDto(type = "unshareScreen", roomType = VIDEO_STREAM_TYPE_SCREEN), + sessionId + ) + } + } + /** * Picture in picture would be shown on top of the shared screen, so the call is just moved to the background * while sharing; the call notification shows that the screen is being shared. @@ -4171,11 +4244,9 @@ class CallActivity : CallBaseActivity() { null } if (peerConnectionWrapper != null) { - if (hasMCU) { - val observer = ScreenSharePublisherPeerConnectionObserver(peerConnectionWrapper) - screenSharePublisherObservers[peerConnectionWrapper] = observer - peerConnectionWrapper.addObserver(observer) - } + val observer = ScreenSharePeerConnectionObserver(peerConnectionWrapper) + screenShareObservers[peerConnectionWrapper] = observer + peerConnectionWrapper.addObserver(observer) screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) } peerConnectionWrapper == null @@ -4196,7 +4267,7 @@ class CallActivity : CallBaseActivity() { private fun closeScreenSharePeerConnection(peerConnectionWrapper: PeerConnectionWrapper) { synchronized(screenShareLock) { - screenSharePublisherObservers.remove(peerConnectionWrapper) + screenShareObservers.remove(peerConnectionWrapper) }?.let { peerConnectionWrapper.removeObserver(it) } peerConnectionWrapper.removePeerConnection() } @@ -4262,6 +4333,7 @@ class CallActivity : CallBaseActivity() { private const val FRAME_RATE: Int = 30 private const val SCREEN_SHARE_FRAME_RATE: Int = 15 private const val SCREEN_SHARE_MAX_DIMENSION: Int = 1920 + private const val SCREEN_SHARE_MAX_RETRIES: Int = 3 private const val WIDTH_16_TO_9_RATIO: Int = 1280 private const val HEIGHT_16_TO_9_RATIO: Int = 720 private const val WIDTH_4_TO_3_RATIO: Int = 640 From 0ea55e06456e291e11a2d4a2a1ef9e6fafdf4d0a Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Mon, 26 Oct 2026 22:18:38 +0100 Subject: [PATCH 10/16] fix(call): enlarge touch target of screen sharing indicator The screen sharing indicator opens the dialog to stop sharing, so it needs the minimum touch target size of 48dp. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- app/src/main/res/layout/call_activity.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/src/main/res/layout/call_activity.xml b/app/src/main/res/layout/call_activity.xml index fcbe1cde97a..f225b2a4c18 100644 --- a/app/src/main/res/layout/call_activity.xml +++ b/app/src/main/res/layout/call_activity.xml @@ -55,6 +55,7 @@ @@ -85,7 +86,9 @@ android:id="@+id/call_screen_sharing_indicator" android:layout_width="wrap_content" android:layout_height="wrap_content" - android:layout_marginStart="@dimen/standard_quarter_margin" + android:minWidth="@dimen/min_size_clickable_area" + android:minHeight="@dimen/min_size_clickable_area" + android:scaleType="center" android:contentDescription="@string/nc_stop_screen_sharing" android:src="@drawable/screen_share_indicator" android:translationZ="2dp" From 98953a2f434240d391239693ed469263e1013039 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Mon, 26 Oct 2026 22:20:30 +0100 Subject: [PATCH 11/16] fix(call): request screen share offers only once per publisher Whether the screen share publisher was connected was checked and set separately from whether the connection was still the current one. If the screen sharing was stopped from the signaling thread in between, the flag was set again for the stopped screen sharing, and the participants already in the call were not requested an offer when the screen was shared the next time. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 1a53ff206f9..21b8077c6ab 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -3495,22 +3495,27 @@ class CallActivity : CallBaseActivity() { override fun onIceConnectionStateChanged(iceConnectionState: IceConnectionState) { runOnUiThread { - // The event may have been queued before the connection was closed, even if sharing started again. - val isCurrentConnection = synchronized(screenShareLock) { - screenSharePeerConnectionWrapperList.contains(peerConnectionWrapper) - } - if (!isCurrentConnection) { - return@runOnUiThread - } if (iceConnectionState == IceConnectionState.CONNECTED) { - synchronized(screenShareLock) { + // The check and the update are done together, as the screen sharing may be stopped from the + // signaling thread in between. + val isFirstPublisherConnection = synchronized(screenShareLock) { + // The event may have been queued before the connection was closed, even if sharing started + // again. + if (!screenSharePeerConnectionWrapperList.contains(peerConnectionWrapper)) { + return@runOnUiThread + } screenShareRetryCounts.remove(peerConnectionWrapper.sessionId) + (hasMCU && !isScreenSharePublisherConnected).also { + if (it) { + isScreenSharePublisherConnected = true + } + } } - if (hasMCU && !isScreenSharePublisherConnected) { - isScreenSharePublisherConnected = true + if (isFirstPublisherConnection) { getParticipantSessionKeys().forEach { requestScreenShareOffer(it) } } } else if (iceConnectionState == IceConnectionState.FAILED) { + // Closed connections are ignored there. retryScreenSharePeerConnection(peerConnectionWrapper) } } @@ -3992,6 +3997,7 @@ class CallActivity : CallBaseActivity() { if (isScreenSharingPending) { localScreenStream = stream isScreenSharingPending = false + isScreenSharePublisherConnected = false } localScreenStream !== stream } From c19c7b9b8d596c25cc2b3e8fe858845bc5ec09e6 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Mon, 26 Oct 2026 22:29:11 +0100 Subject: [PATCH 12/16] refactor(call): move sharing of own screen out of CallActivity The state of sharing the own screen (pending, sharing, suspended while reconnecting and stopped, the connections, their retries and whether the publisher is connected) is now kept in LocalScreenShareController, and the capture of the screen in AndroidScreenCapture. CallActivity provides what is needed from the call through LocalScreenShareController.Host. This makes the state machine unit testable, so tests for it are added. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 541 +++--------------- .../talk/call/LocalScreenShareController.kt | 426 ++++++++++++++ .../com/nextcloud/talk/call/ScreenCapture.kt | 182 ++++++ .../talk/call/AndroidScreenCaptureTest.kt | 31 + .../call/LocalScreenShareControllerTest.kt | 409 +++++++++++++ 5 files changed, 1136 insertions(+), 453 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt create mode 100644 app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt create mode 100644 app/src/test/java/com/nextcloud/talk/call/AndroidScreenCaptureTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 21b8077c6ab..b463d1e2238 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -25,13 +25,11 @@ import android.graphics.Color import android.graphics.Rect import android.graphics.Typeface import android.graphics.drawable.Icon -import android.hardware.display.DisplayManager import android.media.AudioAttributes import android.media.AudioFormat import android.media.AudioRecord import android.media.MediaPlayer import android.media.MediaRecorder -import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.net.Uri import android.os.Build @@ -42,9 +40,7 @@ import android.os.ResultReceiver import android.provider.Settings import android.text.TextUtils import android.text.format.DateUtils -import android.util.DisplayMetrics import android.util.Log -import android.view.Display import android.view.MotionEvent import android.view.OrientationEventListener import android.view.View @@ -79,7 +75,9 @@ import com.nextcloud.talk.R import com.nextcloud.talk.api.NcApi import com.nextcloud.talk.application.NextcloudTalkApplication import com.nextcloud.talk.application.NextcloudTalkApplication.Companion.sharedApplication +import com.nextcloud.talk.call.AndroidScreenCapture import com.nextcloud.talk.call.CallParticipantList +import com.nextcloud.talk.call.LocalScreenShareController import com.nextcloud.talk.call.LocalStateBroadcaster import com.nextcloud.talk.call.LocalStateBroadcasterMcu import com.nextcloud.talk.call.LocalStateBroadcasterNoMcu @@ -91,6 +89,7 @@ import com.nextcloud.talk.call.e2ee.CallEncryption import com.nextcloud.talk.call.e2ee.FrameKeyRing import com.nextcloud.talk.call.MutableLocalCallParticipantModel import com.nextcloud.talk.call.ReactionAnimator +import com.nextcloud.talk.call.ScreenCapture import com.nextcloud.talk.call.components.ParticipantGrid import com.nextcloud.talk.call.components.SelfVideoView import com.nextcloud.talk.call.components.screenshare.ScreenShareComponent @@ -205,7 +204,6 @@ import org.webrtc.PeerConnection import org.webrtc.PeerConnection.IceConnectionState import org.webrtc.PeerConnectionFactory import org.webrtc.RendererCommon -import org.webrtc.ScreenCapturerAndroid import org.webrtc.SoftwareVideoDecoderFactory import org.webrtc.SoftwareVideoEncoderFactory import org.webrtc.SurfaceTextureHelper @@ -219,7 +217,6 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject import kotlin.math.abs -import kotlin.math.max @AutoInjector(NextcloudTalkApplication::class) @Suppress("TooManyFunctions", "ReturnCount", "LargeClass") @@ -299,38 +296,13 @@ class CallActivity : CallBaseActivity() { private val callParticipantMessageListeners: MutableMap = HashMap() private val selfPeerConnectionObserver: PeerConnectionObserver = CallActivitySelfPeerConnectionObserver() - private var screenCapturer: ScreenCapturerAndroid? = null - private var screenSurfaceTextureHelper: SurfaceTextureHelper? = null - private var screenVideoSource: VideoSource? = null - private var screenCaptureSize: Pair? = null - private val screenCaptureDisplayListener = ScreenCaptureDisplayListener() + private val localScreenShareController = LocalScreenShareController(ScreenShareHost()) { runOnUiThread(it) } private var conversationScreenSharePermission = false // The permission from signaling is more recent than the one from the conversation, so it takes precedence. private var signalingScreenSharePermission: Boolean? = null - // Guards the screen sharing state (stream, peer connections and capturer), which is changed both from the main - // thread and from the signaling thread. - private val screenShareLock = Any() - - @Volatile - private var localScreenStream: MediaStream? = null - - // Set from asking for the permission until the screen sharing started, failed or was cancelled. - @Volatile - private var isScreenSharingPending = false - - // Without MCU there is one connection per remote participant, with MCU just the own publisher connection. - private val screenSharePeerConnectionWrapperList: MutableList = ArrayList() - - @Volatile - private var isScreenSharePublisherConnected = false - private val screenShareObservers: MutableMap = HashMap() - - // Failed connections are created again a few times per remote session before giving up. - private val screenShareRetryCounts: MutableMap = HashMap() - // Screen sharing needs the foreground service, which is not started without microphone and notification permission. @Volatile private var isCallForegroundServiceStarted = false @@ -764,7 +736,7 @@ class CallActivity : CallBaseActivity() { .setTitle(R.string.nc_stop_screen_sharing) .setMessage(R.string.nc_stop_screen_sharing_confirm_message) .setPositiveButton(R.string.nc_stop_screen_sharing) { _: DialogInterface?, _: Int -> - stopScreenSharing() + localScreenShareController.stop() } .setNegativeButton(R.string.nc_common_dismiss, null) viewThemeUtils.dialog.colorMaterialAlertDialogBackground(this, dialogBuilder) @@ -2691,7 +2663,7 @@ class CallActivity : CallBaseActivity() { */ private fun setUpOwnPublishers() { getOrCreatePeerConnectionWrapperForSessionIdAndType(webSocketClient!!.sessionId, VIDEO_STREAM_TYPE_VIDEO, true) - addScreenSharePeerConnection(webSocketClient!!.sessionId) + localScreenShareController.ensurePublisher() } private fun removeSessions(sessions: Collection) { @@ -2699,7 +2671,7 @@ class CallActivity : CallBaseActivity() { Log.d(TAG, " session that will be removed is: $session") endPeerConnection(session, "video") endPeerConnection(session, "screen") - endScreenSharePeerConnection(session) + localScreenShareController.onParticipantLeft(session) removeCallParticipant(session) } } @@ -2993,7 +2965,7 @@ class CallActivity : CallBaseActivity() { // and send the local state exactly when the data channel becomes ready. localStateBroadcaster!!.handleCallParticipantAdded(callViewModel.getParticipant(sessionId)!!.uiState) - shareScreenWithJoinedParticipant(sessionId) + localScreenShareController.onParticipantJoined(sessionId) initPipMode() } @@ -3467,61 +3439,6 @@ class CallActivity : CallBaseActivity() { } } - private inner class ScreenCaptureDisplayListener : DisplayManager.DisplayListener { - override fun onDisplayAdded(displayId: Int) { - // unused atm - } - - override fun onDisplayRemoved(displayId: Int) { - // unused atm - } - - override fun onDisplayChanged(displayId: Int) { - if (displayId == Display.DEFAULT_DISPLAY) { - updateScreenCaptureSize() - } - } - } - - private inner class ScreenSharePeerConnectionObserver(private val peerConnectionWrapper: PeerConnectionWrapper) : - PeerConnectionObserver { - override fun onStreamAdded(mediaStream: MediaStream) { - // unused atm - } - - override fun onStreamRemoved(mediaStream: MediaStream) { - // unused atm - } - - override fun onIceConnectionStateChanged(iceConnectionState: IceConnectionState) { - runOnUiThread { - if (iceConnectionState == IceConnectionState.CONNECTED) { - // The check and the update are done together, as the screen sharing may be stopped from the - // signaling thread in between. - val isFirstPublisherConnection = synchronized(screenShareLock) { - // The event may have been queued before the connection was closed, even if sharing started - // again. - if (!screenSharePeerConnectionWrapperList.contains(peerConnectionWrapper)) { - return@runOnUiThread - } - screenShareRetryCounts.remove(peerConnectionWrapper.sessionId) - (hasMCU && !isScreenSharePublisherConnected).also { - if (it) { - isScreenSharePublisherConnected = true - } - } - } - if (isFirstPublisherConnection) { - getParticipantSessionKeys().forEach { requestScreenShareOffer(it) } - } - } else if (iceConnectionState == IceConnectionState.FAILED) { - // Closed connections are ignored there. - retryScreenSharePeerConnection(peerConnectionWrapper) - } - } - } - } - private inner class CallActivitySelfPeerConnectionObserver : PeerConnectionObserver { override fun onStreamAdded(mediaStream: MediaStream) { // unused atm @@ -3539,7 +3456,10 @@ class CallActivity : CallBaseActivity() { // The connection will recover when the activity resumes. // While sharing the screen the call is expected to be in background and may not be resumed for // a long time, so it reconnects instead. - if (!active && localScreenStream == null && currentCallStatus === CallStatus.IN_CONVERSATION) { + if (!active && + !localScreenShareController.isSharing && + currentCallStatus === CallStatus.IN_CONVERSATION + ) { Log.d(TAG, "ICE FAILED while backgrounded, skipping hangup (will recover on resume)") return@runOnUiThread } @@ -3844,18 +3764,18 @@ class CallActivity : CallBaseActivity() { return } initFeaturesVisibility() - if (!isAllowed && localScreenStream != null) { - stopScreenSharing() + if (!isAllowed && localScreenShareController.isSharing) { + localScreenShareController.stop() Snackbar.make(binding!!.root, R.string.nc_screen_sharing_not_allowed, Snackbar.LENGTH_LONG).show() } } fun toggleScreenSharing() { - if (localScreenStream != null) { - stopScreenSharing() + if (localScreenShareController.isSharing) { + localScreenShareController.stop() return } - if (isScreenSharingPending) { + if (localScreenShareController.isPending) { return } if (!isAllowedToShareScreen) { @@ -3871,413 +3791,131 @@ class CallActivity : CallBaseActivity() { return } // The permission dialog moves the call to the background, which must not enter picture in picture. - isScreenSharingPending = true - updatePipAutoEnter() + if (!localScreenShareController.requestStart()) { + return + } val mediaProjectionManager = getSystemService(MediaProjectionManager::class.java) try { screenSharingPermissionLauncher.launch(mediaProjectionManager.createScreenCaptureIntent()) } catch (e: ActivityNotFoundException) { Log.e(TAG, "Screen capture is not supported on this device", e) - cancelPendingScreenSharing() + localScreenShareController.cancelPending() showScreenSharingError() } } - private fun cancelPendingScreenSharing() { - synchronized(screenShareLock) { - isScreenSharingPending = false - } - runOnUiThread { updatePipAutoEnter() } - } - private fun onScreenSharingPermissionResult(result: ActivityResult) { val permissionResultData = result.data if (result.resultCode != RESULT_OK || permissionResultData == null) { Log.d(TAG, "Screen sharing permission was not granted") - cancelPendingScreenSharing() - return - } - if (!isScreenSharingPending || !isCallActiveForScreenSharing()) { - Log.d(TAG, "Screen sharing was cancelled while asking for the permission") - cancelPendingScreenSharing() + localScreenShareController.cancelPending() return } - - // On Android 14+ the media projection can only be started once the foreground service has the media - // projection type, which in turn requires the permission to have been granted. - val resultReceiver = object : ResultReceiver(Handler(Looper.getMainLooper())) { - override fun onReceiveResult(resultCode: Int, resultData: Bundle?) { - if (resultCode == CallForegroundService.RESULT_SCREEN_SHARING_READY) { - startScreenSharing(permissionResultData) - } else { - cancelPendingScreenSharing() - showScreenSharingError() - } - } - } - CallForegroundService.startScreenSharing(applicationContext, resultReceiver) + localScreenShareController.onPermissionGranted(permissionResultData) } private fun isCallActiveForScreenSharing(): Boolean = isConnectionEstablished && currentCallStatus !== CallStatus.LEAVING - private fun startScreenSharing(permissionResultData: Intent) { - if (localScreenStream != null || - !isScreenSharingPending || - !isAllowedToShareScreen || - !isCallActiveForScreenSharing() - ) { - cancelPendingScreenSharing() - CallForegroundService.stopScreenSharing(applicationContext) - return - } - - val (factory, videoSource) = startScreenCapture(permissionResultData) ?: return - publishScreenShare(factory, videoSource) - } - /** - * Returns the factory used and the video source of the started capture, or null if it could not be started. + * While reconnecting the screen keeps being captured, so it is shared again once back in the call. */ - private fun startScreenCapture(permissionResultData: Intent): Pair? { - val capturer = ScreenCapturerAndroid( - permissionResultData, - object : MediaProjection.Callback() { - override fun onStop() { - runOnUiThread { stopScreenSharing() } - } - } - ) - // Both are released when hanging up, which may happen in parallel from the signaling thread. - val eglBase = rootEglBase - val factory = peerConnectionFactory - if (eglBase == null || factory == null) { - cancelPendingScreenSharing() - CallForegroundService.stopScreenSharing(applicationContext) - return null - } - val surfaceTextureHelper = SurfaceTextureHelper.create("ScreenCaptureThread", eglBase.eglBaseContext) - val videoSource = factory.createVideoSource(true) - val captureSize = getScreenCaptureSize() - synchronized(screenShareLock) { - screenCapturer = capturer - screenSurfaceTextureHelper = surfaceTextureHelper - screenVideoSource = videoSource - screenCaptureSize = captureSize - } - - capturer.initialize(surfaceTextureHelper, applicationContext, videoSource.capturerObserver) - // The media projection may fail to start, for example, if the permission expired or was revoked. - val captureError = try { - capturer.startCapture(captureSize.first, captureSize.second, SCREEN_SHARE_FRAME_RATE) - null - } catch (e: SecurityException) { - e - } catch (e: IllegalStateException) { - e - } - if (captureError != null) { - Log.e(TAG, "Failed to start screen capture", captureError) - releaseScreenCapture() - cancelPendingScreenSharing() - CallForegroundService.stopScreenSharing(applicationContext) - showScreenSharingError() - return null - } - getSystemService(DisplayManager::class.java) - .registerDisplayListener(screenCaptureDisplayListener, Handler(Looper.getMainLooper())) - return Pair(factory, videoSource) - } - - private fun publishScreenShare(factory: PeerConnectionFactory, videoSource: VideoSource) { - val stream = factory.createLocalMediaStream("NCSS") - stream.addTrack(factory.createVideoTrack("NCs0", videoSource)) - val isCancelled = synchronized(screenShareLock) { - // stopScreenSharing() may have been called from the signaling thread while the capture was being started. - if (isScreenSharingPending) { - localScreenStream = stream - isScreenSharingPending = false - isScreenSharePublisherConnected = false - } - localScreenStream !== stream - } - if (isCancelled) { - Log.d(TAG, "Screen sharing was cancelled while starting it") - releaseScreenCapture() - stream.dispose() - CallForegroundService.stopScreenSharing(applicationContext) - updatePipAutoEnter() - return - } - callViewModel.setLocalScreenSharing(true) - updatePipAutoEnter() - - if (hasMCU) { - addScreenSharePeerConnection(webSocketClient!!.sessionId) + private fun stopOrSuspendScreenSharing(shutDownView: Boolean) { + val isReconnecting = currentCallStatus === CallStatus.RECONNECTING || + currentCallStatus === CallStatus.PUBLISHER_FAILED + if (shutDownView || !isReconnecting) { + localScreenShareController.stop() } else { - getParticipantSessionKeys().forEach { addScreenSharePeerConnection(it) } + localScreenShareController.suspend() } } /** - * Can be called from any thread; only the first call after starting the screen sharing tears it down. If the - * screen sharing is still being started it is cancelled instead. + * Picture in picture would be shown on top of the shared screen, so the call is just moved to the background + * while sharing; the call notification shows that the screen is being shared. */ - fun stopScreenSharing() { - val stream: MediaStream? - var peerConnectionWrappers: List = emptyList() - var wasPending = false - synchronized(screenShareLock) { - stream = localScreenStream - if (stream == null) { - wasPending = isScreenSharingPending - isScreenSharingPending = false - } else { - localScreenStream = null - isScreenSharePublisherConnected = false - screenShareRetryCounts.clear() - peerConnectionWrappers = screenSharePeerConnectionWrapperList.toList() - screenSharePeerConnectionWrapperList.clear() - } - } - if (stream == null) { - if (wasPending) { - runOnUiThread { updatePipAutoEnter() } - } - return - } + override fun isAutomaticPipModeSuppressed(): Boolean = + localScreenShareController.isSharing || localScreenShareController.isPending - sendUnshareScreen(getParticipantSessionKeys()) + private fun showScreenSharingError() { + Snackbar.make(binding!!.root, R.string.nc_screen_sharing_failed, Snackbar.LENGTH_LONG).show() + } - peerConnectionWrappers.forEach { closeScreenSharePeerConnection(it) } + private inner class ScreenShareHost : LocalScreenShareController.Host { + override val hasMCU: Boolean + get() = this@CallActivity.hasMCU - releaseScreenCapture() - stream.dispose() + override val ownSessionId: String? + get() = if (webSocketClient != null) webSocketClient!!.sessionId else callSession - CallForegroundService.stopScreenSharing(applicationContext) - callViewModel.setLocalScreenSharing(false) - runOnUiThread { updatePipAutoEnter() } - } + override val participantSessionIds: Collection + get() = getParticipantSessionKeys() - /** - * While reconnecting only the connections are closed, so the screen keeps being captured and is shared again - * once back in the call, without asking the user for the permission again. - */ - private fun stopOrSuspendScreenSharing(shutDownView: Boolean) { - val isReconnecting = currentCallStatus === CallStatus.RECONNECTING || - currentCallStatus === CallStatus.PUBLISHER_FAILED - if (shutDownView || !isReconnecting) { - stopScreenSharing() - return - } + override val isCallActive: Boolean + get() = isCallActiveForScreenSharing() - val peerConnectionWrappers = synchronized(screenShareLock) { - isScreenSharePublisherConnected = false - screenShareRetryCounts.clear() - screenSharePeerConnectionWrapperList.toList().also { screenSharePeerConnectionWrapperList.clear() } - } - peerConnectionWrappers.forEach { closeScreenSharePeerConnection(it) } - } + override val isScreenSharingAllowed: Boolean + get() = isAllowedToShareScreen - /** - * Must be called on the main thread. - */ - private fun retryScreenSharePeerConnection(peerConnectionWrapper: PeerConnectionWrapper) { - val sessionId = peerConnectionWrapper.sessionId - val shouldRetry = synchronized(screenShareLock) { - if (!screenSharePeerConnectionWrapperList.remove(peerConnectionWrapper)) { - return - } - if (hasMCU) { - isScreenSharePublisherConnected = false + override fun startForegroundServiceScreenSharing(onResult: (isReady: Boolean) -> Unit) { + val resultReceiver = object : ResultReceiver(Handler(Looper.getMainLooper())) { + override fun onReceiveResult(resultCode: Int, resultData: Bundle?) { + onResult(resultCode == CallForegroundService.RESULT_SCREEN_SHARING_READY) + } } - val retryCount = screenShareRetryCounts[sessionId] ?: 0 - screenShareRetryCounts[sessionId] = retryCount + 1 - retryCount < SCREEN_SHARE_MAX_RETRIES + CallForegroundService.startScreenSharing(applicationContext, resultReceiver) } - closeScreenSharePeerConnection(peerConnectionWrapper) - if (!isCallActiveForScreenSharing()) { - // The connections are set up again once reconnected. - return - } - if (shouldRetry) { - Log.w(TAG, "Screen share connection to $sessionId failed, creating it again") - // The remote participants close the connection receiving the screen, so the new offer is not handled by - // the failed connection. - sendUnshareScreen(if (hasMCU) getParticipantSessionKeys() else listOf(sessionId)) - addScreenSharePeerConnection(sessionId) - } else if (hasMCU) { - Log.e(TAG, "Screen share publisher connection failed") - stopScreenSharing() - showScreenSharingError() - } else { - Log.e(TAG, "Screen share connection to $sessionId failed, not sharing the screen with it anymore") + override fun stopForegroundServiceScreenSharing() { + CallForegroundService.stopScreenSharing(applicationContext) } - } - private fun sendUnshareScreen(sessionIds: Collection) { - sessionIds.forEach { sessionId -> - messageSender?.send( - NCSignalingMessageDto(type = "unshareScreen", roomType = VIDEO_STREAM_TYPE_SCREEN), - sessionId - ) + override fun createScreenCapture(permissionResultData: Intent, onStop: () -> Unit): ScreenCapture? { + // Both are released when hanging up, which may happen in parallel from the signaling thread. + val eglBase = rootEglBase ?: return null + val factory = peerConnectionFactory ?: return null + return AndroidScreenCapture(applicationContext, eglBase, factory, permissionResultData, onStop) } - } - - /** - * Picture in picture would be shown on top of the shared screen, so the call is just moved to the background - * while sharing; the call notification shows that the screen is being shared. - */ - override fun isAutomaticPipModeSuppressed(): Boolean = localScreenStream != null || isScreenSharingPending - - private fun releaseScreenCapture() { - getSystemService(DisplayManager::class.java).unregisterDisplayListener(screenCaptureDisplayListener) - val capturer: ScreenCapturerAndroid? - val videoSource: VideoSource? - val surfaceTextureHelper: SurfaceTextureHelper? - synchronized(screenShareLock) { - capturer = screenCapturer - videoSource = screenVideoSource - surfaceTextureHelper = screenSurfaceTextureHelper - screenCapturer = null - screenVideoSource = null - screenSurfaceTextureHelper = null - screenCaptureSize = null + override fun createPeerConnection(sessionId: String, stream: MediaStream): PeerConnectionWrapper { + val (senderKeyRing, receiverKeyRing) = keyRingsFor(hasMCU, hasMCU, sessionId) + return PeerConnectionWrapper( + peerConnectionFactory, + iceServers, + sdpConstraintsForMCUPublisher, + sessionId, + ownSessionId, + stream, + hasMCU, + hasMCU, + VIDEO_STREAM_TYPE_SCREEN, + signalingMessageReceiver, + signalingMessageSender, + isCallEndToEndEncryptionEnabled, + senderKeyRing, + receiverKeyRing, + true + ) } - capturer?.let { - try { - it.stopCapture() - } catch (e: InterruptedException) { - Log.e(TAG, "Failed to stop screen capture", e) - } - it.dispose() + override fun send(message: NCSignalingMessageDto, sessionId: String) { + messageSender?.send(message, sessionId) } - videoSource?.dispose() - surfaceTextureHelper?.dispose() - } - private fun updateScreenCaptureSize() { - val captureSize = getScreenCaptureSize() - synchronized(screenShareLock) { - val capturer = screenCapturer ?: return - if (captureSize == screenCaptureSize) { - return - } - screenCaptureSize = captureSize - capturer.changeCaptureFormat(captureSize.first, captureSize.second, SCREEN_SHARE_FRAME_RATE) + override fun onStateChanged() { + callViewModel.setLocalScreenSharing(localScreenShareController.isSharing) + updatePipAutoEnter() } - } - - private fun showScreenSharingError() { - Snackbar.make(binding!!.root, R.string.nc_screen_sharing_failed, Snackbar.LENGTH_LONG).show() - } - /** - * Returns the size of the whole display in its current rotation, independently of the size of the call window - * (for example, in picture in picture or split screen) and of whether the call is visible, scaled down to - * [SCREEN_SHARE_MAX_DIMENSION]. - */ - private fun getScreenCaptureSize(): Pair { - val displayMetrics = DisplayMetrics() - // The configuration of the activity is not updated while it is stopped, so the display is queried directly. - @Suppress("DEPRECATION") - getSystemService(DisplayManager::class.java).getDisplay(Display.DEFAULT_DISPLAY).getRealMetrics(displayMetrics) - val displayWidth = displayMetrics.widthPixels - val displayHeight = displayMetrics.heightPixels - - val scale = max(1f, max(displayWidth, displayHeight).toFloat() / SCREEN_SHARE_MAX_DIMENSION) - // Video encoders expect even dimensions. - val width = (displayWidth / scale).toInt() and 1.inv() - val height = (displayHeight / scale).toInt() and 1.inv() - return Pair(width, height) - } - - private fun shareScreenWithJoinedParticipant(sessionId: String) { - if (localScreenStream == null) { - return - } - if (hasMCU) { - // Participants joining before the publisher is connected are requested once it is connected. - if (isScreenSharePublisherConnected) { - requestScreenShareOffer(sessionId) - } - } else { - addScreenSharePeerConnection(sessionId) + override fun onScreenSharingFailed() { + showScreenSharingError() } - } - /** - * With MCU the offer for the screen publisher is sent by the MCU to the remote participant on request. - */ - private fun requestScreenShareOffer(sessionId: String) { - messageSender?.send( - NCSignalingMessageDto(type = "sendoffer", roomType = VIDEO_STREAM_TYPE_SCREEN), - sessionId - ) - } - - private fun addScreenSharePeerConnection(sessionId: String?) { - val isEncryptionFailed = synchronized(screenShareLock) { - val stream = localScreenStream ?: return - if (sessionId == null || screenSharePeerConnectionWrapperList.any { it.sessionId == sessionId }) { - return - } - - val (senderKeyRing, receiverKeyRing) = keyRingsFor(hasMCU, hasMCU, sessionId) - val peerConnectionWrapper = try { - PeerConnectionWrapper( - peerConnectionFactory, - iceServers, - sdpConstraintsForMCUPublisher, - sessionId, - if (webSocketClient != null) webSocketClient!!.sessionId else callSession, - stream, - hasMCU, - hasMCU, - VIDEO_STREAM_TYPE_SCREEN, - signalingMessageReceiver, - signalingMessageSender, - isCallEndToEndEncryptionEnabled, - senderKeyRing, - receiverKeyRing, - true - ) - } catch (e: IllegalStateException) { - Log.e(TAG, "Not creating a screen share connection to $sessionId", e) - null - } - if (peerConnectionWrapper != null) { - val observer = ScreenSharePeerConnectionObserver(peerConnectionWrapper) - screenShareObservers[peerConnectionWrapper] = observer - peerConnectionWrapper.addObserver(observer) - screenSharePeerConnectionWrapperList.add(peerConnectionWrapper) - } - peerConnectionWrapper == null - } - // Hanging up also stops the screen sharing, so it is not done while holding the lock. - if (isEncryptionFailed) { + override fun onEncryptionFailed() { failCallEncryption(R.string.nc_call_e2ee_setup_failed) } } - private fun endScreenSharePeerConnection(sessionId: String?) { - val peerConnectionWrapper = synchronized(screenShareLock) { - screenSharePeerConnectionWrapperList.firstOrNull { it.sessionId == sessionId } - ?.also { screenSharePeerConnectionWrapperList.remove(it) } - } ?: return - closeScreenSharePeerConnection(peerConnectionWrapper) - } - - private fun closeScreenSharePeerConnection(peerConnectionWrapper: PeerConnectionWrapper) { - synchronized(screenShareLock) { - screenShareObservers.remove(peerConnectionWrapper) - }?.let { peerConnectionWrapper.removeObserver(it) } - peerConnectionWrapper.removePeerConnection() - } - val isAllowedToStartOrStopRecording: Boolean get() { val spreedCap = conversationUser.capabilities?.spreedCapability @@ -4337,9 +3975,6 @@ class CallActivity : CallBaseActivity() { private const val MICROPHONE_VALUE_SLEEP: Long = 1000 private const val FRAME_RATE: Int = 30 - private const val SCREEN_SHARE_FRAME_RATE: Int = 15 - private const val SCREEN_SHARE_MAX_DIMENSION: Int = 1920 - private const val SCREEN_SHARE_MAX_RETRIES: Int = 3 private const val WIDTH_16_TO_9_RATIO: Int = 1280 private const val HEIGHT_16_TO_9_RATIO: Int = 720 private const val WIDTH_4_TO_3_RATIO: Int = 640 diff --git a/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt b/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt new file mode 100644 index 00000000000..8d3ccdb811b --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt @@ -0,0 +1,426 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ +package com.nextcloud.talk.call + +import android.content.Intent +import android.util.Log +import com.nextcloud.talk.models.json.signaling.NCSignalingMessageDto +import com.nextcloud.talk.webrtc.PeerConnectionWrapper +import com.nextcloud.talk.webrtc.PeerConnectionWrapper.PeerConnectionObserver +import org.webrtc.MediaStream +import org.webrtc.PeerConnection.IceConnectionState + +/** + * Shares the local screen in a call, from asking for the permission until the screen sharing is stopped. + * + * Without MCU the screen is sent through one connection per remote participant, with MCU through a single publisher + * connection, for which the MCU then sends offers to the remote participants on request. Failed connections are + * created again a few times, and while the call reconnects the screen keeps being captured, so it is shared again + * once back in the call. + * + * The state is changed both from the main thread and from the signaling thread, so the methods can be called from + * any thread unless stated otherwise. + */ +class LocalScreenShareController(private val host: Host, private val runOnMainThread: (Runnable) -> Unit) { + + /** + * Provides what the screen sharing needs from the call. + */ + interface Host { + val hasMCU: Boolean + + /** + * The own signaling session, which the publisher connection is created for with MCU. + */ + val ownSessionId: String? + + val participantSessionIds: Collection + + /** + * Whether the call is joined, so connections can be created. + */ + val isCallActive: Boolean + + val isScreenSharingAllowed: Boolean + + /** + * Adds the media projection type to the call foreground service, which is needed before the screen can be + * captured. + */ + fun startForegroundServiceScreenSharing(onResult: (isReady: Boolean) -> Unit) + + fun stopForegroundServiceScreenSharing() + + /** + * Returns null if the capture can not be created, for example, because the call is being hung up. + */ + fun createScreenCapture(permissionResultData: Intent, onStop: () -> Unit): ScreenCapture? + + /** + * @throws IllegalStateException if the connection has to be end-to-end encrypted but the keys are missing + */ + fun createPeerConnection(sessionId: String, stream: MediaStream): PeerConnectionWrapper + + fun send(message: NCSignalingMessageDto, sessionId: String) + + /** + * Called on the main thread when [isSharing] or [isPending] changed. + */ + fun onStateChanged() + + /** + * Called on the main thread when the screen sharing could not be started or failed. + */ + fun onScreenSharingFailed() + + /** + * Called when a connection could not be created because it could not be end-to-end encrypted. + */ + fun onEncryptionFailed() + } + + // Guards the whole state; host methods that may tear the screen sharing down are not called while holding it. + private val lock = Any() + + @Volatile + private var capture: ScreenCapture? = null + + // Set from asking for the permission until the screen sharing started, failed or was cancelled. + @Volatile + private var isPendingInternal = false + + private val connections: MutableList = ArrayList() + private val observers: MutableMap = HashMap() + + // Failed connections are created again a few times per remote session before giving up. + private val retryCounts: MutableMap = HashMap() + + private var isPublisherConnected = false + + val isSharing: Boolean + get() = capture != null + + val isPending: Boolean + get() = isPendingInternal + + /** + * Marks the screen sharing as being started, which has to be followed by [onPermissionGranted] or + * [cancelPending]. Returns false if the screen is already shared or being started. + */ + fun requestStart(): Boolean { + synchronized(lock) { + if (capture != null || isPendingInternal) { + return false + } + isPendingInternal = true + } + notifyStateChanged() + return true + } + + fun cancelPending() { + synchronized(lock) { + isPendingInternal = false + } + notifyStateChanged() + } + + /** + * Must be called on the main thread. + */ + fun onPermissionGranted(permissionResultData: Intent) { + if (!isPendingInternal || !host.isCallActive) { + Log.d(TAG, "Screen sharing was cancelled while asking for the permission") + cancelPending() + return + } + + // On Android 14+ the media projection can only be started once the foreground service has the media + // projection type, which in turn requires the permission to have been granted. + host.startForegroundServiceScreenSharing { isReady -> + if (isReady) { + start(permissionResultData) + } else { + cancelPending() + host.onScreenSharingFailed() + } + } + } + + private fun start(permissionResultData: Intent) { + val newCapture = startCapture(permissionResultData) ?: return + + val isCancelled = synchronized(lock) { + // stop() may have been called from the signaling thread while the capture was being started. + if (isPendingInternal) { + capture = newCapture + isPendingInternal = false + isPublisherConnected = false + } + capture !== newCapture + } + if (isCancelled) { + Log.d(TAG, "Screen sharing was cancelled while starting it") + newCapture.release() + host.stopForegroundServiceScreenSharing() + notifyStateChanged() + } else { + notifyStateChanged() + if (host.hasMCU) { + addConnection(host.ownSessionId) + } else { + host.participantSessionIds.forEach { addConnection(it) } + } + } + } + + /** + * Returns the started capture, or null if the screen sharing could not be started or was cancelled. + */ + private fun startCapture(permissionResultData: Intent): ScreenCapture? { + val canStart = capture == null && isPendingInternal && host.isScreenSharingAllowed && host.isCallActive + val newCapture = if (canStart) { + host.createScreenCapture(permissionResultData) { runOnMainThread { stop() } } + } else { + null + } + return when { + newCapture == null -> { + cancelPending() + host.stopForegroundServiceScreenSharing() + null + } + + !newCapture.start() -> { + newCapture.release() + cancelPending() + host.stopForegroundServiceScreenSharing() + host.onScreenSharingFailed() + null + } + + else -> newCapture + } + } + + /** + * Only the first call after starting the screen sharing tears it down. If the screen sharing is still being + * started it is cancelled instead. + */ + fun stop() { + val stoppedCapture: ScreenCapture? + var stoppedConnections: List = emptyList() + var wasPending = false + synchronized(lock) { + stoppedCapture = capture + if (stoppedCapture == null) { + wasPending = isPendingInternal + isPendingInternal = false + } else { + capture = null + isPublisherConnected = false + retryCounts.clear() + stoppedConnections = connections.toList() + connections.clear() + } + } + if (stoppedCapture == null) { + if (wasPending) { + notifyStateChanged() + } + return + } + + sendScreenMessage("unshareScreen", host.participantSessionIds) + + stoppedConnections.forEach { closeConnection(it) } + + stoppedCapture.release() + + host.stopForegroundServiceScreenSharing() + notifyStateChanged() + } + + /** + * Closes the connections while the call reconnects, but keeps capturing the screen, so it is shared again once + * back in the call without asking the user for the permission again. + */ + fun suspend() { + val suspendedConnections = synchronized(lock) { + isPublisherConnected = false + retryCounts.clear() + connections.toList().also { connections.clear() } + } + suspendedConnections.forEach { closeConnection(it) } + } + + /** + * Sets up the publisher connection with MCU if the screen is shared, for example, after reconnecting. + */ + fun ensurePublisher() { + if (host.hasMCU) { + addConnection(host.ownSessionId) + } + } + + fun onParticipantJoined(sessionId: String) { + if (capture == null) { + return + } + if (host.hasMCU) { + // Participants joining before the publisher is connected are requested once it is connected. + val isConnected = synchronized(lock) { isPublisherConnected } + if (isConnected) { + sendScreenMessage("sendoffer", listOf(sessionId)) + } + } else { + addConnection(sessionId) + } + } + + fun onParticipantLeft(sessionId: String?) { + val connection = synchronized(lock) { + connections.firstOrNull { it.sessionId == sessionId }?.also { connections.remove(it) } + } ?: return + closeConnection(connection) + } + + private fun addConnection(sessionId: String?) { + val isEncryptionFailed = synchronized(lock) { + val stream = capture?.stream ?: return + if (sessionId == null || connections.any { it.sessionId == sessionId }) { + return + } + + val connection = try { + host.createPeerConnection(sessionId, stream) + } catch (e: IllegalStateException) { + Log.e(TAG, "Not creating a screen share connection to $sessionId", e) + null + } + if (connection != null) { + val observer = ConnectionObserver(connection) + observers[connection] = observer + connection.addObserver(observer) + connections.add(connection) + } + connection == null + } + // Ending the call also stops the screen sharing, so it is not done while holding the lock. + if (isEncryptionFailed) { + host.onEncryptionFailed() + } + } + + private fun closeConnection(connection: PeerConnectionWrapper) { + synchronized(lock) { + observers.remove(connection) + }?.let { connection.removeObserver(it) } + connection.removePeerConnection() + } + + /** + * With MCU "sendoffer" requests the MCU to send the offer for the screen publisher to the remote participant. + */ + private fun sendScreenMessage(type: String, sessionIds: Collection) { + sessionIds.forEach { sessionId -> + host.send(NCSignalingMessageDto(type = type, roomType = ROOM_TYPE_SCREEN), sessionId) + } + } + + private fun notifyStateChanged() { + runOnMainThread { host.onStateChanged() } + } + + private inner class ConnectionObserver(private val connection: PeerConnectionWrapper) : PeerConnectionObserver { + override fun onStreamAdded(mediaStream: MediaStream) { + // unused atm + } + + override fun onStreamRemoved(mediaStream: MediaStream) { + // unused atm + } + + override fun onIceConnectionStateChanged(iceConnectionState: IceConnectionState) { + runOnMainThread { + if (iceConnectionState == IceConnectionState.CONNECTED) { + onConnected() + } else if (iceConnectionState == IceConnectionState.FAILED) { + // Closed connections are ignored there. + onFailed() + } + } + } + + /** + * Must be called on the main thread. + */ + private fun onConnected() { + // The check and the update are done together, as the screen sharing may be stopped from the signaling + // thread in between. + val isFirstPublisherConnection = synchronized(lock) { + // The event may have been queued before the connection was closed, even if sharing started again. + if (!connections.contains(connection)) { + return + } + retryCounts.remove(connection.sessionId) + (host.hasMCU && !isPublisherConnected).also { + if (it) { + isPublisherConnected = true + } + } + } + if (isFirstPublisherConnection) { + sendScreenMessage("sendoffer", host.participantSessionIds) + } + } + + /** + * Must be called on the main thread. + */ + private fun onFailed() { + val sessionId = connection.sessionId + val shouldRetry = synchronized(lock) { + if (!connections.remove(connection)) { + return + } + if (host.hasMCU) { + isPublisherConnected = false + } + val retryCount = retryCounts[sessionId] ?: 0 + retryCounts[sessionId] = retryCount + 1 + retryCount < MAX_RETRIES + } + closeConnection(connection) + + if (!host.isCallActive) { + // The connections are set up again once reconnected. + return + } + if (shouldRetry) { + Log.w(TAG, "Screen share connection to $sessionId failed, creating it again") + // The remote participants close the connection receiving the screen, so the new offer is not handled + // by the failed connection. + sendScreenMessage("unshareScreen", if (host.hasMCU) host.participantSessionIds else listOf(sessionId)) + addConnection(sessionId) + } else if (host.hasMCU) { + Log.e(TAG, "Screen share publisher connection failed") + stop() + host.onScreenSharingFailed() + } else { + Log.e(TAG, "Screen share connection to $sessionId failed, not sharing the screen with it anymore") + } + } + } + + companion object { + private val TAG = LocalScreenShareController::class.java.simpleName + + private const val ROOM_TYPE_SCREEN = "screen" + private const val MAX_RETRIES = 3 + } +} diff --git a/app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt b/app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt new file mode 100644 index 00000000000..55b5caf3f97 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt @@ -0,0 +1,182 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ +package com.nextcloud.talk.call + +import android.content.Context +import android.content.Intent +import android.hardware.display.DisplayManager +import android.media.projection.MediaProjection +import android.os.Handler +import android.os.Looper +import android.util.DisplayMetrics +import android.util.Log +import android.view.Display +import org.webrtc.EglBase +import org.webrtc.MediaStream +import org.webrtc.PeerConnectionFactory +import org.webrtc.ScreenCapturerAndroid +import org.webrtc.SurfaceTextureHelper +import kotlin.math.max + +/** + * Capture of the local screen, provided as a stream with a single video track. + */ +interface ScreenCapture { + val stream: MediaStream + + /** + * Returns false if the capture could not be started. + */ + fun start(): Boolean + + /** + * Stops the capture and releases it, including the stream. Can be called from any thread. + */ + fun release() +} + +/** + * Captures the whole display through a media projection, independently of the size of the call window (for example, + * in picture in picture or split screen) and of whether the call is visible. + * + * @param permissionResultData the result of the media projection permission dialog + * @param onStop called when the media projection is stopped by the system or the user + */ +class AndroidScreenCapture( + private val context: Context, + eglBase: EglBase, + factory: PeerConnectionFactory, + permissionResultData: Intent, + onStop: () -> Unit +) : ScreenCapture { + + private val capturer = ScreenCapturerAndroid( + permissionResultData, + object : MediaProjection.Callback() { + override fun onStop() { + onStop() + } + } + ) + private val surfaceTextureHelper = SurfaceTextureHelper.create("ScreenCaptureThread", eglBase.eglBaseContext) + private val videoSource = factory.createVideoSource(true) + + override val stream: MediaStream = factory.createLocalMediaStream("NCSS").apply { + addTrack(factory.createVideoTrack("NCs0", videoSource)) + } + + private val displayManager = context.getSystemService(DisplayManager::class.java) + private val displayListener = object : DisplayManager.DisplayListener { + override fun onDisplayAdded(displayId: Int) { + // unused atm + } + + override fun onDisplayRemoved(displayId: Int) { + // unused atm + } + + override fun onDisplayChanged(displayId: Int) { + if (displayId == Display.DEFAULT_DISPLAY) { + updateCaptureSize() + } + } + } + + // Guards the capturer, as the size is changed from the main thread while it may be released from any thread. + private val lock = Any() + private var captureSize: Pair? = null + private var isReleased = false + + override fun start(): Boolean { + val size = getCaptureSize() + val isStarted = synchronized(lock) { + !isReleased && startCapture(size) + } + if (isStarted) { + displayManager.registerDisplayListener(displayListener, Handler(Looper.getMainLooper())) + } + return isStarted + } + + /** + * Must be called while holding the lock. + */ + private fun startCapture(size: Pair): Boolean { + captureSize = size + capturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) + // The media projection may fail to start, for example, if the permission expired or was revoked. + return try { + capturer.startCapture(size.first, size.second, FRAME_RATE) + true + } catch (e: SecurityException) { + Log.e(TAG, "Failed to start screen capture", e) + false + } catch (e: IllegalStateException) { + Log.e(TAG, "Failed to start screen capture", e) + false + } + } + + override fun release() { + displayManager.unregisterDisplayListener(displayListener) + + synchronized(lock) { + if (isReleased) { + return + } + isReleased = true + if (captureSize != null) { + try { + capturer.stopCapture() + } catch (e: InterruptedException) { + Log.e(TAG, "Failed to stop screen capture", e) + } + } + capturer.dispose() + } + videoSource.dispose() + surfaceTextureHelper.dispose() + stream.dispose() + } + + private fun updateCaptureSize() { + val size = getCaptureSize() + synchronized(lock) { + if (isReleased || captureSize == null || size == captureSize) { + return + } + captureSize = size + capturer.changeCaptureFormat(size.first, size.second, FRAME_RATE) + } + } + + private fun getCaptureSize(): Pair { + val displayMetrics = DisplayMetrics() + // The configuration of the activity is not updated while it is stopped, so the display is queried directly. + @Suppress("DEPRECATION") + displayManager.getDisplay(Display.DEFAULT_DISPLAY).getRealMetrics(displayMetrics) + return scaleCaptureSize(displayMetrics.widthPixels, displayMetrics.heightPixels) + } + + companion object { + private val TAG = AndroidScreenCapture::class.java.simpleName + + private const val FRAME_RATE = 15 + private const val MAX_DIMENSION = 1920 + + /** + * Returns the given display size in its current rotation scaled down to [MAX_DIMENSION]. + */ + fun scaleCaptureSize(displayWidth: Int, displayHeight: Int): Pair { + val scale = max(1f, max(displayWidth, displayHeight).toFloat() / MAX_DIMENSION) + // Video encoders expect even dimensions. + val width = (displayWidth / scale).toInt() and 1.inv() + val height = (displayHeight / scale).toInt() and 1.inv() + return Pair(width, height) + } + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/AndroidScreenCaptureTest.kt b/app/src/test/java/com/nextcloud/talk/call/AndroidScreenCaptureTest.kt new file mode 100644 index 00000000000..b8c01807e1a --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/AndroidScreenCaptureTest.kt @@ -0,0 +1,31 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ +package com.nextcloud.talk.call + +import org.junit.Assert.assertEquals +import org.junit.Test + +class AndroidScreenCaptureTest { + + @Test + fun testSmallDisplayIsNotScaled() { + assertEquals(Pair(1080, 1920), AndroidScreenCapture.scaleCaptureSize(1080, 1920)) + assertEquals(Pair(720, 1280), AndroidScreenCapture.scaleCaptureSize(720, 1280)) + } + + @Test + fun testLargeDisplayIsScaledInBothRotations() { + assertEquals(Pair(1920, 864), AndroidScreenCapture.scaleCaptureSize(3840, 1728)) + assertEquals(Pair(864, 1920), AndroidScreenCapture.scaleCaptureSize(1728, 3840)) + } + + @Test + fun testDimensionsAreEven() { + assertEquals(Pair(1920, 886), AndroidScreenCapture.scaleCaptureSize(2400, 1108)) + assertEquals(Pair(720, 1280), AndroidScreenCapture.scaleCaptureSize(721, 1281)) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt b/app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt new file mode 100644 index 00000000000..bfb833acd90 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt @@ -0,0 +1,409 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ +package com.nextcloud.talk.call + +import android.content.Intent +import com.nextcloud.talk.models.json.signaling.NCSignalingMessageDto +import com.nextcloud.talk.webrtc.PeerConnectionWrapper +import com.nextcloud.talk.webrtc.PeerConnectionWrapper.PeerConnectionObserver +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.mockito.ArgumentCaptor +import org.mockito.Mockito +import org.mockito.Mockito.verify +import org.webrtc.MediaStream +import org.webrtc.PeerConnection.IceConnectionState + +@Suppress("TooManyFunctions") +class LocalScreenShareControllerTest { + + private class FakeScreenCapture(private val canStart: Boolean = true) : ScreenCapture { + override val stream: MediaStream = Mockito.mock(MediaStream::class.java) + var isStarted = false + var isReleased = false + + override fun start(): Boolean { + isStarted = canStart + return canStart + } + + override fun release() { + isReleased = true + } + } + + private class FakeHost : LocalScreenShareController.Host { + override var hasMCU = false + override var ownSessionId: String? = OWN_SESSION_ID + override var participantSessionIds: Collection = listOf(SESSION_ID_1, SESSION_ID_2) + override var isCallActive = true + override var isScreenSharingAllowed = true + + var foregroundServiceResult: ((Boolean) -> Unit)? = null + var isForegroundServiceScreenSharing = false + var canCreateConnections = true + var capture: FakeScreenCapture? = FakeScreenCapture() + var onCaptureStop: (() -> Unit)? = null + val connections = ArrayList() + val sentMessages = ArrayList>() + var stateChangedCount = 0 + var failedCount = 0 + var encryptionFailedCount = 0 + + override fun startForegroundServiceScreenSharing(onResult: (isReady: Boolean) -> Unit) { + isForegroundServiceScreenSharing = true + foregroundServiceResult = onResult + } + + override fun stopForegroundServiceScreenSharing() { + isForegroundServiceScreenSharing = false + } + + override fun createScreenCapture(permissionResultData: Intent, onStop: () -> Unit): ScreenCapture? { + onCaptureStop = onStop + return capture + } + + override fun createPeerConnection(sessionId: String, stream: MediaStream): PeerConnectionWrapper { + check(canCreateConnections) { "No keys" } + val connection = Mockito.mock(PeerConnectionWrapper::class.java) + Mockito.`when`(connection.sessionId).thenReturn(sessionId) + connections.add(connection) + return connection + } + + override fun send(message: NCSignalingMessageDto, sessionId: String) { + assertEquals("screen", message.roomType) + sentMessages.add(Pair(message.type, sessionId)) + } + + override fun onStateChanged() { + stateChangedCount++ + } + + override fun onScreenSharingFailed() { + failedCount++ + } + + override fun onEncryptionFailed() { + encryptionFailedCount++ + } + + fun connectionTo(sessionId: String): PeerConnectionWrapper = connections.last { it.sessionId == sessionId } + + fun messages(type: String): List = sentMessages.filter { it.first == type }.map { it.second } + } + + private lateinit var host: FakeHost + private lateinit var controller: LocalScreenShareController + private val permissionResultData = Mockito.mock(Intent::class.java) + + @Before + fun setUp() { + host = FakeHost() + controller = LocalScreenShareController(host) { it.run() } + } + + private fun startSharing() { + assertTrue(controller.requestStart()) + controller.onPermissionGranted(permissionResultData) + host.foregroundServiceResult!!.invoke(true) + } + + private fun setIceConnectionState(connection: PeerConnectionWrapper, state: IceConnectionState) { + val captor = ArgumentCaptor.forClass(PeerConnectionObserver::class.java) + verify(connection).addObserver(captor.capture()) + captor.value.onIceConnectionStateChanged(state) + } + + @Test + fun testStartWithoutMcuConnectsToEachParticipant() { + startSharing() + + assertTrue(controller.isSharing) + assertFalse(controller.isPending) + assertTrue(host.capture!!.isStarted) + assertTrue(host.isForegroundServiceScreenSharing) + assertEquals(listOf(SESSION_ID_1, SESSION_ID_2), host.connections.map { it.sessionId }) + } + + @Test + fun testStartWithMcuConnectsToOwnPublisher() { + host.hasMCU = true + + startSharing() + + assertEquals(listOf(OWN_SESSION_ID), host.connections.map { it.sessionId }) + } + + @Test + fun testRequestStartIsRefusedWhilePendingOrSharing() { + assertTrue(controller.requestStart()) + assertFalse(controller.requestStart()) + + controller.onPermissionGranted(permissionResultData) + host.foregroundServiceResult!!.invoke(true) + + assertFalse(controller.requestStart()) + } + + @Test + fun testPermissionGrantedAfterStopIsIgnored() { + controller.requestStart() + controller.stop() + + controller.onPermissionGranted(permissionResultData) + + assertFalse(controller.isPending) + assertFalse(controller.isSharing) + assertFalse(host.isForegroundServiceScreenSharing) + } + + @Test + fun testForegroundServiceFailure() { + controller.requestStart() + controller.onPermissionGranted(permissionResultData) + + host.foregroundServiceResult!!.invoke(false) + + assertFalse(controller.isPending) + assertFalse(controller.isSharing) + assertEquals(1, host.failedCount) + } + + @Test + fun testCaptureFailure() { + host.capture = FakeScreenCapture(canStart = false) + + startSharing() + + assertFalse(controller.isPending) + assertFalse(controller.isSharing) + assertTrue(host.capture!!.isReleased) + assertFalse(host.isForegroundServiceScreenSharing) + assertEquals(1, host.failedCount) + assertTrue(host.connections.isEmpty()) + } + + @Test + fun testStopWhileWaitingForForegroundService() { + controller.requestStart() + controller.onPermissionGranted(permissionResultData) + controller.stop() + + host.foregroundServiceResult!!.invoke(true) + + assertFalse(controller.isSharing) + assertFalse(host.capture!!.isStarted) + assertFalse(host.isForegroundServiceScreenSharing) + } + + @Test + fun testStartIsAbortedWhenNotAllowedAnymore() { + controller.requestStart() + controller.onPermissionGranted(permissionResultData) + host.isScreenSharingAllowed = false + + host.foregroundServiceResult!!.invoke(true) + + assertFalse(controller.isSharing) + assertFalse(controller.isPending) + assertFalse(host.isForegroundServiceScreenSharing) + } + + @Test + fun testStop() { + startSharing() + val connections = host.connections.toList() + val stateChangedCount = host.stateChangedCount + + controller.stop() + controller.stop() + + assertFalse(controller.isSharing) + assertTrue(host.capture!!.isReleased) + assertFalse(host.isForegroundServiceScreenSharing) + assertEquals(listOf(SESSION_ID_1, SESSION_ID_2), host.messages("unshareScreen")) + connections.forEach { verify(it).removePeerConnection() } + assertEquals(stateChangedCount + 1, host.stateChangedCount) + } + + @Test + fun testStopWhenMediaProjectionStops() { + startSharing() + + host.onCaptureStop!!.invoke() + + assertFalse(controller.isSharing) + assertTrue(host.capture!!.isReleased) + } + + @Test + fun testOffersAreRequestedOnceThePublisherIsConnected() { + host.hasMCU = true + startSharing() + val publisher = host.connectionTo(OWN_SESSION_ID) + + controller.onParticipantJoined(SESSION_ID_3) + assertTrue(host.messages("sendoffer").isEmpty()) + + setIceConnectionState(publisher, IceConnectionState.CONNECTED) + setIceConnectionState(publisher, IceConnectionState.CONNECTED) + assertEquals(listOf(SESSION_ID_1, SESSION_ID_2), host.messages("sendoffer")) + + controller.onParticipantJoined(SESSION_ID_3) + assertEquals(listOf(SESSION_ID_1, SESSION_ID_2, SESSION_ID_3), host.messages("sendoffer")) + } + + @Test + fun testConnectedEventAfterStopDoesNotAffectNextScreenSharing() { + host.hasMCU = true + startSharing() + val oldPublisher = host.connectionTo(OWN_SESSION_ID) + controller.stop() + + setIceConnectionState(oldPublisher, IceConnectionState.CONNECTED) + assertTrue(host.messages("sendoffer").isEmpty()) + + host.capture = FakeScreenCapture() + startSharing() + setIceConnectionState(host.connectionTo(OWN_SESSION_ID), IceConnectionState.CONNECTED) + assertEquals(listOf(SESSION_ID_1, SESSION_ID_2), host.messages("sendoffer")) + } + + @Test + fun testJoinedAndLeftParticipantsWithoutMcu() { + startSharing() + + controller.onParticipantJoined(SESSION_ID_3) + val connection = host.connectionTo(SESSION_ID_3) + controller.onParticipantLeft(SESSION_ID_3) + + verify(connection).removePeerConnection() + assertEquals(3, host.connections.size) + } + + @Test + fun testParticipantsJoiningWhileNotSharingAreIgnored() { + controller.onParticipantJoined(SESSION_ID_3) + + assertTrue(host.connections.isEmpty()) + } + + @Test + fun testFailedConnectionWithoutMcuIsRetriedBeforeGivingUp() { + startSharing() + + repeat(MAX_RETRIES) { + val connection = host.connectionTo(SESSION_ID_1) + setIceConnectionState(connection, IceConnectionState.FAILED) + verify(connection).removePeerConnection() + } + assertEquals(List(MAX_RETRIES) { SESSION_ID_1 }, host.messages("unshareScreen")) + + setIceConnectionState(host.connectionTo(SESSION_ID_1), IceConnectionState.FAILED) + + // The other participants still get the screen. + assertTrue(controller.isSharing) + assertEquals(0, host.failedCount) + assertEquals(2 + MAX_RETRIES, host.connections.size) + } + + @Test + fun testSuccessfulConnectionResetsRetries() { + startSharing() + + repeat(MAX_RETRIES) { + setIceConnectionState(host.connectionTo(SESSION_ID_1), IceConnectionState.FAILED) + } + setIceConnectionState(host.connectionTo(SESSION_ID_1), IceConnectionState.CONNECTED) + setIceConnectionState(host.connectionTo(SESSION_ID_1), IceConnectionState.FAILED) + + assertEquals(2 + MAX_RETRIES + 1, host.connections.size) + } + + @Test + fun testFailedPublisherStopsScreenSharingAfterRetries() { + host.hasMCU = true + startSharing() + + repeat(MAX_RETRIES + 1) { + setIceConnectionState(host.connectionTo(OWN_SESSION_ID), IceConnectionState.FAILED) + } + + assertFalse(controller.isSharing) + assertEquals(1, host.failedCount) + assertEquals(1 + MAX_RETRIES, host.connections.size) + } + + @Test + fun testFailedConnectionIsNotRetriedWhileReconnecting() { + startSharing() + host.isCallActive = false + + setIceConnectionState(host.connectionTo(SESSION_ID_1), IceConnectionState.FAILED) + + assertEquals(2, host.connections.size) + assertTrue(controller.isSharing) + } + + @Test + fun testSuspendKeepsCapturingAndSharesAgainWithMcu() { + host.hasMCU = true + startSharing() + val publisher = host.connectionTo(OWN_SESSION_ID) + + controller.suspend() + + verify(publisher).removePeerConnection() + assertTrue(controller.isSharing) + assertFalse(host.capture!!.isReleased) + assertTrue(host.isForegroundServiceScreenSharing) + + host.ownSessionId = NEW_OWN_SESSION_ID + controller.ensurePublisher() + + assertEquals(NEW_OWN_SESSION_ID, host.connections.last().sessionId) + } + + @Test + fun testSuspendSharesAgainWithJoinedParticipantsWithoutMcu() { + startSharing() + + controller.suspend() + controller.ensurePublisher() + assertEquals(2, host.connections.size) + + controller.onParticipantJoined(SESSION_ID_1) + assertEquals(SESSION_ID_1, host.connections.last().sessionId) + assertEquals(3, host.connections.size) + } + + @Test + fun testEncryptionFailure() { + host.canCreateConnections = false + + startSharing() + + assertEquals(2, host.encryptionFailedCount) + assertTrue(host.connections.isEmpty()) + // Ending the call is up to the host. + assertTrue(controller.isSharing) + } + + companion object { + private const val OWN_SESSION_ID = "own-session-id" + private const val NEW_OWN_SESSION_ID = "new-own-session-id" + private const val SESSION_ID_1 = "session-id-1" + private const val SESSION_ID_2 = "session-id-2" + private const val SESSION_ID_3 = "session-id-3" + private const val MAX_RETRIES = 3 + } +} From e3e3ef784a3d72ae8424766832d2caff772bea0a Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 27 Oct 2026 08:02:08 +0100 Subject: [PATCH 13/16] fix(call): reset screen share retries when a participant leaves Without MCU a participant whose screen share connection failed too often did not get any retry once it joined again. Besides that, the own session is resolved in a single place and the screen sharing permission fields are declared next to the other screen sharing fields. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 45 ++++++++++--------- .../talk/call/LocalScreenShareController.kt | 2 + .../call/LocalScreenShareControllerTest.kt | 14 ++++++ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index b463d1e2238..57be51a1fd6 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -303,6 +303,24 @@ class CallActivity : CallBaseActivity() { // The permission from signaling is more recent than the one from the conversation, so it takes precedence. private var signalingScreenSharePermission: Boolean? = null + @Volatile + var isAllowedToShareScreen = false + private set + + private val screenSharePermissionListener = object : SignalingMessageReceiver.ParticipantListMessageListener { + override fun onUsersInRoom(participants: MutableList) { + updateScreenSharePermission(participants) + } + + override fun onParticipantsUpdate(participants: MutableList) { + updateScreenSharePermission(participants) + } + + override fun onAllParticipantsUpdate(inCall: Long) { + // unused atm + } + } + // Screen sharing needs the foreground service, which is not started without microphone and notification permission. @Volatile private var isCallForegroundServiceStarted = false @@ -3718,26 +3736,8 @@ class CallActivity : CallBaseActivity() { eventBus.post(ConfigurationChangeEvent()) } - @Volatile - var isAllowedToShareScreen = false - private set - - private val screenSharePermissionListener = object : SignalingMessageReceiver.ParticipantListMessageListener { - override fun onUsersInRoom(participants: MutableList) { - updateScreenSharePermission(participants) - } - - override fun onParticipantsUpdate(participants: MutableList) { - updateScreenSharePermission(participants) - } - - override fun onAllParticipantsUpdate(inCall: Long) { - // unused atm - } - } - private fun updateScreenSharePermission(participants: List) { - val ownSessionId = if (webSocketClient != null) webSocketClient!!.sessionId else callSession + val ownSessionId = getOwnSessionId() val permissions = participants.firstOrNull { it?.sessionId == ownSessionId }?.participantPermissions ?: return val isAllowed = ParticipantPermissions.canPublishScreen( conversationUser.capabilities?.spreedCapability, @@ -3814,6 +3814,11 @@ class CallActivity : CallBaseActivity() { localScreenShareController.onPermissionGranted(permissionResultData) } + /** + * The own signaling session, which differs from the call session with external signaling. + */ + private fun getOwnSessionId(): String? = if (webSocketClient != null) webSocketClient!!.sessionId else callSession + private fun isCallActiveForScreenSharing(): Boolean = isConnectionEstablished && currentCallStatus !== CallStatus.LEAVING @@ -3846,7 +3851,7 @@ class CallActivity : CallBaseActivity() { get() = this@CallActivity.hasMCU override val ownSessionId: String? - get() = if (webSocketClient != null) webSocketClient!!.sessionId else callSession + get() = getOwnSessionId() override val participantSessionIds: Collection get() = getParticipantSessionKeys() diff --git a/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt b/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt index 8d3ccdb811b..3caccf84342 100644 --- a/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt +++ b/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt @@ -284,6 +284,8 @@ class LocalScreenShareController(private val host: Host, private val runOnMainTh fun onParticipantLeft(sessionId: String?) { val connection = synchronized(lock) { + // The connection is created again from scratch if the participant joins again. + retryCounts.remove(sessionId) connections.firstOrNull { it.sessionId == sessionId }?.also { connections.remove(it) } } ?: return closeConnection(connection) diff --git a/app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt b/app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt index bfb833acd90..611e304d54e 100644 --- a/app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt +++ b/app/src/test/java/com/nextcloud/talk/call/LocalScreenShareControllerTest.kt @@ -316,6 +316,20 @@ class LocalScreenShareControllerTest { assertEquals(2 + MAX_RETRIES, host.connections.size) } + @Test + fun testRejoiningParticipantResetsRetries() { + startSharing() + repeat(MAX_RETRIES + 1) { + setIceConnectionState(host.connectionTo(SESSION_ID_1), IceConnectionState.FAILED) + } + + controller.onParticipantLeft(SESSION_ID_1) + controller.onParticipantJoined(SESSION_ID_1) + setIceConnectionState(host.connectionTo(SESSION_ID_1), IceConnectionState.FAILED) + + assertEquals(2 + MAX_RETRIES + 2, host.connections.size) + } + @Test fun testSuccessfulConnectionResetsRetries() { startSharing() From c4bbd17f532462a8497286beb9b275fc529215d1 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 27 Oct 2026 16:03:59 +0100 Subject: [PATCH 14/16] fix(call): keep the call running whenever it is in background The call task is excluded from recents, and the system removes such tasks once they are in background and another task is used, which destroyed the activity and ended the call. This happened when sharing the screen, as the call is then moved to the background instead of entering picture in picture, but also when picture in picture is not possible or was closed. Therefore the call is shown in recents whenever it is in background. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 57be51a1fd6..16145bd82ae 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -943,6 +943,7 @@ class CallActivity : CallBaseActivity() { override fun onStart() { super.onStart() active = true + updateExcludeFromRecents(true) initFeaturesVisibility() try { cache!!.evictAll() @@ -951,6 +952,33 @@ class CallActivity : CallBaseActivity() { } } + /** + * The call task is excluded from recents, so it is not shown there next to the conversations while the call is on + * screen or in picture in picture. Once in background, for example, when sharing the screen or when picture in + * picture is not possible or was closed, the system removes background tasks excluded from recents as soon as + * another task is used, which destroys the activity and ends the call. Therefore the call is shown in recents while + * in background, so it keeps running until it is ended or removed from recents. + */ + private fun updateExcludeFromRecents(isExcluded: Boolean) { + if (isFinishing || isDestroyed) { + return + } + try { + getSystemService(ActivityManager::class.java).appTasks + .firstOrNull { it.taskInfo?.let { taskInfo -> taskIdOf(taskInfo) } == taskId } + ?.setExcludeFromRecents(isExcluded) + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Failed to update whether the call is excluded from recents", e) + } + } + + /** + * TaskInfo.taskId is only available since API 29, before that the task ID is provided as persistentId. + */ + @Suppress("DEPRECATION") + private fun taskIdOf(taskInfo: ActivityManager.RecentTaskInfo): Int = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) taskInfo.taskId else taskInfo.persistentId + override fun onStop() { super.onStop() Log.d( @@ -959,6 +987,7 @@ class CallActivity : CallBaseActivity() { " isFinishing=$isFinishing isChangingConfigurations=$isChangingConfigurations" ) active = false + updateExcludeFromRecents(false) if (isMicInputAudioThreadRunning) { stopMicInputDetection() From f1c58b0917181bddb798f7e62ca62a0a29ccde78 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Wed, 28 Oct 2026 20:02:55 +0100 Subject: [PATCH 15/16] refactor(call): use taskIdCompat to read the task id of the call The task id is read from persistentId before API 29 by the existing taskIdCompat helper, so the separate taskIdOf helper is not needed. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Marcel Hibbe --- .../java/com/nextcloud/talk/activities/CallActivity.kt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 16145bd82ae..54f32418158 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -965,20 +965,13 @@ class CallActivity : CallBaseActivity() { } try { getSystemService(ActivityManager::class.java).appTasks - .firstOrNull { it.taskInfo?.let { taskInfo -> taskIdOf(taskInfo) } == taskId } + .firstOrNull { it.taskInfo?.taskIdCompat == taskId } ?.setExcludeFromRecents(isExcluded) } catch (e: IllegalArgumentException) { Log.w(TAG, "Failed to update whether the call is excluded from recents", e) } } - /** - * TaskInfo.taskId is only available since API 29, before that the task ID is provided as persistentId. - */ - @Suppress("DEPRECATION") - private fun taskIdOf(taskInfo: ActivityManager.RecentTaskInfo): Int = - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) taskInfo.taskId else taskInfo.persistentId - override fun onStop() { super.onStop() Log.d( From ca58d7e0bc0e811e216def69ff7a835dbccc1ccb Mon Sep 17 00:00:00 2001 From: rapterjet2004 Date: Tue, 3 Nov 2026 13:04:27 -0600 Subject: [PATCH 16/16] fix(call): harden screen sharing teardown and dialog reuse Address review findings on the screen sharing changes: - ScreenCapture: only stop the capturer in release() if it was actually started, so a failed startCapture() no longer calls stopCapture() on a capturer whose media projection was never set up. - CallForegroundService: also catch IllegalStateException (e.g. ForegroundServiceStartNotAllowedException) when reconfiguring the foreground service type, which can be thrown while in background. - CallActivity: always create a fresh MoreCallActionsDialog from the recording indicator, as reusing a dismissed instance leaves its lifecycle-scoped observers inactive. - LocalScreenShareController: only notify a state change in cancelPending() when it actually was pending, mirroring stop(). Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Julius Linus Signed-off-by: rapterjet2004 --- .../nextcloud/talk/activities/CallActivity.kt | 6 ++--- .../talk/call/LocalScreenShareController.kt | 8 ++++--- .../com/nextcloud/talk/call/ScreenCapture.kt | 6 ++++- .../talk/services/CallForegroundService.kt | 22 +++++++++++++------ 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 54f32418158..b25e4cb6a37 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -1050,9 +1050,9 @@ class CallActivity : CallBaseActivity() { binding!!.callRecordingIndicator.setOnClickListener { if (isAllowedToStartOrStopRecording) { if (callRecordingViewModel!!.viewState.value is RecordingStartingState) { - if (moreCallActionsDialog == null) { - moreCallActionsDialog = MoreCallActionsDialog(this) - } + // A fresh instance is created every time, as the dialog's lifecycle is destroyed on dismiss and + // reusing the instance would leave its lifecycle-scoped observers inactive. + moreCallActionsDialog = MoreCallActionsDialog(this) moreCallActionsDialog!!.show() } else { callRecordingViewModel!!.clickRecordButton() diff --git a/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt b/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt index 3caccf84342..3b3910296be 100644 --- a/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt +++ b/app/src/main/java/com/nextcloud/talk/call/LocalScreenShareController.kt @@ -123,10 +123,12 @@ class LocalScreenShareController(private val host: Host, private val runOnMainTh } fun cancelPending() { - synchronized(lock) { - isPendingInternal = false + val wasPending = synchronized(lock) { + isPendingInternal.also { isPendingInternal = false } + } + if (wasPending) { + notifyStateChanged() } - notifyStateChanged() } /** diff --git a/app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt b/app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt index 55b5caf3f97..5fc582e5a2a 100644 --- a/app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt +++ b/app/src/main/java/com/nextcloud/talk/call/ScreenCapture.kt @@ -89,6 +89,7 @@ class AndroidScreenCapture( // Guards the capturer, as the size is changed from the main thread while it may be released from any thread. private val lock = Any() private var captureSize: Pair? = null + private var isCapturing = false private var isReleased = false override fun start(): Boolean { @@ -111,6 +112,7 @@ class AndroidScreenCapture( // The media projection may fail to start, for example, if the permission expired or was revoked. return try { capturer.startCapture(size.first, size.second, FRAME_RATE) + isCapturing = true true } catch (e: SecurityException) { Log.e(TAG, "Failed to start screen capture", e) @@ -129,7 +131,9 @@ class AndroidScreenCapture( return } isReleased = true - if (captureSize != null) { + // Only stop the capturer if it was actually started; stopping a capturer that failed to start (so its + // media projection was never set up) would throw. + if (isCapturing) { try { capturer.stopCapture() } catch (e: InterruptedException) { diff --git a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt index 5d62ab2ab7a..816dc5196ea 100644 --- a/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt +++ b/app/src/main/java/com/nextcloud/talk/services/CallForegroundService.kt @@ -165,18 +165,26 @@ class CallForegroundService : Service() { startForeground(NOTIFICATION_ID, notification) } } catch (e: SecurityException) { - Log.e(TAG, "Failed to update foreground service type for screen sharing", e) - isScreenSharing = false - resultCode = RESULT_SCREEN_SHARING_FAILED - // The type may not be changed while in background, but the notification must not keep stating that the - // screen is shared. - getSystemService(NotificationManager::class.java) - .notify(NOTIFICATION_ID, buildNotification(currentConversationName, currentCallExtras)) + resultCode = handleScreenSharingTypeUpdateFailure(e) + } catch (e: IllegalStateException) { + // Reconfiguring the foreground service while in background may not be allowed, which throws a + // ForegroundServiceStartNotAllowedException (an IllegalStateException), not a SecurityException. + resultCode = handleScreenSharingTypeUpdateFailure(e) } resultReceiver?.send(resultCode, null) } + private fun handleScreenSharingTypeUpdateFailure(e: Exception): Int { + Log.e(TAG, "Failed to update foreground service type for screen sharing", e) + isScreenSharing = false + // The type may not be changed while in background, but the notification must not keep stating that the + // screen is shared. + getSystemService(NotificationManager::class.java) + .notify(NOTIFICATION_ID, buildNotification(currentConversationName, currentCallExtras)) + return RESULT_SCREEN_SHARING_FAILED + } + override fun onDestroy() { Log.d(TAG, "onDestroy called") handler.removeCallbacksAndMessages(null)