From a8415a34843315b4a6510b77fbdc9461313671fb Mon Sep 17 00:00:00 2001 From: Cody Henthorne Date: Fri, 18 Sep 2020 15:51:17 -0400 Subject: [PATCH] Add pre-join vanity view for 1:1 video calls. --- .../securesms/WebRtcCallActivity.java | 76 ++++----- .../webrtc/CallParticipantsState.java | 8 +- .../webrtc/TextureViewRenderer.java | 10 +- .../components/webrtc/WebRtcCallView.java | 147 ++++++++++++------ .../webrtc/WebRtcCallViewModel.java | 7 +- .../components/webrtc/WebRtcControls.java | 15 +- .../webrtc/WebRtcLocalRenderState.java | 3 +- .../securesms/events/WebRtcViewModel.java | 1 + .../securesms/ringrtc/Camera.java | 26 ++-- .../securesms/ringrtc/CameraState.java | 17 +- .../securesms/service/WebRtcCallService.java | 123 ++++++++++++--- .../securesms/util/CommunicationActions.java | 15 +- app/src/main/res/layout/webrtc_call_view.xml | 39 +++-- .../webrtc_call_view_large_local_render.xml | 32 ++++ app/src/main/res/values/strings.xml | 1 + 15 files changed, 353 insertions(+), 167 deletions(-) create mode 100644 app/src/main/res/layout/webrtc_call_view_large_local_render.xml diff --git a/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java b/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java index c5d63a8c8..f0a3ef251 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java +++ b/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java @@ -19,8 +19,6 @@ package org.thoughtcrime.securesms; import android.Manifest; import android.app.PictureInPictureParams; -import android.content.DialogInterface; -import android.content.DialogInterface.OnClickListener; import android.content.Intent; import android.content.pm.PackageManager; import android.content.res.Configuration; @@ -42,10 +40,10 @@ import org.greenrobot.eventbus.Subscribe; import org.greenrobot.eventbus.ThreadMode; import org.thoughtcrime.securesms.components.TooltipPopup; import org.thoughtcrime.securesms.components.webrtc.CallParticipantsState; -import org.thoughtcrime.securesms.components.webrtc.participantslist.CallParticipantsListDialog; import org.thoughtcrime.securesms.components.webrtc.WebRtcAudioOutput; import org.thoughtcrime.securesms.components.webrtc.WebRtcCallView; import org.thoughtcrime.securesms.components.webrtc.WebRtcCallViewModel; +import org.thoughtcrime.securesms.components.webrtc.participantslist.CallParticipantsListDialog; import org.thoughtcrime.securesms.conversation.ui.error.SafetyNumberChangeDialog; import org.thoughtcrime.securesms.events.WebRtcViewModel; import org.thoughtcrime.securesms.logging.Log; @@ -57,7 +55,6 @@ import org.thoughtcrime.securesms.service.WebRtcCallService; import org.thoughtcrime.securesms.sms.MessageSender; import org.thoughtcrime.securesms.util.EllapsedTimeFormatter; import org.thoughtcrime.securesms.util.TextSecurePreferences; -import org.thoughtcrime.securesms.util.ViewUtil; import org.whispersystems.libsignal.IdentityKey; import org.whispersystems.signalservice.api.messages.calls.HangupMessage; import org.whispersystems.signalservice.api.messages.calls.OfferMessage; @@ -89,6 +86,7 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe requestWindowFeature(Window.FEATURE_NO_TITLE); setContentView(R.layout.webrtc_call_activity); + //noinspection ConstantConditions getSupportActionBar().hide(); setVolumeControlStream(AudioManager.STREAM_VOICE_CALL); @@ -136,11 +134,13 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe super.onStop(); EventBus.getDefault().unregister(this); - } - @Override - public void onConfigurationChanged(Configuration newConfiguration) { - super.onConfigurationChanged(newConfiguration); + CallParticipantsState state = viewModel.getCallParticipantsState().getValue(); + if (state != null && state.getCallState() == WebRtcViewModel.State.CALL_PRE_JOIN) { + Intent intent = new Intent(this, WebRtcCallService.class); + intent.setAction(WebRtcCallService.ACTION_CANCEL_PRE_JOIN_CALL); + startService(intent); + } } @Override @@ -200,7 +200,7 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe } private void initializeResources() { - callScreen = ViewUtil.findById(this, R.id.callScreen); + callScreen = findViewById(R.id.callScreen); callScreen.setControlsListener(new ControlsListener()); } @@ -376,7 +376,7 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe startService(intent); } - private void handleOutgoingCall(@NonNull WebRtcViewModel event) { + private void handleOutgoingCall() { callScreen.setStatus(getString(R.string.WebRtcCallActivity__calling)); } @@ -393,27 +393,27 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe delayedFinish(); } - private void handleCallRinging(@NonNull WebRtcViewModel event) { + private void handleCallRinging() { callScreen.setStatus(getString(R.string.RedPhone_ringing)); } - private void handleCallBusy(@NonNull WebRtcViewModel event) { + private void handleCallBusy() { EventBus.getDefault().removeStickyEvent(WebRtcViewModel.class); callScreen.setStatus(getString(R.string.RedPhone_busy)); delayedFinish(WebRtcCallService.BUSY_TONE_LENGTH); } - private void handleCallConnected(@NonNull WebRtcViewModel event) { + private void handleCallConnected() { getWindow().addFlags(WindowManager.LayoutParams.FLAG_IGNORE_CHEEK_PRESSES); } - private void handleRecipientUnavailable(@NonNull WebRtcViewModel event) { + private void handleRecipientUnavailable() { EventBus.getDefault().removeStickyEvent(WebRtcViewModel.class); callScreen.setStatus(getString(R.string.RedPhone_recipient_unavailable)); delayedFinish(); } - private void handleServerFailure(@NonNull WebRtcViewModel event) { + private void handleServerFailure() { EventBus.getDefault().removeStickyEvent(WebRtcViewModel.class); callScreen.setStatus(getString(R.string.RedPhone_network_failed)); delayedFinish(); @@ -421,24 +421,14 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe private void handleNoSuchUser(final @NonNull WebRtcViewModel event) { if (isFinishing()) return; // XXX Stuart added this check above, not sure why, so I'm repeating in ignorance. - moxie - AlertDialog.Builder dialog = new AlertDialog.Builder(this); - dialog.setTitle(R.string.RedPhone_number_not_registered); - dialog.setIconAttribute(R.attr.dialog_alert_icon); - dialog.setMessage(R.string.RedPhone_the_number_you_dialed_does_not_support_secure_voice); - dialog.setCancelable(true); - dialog.setPositiveButton(R.string.RedPhone_got_it, new OnClickListener() { - @Override - public void onClick(DialogInterface dialog, int which) { - WebRtcCallActivity.this.handleTerminate(event.getRecipient(), HangupMessage.Type.NORMAL); - } - }); - dialog.setOnCancelListener(new DialogInterface.OnCancelListener() { - @Override - public void onCancel(DialogInterface dialog) { - WebRtcCallActivity.this.handleTerminate(event.getRecipient(), HangupMessage.Type.NORMAL); - } - }); - dialog.show(); + new AlertDialog.Builder(this) + .setTitle(R.string.RedPhone_number_not_registered) + .setIconAttribute(R.attr.dialog_alert_icon) + .setMessage(R.string.RedPhone_the_number_you_dialed_does_not_support_secure_voice) + .setCancelable(true) + .setPositiveButton(R.string.RedPhone_got_it, (d, w) -> handleTerminate(event.getRecipient(), HangupMessage.Type.NORMAL)) + .setOnCancelListener(d -> handleTerminate(event.getRecipient(), HangupMessage.Type.NORMAL)) + .show(); } private void handleUntrustedIdentity(@NonNull WebRtcViewModel event) { @@ -490,18 +480,18 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe callScreen.setRecipient(event.getRecipient()); switch (event.getState()) { - case CALL_CONNECTED: handleCallConnected(event); break; - case NETWORK_FAILURE: handleServerFailure(event); break; - case CALL_RINGING: handleCallRinging(event); break; + case CALL_CONNECTED: handleCallConnected(); break; + case NETWORK_FAILURE: handleServerFailure(); break; + case CALL_RINGING: handleCallRinging(); break; case CALL_DISCONNECTED: handleTerminate(event.getRecipient(), HangupMessage.Type.NORMAL); break; case CALL_ACCEPTED_ELSEWHERE: handleTerminate(event.getRecipient(), HangupMessage.Type.ACCEPTED); break; case CALL_DECLINED_ELSEWHERE: handleTerminate(event.getRecipient(), HangupMessage.Type.DECLINED); break; case CALL_ONGOING_ELSEWHERE: handleTerminate(event.getRecipient(), HangupMessage.Type.BUSY); break; case CALL_NEEDS_PERMISSION: handleTerminate(event.getRecipient(), HangupMessage.Type.NEED_PERMISSION); break; case NO_SUCH_USER: handleNoSuchUser(event); break; - case RECIPIENT_UNAVAILABLE: handleRecipientUnavailable(event); break; - case CALL_OUTGOING: handleOutgoingCall(event); break; - case CALL_BUSY: handleCallBusy(event); break; + case RECIPIENT_UNAVAILABLE: handleRecipientUnavailable(); break; + case CALL_OUTGOING: handleOutgoingCall(); break; + case CALL_BUSY: handleCallBusy(); break; case UNTRUSTED_IDENTITY: handleUntrustedIdentity(event); break; } @@ -518,12 +508,14 @@ public class WebRtcCallActivity extends AppCompatActivity implements SafetyNumbe private final class ControlsListener implements WebRtcCallView.ControlsListener { @Override - public void onStartCall() { + public void onStartCall(boolean isVideoCall) { + enableVideoIfAvailable = isVideoCall; + Intent intent = new Intent(WebRtcCallActivity.this, WebRtcCallService.class); intent.setAction(WebRtcCallService.ACTION_OUTGOING_CALL) .putExtra(WebRtcCallService.EXTRA_REMOTE_PEER, new RemotePeer(viewModel.getRecipient().getId())) - .putExtra(WebRtcCallService.EXTRA_OFFER_TYPE, OfferMessage.Type.VIDEO_CALL.getCode()); - WebRtcCallActivity.this.startService(intent); + .putExtra(WebRtcCallService.EXTRA_OFFER_TYPE, (isVideoCall ? OfferMessage.Type.VIDEO_CALL : OfferMessage.Type.AUDIO_CALL).getCode()); + startService(intent); MessageSender.onMessageSent(); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantsState.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantsState.java index 297999946..0981b9dd5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantsState.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantsState.java @@ -57,6 +57,10 @@ public final class CallParticipantsState { this.isViewingFocusedParticipant = isViewingFocusedParticipant; } + public @NonNull WebRtcViewModel.State getCallState() { + return callState; + } + public @NonNull List getGridParticipants() { if (getAllRemoteParticipants().size() > SMALL_GROUP_MAX) { return getAllRemoteParticipants().subList(0, SMALL_GROUP_MAX); @@ -194,9 +198,11 @@ public final class CallParticipantsState { } else { localRenderState = WebRtcLocalRenderState.SMALL_RECTANGLE; } - } else { + } else if (callState != WebRtcViewModel.State.CALL_DISCONNECTED) { localRenderState = WebRtcLocalRenderState.LARGE; } + } else if (callState == WebRtcViewModel.State.CALL_PRE_JOIN) { + localRenderState = WebRtcLocalRenderState.LARGE_NO_VIDEO; } return localRenderState; diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/TextureViewRenderer.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/TextureViewRenderer.java index 5415152a8..199a81b82 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/TextureViewRenderer.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/TextureViewRenderer.java @@ -70,12 +70,18 @@ public class TextureViewRenderer extends TextureView implements TextureView.Surf } public void attachBroadcastVideoSink(@Nullable BroadcastVideoSink videoSink) { + if (attachedVideoSink == videoSink) { + return; + } + if (attachedVideoSink != null) { attachedVideoSink.removeSink(this); } if (videoSink != null) { videoSink.addSink(this); + } else { + clearImage(); } attachedVideoSink = videoSink; @@ -232,9 +238,7 @@ public class TextureViewRenderer extends TextureView implements TextureView.Surf @Override public void onFrame(VideoFrame videoFrame) { - if (isShown()) { - eglRenderer.onFrame(videoFrame); - } + eglRenderer.onFrame(videoFrame); } @Override diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java index 95e85e53f..826d7a7fa 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java @@ -1,6 +1,8 @@ package org.thoughtcrime.securesms.components.webrtc; import android.content.Context; +import android.graphics.ColorMatrix; +import android.graphics.ColorMatrixColorFilter; import android.util.AttributeSet; import android.view.LayoutInflater; import android.view.View; @@ -27,14 +29,20 @@ import androidx.transition.TransitionSet; import androidx.viewpager2.widget.MarginPageTransformer; import androidx.viewpager2.widget.ViewPager2; +import com.bumptech.glide.load.engine.DiskCacheStrategy; +import com.bumptech.glide.load.resource.bitmap.CenterCrop; + import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.animation.ResizeAnimation; import org.thoughtcrime.securesms.components.AccessibleToggleButton; +import org.thoughtcrime.securesms.contacts.avatars.ProfileContactPhoto; import org.thoughtcrime.securesms.events.CallParticipant; import org.thoughtcrime.securesms.mediasend.SimpleAnimationListener; +import org.thoughtcrime.securesms.mms.GlideApp; import org.thoughtcrime.securesms.recipients.Recipient; import org.thoughtcrime.securesms.recipients.RecipientId; import org.thoughtcrime.securesms.ringrtc.CameraState; +import org.thoughtcrime.securesms.util.BlurTransformation; import org.thoughtcrime.securesms.util.SetUtil; import org.thoughtcrime.securesms.util.ViewUtil; import org.webrtc.RendererCommon; @@ -58,10 +66,12 @@ public class WebRtcCallView extends FrameLayout { private WebRtcAudioOutputToggleButton audioToggle; private AccessibleToggleButton videoToggle; private AccessibleToggleButton micToggle; - private ViewGroup localRenderPipFrame; + private ViewGroup smallLocalRenderFrame; private TextureViewRenderer smallLocalRender; private View largeLocalRenderFrame; private TextureViewRenderer largeLocalRender; + private View largeLocalRenderNoVideo; + private ImageView largeLocalRenderNoVideoAvatar; private TextView recipientName; private TextView status; private ConstraintLayout parent; @@ -74,7 +84,7 @@ public class WebRtcCallView extends FrameLayout { private ImageView hangup; private View answerWithAudio; private View answerWithAudioLabel; - private View ongoingFooterGradient; + private View footerGradient; private View startCallControls; private ViewPager2 callParticipantsPager; private RecyclerView callParticipantsRecycler; @@ -110,33 +120,34 @@ public class WebRtcCallView extends FrameLayout { protected void onFinishInflate() { super.onFinishInflate(); - audioToggle = findViewById(R.id.call_screen_speaker_toggle); - videoToggle = findViewById(R.id.call_screen_video_toggle); - micToggle = findViewById(R.id.call_screen_audio_mic_toggle); - localRenderPipFrame = findViewById(R.id.call_screen_pip); - smallLocalRender = findViewById(R.id.call_screen_small_local_renderer); - largeLocalRenderFrame = findViewById(R.id.call_screen_large_local_renderer_frame); - largeLocalRender = findViewById(R.id.call_screen_large_local_renderer); - recipientName = findViewById(R.id.call_screen_recipient_name); - status = findViewById(R.id.call_screen_status); - parent = findViewById(R.id.call_screen); - participantsParent = findViewById(R.id.call_screen_participants_parent); - answer = findViewById(R.id.call_screen_answer_call); - cameraDirectionToggle = findViewById(R.id.call_screen_camera_direction_toggle); - hangup = findViewById(R.id.call_screen_end_call); - answerWithAudio = findViewById(R.id.call_screen_answer_with_audio); - answerWithAudioLabel = findViewById(R.id.call_screen_answer_with_audio_label); - ongoingFooterGradient = findViewById(R.id.call_screen_ongoing_footer_gradient); - startCallControls = findViewById(R.id.call_screen_start_call_controls); - callParticipantsPager = findViewById(R.id.call_screen_participants_pager); - callParticipantsRecycler = findViewById(R.id.call_screen_participants_recycler); - toolbar = findViewById(R.id.call_screen_toolbar); + audioToggle = findViewById(R.id.call_screen_speaker_toggle); + videoToggle = findViewById(R.id.call_screen_video_toggle); + micToggle = findViewById(R.id.call_screen_audio_mic_toggle); + smallLocalRenderFrame = findViewById(R.id.call_screen_pip); + smallLocalRender = findViewById(R.id.call_screen_small_local_renderer); + largeLocalRenderFrame = findViewById(R.id.call_screen_large_local_renderer_frame); + largeLocalRender = findViewById(R.id.call_screen_large_local_renderer); + largeLocalRenderNoVideo = findViewById(R.id.call_screen_large_local_video_off); + largeLocalRenderNoVideoAvatar = findViewById(R.id.call_screen_large_local_video_off_avatar); + recipientName = findViewById(R.id.call_screen_recipient_name); + status = findViewById(R.id.call_screen_status); + parent = findViewById(R.id.call_screen); + participantsParent = findViewById(R.id.call_screen_participants_parent); + answer = findViewById(R.id.call_screen_answer_call); + cameraDirectionToggle = findViewById(R.id.call_screen_camera_direction_toggle); + hangup = findViewById(R.id.call_screen_end_call); + answerWithAudio = findViewById(R.id.call_screen_answer_with_audio); + answerWithAudioLabel = findViewById(R.id.call_screen_answer_with_audio_label); + footerGradient = findViewById(R.id.call_screen_footer_gradient); + startCallControls = findViewById(R.id.call_screen_start_call_controls); + callParticipantsPager = findViewById(R.id.call_screen_participants_pager); + callParticipantsRecycler = findViewById(R.id.call_screen_participants_recycler); + toolbar = findViewById(R.id.call_screen_toolbar); View topGradient = findViewById(R.id.call_screen_header_gradient); View decline = findViewById(R.id.call_screen_decline_call); View answerLabel = findViewById(R.id.call_screen_answer_call_label); View declineLabel = findViewById(R.id.call_screen_decline_call_label); - View incomingFooterGradient = findViewById(R.id.call_screen_incoming_footer_gradient); Guideline statusBarGuideline = findViewById(R.id.call_screen_status_bar_guideline); View startCall = findViewById(R.id.call_screen_start_call_start_call); View cancelStartCall = findViewById(R.id.call_screen_start_call_cancel); @@ -163,7 +174,7 @@ public class WebRtcCallView extends FrameLayout { incomingCallViews.add(answerLabel); incomingCallViews.add(decline); incomingCallViews.add(declineLabel); - incomingCallViews.add(incomingFooterGradient); + incomingCallViews.add(footerGradient); adjustableMarginsSet.add(micToggle); adjustableMarginsSet.add(cameraDirectionToggle); @@ -190,11 +201,16 @@ public class WebRtcCallView extends FrameLayout { answer.setOnClickListener(v -> runIfNonNull(controlsListener, ControlsListener::onAcceptCallPressed)); answerWithAudio.setOnClickListener(v -> runIfNonNull(controlsListener, ControlsListener::onAcceptCallWithVoiceOnlyPressed)); - pictureInPictureGestureHelper = PictureInPictureGestureHelper.applyTo(localRenderPipFrame); + pictureInPictureGestureHelper = PictureInPictureGestureHelper.applyTo(smallLocalRenderFrame); - startCall.setOnClickListener(v -> runIfNonNull(controlsListener, ControlsListener::onStartCall)); + startCall.setOnClickListener(v -> runIfNonNull(controlsListener, listener -> listener.onStartCall(videoToggle.isChecked()))); cancelStartCall.setOnClickListener(v -> runIfNonNull(controlsListener, ControlsListener::onCancelStartCall)); + ColorMatrix greyScaleMatrix = new ColorMatrix(); + greyScaleMatrix.setSaturation(0); + largeLocalRenderNoVideoAvatar.setAlpha(0.6f); + largeLocalRenderNoVideoAvatar.setColorFilter(new ColorMatrixColorFilter(greyScaleMatrix)); + int statusBarHeight = ViewUtil.getStatusBarHeight(this); statusBarGuideline.setGuidelineBegin(statusBarHeight); } @@ -245,8 +261,6 @@ public class WebRtcCallView extends FrameLayout { } public void updateLocalCallParticipant(@NonNull WebRtcLocalRenderState state, @NonNull CallParticipant localCallParticipant) { - videoToggle.setChecked(state != WebRtcLocalRenderState.GONE, false); - smallLocalRender.setMirror(localCallParticipant.getCameraDirection() == CameraState.Direction.FRONT); largeLocalRender.setMirror(localCallParticipant.getCameraDirection() == CameraState.Direction.FRONT); @@ -258,27 +272,65 @@ public class WebRtcCallView extends FrameLayout { largeLocalRender.init(localCallParticipant.getVideoSink().getEglBase()); } - smallLocalRender.attachBroadcastVideoSink(localCallParticipant.getVideoSink()); - largeLocalRender.attachBroadcastVideoSink(localCallParticipant.getVideoSink()); - switch (state) { - case LARGE: - largeLocalRenderFrame.setVisibility(View.VISIBLE); - localRenderPipFrame.setVisibility(View.GONE); - break; case GONE: + largeLocalRender.attachBroadcastVideoSink(null); largeLocalRenderFrame.setVisibility(View.GONE); - localRenderPipFrame.setVisibility(View.GONE); + smallLocalRender.attachBroadcastVideoSink(null); + smallLocalRenderFrame.setVisibility(View.GONE); + + videoToggle.setChecked(false, false); break; case SMALL_RECTANGLE: - largeLocalRenderFrame.setVisibility(View.GONE); - localRenderPipFrame.setVisibility(View.VISIBLE); + smallLocalRenderFrame.setVisibility(View.VISIBLE); + smallLocalRender.attachBroadcastVideoSink(localCallParticipant.getVideoSink()); animatePipToRectangle(); + + largeLocalRender.attachBroadcastVideoSink(null); + largeLocalRenderFrame.setVisibility(View.GONE); + + videoToggle.setChecked(true, false); break; case SMALL_SQUARE: - largeLocalRenderFrame.setVisibility(View.GONE); - localRenderPipFrame.setVisibility(View.VISIBLE); + smallLocalRenderFrame.setVisibility(View.VISIBLE); + smallLocalRender.attachBroadcastVideoSink(localCallParticipant.getVideoSink()); animatePipToSquare(); + + largeLocalRender.attachBroadcastVideoSink(null); + largeLocalRenderFrame.setVisibility(View.GONE); + + videoToggle.setChecked(true, false); + break; + case LARGE: + largeLocalRender.attachBroadcastVideoSink(localCallParticipant.getVideoSink()); + largeLocalRenderFrame.setVisibility(View.VISIBLE); + + largeLocalRenderNoVideo.setVisibility(View.GONE); + largeLocalRenderNoVideoAvatar.setVisibility(View.GONE); + + smallLocalRender.attachBroadcastVideoSink(null); + smallLocalRenderFrame.setVisibility(View.GONE); + + videoToggle.setChecked(true, false); + break; + case LARGE_NO_VIDEO: + largeLocalRender.attachBroadcastVideoSink(null); + largeLocalRenderFrame.setVisibility(View.VISIBLE); + + largeLocalRenderNoVideo.setVisibility(View.VISIBLE); + largeLocalRenderNoVideoAvatar.setVisibility(View.VISIBLE); + + GlideApp.with(getContext().getApplicationContext()) + .load(new ProfileContactPhoto(localCallParticipant.getRecipient(), localCallParticipant.getRecipient().getProfileAvatar())) + .transform(new CenterCrop(), new BlurTransformation(getContext(), 0.25f, BlurTransformation.MAX_RADIUS)) + .diskCacheStrategy(DiskCacheStrategy.ALL) + .into(largeLocalRenderNoVideoAvatar); + + smallLocalRender.attachBroadcastVideoSink(null); + smallLocalRenderFrame.setVisibility(View.GONE); + + videoToggle.setChecked(false, false); + break; } } @@ -330,6 +382,7 @@ public class WebRtcCallView extends FrameLayout { visibleViewSet.clear(); if (webRtcControls.displayStartCallControls()) { + visibleViewSet.add(footerGradient); visibleViewSet.add(startCallControls); } @@ -367,7 +420,7 @@ public class WebRtcCallView extends FrameLayout { if (webRtcControls.displayEndCall()) { visibleViewSet.add(hangup); - visibleViewSet.add(ongoingFooterGradient); + visibleViewSet.add(footerGradient); } if (webRtcControls.displayMuteAudio()) { @@ -411,7 +464,7 @@ public class WebRtcCallView extends FrameLayout { } private void animatePipToRectangle() { - ResizeAnimation animation = new ResizeAnimation(localRenderPipFrame, ViewUtil.dpToPx(90), ViewUtil.dpToPx(160)); + ResizeAnimation animation = new ResizeAnimation(smallLocalRenderFrame, ViewUtil.dpToPx(90), ViewUtil.dpToPx(160)); animation.setDuration(PIP_RESIZE_DURATION); animation.setAnimationListener(new SimpleAnimationListener() { @Override @@ -421,14 +474,14 @@ public class WebRtcCallView extends FrameLayout { } }); - localRenderPipFrame.startAnimation(animation); + smallLocalRenderFrame.startAnimation(animation); } private void animatePipToSquare() { pictureInPictureGestureHelper.lockToBottomEnd(); pictureInPictureGestureHelper.performAfterFling(() -> { - ResizeAnimation animation = new ResizeAnimation(localRenderPipFrame, ViewUtil.dpToPx(72), ViewUtil.dpToPx(72)); + ResizeAnimation animation = new ResizeAnimation(smallLocalRenderFrame, ViewUtil.dpToPx(72), ViewUtil.dpToPx(72)); animation.setDuration(PIP_RESIZE_DURATION); animation.setAnimationListener(new SimpleAnimationListener() { @Override @@ -437,7 +490,7 @@ public class WebRtcCallView extends FrameLayout { } }); - localRenderPipFrame.startAnimation(animation); + smallLocalRenderFrame.startAnimation(animation); }); } @@ -580,7 +633,7 @@ public class WebRtcCallView extends FrameLayout { } public interface ControlsListener { - void onStartCall(); + void onStartCall(boolean isVideoCall); void onCancelStartCall(); void onControlsFadeOut(); void onAudioOutputChanged(@NonNull WebRtcAudioOutput audioOutput); diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallViewModel.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallViewModel.java index b428ed7cf..5acc4dbdd 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallViewModel.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallViewModel.java @@ -94,7 +94,7 @@ public class WebRtcCallViewModel extends ViewModel { @MainThread public void updateFromWebRtcViewModel(@NonNull WebRtcViewModel webRtcViewModel, boolean enableVideo) { - canEnterPipMode = true; + canEnterPipMode = webRtcViewModel.getState() != WebRtcViewModel.State.CALL_PRE_JOIN; CallParticipant localParticipant = webRtcViewModel.getLocalParticipant(); @@ -143,6 +143,9 @@ public class WebRtcCallViewModel extends ViewModel { final WebRtcControls.CallState callState; switch (state) { + case CALL_PRE_JOIN: + callState = WebRtcControls.CallState.PRE_JOIN; + break; case CALL_INCOMING: callState = WebRtcControls.CallState.INCOMING; answerWithVideoAvailable = isRemoteVideoOffer; @@ -167,7 +170,7 @@ public class WebRtcCallViewModel extends ViewModel { isRemoteVideoEnabled || isRemoteVideoOffer, isMoreThanOneCameraAvailable, isBluetoothAvailable, - isInPipMode.getValue() == Boolean.TRUE, + Boolean.TRUE.equals(isInPipMode.getValue()), callState, audioOutput)); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcControls.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcControls.java index 2c1732f2c..7f3c252a6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcControls.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcControls.java @@ -37,7 +37,7 @@ public final class WebRtcControls { } boolean displayStartCallControls() { - return false; + return isPreJoin(); } boolean displayEndCall() { @@ -45,19 +45,19 @@ public final class WebRtcControls { } boolean displayMuteAudio() { - return isAtLeastOutgoing(); + return isPreJoin() || isAtLeastOutgoing(); } boolean displayVideoToggle() { - return isAtLeastOutgoing(); + return isPreJoin() || isAtLeastOutgoing(); } boolean displayAudioToggle() { - return isAtLeastOutgoing() && (!isLocalVideoEnabled || isBluetoothAvailable); + return (isPreJoin() || isAtLeastOutgoing()) && (!isLocalVideoEnabled || isBluetoothAvailable); } boolean displayCameraToggle() { - return isAtLeastOutgoing() && isLocalVideoEnabled && isMoreThanOneCameraAvailable; + return (isPreJoin() || isAtLeastOutgoing()) && isLocalVideoEnabled && isMoreThanOneCameraAvailable; } boolean displayRemoteVideoRecycler() { @@ -100,6 +100,10 @@ public final class WebRtcControls { return audioOutput; } + private boolean isPreJoin() { + return callState == CallState.PRE_JOIN; + } + private boolean isOngoing() { return callState == CallState.ONGOING; } @@ -114,6 +118,7 @@ public final class WebRtcControls { public enum CallState { NONE, + PRE_JOIN, INCOMING, OUTGOING, ONGOING, diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcLocalRenderState.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcLocalRenderState.java index 13f5fa2d9..60fa4494d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcLocalRenderState.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcLocalRenderState.java @@ -4,5 +4,6 @@ public enum WebRtcLocalRenderState { GONE, SMALL_RECTANGLE, SMALL_SQUARE, - LARGE + LARGE, + LARGE_NO_VIDEO } diff --git a/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java b/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java index 08dfdff91..608ee5af0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java +++ b/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java @@ -14,6 +14,7 @@ public class WebRtcViewModel { public enum State { // Normal states + CALL_PRE_JOIN, CALL_INCOMING, CALL_OUTGOING, CALL_CONNECTED, diff --git a/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java b/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java index 1034bd54d..083011a7f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java +++ b/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java @@ -6,19 +6,14 @@ import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CameraMetadata; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import com.annimon.stream.Stream; -import java.io.IOException; -import java.util.List; -import java.util.LinkedList; - import org.signal.ringrtc.CameraControl; - import org.thoughtcrime.securesms.logging.Log; - import org.webrtc.Camera1Enumerator; import org.webrtc.Camera2Capturer; import org.webrtc.Camera2Enumerator; @@ -28,6 +23,9 @@ import org.webrtc.CapturerObserver; import org.webrtc.EglBase; import org.webrtc.SurfaceTextureHelper; +import java.util.LinkedList; +import java.util.List; + import static org.thoughtcrime.securesms.ringrtc.CameraState.Direction.BACK; import static org.thoughtcrime.securesms.ringrtc.CameraState.Direction.FRONT; import static org.thoughtcrime.securesms.ringrtc.CameraState.Direction.NONE; @@ -48,9 +46,10 @@ public class Camera implements CameraControl, CameraVideoCapturer.CameraSwitchHa @NonNull private CameraState.Direction activeDirection; private boolean enabled; - public Camera(@NonNull Context context, + public Camera(@NonNull Context context, @NonNull CameraEventListener cameraEventListener, - @NonNull EglBase eglBase) + @NonNull EglBase eglBase, + @NonNull CameraState.Direction desiredCameraDirection) { this.context = context; this.cameraEventListener = cameraEventListener; @@ -58,13 +57,16 @@ public class Camera implements CameraControl, CameraVideoCapturer.CameraSwitchHa CameraEnumerator enumerator = getCameraEnumerator(context); cameraCount = enumerator.getDeviceNames().length; - CameraVideoCapturer capturerCandidate = createVideoCapturer(enumerator, FRONT); + CameraState.Direction firstChoice = desiredCameraDirection.isUsable() ? desiredCameraDirection : FRONT; + + CameraVideoCapturer capturerCandidate = createVideoCapturer(enumerator, firstChoice); if (capturerCandidate != null) { - activeDirection = FRONT; + activeDirection = firstChoice; } else { - capturerCandidate = createVideoCapturer(enumerator, BACK); + CameraState.Direction secondChoice = firstChoice.switchDirection(); + capturerCandidate = createVideoCapturer(enumerator, secondChoice); if (capturerCandidate != null) { - activeDirection = BACK; + activeDirection = secondChoice; } else { activeDirection = NONE; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/ringrtc/CameraState.java b/app/src/main/java/org/thoughtcrime/securesms/ringrtc/CameraState.java index 45f3720b6..1e96643a9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/ringrtc/CameraState.java +++ b/app/src/main/java/org/thoughtcrime/securesms/ringrtc/CameraState.java @@ -32,6 +32,21 @@ public class CameraState { } public enum Direction { - FRONT, BACK, NONE, PENDING + FRONT, BACK, NONE, PENDING; + + public boolean isUsable() { + return this == FRONT || this == BACK; + } + + public Direction switchDirection() { + switch (this) { + case FRONT: + return BACK; + case BACK: + return FRONT; + default: + return this; + } + } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java index 5232c0713..5ed306c26 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java @@ -57,13 +57,15 @@ import org.thoughtcrime.securesms.webrtc.audio.BluetoothStateManager; import org.thoughtcrime.securesms.webrtc.audio.OutgoingRinger; import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager; import org.thoughtcrime.securesms.webrtc.locks.LockManager; +import org.webrtc.CapturerObserver; import org.webrtc.EglBase; import org.webrtc.PeerConnection; -import org.whispersystems.libsignal.ecc.ECPublicKey; -import org.whispersystems.libsignal.util.Pair; +import org.webrtc.VideoFrame; import org.whispersystems.libsignal.InvalidKeyException; import org.whispersystems.libsignal.ecc.Curve; import org.whispersystems.libsignal.ecc.DjbECPublicKey; +import org.whispersystems.libsignal.ecc.ECPublicKey; +import org.whispersystems.libsignal.util.Pair; import org.whispersystems.signalservice.api.SignalServiceAccountManager; import org.whispersystems.signalservice.api.SignalServiceMessageSender; import org.whispersystems.signalservice.api.crypto.UntrustedIdentityException; @@ -128,6 +130,8 @@ public class WebRtcCallService extends Service implements CallManager.Observer, public static final String EXTRA_BROADCAST = "broadcast"; public static final String EXTRA_ANSWER_WITH_VIDEO = "enable_video"; + public static final String ACTION_PRE_JOIN_CALL = "CALL_PRE_JOIN"; + public static final String ACTION_CANCEL_PRE_JOIN_CALL = "CANCEL_PRE_JOIN_CALL"; public static final String ACTION_OUTGOING_CALL = "CALL_OUTGOING"; public static final String ACTION_DENY_CALL = "DENY_CALL"; public static final String ACTION_LOCAL_HANGUP = "LOCAL_HANGUP"; @@ -196,6 +200,7 @@ public class WebRtcCallService extends Service implements CallManager.Observer, @Nullable private CallManager callManager; @Nullable private RemotePeer activePeer; @Nullable private RemotePeer busyPeer; + @Nullable private RemotePeer preJoinPeer; @Nullable private SparseArray peerMap; @Nullable private EglBase eglBase; @@ -232,6 +237,8 @@ public class WebRtcCallService extends Service implements CallManager.Observer, serviceExecutor.execute(() -> { if (intent.getAction().equals(ACTION_RECEIVE_OFFER)) handleReceivedOffer(intent); else if (intent.getAction().equals(ACTION_RECEIVE_BUSY)) handleReceivedBusy(intent); + else if (intent.getAction().equals(ACTION_PRE_JOIN_CALL)) handlePreJoinCall(intent); + else if (intent.getAction().equals(ACTION_CANCEL_PRE_JOIN_CALL)) handleCancelPreJoinCall(); else if (intent.getAction().equals(ACTION_OUTGOING_CALL) && isIdle()) handleOutgoingCall(intent); else if (intent.getAction().equals(ACTION_DENY_CALL)) handleDenyCall(intent); else if (intent.getAction().equals(ACTION_LOCAL_HANGUP)) handleLocalHangup(intent); @@ -334,6 +341,8 @@ public class WebRtcCallService extends Service implements CallManager.Observer, localCameraState = newCameraState; if (activePeer != null) { sendMessage(viewModelStateFor(activePeer), activePeer, localCameraState, bluetoothAvailable, microphoneEnabled, isRemoteVideoOffer); + } else if (preJoinPeer != null) { + sendMessage(WebRtcViewModel.State.CALL_PRE_JOIN, preJoinPeer, localCameraState, bluetoothAvailable, microphoneEnabled, isRemoteVideoOffer); } } @@ -447,6 +456,59 @@ public class WebRtcCallService extends Service implements CallManager.Observer, } } + private void handlePreJoinCall(Intent intent) { + Log.i(TAG, "handlePreJoinCall():"); + + RemotePeer remotePeer = getRemotePeer(intent); + + if (remotePeer.getState() != CallState.IDLE) { + throw new IllegalStateException("Dialing from non-idle?"); + } + + preJoinPeer = remotePeer; + + initializeVideo(); + + localCameraState = initializeVanityCamera(); + + EventBus.getDefault().removeStickyEvent(WebRtcViewModel.class); + + sendMessage(WebRtcViewModel.State.CALL_PRE_JOIN, + remotePeer, + localCameraState, + bluetoothAvailable, + microphoneEnabled, + false); + } + + private @NonNull CameraState initializeVanityCamera() { + if (camera == null || localSink == null) { + return CameraState.UNKNOWN; + } + + if (camera.hasCapturer()) { + camera.initCapturer(new CapturerObserver() { + @Override + public void onFrameCaptured(VideoFrame videoFrame) { + localSink.onFrame(videoFrame); + } + + @Override + public void onCapturerStarted(boolean success) {} + + @Override + public void onCapturerStopped() {} + }); + camera.setEnabled(true); + } + return camera.getCameraState(); + } + + private void handleCancelPreJoinCall() { + cleanupVideo(); + preJoinPeer = null; + } + private void handleOutgoingCall(Intent intent) { Log.i(TAG, "handleOutgoingCall():"); @@ -456,6 +518,8 @@ public class WebRtcCallService extends Service implements CallManager.Observer, throw new IllegalStateException("Dialing from non-idle?"); } + preJoinPeer = null; + EventBus.getDefault().removeStickyEvent(WebRtcViewModel.class); peerMap.append(remotePeer.hashCode(), remotePeer); @@ -575,6 +639,8 @@ public class WebRtcCallService extends Service implements CallManager.Observer, localCameraState = camera.getCameraState(); if (activePeer != null) { sendMessage(viewModelStateFor(activePeer), activePeer, localCameraState, bluetoothAvailable, microphoneEnabled, isRemoteVideoOffer); + } else if (preJoinPeer != null) { + sendMessage(WebRtcViewModel.State.CALL_PRE_JOIN, preJoinPeer, localCameraState, bluetoothAvailable, microphoneEnabled, isRemoteVideoOffer); } } } @@ -1016,7 +1082,15 @@ public class WebRtcCallService extends Service implements CallManager.Observer, AudioManager audioManager = ServiceUtil.getAudioManager(this); if (activePeer == null) { - Log.w(TAG, "handleSetEnableVideo(): Ignoring for inactive call."); + if (preJoinPeer != null) { + Log.w(TAG, "handleSetEnableVideo(): Changing for pre-join call."); + camera.setEnabled(enable); + enableVideoOnCreate = enable; + localCameraState = camera.getCameraState(); + sendMessage(WebRtcViewModel.State.CALL_PRE_JOIN, preJoinPeer, localCameraState, bluetoothAvailable, microphoneEnabled, isRemoteVideoOffer); + } else { + Log.w(TAG, "handleSetEnableVideo(): Ignoring for inactive call."); + } return; } @@ -1300,13 +1374,35 @@ public class WebRtcCallService extends Service implements CallManager.Observer, private void initializeVideo() { Util.runOnMainSync(() -> { - eglBase = EglBase.create(); - localSink = new BroadcastVideoSink(eglBase); - camera = new Camera(WebRtcCallService.this, WebRtcCallService.this, eglBase); + if (eglBase == null) { + eglBase = EglBase.create(); + localSink = new BroadcastVideoSink(eglBase); + } + + if (camera != null) { + camera.setEnabled(false); + camera.dispose(); + } + + camera = new Camera(WebRtcCallService.this, WebRtcCallService.this, eglBase, localCameraState.getActiveDirection()); localCameraState = camera.getCameraState(); }); } + private void cleanupVideo() { + if (camera != null) { + camera.dispose(); + camera = null; + } + + if (eglBase != null) { + eglBase.release(); + eglBase = null; + } + + localCameraState = CameraState.UNKNOWN; + } + private void setCallInProgressNotification(int type, RemotePeer remotePeer) { startForeground(CallNotificationBuilder.getNotificationId(getApplicationContext(), type), CallNotificationBuilder.getCallInProgressNotification(this, type, remotePeer.getRecipient())); @@ -1334,26 +1430,15 @@ public class WebRtcCallService extends Service implements CallManager.Observer, audioManager.stop(playDisconnectSound); bluetoothStateManager.setWantsConnection(false); - if (camera != null) { - camera.dispose(); - camera = null; - } + cleanupVideo(); - if (eglBase != null) { - eglBase.release(); - eglBase = null; - } - - this.localCameraState = CameraState.UNKNOWN; this.microphoneEnabled = true; this.enableVideoOnCreate = false; Log.i(TAG, "clear activePeer callId: " + activePeer.getCallId() + " key: " + activePeer.hashCode()); this.activePeer = null; - for (CallParticipant participant : remoteParticipantMap.values()) { - remoteParticipantMap.put(participant.getRecipient(), participant.withVideoEnabled(false)); - } + remoteParticipantMap.clear(); lockManager.updatePhoneState(LockManager.PhoneState.IDLE); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/CommunicationActions.java b/app/src/main/java/org/thoughtcrime/securesms/util/CommunicationActions.java index 58ceab0be..511ae27df 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/CommunicationActions.java +++ b/app/src/main/java/org/thoughtcrime/securesms/util/CommunicationActions.java @@ -81,16 +81,7 @@ public class CommunicationActions { WebRtcCallService.isCallActive(activity, new ResultReceiver(new Handler(Looper.getMainLooper())) { @Override protected void onReceiveResult(int resultCode, Bundle resultData) { - if (resultCode == 1) { - startCallInternal(activity, recipient, false); - } else { - new AlertDialog.Builder(activity) - .setMessage(R.string.CommunicationActions_start_video_call) - .setPositiveButton(R.string.CommunicationActions_call, (d, w) -> startCallInternal(activity, recipient, true)) - .setNegativeButton(R.string.CommunicationActions_cancel, (d, w) -> d.dismiss()) - .setCancelable(true) - .show(); - } + startCallInternal(activity, recipient, resultCode != 1); } }); } @@ -268,13 +259,11 @@ public class CommunicationActions { .withPermanentDenialDialog(activity.getString(R.string.ConversationActivity_signal_needs_the_microphone_and_camera_permissions_in_order_to_call_s, recipient.getDisplayName(activity))) .onAllGranted(() -> { Intent intent = new Intent(activity, WebRtcCallService.class); - intent.setAction(WebRtcCallService.ACTION_OUTGOING_CALL) + intent.setAction(WebRtcCallService.ACTION_PRE_JOIN_CALL) .putExtra(WebRtcCallService.EXTRA_REMOTE_PEER, new RemotePeer(recipient.getId())) .putExtra(WebRtcCallService.EXTRA_OFFER_TYPE, OfferMessage.Type.VIDEO_CALL.getCode()); activity.startService(intent); - MessageSender.onMessageSent(); - Intent activityIntent = new Intent(activity, WebRtcCallActivity.class); activityIntent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) diff --git a/app/src/main/res/layout/webrtc_call_view.xml b/app/src/main/res/layout/webrtc_call_view.xml index 3569ebb5e..c72af490f 100644 --- a/app/src/main/res/layout/webrtc_call_view.xml +++ b/app/src/main/res/layout/webrtc_call_view.xml @@ -27,18 +27,10 @@ - - - - - + android:layout_height="match_parent" /> - + android:layout_height="30dp" + app:layout_constraintBottom_toBottomOf="@id/call_screen_footer_gradient_barrier"/> + + \ No newline at end of file diff --git a/app/src/main/res/layout/webrtc_call_view_large_local_render.xml b/app/src/main/res/layout/webrtc_call_view_large_local_render.xml new file mode 100644 index 000000000..4817c9887 --- /dev/null +++ b/app/src/main/res/layout/webrtc_call_view_large_local_render.xml @@ -0,0 +1,32 @@ + + + + + + + + + + diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index eb47652b4..ff0c6bff2 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1207,6 +1207,7 @@ Start Call Group Call View participants + Your video is off