blallo
/
Signal-Android
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Audio wave forms on voice notes.

master
Alan Evans 2020-05-28 16:50:27 -03:00 committed by Greyson Parrelli
parent 69adcd1d69
commit daace9bd1a
11 changed files with 464 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

232
app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForm.java 100644
View File

@ -0,0 +1,232 @@
package org.thoughtcrime.securesms.audio;
import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.net.Uri;
import android.os.Build;
import android.util.LruCache;
import androidx.annotation.AnyThread;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import androidx.annotation.WorkerThread;
import androidx.core.util.Consumer;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.mms.AudioSlide;
import org.thoughtcrime.securesms.util.Util;
import org.thoughtcrime.securesms.util.concurrent.SignalExecutors;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@RequiresApi(api = Build.VERSION_CODES.M)
public final class AudioWaveForm {
private static final String TAG = Log.tag(AudioWaveForm.class);
private static final int BARS = 46;
private static final int SAMPLES_PER_BAR = 4;
private final Context context;
private final AudioSlide slide;
public AudioWaveForm(@NonNull Context context, @NonNull AudioSlide slide) {
this.context = context.getApplicationContext();
this.slide = slide;
}
private static final LruCache<Uri, AudioFileInfo> WAVE_FORM_CACHE = new LruCache<>(200);
private static final Executor AUDIO_DECODER_EXECUTOR = SignalExecutors.BOUNDED;
@AnyThread
public void generateWaveForm(@NonNull Consumer<AudioFileInfo> onSuccess, @NonNull Consumer<IOException> onFailure) {
AUDIO_DECODER_EXECUTOR.execute(() -> {
try {
long startTime = System.currentTimeMillis();
Uri uri = slide.getUri();
if (uri == null) {
Util.runOnMain(() -> onFailure.accept(null));
return;
}
AudioFileInfo cached = WAVE_FORM_CACHE.get(uri);
if (cached != null) {
Util.runOnMain(() -> onSuccess.accept(cached));
return;
}
AudioFileInfo fileInfo = generateWaveForm(uri);
WAVE_FORM_CACHE.put(uri, fileInfo);
Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms", System.currentTimeMillis() - startTime));
Util.runOnMain(() -> onSuccess.accept(fileInfo));
} catch (IOException e) {
Log.e(TAG, "", e);
onFailure.accept(e);
}
});
}
/**
* Based on decode sample from:
* <p>
* https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/DecoderTest.java
*/
@WorkerThread
@RequiresApi(api = 23)
private AudioFileInfo generateWaveForm(@NonNull Uri uri) throws IOException {
try (MediaInput dataSource = DecryptableUriMediaInput.createForUri(context, uri)) {
long[] wave = new long[BARS];
int[] waveSamples = new int[BARS];
int[] inputSamples = new int[BARS * SAMPLES_PER_BAR];
MediaExtractor extractor = dataSource.createExtractor();
MediaFormat format = extractor.getTrackFormat(0);
long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION);
String mime = requireAudio(format.getString(MediaFormat.KEY_MIME));
MediaCodec codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, null, null, 0);
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(0);
long kTimeOutUs = 5000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
int noOutputCounter = 0;
while (!sawOutputEOS && noOutputCounter < 50) {
noOutputCounter++;
if (!sawInputEOS) {
int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeUs = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(
inputBufIndex,
0,
sampleSize,
presentationTimeUs,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
int barSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
inputSamples[barSampleIndex]++;
sawInputEOS = !extractor.advance();
if (inputSamples[barSampleIndex] > 0) {
int nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
while (!sawInputEOS && nextBarSampleIndex == barSampleIndex) {
sawInputEOS = !extractor.advance();
if (!sawInputEOS) {
nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
}
}
}
}
}
}
int outputBufferIndex;
do {
outputBufferIndex = codec.dequeueOutputBuffer(info, kTimeOutUs);
if (outputBufferIndex >= 0) {
if (info.size > 0) {
noOutputCounter = 0;
}
ByteBuffer buf = codecOutputBuffers[outputBufferIndex];
int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs) - 1;
long total = 0;
for (int i = 0; i < info.size; i += 2 * 4) {
short aShort = buf.getShort(i);
total += Math.abs(aShort);
}
if (barIndex > 0) {
wave[barIndex] += total;
waveSamples[barIndex] += info.size / 2;
}
codec.releaseOutputBuffer(outputBufferIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "saw output EOS.");
sawOutputEOS = true;
}
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
Log.d(TAG, "output buffers have changed.");
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat oformat = codec.getOutputFormat();
Log.d(TAG, "output format has changed to " + oformat);
} else {
Log.d(TAG, "dequeueOutputBuffer returned " + outputBufferIndex);
}
} while (outputBufferIndex >= 0);
}
codec.stop();
codec.release();
extractor.release();
float[] floats = new float[AudioWaveForm.BARS];
float max = 0;
for (int i = 0; i < AudioWaveForm.BARS; i++) {
floats[i] = wave[i] / (float) waveSamples[i];
if (floats[i] > max) {
max = floats[i];
}
}
for (int i = 0; i < AudioWaveForm.BARS; i++) {
floats[i] /= max;
}
return new AudioFileInfo(totalDurationUs, floats);
}
}
private static @NonNull String requireAudio(@NonNull String mime) {
if (!mime.startsWith("audio/")) {
throw new AssertionError();
}
return mime;
}
public static class AudioFileInfo {
private final long durationUs;
private final float[] waveForm;
private AudioFileInfo(long durationUs, float[] waveForm) {
this.durationUs = durationUs;
this.waveForm = waveForm;
}
public long getDuration(@NonNull TimeUnit timeUnit) {
return timeUnit.convert(durationUs, TimeUnit.MICROSECONDS);
}
public float[] getWaveForm() {
return waveForm;
}
}
}

36
app/src/main/java/org/thoughtcrime/securesms/components/AudioView.java
View File

@ -14,6 +14,7 @@ import android.widget.ImageView;
import android.widget.SeekBar;
import android.widget.TextView;
import androidx.annotation.ColorInt;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
@ -29,6 +30,7 @@ import org.greenrobot.eventbus.Subscribe;
import org.greenrobot.eventbus.ThreadMode;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.audio.AudioSlidePlayer;
import org.thoughtcrime.securesms.audio.AudioWaveForm;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
import org.thoughtcrime.securesms.events.PartProgressEvent;
import org.thoughtcrime.securesms.logging.Log;
@ -57,6 +59,10 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis
private final boolean autoRewind;
@Nullable private final TextView timestamp;
@Nullable private final TextView duration;
@ColorInt private final int waveFormPlayedBarsColor;
@ColorInt private final int waveFormUnplayedBarsColor;
@Nullable private SlideClickListener downloadListener;
@Nullable private AudioSlidePlayer audioSlidePlayer;
@ -91,6 +97,7 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis
this.circleProgress = findViewById(R.id.circle_progress);
this.seekBar = findViewById(R.id.seek);
this.timestamp = findViewById(R.id.timestamp);
this.duration = findViewById(R.id.duration);
lottieDirection = REVERSE;
this.playPauseButton.setOnClickListener(new PlayPauseClickedListener());
@ -98,6 +105,10 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis
setTint(typedArray.getColor(R.styleable.AudioView_foregroundTintColor, Color.WHITE),
typedArray.getColor(R.styleable.AudioView_backgroundTintColor, Color.WHITE));
this.waveFormPlayedBarsColor = typedArray.getColor(R.styleable.AudioView_waveformPlayedBarsColor, Color.WHITE);
this.waveFormUnplayedBarsColor = typedArray.getColor(R.styleable.AudioView_waveformUnplayedBarsColor, Color.WHITE);
container.setBackgroundColor(typedArray.getColor(R.styleable.AudioView_widgetBackground, Color.TRANSPARENT));
} finally {
if (typedArray != null) {
@ -141,6 +152,28 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis
}
this.audioSlidePlayer = AudioSlidePlayer.createFor(getContext(), audio, this);
if (seekBar instanceof WaveFormSeekBarView) {
WaveFormSeekBarView waveFormView = (WaveFormSeekBarView) seekBar;
waveFormView.setColors(waveFormPlayedBarsColor, waveFormUnplayedBarsColor);
if (android.os.Build.VERSION.SDK_INT >= 23) {
new AudioWaveForm(getContext(), audio).generateWaveForm(
data -> {
waveFormView.setWaveData(data.getWaveForm());
if (duration != null) {
long durationSecs = data.getDuration(TimeUnit.SECONDS);
duration.setText(getContext().getResources().getString(R.string.AudioView_duration, durationSecs / 60, durationSecs % 60));
duration.setVisibility(VISIBLE);
}
},
e -> waveFormView.setWaveMode(false));
} else {
waveFormView.setWaveMode(false);
if (duration != null) {
duration.setVisibility(GONE);
}
}
}
}
public void cleanup() {
@ -232,6 +265,9 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis
if (this.timestamp != null) {
this.timestamp.setTextColor(foregroundTint);
}
if (this.duration != null) {
this.duration.setTextColor(foregroundTint);
}
this.seekBar.getProgressDrawable().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN);
this.seekBar.getThumb().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN);
}

155
app/src/main/java/org/thoughtcrime/securesms/components/WaveFormSeekBarView.java 100644
View File

@ -0,0 +1,155 @@
package org.thoughtcrime.securesms.components;
import android.content.Context;
import android.graphics.Canvas;
import android.graphics.Paint;
import android.graphics.drawable.Drawable;
import android.util.AttributeSet;
import android.view.animation.Interpolator;
import android.view.animation.OvershootInterpolator;
import androidx.annotation.ColorInt;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.Px;
import androidx.appcompat.widget.AppCompatSeekBar;
import org.thoughtcrime.securesms.R;
import java.util.Arrays;
public final class WaveFormSeekBarView extends AppCompatSeekBar {
private static final int ANIM_DURATION = 450;
private static final int ANIM_BAR_OFF_SET_DURATION = 12;
private final Interpolator overshoot = new OvershootInterpolator();
private final Paint paint = new Paint();
private float[] data = new float[0];
private long dataSetTime;
private Drawable progressDrawable;
private boolean waveMode;
@ColorInt private int playedBarColor = 0xffffffff;
@ColorInt private int unplayedBarColor = 0x7fffffff;
@Px private int barWidth;
public WaveFormSeekBarView(Context context) {
super(context);
init();
}
public WaveFormSeekBarView(Context context, @Nullable AttributeSet attrs) {
super(context, attrs);
init();
}
public WaveFormSeekBarView(Context context, @Nullable AttributeSet attrs, int defStyleAttr) {
super(context, attrs, defStyleAttr);
init();
}
private void init() {
setWillNotDraw(false);
paint.setStrokeCap(Paint.Cap.ROUND);
paint.setAntiAlias(true);
progressDrawable = super.getProgressDrawable();
if (isInEditMode()) {
setWaveData(sinusoidalExampleData());
dataSetTime = 0;
}
barWidth = getResources().getDimensionPixelSize(R.dimen.wave_form_bar_width);
}
public void setColors(@ColorInt int playedBarColor, @ColorInt int unplayedBarColor) {
this.playedBarColor = playedBarColor;
this.unplayedBarColor = unplayedBarColor;
invalidate();
}
@Override
public void setProgressDrawable(Drawable progressDrawable) {
this.progressDrawable = progressDrawable;
if (!waveMode) {
super.setProgressDrawable(progressDrawable);
}
}
@Override
public Drawable getProgressDrawable() {
return progressDrawable;
}
public void setWaveData(@NonNull float[] data) {
if (!Arrays.equals(data, this.data)) {
this.data = data;
this.dataSetTime = System.currentTimeMillis();
setWaveMode(data.length > 0);
invalidate();
}
}
public void setWaveMode(boolean waveMode) {
this.waveMode = waveMode;
super.setProgressDrawable(this.waveMode ? null : progressDrawable);
invalidate();
}
@Override
protected void onDraw(Canvas canvas) {
drawWave(canvas);
super.onDraw(canvas);
}
private void drawWave(Canvas canvas) {
paint.setStrokeWidth(barWidth);
int usableHeight = getHeight() - getPaddingTop() - getPaddingBottom();
int usableWidth = getWidth() - getPaddingLeft() - getPaddingRight();
float midpoint = usableHeight / 2f;
float maxHeight = usableHeight / 2f - barWidth;
float barGap = (usableWidth - data.length * barWidth) / (float) (data.length - 1);
boolean hasMoreFrames = false;
canvas.save();
canvas.translate(getPaddingLeft(), getPaddingTop());
for (int bar = 0; bar < data.length; bar++) {
float x = bar * (barWidth + barGap) + barWidth / 2f;
float y = data[bar] * maxHeight;
float progress = x / usableWidth;
paint.setColor(progress * getMax() < getProgress() ? playedBarColor : unplayedBarColor);
long time = System.currentTimeMillis() - bar * ANIM_BAR_OFF_SET_DURATION - dataSetTime;
float timeX = Math.max(0, Math.min(1, time / (float) ANIM_DURATION));
float interpolatedTime = overshoot.getInterpolation(timeX);
float interpolatedY = y * interpolatedTime;
canvas.drawLine(x, midpoint - interpolatedY, x, midpoint + interpolatedY, paint);
if (time < ANIM_DURATION) {
hasMoreFrames = true;
}
}
canvas.restore();
if (hasMoreFrames) {
invalidate();
}
}
private static float[] sinusoidalExampleData() {
float[] data = new float[21];
for (int i = 0; i < data.length; i++) {
data[i] = (float) Math.sin(i / (float) (data.length - 1) * 2 * Math.PI);
}
return data;
}
}

25
app/src/main/res/layout/audio_view.xml
View File

@ -17,11 +17,30 @@
<include layout="@layout/audio_view_circle" />
<SeekBar
<org.thoughtcrime.securesms.components.WaveFormSeekBarView
android:id="@+id/seek"
android:layout_width="fill_parent"
android:layout_width="0dp"
android:layout_weight="1"
android:layout_height="match_parent"
android:layout_gravity="center_vertical"
android:paddingTop="10dp"
android:paddingBottom="10dp"
android:paddingStart="12dp"
android:paddingEnd="8dp"
tools:progress="50" />
<TextView
android:id="@+id/duration"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="center_vertical" />
android:layout_gravity="center_vertical"
android:fontFamily="sans-serif-light"
android:textAppearance="?android:attr/textAppearanceMedium"
android:textColor="?conversation_item_sent_text_secondary_color"
android:textSize="@dimen/conversation_item_date_text_size"
android:visibility="gone"
tools:text="00:30"
tools:visibility="visible" />
</LinearLayout>

4
app/src/main/res/layout/conversation_item_received.xml
View File

@ -149,9 +149,9 @@
<ViewStub
android:id="@+id/audio_view_stub"
android:layout="@layout/conversation_item_received_audio"
android:layout_width="210dp"
android:layout_width="@dimen/message_audio_width"
android:layout_height="wrap_content"
android:layout_marginTop="@dimen/message_bubble_top_padding"
android:layout_marginTop="@dimen/message_bubble_top_padding_audio"
android:layout_marginBottom="@dimen/message_bubble_collapsed_footer_padding"
android:layout_marginStart="@dimen/message_bubble_horizontal_padding"
android:layout_marginEnd="@dimen/message_bubble_horizontal_padding" />

4
app/src/main/res/layout/conversation_item_received_audio.xml
View File

@ -4,9 +4,11 @@
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:id="@+id/audio_view"
android:layout_width="210dp"
android:layout_width="@dimen/message_audio_width"
android:layout_height="wrap_content"
android:visibility="gone"
app:foregroundTintColor="@color/white"
app:backgroundTintColor="@color/blue_500"
app:waveformPlayedBarsColor="@color/core_white"
app:waveformUnplayedBarsColor="@color/transparent_white_40"
tools:visibility="visible"/>

4
app/src/main/res/layout/conversation_item_sent.xml
View File

@ -86,10 +86,10 @@
<ViewStub
android:id="@+id/audio_view_stub"
android:layout_width="210dp"
android:layout_width="@dimen/message_audio_width"
android:layout_height="wrap_content"
android:layout_marginStart="@dimen/message_bubble_horizontal_padding"
android:layout_marginTop="@dimen/message_bubble_top_padding"
android:layout_marginTop="@dimen/message_bubble_top_padding_audio"
android:layout_marginEnd="@dimen/message_bubble_horizontal_padding"
android:layout_marginBottom="@dimen/message_bubble_collapsed_footer_padding"
android:layout="@layout/conversation_item_sent_audio" />

4
app/src/main/res/layout/conversation_item_sent_audio.xml
View File

@ -3,8 +3,10 @@
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
android:id="@+id/audio_view"
android:layout_width="210dp"
android:layout_width="@dimen/message_audio_width"
android:layout_height="wrap_content"
app:foregroundTintColor="@color/grey_500"
app:backgroundTintColor="@color/white"
app:waveformPlayedBarsColor="@color/core_ultramarine_light"
app:waveformUnplayedBarsColor="@color/core_grey_25"
android:visibility="gone"/>

2
app/src/main/res/values/attrs.xml
View File

@ -339,6 +339,8 @@
<attr name="widgetBackground" format="color"/>
<attr name="foregroundTintColor" format="color" />
<attr name="backgroundTintColor" format="color" />
<attr name="waveformPlayedBarsColor" format="color" />
<attr name="waveformUnplayedBarsColor" format="color" />
<attr name="small" format="boolean" />
<attr name="autoRewind" format="boolean" />
</declare-styleable>

4
app/src/main/res/values/dimens.xml
View File

@ -34,6 +34,7 @@
<dimen name="message_bubble_shadow_distance">1.5dp</dimen>
<dimen name="message_bubble_horizontal_padding">12dp</dimen>
<dimen name="message_bubble_top_padding">6dp</dimen>
<dimen name="message_bubble_top_padding_audio">12dp</dimen>
<dimen name="message_bubble_collapsed_footer_padding">6dp</dimen>
<dimen name="message_bubble_edge_margin">32dp</dimen>
<dimen name="message_bubble_bottom_padding">8dp</dimen>
@ -45,6 +46,7 @@
<dimen name="media_bubble_min_height">100dp</dimen>
<dimen name="media_bubble_max_height">320dp</dimen>
<dimen name="media_bubble_sticker_dimens">175dp</dimen>
<dimen name="message_audio_width">260dp</dimen>
<dimen name="reactions_bubble_margin">-37dp</dimen>
<dimen name="reactions_bubble_size">32dp</dimen>
@ -162,4 +164,6 @@
<dimen name="group_manage_fragment_card_vertical_padding">12dp</dimen>
<dimen name="group_manage_fragment_row_height">52dp</dimen>
<dimen name="group_manage_fragment_row_horizontal_padding">16dp</dimen>
<dimen name="wave_form_bar_width">2dp</dimen>
</resources>

3
app/src/main/res/values/strings.xml
View File

@ -1166,6 +1166,9 @@
<!-- ViewOnceMessageActivity -->
<string name="ViewOnceMessageActivity_video_duration" translatable="false">%1$02d:%2$02d</string>
<!-- AudioView -->
<string name="AudioView_duration" translatable="false">%1$02d:%2$02d</string>
<!-- MessageDisplayHelper -->
<string name="MessageDisplayHelper_bad_encrypted_message">Bad encrypted message</string>
<string name="MessageDisplayHelper_message_encrypted_for_non_existing_session">Message encrypted for non-existing session</string>