diff --git a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp index ef86ac8e3..27ce43096 100644 --- a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp +++ b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwner (JNIEnv *env, jclass clazz, jobject fileDescriptor) @@ -28,4 +30,16 @@ jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwn } return stat_struct.st_uid; -} \ No newline at end of file +} + +JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_createMemoryFileDescriptor + (JNIEnv *env, jclass clazz, jstring jname) +{ + const char *name = env->GetStringUTFChars(jname, NULL); + + int fd = syscall(SYS_memfd_create, name, MFD_CLOEXEC); + + env->ReleaseStringUTFChars(jname, name); + + return fd; +} diff --git a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h index abf13b35b..12faa685e 100644 --- a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h +++ b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h @@ -15,6 +15,14 @@ extern "C" { JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwner (JNIEnv *, jclass, jobject); +/* + * Class: org_thoughtcrime_securesms_util_FileUtils + * Method: createMemoryFileDescriptor + * Signature: (Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_createMemoryFileDescriptor + (JNIEnv *, jclass, jstring); + #ifdef __cplusplus } #endif diff --git a/libs/arm64-v8a/libnative-utils.so b/libs/arm64-v8a/libnative-utils.so index a6af30560..0d84ac84e 100755 Binary files a/libs/arm64-v8a/libnative-utils.so and b/libs/arm64-v8a/libnative-utils.so differ diff --git a/libs/armeabi-v7a/libnative-utils.so b/libs/armeabi-v7a/libnative-utils.so index 73202fc83..a6da60b85 100755 Binary files a/libs/armeabi-v7a/libnative-utils.so and b/libs/armeabi-v7a/libnative-utils.so differ diff --git a/libs/x86/libnative-utils.so b/libs/x86/libnative-utils.so index c428cbe6c..882e67e57 100755 Binary files a/libs/x86/libnative-utils.so and b/libs/x86/libnative-utils.so differ diff --git a/libs/x86_64/libnative-utils.so b/libs/x86_64/libnative-utils.so index 27d3c63f4..fc03a67d7 100755 Binary files a/libs/x86_64/libnative-utils.so and b/libs/x86_64/libnative-utils.so differ diff --git a/res/values/strings.xml b/res/values/strings.xml index 5f4064343..adbf618de 100644 --- a/res/values/strings.xml +++ b/res/values/strings.xml @@ -72,6 +72,7 @@ Uploading media... + Compressing video... Error playing audio! diff --git a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java index 098de8d9d..ee359994c 100644 --- a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java +++ b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java @@ -16,20 +16,22 @@ */ package org.thoughtcrime.securesms.database; -import android.annotation.SuppressLint; import android.content.ContentValues; import android.content.Context; import android.database.Cursor; import android.graphics.Bitmap; +import android.media.MediaDataSource; import android.media.MediaMetadataRetriever; import android.net.Uri; import android.os.Build; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.VisibleForTesting; import android.text.TextUtils; import android.util.Pair; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.annotation.VisibleForTesting; + import com.bumptech.glide.Glide; import net.sqlcipher.database.SQLiteDatabase; @@ -495,13 +497,12 @@ public class AttachmentDatabase extends Database { return insertedAttachments; } - public @NonNull Attachment updateAttachmentData(@NonNull Attachment attachment, - @NonNull MediaStream mediaStream) + public @NonNull DatabaseAttachment updateAttachmentData(@NonNull DatabaseAttachment databaseAttachment, + @NonNull MediaStream mediaStream) throws MmsException { - SQLiteDatabase database = databaseHelper.getWritableDatabase(); - DatabaseAttachment databaseAttachment = (DatabaseAttachment) attachment; - DataInfo dataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA); + SQLiteDatabase database = databaseHelper.getWritableDatabase(); + DataInfo dataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA); if (dataInfo == null) { throw new MmsException("No attachment data found!"); @@ -839,8 +840,9 @@ public class AttachmentDatabase extends Database { Bitmap bitmap = MediaUtil.getVideoThumbnail(context, attachment.getDataUri()); if (bitmap != null) { - ThumbnailData thumbnailData = new ThumbnailData(bitmap); - updateAttachmentThumbnail(attachmentId, thumbnailData.toDataStream(), thumbnailData.getAspectRatio()); + try (ThumbnailData thumbnailData = new ThumbnailData(bitmap)) { + updateAttachmentThumbnail(attachmentId, thumbnailData.toDataStream(), thumbnailData.getAspectRatio()); + } } else { Log.w(TAG, "Retrieving video thumbnail failed, submitting thumbnail generation job..."); thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId)); @@ -912,46 +914,53 @@ public class AttachmentDatabase extends Database { return null; } - ThumbnailData data = null; - if (MediaUtil.isVideoType(attachment.getContentType())) { - data = generateVideoThumbnail(attachmentId); + + try (ThumbnailData data = generateVideoThumbnail(attachmentId)) { + + if (data != null) { + updateAttachmentThumbnail(attachmentId, data.toDataStream(), data.getAspectRatio()); + + return getDataStream(attachmentId, THUMBNAIL, 0); + } + } } - if (data == null) { - return null; - } - - updateAttachmentThumbnail(attachmentId, data.toDataStream(), data.getAspectRatio()); - - return getDataStream(attachmentId, THUMBNAIL, 0); + return null; } - @SuppressLint("NewApi") - private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId) throws IOException { + if (Build.VERSION.SDK_INT < 23) { Log.w(TAG, "Video thumbnails not supported..."); return null; } - DataInfo dataInfo = getAttachmentDataFileInfo(attachmentId, DATA); + try (MediaDataSource dataSource = mediaDataSourceFor(attachmentId)) { + if (dataSource == null) return null; - if (dataInfo == null) { - Log.w(TAG, "No data file found for video thumbnail..."); - return null; + MediaMetadataRetriever retriever = new MediaMetadataRetriever(); + retriever.setDataSource(dataSource); + + Bitmap bitmap = retriever.getFrameAtTime(1000); + + Log.i(TAG, "Generated video thumbnail..."); + return bitmap != null ? new ThumbnailData(bitmap) : null; } - - EncryptedMediaDataSource dataSource = new EncryptedMediaDataSource(attachmentSecret, dataInfo.file, dataInfo.random, dataInfo.length); - MediaMetadataRetriever retriever = new MediaMetadataRetriever(); - retriever.setDataSource(dataSource); - - Bitmap bitmap = retriever.getFrameAtTime(1000); - - Log.i(TAG, "Generated video thumbnail..."); - return new ThumbnailData(bitmap); } } + @RequiresApi(23) + public @Nullable MediaDataSource mediaDataSourceFor(@NonNull AttachmentId attachmentId) { + DataInfo dataInfo = getAttachmentDataFileInfo(attachmentId, DATA); + + if (dataInfo == null) { + Log.w(TAG, "No data file found for video attachment..."); + return null; + } + + return EncryptedMediaDataSource.createFor(attachmentSecret, dataInfo.file, dataInfo.random, dataInfo.length); + } + private static class DataInfo { private final File file; private final long length; diff --git a/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java b/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java index d92306c94..3addf9c4a 100644 --- a/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java +++ b/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java @@ -18,8 +18,6 @@ import org.thoughtcrime.securesms.jobmanager.Job; import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.MediaConstraints; -import org.thoughtcrime.securesms.mms.MediaStream; -import org.thoughtcrime.securesms.mms.MmsException; import org.thoughtcrime.securesms.mms.PartAuthority; import org.thoughtcrime.securesms.service.GenericForegroundService; import org.thoughtcrime.securesms.service.NotificationController; @@ -48,13 +46,26 @@ public class AttachmentUploadJob extends BaseJob { */ private static final int FOREGROUND_LIMIT = 10 * 1024 * 1024; - private AttachmentId attachmentId; + /** + * The {@link PartProgressEvent} on the {@link EventBus} is shared between transcoding and uploading. + *

+ * This number is the ratio that represents the transcoding effort, after which it will hand + * over to the to complete the progress. + */ + private static final double ENCODING_PROGRESS_RATIO = 0.75; - public AttachmentUploadJob(AttachmentId attachmentId) { + private final AttachmentId attachmentId; + + public static AttachmentUploadJob fromAttachment(DatabaseAttachment databaseAttachment) { + return new AttachmentUploadJob(databaseAttachment.getAttachmentId(), MediaUtil.isVideo(databaseAttachment) && MediaConstraints.isVideoTranscodeAvailable()); + } + + private AttachmentUploadJob(AttachmentId attachmentId, boolean isVideoTranscode) { this(new Job.Parameters.Builder() .addConstraint(NetworkConstraint.KEY) .setLifespan(TimeUnit.DAYS.toMillis(1)) .setMaxAttempts(Parameters.UNLIMITED) + .setQueue(isVideoTranscode ? "VIDEO_TRANSCODE" : null) .build(), attachmentId); } @@ -86,11 +97,13 @@ public class AttachmentUploadJob extends BaseJob { throw new IllegalStateException("Cannot find the specified attachment."); } - MediaConstraints mediaConstraints = MediaConstraints.getPushMediaConstraints(); - Attachment scaledAttachment = scaleAndStripExif(database, mediaConstraints, databaseAttachment); + MediaConstraints mediaConstraints = MediaConstraints.getPushMediaConstraints(); + Attachment scaledAttachment = scaleAndStripExif(database, mediaConstraints, databaseAttachment); + boolean videoTranscodeOccurred = databaseAttachment != scaledAttachment && MediaUtil.isVideo(scaledAttachment); + double progressStartPoint = videoTranscodeOccurred ? ENCODING_PROGRESS_RATIO : 0; try (NotificationController notification = getNotificationForAttachment(scaledAttachment)) { - SignalServiceAttachment localAttachment = getAttachmentFor(scaledAttachment, notification); + SignalServiceAttachment localAttachment = getAttachmentFor(scaledAttachment, notification, progressStartPoint); SignalServiceAttachmentPointer remoteAttachment = messageSender.uploadAttachment(localAttachment.asStream(), databaseAttachment.isSticker()); Attachment attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.getFastPreflightId()).get(); @@ -114,7 +127,12 @@ public class AttachmentUploadJob extends BaseJob { return exception instanceof IOException; } - private SignalServiceAttachment getAttachmentFor(Attachment attachment, @Nullable NotificationController notification) { + /** + * @param progressStartPoint A value from 0..1 that represents any progress already shown. + * The {@link PartProgressEvent} of this task will fit in the remaining + * 1 - progressStartPoint. + */ + private SignalServiceAttachment getAttachmentFor(Attachment attachment, @Nullable NotificationController notification, double progressStartPoint) { try { if (attachment.getDataUri() == null || attachment.getSize() == 0) throw new IOException("Assertion failed, outgoing attachment has no data!"); InputStream is = PartAuthority.getAttachmentStream(context, attachment.getDataUri()); @@ -128,7 +146,8 @@ public class AttachmentUploadJob extends BaseJob { .withHeight(attachment.getHeight()) .withCaption(attachment.getCaption()) .withListener((total, progress) -> { - EventBus.getDefault().postSticky(new PartProgressEvent(attachment, total, progress)); + long cumulativeProgress = (long) ((1.0 - progressStartPoint) * progress + total * progressStartPoint); + EventBus.getDefault().postSticky(new PartProgressEvent(attachment, total, cumulativeProgress)); if (notification != null) { notification.setProgress(total, progress); } @@ -142,26 +161,21 @@ public class AttachmentUploadJob extends BaseJob { private Attachment scaleAndStripExif(@NonNull AttachmentDatabase attachmentDatabase, @NonNull MediaConstraints constraints, - @NonNull Attachment attachment) + @NonNull DatabaseAttachment attachment) throws UndeliverableMessageException { - try { - if (constraints.isSatisfied(context, attachment)) { - if (MediaUtil.isJpeg(attachment)) { - MediaStream stripped = constraints.getResizedMedia(context, attachment); - return attachmentDatabase.updateAttachmentData(attachment, stripped); - } else { - return attachment; - } - } else if (constraints.canResize(attachment)) { - MediaStream resized = constraints.getResizedMedia(context, attachment); - return attachmentDatabase.updateAttachmentData(attachment, resized); - } else { - throw new UndeliverableMessageException("Size constraints could not be met!"); - } - } catch (IOException | MmsException e) { - throw new UndeliverableMessageException(e); - } + MediaResizer mediaResizer = new MediaResizer(context, constraints); + + MediaResizer.ProgressListener progressListener = (progress, total) -> { + PartProgressEvent event = new PartProgressEvent(attachment, + total, + (long) (progress * ENCODING_PROGRESS_RATIO)); + EventBus.getDefault().postSticky(event); + }; + + return mediaResizer.scaleAndStripExifToDatabase(attachmentDatabase, + attachment, + progressListener); } public static final class Factory implements Job.Factory { diff --git a/src/org/thoughtcrime/securesms/jobs/BaseJob.java b/src/org/thoughtcrime/securesms/jobs/BaseJob.java index 73053b047..befe91d65 100644 --- a/src/org/thoughtcrime/securesms/jobs/BaseJob.java +++ b/src/org/thoughtcrime/securesms/jobs/BaseJob.java @@ -1,6 +1,7 @@ package org.thoughtcrime.securesms.jobs; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import org.thoughtcrime.securesms.jobmanager.Job; import org.thoughtcrime.securesms.jobmanager.JobLogger; @@ -33,4 +34,20 @@ public abstract class BaseJob extends Job { protected abstract void onRun() throws Exception; protected abstract boolean onShouldRetry(@NonNull Exception e); + + protected void log(@NonNull String tag, @NonNull String message) { + Log.i(tag, JobLogger.format(this, message)); + } + + protected void warn(@NonNull String tag, @NonNull String message) { + warn(tag, message, null); + } + + protected void warn(@NonNull String tag, @Nullable Throwable t) { + warn(tag, "", t); + } + + protected void warn(@NonNull String tag, @NonNull String message, @Nullable Throwable t) { + Log.w(tag, JobLogger.format(this, message), t); + } } diff --git a/src/org/thoughtcrime/securesms/jobs/MediaResizer.java b/src/org/thoughtcrime/securesms/jobs/MediaResizer.java new file mode 100644 index 000000000..860187ed1 --- /dev/null +++ b/src/org/thoughtcrime/securesms/jobs/MediaResizer.java @@ -0,0 +1,142 @@ +package org.thoughtcrime.securesms.jobs; + +import android.content.Context; +import android.media.MediaDataSource; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.R; +import org.thoughtcrime.securesms.attachments.Attachment; +import org.thoughtcrime.securesms.attachments.DatabaseAttachment; +import org.thoughtcrime.securesms.database.AttachmentDatabase; +import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader; +import org.thoughtcrime.securesms.mms.MediaConstraints; +import org.thoughtcrime.securesms.mms.MediaStream; +import org.thoughtcrime.securesms.mms.MmsException; +import org.thoughtcrime.securesms.service.GenericForegroundService; +import org.thoughtcrime.securesms.service.NotificationController; +import org.thoughtcrime.securesms.transport.UndeliverableMessageException; +import org.thoughtcrime.securesms.util.BitmapDecodingException; +import org.thoughtcrime.securesms.util.BitmapUtil; +import org.thoughtcrime.securesms.util.MediaUtil; +import org.thoughtcrime.securesms.video.InMemoryTranscoder; +import org.thoughtcrime.securesms.video.videoconverter.BadVideoException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +final class MediaResizer { + + @NonNull private final Context context; + @NonNull private final MediaConstraints constraints; + + MediaResizer(@NonNull Context context, + @NonNull MediaConstraints constraints) + { + this.context = context; + this.constraints = constraints; + } + + List scaleAndStripExifToDatabase(@NonNull AttachmentDatabase attachmentDatabase, + @NonNull List attachments) + throws UndeliverableMessageException + { + List results = new ArrayList<>(attachments.size()); + + for (Attachment attachment : attachments) { + results.add(scaleAndStripExifToDatabase(attachmentDatabase, (DatabaseAttachment) attachment, null)); + } + + return results; + } + + DatabaseAttachment scaleAndStripExifToDatabase(@NonNull AttachmentDatabase attachmentDatabase, + @NonNull DatabaseAttachment attachment, + @Nullable ProgressListener transcodeProgressListener) + throws UndeliverableMessageException + { + try { + if (MediaUtil.isVideo(attachment) && MediaConstraints.isVideoTranscodeAvailable()) { + return transcodeVideoIfNeededToDatabase(attachmentDatabase, attachment, transcodeProgressListener); + } else if (constraints.isSatisfied(context, attachment)) { + if (MediaUtil.isJpeg(attachment)) { + MediaStream stripped = getResizedMedia(context, attachment); + return attachmentDatabase.updateAttachmentData(attachment, stripped); + } else { + return attachment; + } + } else if (constraints.canResize(attachment)) { + MediaStream resized = getResizedMedia(context, attachment); + return attachmentDatabase.updateAttachmentData(attachment, resized); + } else { + throw new UndeliverableMessageException("Size constraints could not be met!"); + } + } catch (IOException | MmsException e) { + throw new UndeliverableMessageException(e); + } + } + + @RequiresApi(26) + private @NonNull DatabaseAttachment transcodeVideoIfNeededToDatabase(@NonNull AttachmentDatabase attachmentDatabase, + @NonNull DatabaseAttachment attachment, + @Nullable ProgressListener progressListener) + throws UndeliverableMessageException + { + try (NotificationController notification = GenericForegroundService.startForegroundTask(context, context.getString(R.string.AttachmentUploadJob_compressing_video_start))) { + + notification.setIndeterminateProgress(); + + try (MediaDataSource dataSource = attachmentDatabase.mediaDataSourceFor(attachment.getAttachmentId())) { + + if (dataSource == null) { + throw new UndeliverableMessageException("Cannot get media data source for attachment."); + } + + try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, constraints.getCompressedVideoMaxSize(context))) { + + if (transcoder.isTranscodeRequired()) { + + MediaStream mediaStream = transcoder.transcode(percent -> { + notification.setProgress(100, percent); + + if (progressListener != null) { + progressListener.onProgress(percent, 100); + } + }); + + return attachmentDatabase.updateAttachmentData(attachment, mediaStream); + } else { + return attachment; + } + } + } + } catch (IOException | MmsException | BadVideoException e) { + throw new UndeliverableMessageException("Failed to transcode", e); + } + } + + private MediaStream getResizedMedia(@NonNull Context context, @NonNull Attachment attachment) + throws IOException + { + if (!constraints.canResize(attachment)) { + throw new UnsupportedOperationException("Cannot resize this content type"); + } + + try { + // XXX - This is loading everything into memory! We want the send path to be stream-like. + BitmapUtil.ScaleResult scaleResult = BitmapUtil.createScaledBytes(context, new DecryptableStreamUriLoader.DecryptableUri(attachment.getDataUri()), constraints); + return new MediaStream(new ByteArrayInputStream(scaleResult.getBitmap()), MediaUtil.IMAGE_JPEG, scaleResult.getWidth(), scaleResult.getHeight()); + } catch (BitmapDecodingException e) { + throw new IOException(e); + } + } + + public interface ProgressListener { + + void onProgress(long progress, long total); + } +} diff --git a/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java b/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java index 2a75ad07d..6077c4e06 100644 --- a/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java +++ b/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java @@ -94,7 +94,7 @@ public class PushGroupSendJob extends PushSendJob { attachments.addAll(Stream.of(message.getLinkPreviews()).filter(p -> p.getThumbnail().isPresent()).map(p -> p.getThumbnail().get()).toList()); attachments.addAll(Stream.of(message.getSharedContacts()).filter(c -> c.getAvatar() != null).map(c -> c.getAvatar().getAttachment()).withoutNulls().toList()); - List attachmentJobs = Stream.of(attachments).map(a -> new AttachmentUploadJob(((DatabaseAttachment) a).getAttachmentId())).toList(); + List attachmentJobs = Stream.of(attachments).map(a -> AttachmentUploadJob.fromAttachment((DatabaseAttachment) a)).toList(); if (attachmentJobs.isEmpty()) { jobManager.add(new PushGroupSendJob(messageId, destination, filterAddress)); diff --git a/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java b/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java index adc627a76..fc2b73ddb 100644 --- a/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java +++ b/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java @@ -81,7 +81,7 @@ public class PushMediaSendJob extends PushSendJob { attachments.addAll(Stream.of(message.getLinkPreviews()).filter(p -> p.getThumbnail().isPresent()).map(p -> p.getThumbnail().get()).toList()); attachments.addAll(Stream.of(message.getSharedContacts()).filter(c -> c.getAvatar() != null).map(c -> c.getAvatar().getAttachment()).withoutNulls().toList()); - List attachmentJobs = Stream.of(attachments).map(a -> new AttachmentUploadJob(((DatabaseAttachment) a).getAttachmentId())).toList(); + List attachmentJobs = Stream.of(attachments).map(a -> AttachmentUploadJob.fromAttachment((DatabaseAttachment) a)).toList(); if (attachmentJobs.isEmpty()) { jobManager.add(new PushMediaSendJob(messageId, destination)); diff --git a/src/org/thoughtcrime/securesms/jobs/SendJob.java b/src/org/thoughtcrime/securesms/jobs/SendJob.java index 525d41229..1775da3a4 100644 --- a/src/org/thoughtcrime/securesms/jobs/SendJob.java +++ b/src/org/thoughtcrime/securesms/jobs/SendJob.java @@ -1,7 +1,6 @@ package org.thoughtcrime.securesms.jobs; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import org.thoughtcrime.securesms.BuildConfig; import org.thoughtcrime.securesms.TextSecureExpiredException; @@ -9,17 +8,11 @@ import org.thoughtcrime.securesms.attachments.Attachment; import org.thoughtcrime.securesms.database.AttachmentDatabase; import org.thoughtcrime.securesms.database.DatabaseFactory; import org.thoughtcrime.securesms.jobmanager.Job; -import org.thoughtcrime.securesms.jobmanager.JobLogger; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.MediaConstraints; -import org.thoughtcrime.securesms.mms.MediaStream; -import org.thoughtcrime.securesms.mms.MmsException; import org.thoughtcrime.securesms.transport.UndeliverableMessageException; -import org.thoughtcrime.securesms.util.MediaUtil; import org.thoughtcrime.securesms.util.Util; -import java.io.IOException; -import java.util.LinkedList; import java.util.List; public abstract class SendJob extends BaseJob { @@ -58,45 +51,9 @@ public abstract class SendJob extends BaseJob { @NonNull List attachments) throws UndeliverableMessageException { + MediaResizer mediaResizer = new MediaResizer(context, constraints); AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context); - List results = new LinkedList<>(); - for (Attachment attachment : attachments) { - try { - if (constraints.isSatisfied(context, attachment)) { - if (MediaUtil.isJpeg(attachment)) { - MediaStream stripped = constraints.getResizedMedia(context, attachment); - results.add(attachmentDatabase.updateAttachmentData(attachment, stripped)); - } else { - results.add(attachment); - } - } else if (constraints.canResize(attachment)) { - MediaStream resized = constraints.getResizedMedia(context, attachment); - results.add(attachmentDatabase.updateAttachmentData(attachment, resized)); - } else { - throw new UndeliverableMessageException("Size constraints could not be met!"); - } - } catch (IOException | MmsException e) { - throw new UndeliverableMessageException(e); - } - } - - return results; - } - - protected void log(@NonNull String tag, @NonNull String message) { - Log.i(tag, JobLogger.format(this, message)); - } - - protected void warn(@NonNull String tag, @NonNull String message) { - warn(tag, message, null); - } - - protected void warn(@NonNull String tag, @Nullable Throwable t) { - warn(tag, "", t); - } - - protected void warn(@NonNull String tag, @NonNull String message, @Nullable Throwable t) { - Log.w(tag, JobLogger.format(this, message), t); + return mediaResizer.scaleAndStripExifToDatabase(attachmentDatabase, attachments); } } diff --git a/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java b/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java index 1834a197e..6d642c0c9 100644 --- a/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java +++ b/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java @@ -466,7 +466,7 @@ class MediaSendViewModel extends ViewModel { .filter(m -> { return (MediaUtil.isImageType(m.getMimeType()) && !MediaUtil.isGif(m.getMimeType())) || (MediaUtil.isGif(m.getMimeType()) && m.getSize() < mediaConstraints.getGifMaxSize(context)) || - (MediaUtil.isVideoType(m.getMimeType()) && m.getSize() < mediaConstraints.getVideoMaxSize(context)); + (MediaUtil.isVideoType(m.getMimeType()) && m.getSize() < mediaConstraints.getUncompressedVideoMaxSize(context)); }).toList(); } diff --git a/src/org/thoughtcrime/securesms/mms/MediaConstraints.java b/src/org/thoughtcrime/securesms/mms/MediaConstraints.java index 37ffb87e8..dab8c05fa 100644 --- a/src/org/thoughtcrime/securesms/mms/MediaConstraints.java +++ b/src/org/thoughtcrime/securesms/mms/MediaConstraints.java @@ -2,18 +2,18 @@ package org.thoughtcrime.securesms.mms; import android.content.Context; import android.net.Uri; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import org.thoughtcrime.securesms.logging.Log; +import android.os.Build; import android.util.Pair; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + import org.thoughtcrime.securesms.attachments.Attachment; -import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader.DecryptableUri; +import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.util.BitmapDecodingException; import org.thoughtcrime.securesms.util.BitmapUtil; import org.thoughtcrime.securesms.util.MediaUtil; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -34,6 +34,15 @@ public abstract class MediaConstraints { public abstract int getGifMaxSize(Context context); public abstract int getVideoMaxSize(Context context); + + public int getUncompressedVideoMaxSize(Context context) { + return getVideoMaxSize(context); + } + + public int getCompressedVideoMaxSize(Context context) { + return getVideoMaxSize(context); + } + public abstract int getAudioMaxSize(Context context); public abstract int getDocumentMaxSize(Context context); @@ -61,23 +70,12 @@ public abstract class MediaConstraints { } } - public boolean canResize(@Nullable Attachment attachment) { - return attachment != null && MediaUtil.isImage(attachment) && !MediaUtil.isGif(attachment); + public boolean canResize(@NonNull Attachment attachment) { + return MediaUtil.isImage(attachment) && !MediaUtil.isGif(attachment) || + MediaUtil.isVideo(attachment) && isVideoTranscodeAvailable(); } - public MediaStream getResizedMedia(@NonNull Context context, @NonNull Attachment attachment) - throws IOException - { - if (!canResize(attachment)) { - throw new UnsupportedOperationException("Cannot resize this content type"); - } - - try { - // XXX - This is loading everything into memory! We want the send path to be stream-like. - BitmapUtil.ScaleResult scaleResult = BitmapUtil.createScaledBytes(context, new DecryptableUri(attachment.getDataUri()), this); - return new MediaStream(new ByteArrayInputStream(scaleResult.getBitmap()), MediaUtil.IMAGE_JPEG, scaleResult.getWidth(), scaleResult.getHeight()); - } catch (BitmapDecodingException e) { - throw new IOException(e); - } + public static boolean isVideoTranscodeAvailable() { + return Build.VERSION.SDK_INT >= 26; } } diff --git a/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java b/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java index d7b6c9666..892c610b0 100644 --- a/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java +++ b/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java @@ -39,6 +39,11 @@ final class MmsMediaConstraints extends MediaConstraints { return getMaxMessageSize(context); } + @Override + public int getUncompressedVideoMaxSize(Context context) { + return Math.max(getVideoMaxSize(context), 15 * 1024 * 1024); + } + @Override public int getAudioMaxSize(Context context) { return getMaxMessageSize(context); diff --git a/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java b/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java index 30a6b1ea1..ef936dfeb 100644 --- a/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java +++ b/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java @@ -36,6 +36,18 @@ public class PushMediaConstraints extends MediaConstraints { return 100 * MB; } + @Override + public int getUncompressedVideoMaxSize(Context context) { + return isVideoTranscodeAvailable() ? 200 * MB + : getVideoMaxSize(context); + } + + @Override + public int getCompressedVideoMaxSize(Context context) { + return Util.isLowMemory(context) ? 30 * MB + : 50 * MB; + } + @Override public int getAudioMaxSize(Context context) { return 100 * MB; diff --git a/src/org/thoughtcrime/securesms/mms/Slide.java b/src/org/thoughtcrime/securesms/mms/Slide.java index 2c5dc1cb1..a7743f972 100644 --- a/src/org/thoughtcrime/securesms/mms/Slide.java +++ b/src/org/thoughtcrime/securesms/mms/Slide.java @@ -105,7 +105,7 @@ public abstract class Slide { public @NonNull String getContentDescription() { return ""; } - public Attachment asAttachment() { + public @NonNull Attachment asAttachment() { return attachment; } diff --git a/src/org/thoughtcrime/securesms/sms/MessageSender.java b/src/org/thoughtcrime/securesms/sms/MessageSender.java index 5b4b4da09..97f865c55 100644 --- a/src/org/thoughtcrime/securesms/sms/MessageSender.java +++ b/src/org/thoughtcrime/securesms/sms/MessageSender.java @@ -20,6 +20,8 @@ import android.app.Application; import android.content.Context; import androidx.annotation.NonNull; +import com.annimon.stream.Stream; + import org.thoughtcrime.securesms.attachments.AttachmentId; import org.thoughtcrime.securesms.attachments.DatabaseAttachment; import org.thoughtcrime.securesms.database.MessagingDatabase.SyncMessageId; @@ -133,14 +135,14 @@ public class MessageSender { return; } - ThreadDatabase threadDatabase = DatabaseFactory.getThreadDatabase(context); - MmsDatabase mmsDatabase = DatabaseFactory.getMmsDatabase(context); - AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context); - List> attachmentIds = new ArrayList<>(messages.get(0).getAttachments().size()); - List messageIds = new ArrayList<>(messages.size()); + ThreadDatabase threadDatabase = DatabaseFactory.getThreadDatabase(context); + MmsDatabase mmsDatabase = DatabaseFactory.getMmsDatabase(context); + AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context); + List> databaseAttachments = new ArrayList<>(messages.get(0).getAttachments().size()); + List messageIds = new ArrayList<>(messages.size()); for (int i = 0; i < messages.get(0).getAttachments().size(); i++) { - attachmentIds.add(new ArrayList<>(messages.size())); + databaseAttachments.add(new ArrayList<>(messages.size())); } try { @@ -152,13 +154,13 @@ public class MessageSender { long messageId = mmsDatabase.insertMessageOutbox(message, allocatedThreadId, false, null); List attachments = attachmentDatabase.getAttachmentsForMessage(messageId); - if (attachments.size() != attachmentIds.size()) { - Log.w(TAG, "Got back an attachment list that was a different size than expected. Expected: " + attachmentIds.size() + " Actual: "+ attachments.size()); + if (attachments.size() != databaseAttachments.size()) { + Log.w(TAG, "Got back an attachment list that was a different size than expected. Expected: " + databaseAttachments.size() + " Actual: "+ attachments.size()); return; } for (int i = 0; i < attachments.size(); i++) { - attachmentIds.get(i).add(attachments.get(i).getAttachmentId()); + databaseAttachments.get(i).add(attachments.get(i)); } messageIds.add(messageId); @@ -169,16 +171,20 @@ public class MessageSender { mmsDatabase.endTransaction(); } - List uploadJobs = new ArrayList<>(attachmentIds.size()); - List copyJobs = new ArrayList<>(attachmentIds.size()); - List messageJobs = new ArrayList<>(attachmentIds.get(0).size()); + List uploadJobs = new ArrayList<>(databaseAttachments.size()); + List copyJobs = new ArrayList<>(databaseAttachments.size()); + List messageJobs = new ArrayList<>(databaseAttachments.get(0).size()); - for (List idList : attachmentIds) { - uploadJobs.add(new AttachmentUploadJob(idList.get(0))); + for (List attachmentList : databaseAttachments) { + DatabaseAttachment source = attachmentList.get(0); - if (idList.size() > 1) { - AttachmentId sourceId = idList.get(0); - List destinationIds = idList.subList(1, idList.size()); + uploadJobs.add(AttachmentUploadJob.fromAttachment(source)); + + if (attachmentList.size() > 1) { + AttachmentId sourceId = source.getAttachmentId(); + List destinationIds = Stream.of(attachmentList.subList(1, attachmentList.size())) + .map(DatabaseAttachment::getAttachmentId) + .toList(); copyJobs.add(new AttachmentCopyJob(sourceId, destinationIds)); } diff --git a/src/org/thoughtcrime/securesms/util/FileUtils.java b/src/org/thoughtcrime/securesms/util/FileUtils.java index e437edd84..7b516a9f9 100644 --- a/src/org/thoughtcrime/securesms/util/FileUtils.java +++ b/src/org/thoughtcrime/securesms/util/FileUtils.java @@ -7,7 +7,7 @@ import java.io.IOException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -public class FileUtils { +public final class FileUtils { static { System.loadLibrary("native-utils"); @@ -15,6 +15,8 @@ public class FileUtils { public static native int getFileDescriptorOwner(FileDescriptor fileDescriptor); + static native int createMemoryFileDescriptor(String name); + public static byte[] getFileDigest(FileInputStream fin) throws IOException { try { MessageDigest digest = MessageDigest.getInstance("SHA256"); diff --git a/src/org/thoughtcrime/securesms/util/MediaUtil.java b/src/org/thoughtcrime/securesms/util/MediaUtil.java index 625337f3d..7d510b8fc 100644 --- a/src/org/thoughtcrime/securesms/util/MediaUtil.java +++ b/src/org/thoughtcrime/securesms/util/MediaUtil.java @@ -280,16 +280,17 @@ public class MediaUtil { return sections.length > 1 ? sections[0] : null; } - public static class ThumbnailData { - Bitmap bitmap; - float aspectRatio; + public static class ThumbnailData implements AutoCloseable { - public ThumbnailData(Bitmap bitmap) { + @NonNull private final Bitmap bitmap; + private final float aspectRatio; + + public ThumbnailData(@NonNull Bitmap bitmap) { this.bitmap = bitmap; this.aspectRatio = (float) bitmap.getWidth() / (float) bitmap.getHeight(); } - public Bitmap getBitmap() { + public @NonNull Bitmap getBitmap() { return bitmap; } @@ -300,5 +301,10 @@ public class MediaUtil { public InputStream toDataStream() { return BitmapUtil.toCompressedJpeg(bitmap); } + + @Override + public void close() { + bitmap.recycle(); + } } } diff --git a/src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java b/src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java new file mode 100644 index 000000000..dcc6ea278 --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java @@ -0,0 +1,158 @@ +package org.thoughtcrime.securesms.util; + +import android.app.ActivityManager; +import android.content.Context; +import android.os.ParcelFileDescriptor; + +import androidx.annotation.NonNull; + +import org.thoughtcrime.securesms.logging.Log; + +import java.io.Closeable; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.text.NumberFormat; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicLong; + +public final class MemoryFileDescriptor implements Closeable { + + private static final String TAG = Log.tag(MemoryFileDescriptor.class); + + private final ParcelFileDescriptor parcelFileDescriptor; + private final AtomicLong sizeEstimate; + + /** + * memfd files do not show on the available RAM, so we must track our allocations in addition. + */ + private static long sizeOfAllMemoryFileDescriptors; + + private MemoryFileDescriptor(@NonNull ParcelFileDescriptor parcelFileDescriptor, long sizeEstimate) { + this.parcelFileDescriptor = parcelFileDescriptor; + this.sizeEstimate = new AtomicLong(sizeEstimate); + } + + /** + * @param debugName The name supplied in name is used as a filename and will be displayed + * as the target of the corresponding symbolic link in the directory + * /proc/self/fd/. The displayed name is always prefixed with memfd: + * and serves only for debugging purposes. Names do not affect the + * behavior of the file descriptor, and as such multiple files can have + * the same name without any side effects. + * @param sizeEstimate An estimated upper bound on this file. This is used to check there will be + * enough RAM available and to register with a global counter of reservations. + * Use zero to avoid RAM check. + * @return MemoryFileDescriptor + * @throws MemoryLimitException If there is not enough available RAM to comfortably fit this file. + * @throws IOException If fails to create a memory file descriptor. + */ + public static MemoryFileDescriptor newMemoryFileDescriptor(@NonNull Context context, + @NonNull String debugName, + long sizeEstimate) + throws MemoryLimitException, IOException + { + if (sizeEstimate < 0) throw new IllegalArgumentException(); + + if (sizeEstimate > 0) { + ActivityManager activityManager = ServiceUtil.getActivityManager(context); + ActivityManager.MemoryInfo memoryInfo = new ActivityManager.MemoryInfo(); + + synchronized (MemoryFileDescriptor.class) { + activityManager.getMemoryInfo(memoryInfo); + + long remainingRam = memoryInfo.availMem - memoryInfo.threshold - sizeEstimate - sizeOfAllMemoryFileDescriptors; + + if (remainingRam <= 0) { + NumberFormat numberFormat = NumberFormat.getInstance(Locale.US); + Log.w(TAG, String.format("Not enough RAM available without taking the system into a low memory state.%n" + + "Available: %s%n" + + "Low memory threshold: %s%n" + + "Requested: %s%n" + + "Total MemoryFileDescriptor limit: %s%n" + + "Shortfall: %s", + numberFormat.format(memoryInfo.availMem), + numberFormat.format(memoryInfo.threshold), + numberFormat.format(sizeEstimate), + numberFormat.format(sizeOfAllMemoryFileDescriptors), + numberFormat.format(remainingRam) + )); + throw new MemoryLimitException(); + } + + sizeOfAllMemoryFileDescriptors += sizeEstimate; + } + } + + int fileDescriptor = FileUtils.createMemoryFileDescriptor(debugName); + + if (fileDescriptor < 0) { + throw new IOException("Failed to create a memory file descriptor " + fileDescriptor); + } + + return new MemoryFileDescriptor(ParcelFileDescriptor.adoptFd(fileDescriptor), sizeEstimate); + } + + @Override + public void close() throws IOException { + try { + clearAndRemoveAllocation(); + } catch (Exception e) { + Log.w(TAG, "Failed to clear data in MemoryFileDescriptor", e); + } finally { + parcelFileDescriptor.close(); + } + } + + private void clearAndRemoveAllocation() throws IOException { + clear(); + + long oldEstimate = sizeEstimate.getAndSet(0); + + synchronized (MemoryFileDescriptor.class) { + sizeOfAllMemoryFileDescriptors -= oldEstimate; + } + } + + /** Rewinds and clears all bytes. */ + private void clear() throws IOException { + long size; + try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) { + FileChannel channel = fileInputStream.getChannel(); + size = channel.size(); + + if (size == 0) return; + + channel.position(0); + } + byte[] zeros = new byte[16 * 1024]; + + try (FileOutputStream output = new FileOutputStream(getFileDescriptor())) { + while (size > 0) { + int limit = (int) Math.min(size, zeros.length); + + output.write(zeros, 0, limit); + + size -= limit; + } + } + } + + public FileDescriptor getFileDescriptor() { + return parcelFileDescriptor.getFileDescriptor(); + } + + public void seek(long position) throws IOException { + try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) { + fileInputStream.getChannel().position(position); + } + } + + public long size() throws IOException { + try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) { + return fileInputStream.getChannel().size(); + } + } +} diff --git a/src/org/thoughtcrime/securesms/util/MemoryLimitException.java b/src/org/thoughtcrime/securesms/util/MemoryLimitException.java new file mode 100644 index 000000000..2dfe9d0d8 --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/MemoryLimitException.java @@ -0,0 +1,6 @@ +package org.thoughtcrime.securesms.util; + +import java.io.IOException; + +public final class MemoryLimitException extends IOException { +} diff --git a/src/org/thoughtcrime/securesms/util/ServiceUtil.java b/src/org/thoughtcrime/securesms/util/ServiceUtil.java index 76a413b7f..a86d06597 100644 --- a/src/org/thoughtcrime/securesms/util/ServiceUtil.java +++ b/src/org/thoughtcrime/securesms/util/ServiceUtil.java @@ -1,6 +1,7 @@ package org.thoughtcrime.securesms.util; import android.app.Activity; +import android.app.ActivityManager; import android.app.AlarmManager; import android.app.NotificationManager; import android.app.job.JobScheduler; @@ -69,4 +70,8 @@ public class ServiceUtil { public static @Nullable SubscriptionManager getSubscriptionManager(@NonNull Context context) { return (SubscriptionManager) context.getSystemService(Context.TELEPHONY_SUBSCRIPTION_SERVICE); } + + public static ActivityManager getActivityManager(@NonNull Context context) { + return (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + } } diff --git a/src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java b/src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java new file mode 100644 index 000000000..f4a019978 --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java @@ -0,0 +1,54 @@ +package org.thoughtcrime.securesms.video; + +import android.media.MediaDataSource; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.crypto.AttachmentSecret; +import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream; +import org.thoughtcrime.securesms.util.Util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +@RequiresApi(23) +final class ClassicEncryptedMediaDataSource extends MediaDataSource { + + private final AttachmentSecret attachmentSecret; + private final File mediaFile; + private final long length; + + ClassicEncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, long length) { + this.attachmentSecret = attachmentSecret; + this.mediaFile = mediaFile; + this.length = length; + } + + @Override + public int readAt(long position, byte[] bytes, int offset, int length) throws IOException { + try (InputStream inputStream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, mediaFile)) { + byte[] buffer = new byte[4096]; + long headerRemaining = position; + + while (headerRemaining > 0) { + int read = inputStream.read(buffer, 0, Util.toIntExact(Math.min((long)buffer.length, headerRemaining))); + + if (read == -1) return -1; + + headerRemaining -= read; + } + + return inputStream.read(bytes, offset, length); + } + } + + @Override + public long getSize() { + return length; + } + + @Override + public void close() {} +} diff --git a/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java b/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java index f2e036ed8..d4af426dc 100644 --- a/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java +++ b/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java @@ -1,78 +1,23 @@ package org.thoughtcrime.securesms.video; - -import android.annotation.TargetApi; import android.media.MediaDataSource; -import android.os.Build; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; import org.thoughtcrime.securesms.crypto.AttachmentSecret; -import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream; -import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream; -import org.thoughtcrime.securesms.util.Util; import java.io.File; -import java.io.IOException; -import java.io.InputStream; -@TargetApi(Build.VERSION_CODES.M) -public class EncryptedMediaDataSource extends MediaDataSource { +@RequiresApi(23) +public final class EncryptedMediaDataSource { - private final AttachmentSecret attachmentSecret; - private final File mediaFile; - private final byte[] random; - private final long length; - - public EncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @Nullable byte[] random, long length) { - this.attachmentSecret = attachmentSecret; - this.mediaFile = mediaFile; - this.random = random; - this.length = length; - } - - @Override - public int readAt(long position, byte[] bytes, int offset, int length) throws IOException { - if (random == null) return readAtClassic(position, bytes, offset, length); - else return readAtModern(position, bytes, offset, length); - } - - private int readAtClassic(long position, byte[] bytes, int offset, int length) throws IOException { - InputStream inputStream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, mediaFile); - byte[] buffer = new byte[4096]; - long headerRemaining = position; - - while (headerRemaining > 0) { - int read = inputStream.read(buffer, 0, Util.toIntExact(Math.min((long)buffer.length, headerRemaining))); - - if (read == -1) return -1; - - headerRemaining -= read; + public static MediaDataSource createFor(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @Nullable byte[] random, long length) { + if (random == null) { + return new ClassicEncryptedMediaDataSource(attachmentSecret, mediaFile, length); + } else { + return new ModernEncryptedMediaDataSource(attachmentSecret, mediaFile, random, length); } - - int returnValue = inputStream.read(bytes, offset, length); - inputStream.close(); - return returnValue; - } - - private int readAtModern(long position, byte[] bytes, int offset, int length) throws IOException { - assert(random != null); - - InputStream inputStream = ModernDecryptingPartInputStream.createFor(attachmentSecret, random, mediaFile, position); - int returnValue = inputStream.read(bytes, offset, length); - - inputStream.close(); - - return returnValue; - } - - @Override - public long getSize() throws IOException { - return length; - } - - @Override - public void close() throws IOException { - } } diff --git a/src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java b/src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java new file mode 100644 index 000000000..4863e6ffe --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java @@ -0,0 +1,201 @@ +package org.thoughtcrime.securesms.video; + +import android.content.Context; +import android.media.MediaDataSource; +import android.media.MediaMetadataRetriever; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import com.google.android.exoplayer2.util.MimeTypes; + +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.mms.MediaStream; +import org.thoughtcrime.securesms.transport.UndeliverableMessageException; +import org.thoughtcrime.securesms.util.MemoryFileDescriptor; +import org.thoughtcrime.securesms.video.videoconverter.BadVideoException; +import org.thoughtcrime.securesms.video.videoconverter.MediaConverter; + +import java.io.Closeable; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.IOException; +import java.text.NumberFormat; +import java.util.Locale; + +@RequiresApi(26) +public final class InMemoryTranscoder implements Closeable { + + private static final String TAG = Log.tag(InMemoryTranscoder.class); + + private static final int MAXIMUM_TARGET_VIDEO_BITRATE = 2_000_000; + private static final int LOW_RES_TARGET_VIDEO_BITRATE = 1_750_000; + private static final int MINIMUM_TARGET_VIDEO_BITRATE = 500_000; + private static final int AUDIO_BITRATE = 192_000; + private static final int OUTPUT_FORMAT = 720; + private static final int LOW_RES_OUTPUT_FORMAT = 480; + + private final Context context; + private final MediaDataSource dataSource; + private final long upperSizeLimit; + private final long inSize; + private final long duration; + private final int inputBitRate; + private final int targetVideoBitRate; + private final long memoryFileEstimate; + private final boolean transcodeRequired; + private final long fileSizeEstimate; + private final int outputFormat; + + private @Nullable MemoryFileDescriptor memoryFile; + + /** + * @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller. + */ + public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, long upperSizeLimit) throws IOException { + this.context = context; + this.dataSource = dataSource; + + final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever(); + mediaMetadataRetriever.setDataSource(dataSource); + + long upperSizeLimitWithMargin = (long) (upperSizeLimit / 1.1); + + this.inSize = dataSource.getSize(); + this.duration = getDuration(mediaMetadataRetriever); + this.inputBitRate = bitRate(inSize, duration); + this.targetVideoBitRate = getTargetVideoBitRate(upperSizeLimitWithMargin, duration); + this.upperSizeLimit = upperSizeLimit; + + this.transcodeRequired = inputBitRate >= targetVideoBitRate * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever); + if (!transcodeRequired) { + Log.i(TAG, "Video is within 20% of target bitrate, below the size limit and contained no location metadata."); + } + + this.fileSizeEstimate = (targetVideoBitRate + AUDIO_BITRATE) * duration / 8000; + this.memoryFileEstimate = (long) (fileSizeEstimate * 1.1); + this.outputFormat = targetVideoBitRate < LOW_RES_TARGET_VIDEO_BITRATE + ? LOW_RES_OUTPUT_FORMAT + : OUTPUT_FORMAT; + } + + public @NonNull MediaStream transcode(@NonNull Progress progress) throws IOException, UndeliverableMessageException, BadVideoException { + if (memoryFile != null) throw new AssertionError("Not expecting to reuse transcoder"); + + float durationSec = duration / 1000f; + + NumberFormat numberFormat = NumberFormat.getInstance(Locale.US); + + Log.i(TAG, String.format(Locale.US, + "Transcoding:\n" + + "Target bitrate : %s + %s = %s\n" + + "Target format : %dp\n" + + "Video duration : %.1fs\n" + + "Size limit : %s kB\n" + + "Estimate : %s kB\n" + + "Input size : %s kB\n" + + "Input bitrate : %s bps", + numberFormat.format(targetVideoBitRate), + numberFormat.format(AUDIO_BITRATE), + numberFormat.format(targetVideoBitRate + AUDIO_BITRATE), + outputFormat, + durationSec, + numberFormat.format(upperSizeLimit / 1024), + numberFormat.format(fileSizeEstimate / 1024), + numberFormat.format(inSize / 1024), + numberFormat.format(inputBitRate))); + + if (fileSizeEstimate > upperSizeLimit) { + throw new UndeliverableMessageException("Size constraints could not be met!"); + } + + memoryFile = MemoryFileDescriptor.newMemoryFileDescriptor(context, + "TRANSCODE", + memoryFileEstimate); + final long startTime = System.currentTimeMillis(); + + final FileDescriptor memoryFileFileDescriptor = memoryFile.getFileDescriptor(); + + final MediaConverter converter = new MediaConverter(); + + converter.setInput(dataSource); + converter.setOutput(memoryFileFileDescriptor); + converter.setVideoResolution(outputFormat); + converter.setVideoBitrate(targetVideoBitRate); + converter.setAudioBitrate(AUDIO_BITRATE); + + converter.setListener(percent -> { + progress.onProgress(percent); + return false; + }); + + converter.convert(); + + // output details of the transcoding + long outSize = memoryFile.size(); + float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f; + + Log.i(TAG, String.format(Locale.US, + "Transcoding complete:\n" + + "Transcode time : %.1fs (%.1fx)\n" + + "Output size : %s kB\n" + + " of Original : %.1f%%\n" + + " of Estimate : %.1f%%\n" + + " of Memory : %.1f%%\n" + + "Output bitrate : %s bps", + encodeDurationSec, + durationSec / encodeDurationSec, + numberFormat.format(outSize / 1024), + (outSize * 100d) / inSize, + (outSize * 100d) / fileSizeEstimate, + (outSize * 100d) / memoryFileEstimate, + numberFormat.format(bitRate(outSize, duration)))); + + if (outSize > upperSizeLimit) { + throw new UndeliverableMessageException("Size constraints could not be met!"); + } + + memoryFile.seek(0); + + return new MediaStream(new FileInputStream(memoryFileFileDescriptor), MimeTypes.VIDEO_MP4, 0, 0); + } + + public boolean isTranscodeRequired() { + return transcodeRequired; + } + + @Override + public void close() throws IOException { + if (memoryFile != null) { + memoryFile.close(); + } + } + + private static int bitRate(long bytes, long duration) { + return (int) (bytes * 8 / (duration / 1000f)); + } + + private static int getTargetVideoBitRate(long sizeGuideBytes, long duration) { + sizeGuideBytes -= (duration / 1000d) * AUDIO_BITRATE / 8; + + double targetAttachmentSizeBits = sizeGuideBytes * 8L; + + double bitRateToFixTarget = targetAttachmentSizeBits / (duration / 1000d); + return Math.max(MINIMUM_TARGET_VIDEO_BITRATE, Math.min(MAXIMUM_TARGET_VIDEO_BITRATE, (int) bitRateToFixTarget)); + } + + private static long getDuration(MediaMetadataRetriever mediaMetadataRetriever) { + return Long.parseLong(mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)); + } + + private static boolean containsLocation(MediaMetadataRetriever mediaMetadataRetriever) { + String locationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION); + return locationString != null; + } + + public interface Progress { + + void onProgress(int percent); + } +} diff --git a/src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java b/src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java new file mode 100644 index 000000000..738377a96 --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java @@ -0,0 +1,71 @@ +package org.thoughtcrime.securesms.video; + +import android.media.MediaDataSource; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.crypto.AttachmentSecret; +import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +/** + * Create via {@link EncryptedMediaDataSource}. + *

+ * A {@link MediaDataSource} that points to an encrypted file. + *

+ * It is "modern" compared to the {@link ClassicEncryptedMediaDataSource}. And "modern" refers to + * the presence of a random part of the key supplied in the constructor. + */ +@RequiresApi(23) +final class ModernEncryptedMediaDataSource extends MediaDataSource { + + private final AttachmentSecret attachmentSecret; + private final File mediaFile; + private final byte[] random; + private final long length; + + ModernEncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @NonNull byte[] random, long length) { + this.attachmentSecret = attachmentSecret; + this.mediaFile = mediaFile; + this.random = random; + this.length = length; + } + + @Override + public int readAt(long position, byte[] bytes, int offset, int length) throws IOException { + try (InputStream inputStream = ModernDecryptingPartInputStream.createFor(attachmentSecret, random, mediaFile, position)) { + int totalRead = 0; + + while (length > 0) { + int read = inputStream.read(bytes, offset, length); + + if (read == -1) { + if (totalRead == 0) { + return -1; + } else { + return totalRead; + } + } + + length -= read; + offset += read; + totalRead += read; + } + + return totalRead; + } + } + + @Override + public long getSize() { + return length; + } + + @Override + public void close() { + } +}