From e8e80e5d0558116de0d71b58508fe30d6dd6eba8 Mon Sep 17 00:00:00 2001 From: Alan Evans Date: Thu, 25 Jul 2019 15:29:31 -0400 Subject: [PATCH] Transcode video during attachment upload. --- ..._thoughtcrime_securesms_util_FileUtils.cpp | 16 +- ...rg_thoughtcrime_securesms_util_FileUtils.h | 8 + libs/arm64-v8a/libnative-utils.so | Bin 5912 -> 10008 bytes libs/armeabi-v7a/libnative-utils.so | Bin 13948 -> 13948 bytes libs/x86/libnative-utils.so | Bin 5596 -> 5596 bytes libs/x86_64/libnative-utils.so | Bin 6184 -> 10280 bytes res/values/strings.xml | 1 + .../database/AttachmentDatabase.java | 83 ++++---- .../securesms/jobs/AttachmentUploadJob.java | 68 +++--- .../thoughtcrime/securesms/jobs/BaseJob.java | 17 ++ .../securesms/jobs/MediaResizer.java | 142 +++++++++++++ .../securesms/jobs/PushGroupSendJob.java | 2 +- .../securesms/jobs/PushMediaSendJob.java | 2 +- .../thoughtcrime/securesms/jobs/SendJob.java | 47 +--- .../mediasend/MediaSendViewModel.java | 2 +- .../securesms/mms/MediaConstraints.java | 40 ++-- .../securesms/mms/MmsMediaConstraints.java | 5 + .../securesms/mms/PushMediaConstraints.java | 12 ++ src/org/thoughtcrime/securesms/mms/Slide.java | 2 +- .../securesms/sms/MessageSender.java | 40 ++-- .../securesms/util/FileUtils.java | 4 +- .../securesms/util/MediaUtil.java | 16 +- .../securesms/util/MemoryFileDescriptor.java | 158 ++++++++++++++ .../securesms/util/MemoryLimitException.java | 6 + .../securesms/util/ServiceUtil.java | 5 + .../ClassicEncryptedMediaDataSource.java | 54 +++++ .../video/EncryptedMediaDataSource.java | 73 +------ .../securesms/video/InMemoryTranscoder.java | 201 ++++++++++++++++++ .../video/ModernEncryptedMediaDataSource.java | 71 +++++++ 29 files changed, 853 insertions(+), 222 deletions(-) create mode 100644 src/org/thoughtcrime/securesms/jobs/MediaResizer.java create mode 100644 src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java create mode 100644 src/org/thoughtcrime/securesms/util/MemoryLimitException.java create mode 100644 src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java create mode 100644 src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java create mode 100644 src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java diff --git a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp index ef86ac8e3..27ce43096 100644 --- a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp +++ b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwner (JNIEnv *env, jclass clazz, jobject fileDescriptor) @@ -28,4 +30,16 @@ jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwn } return stat_struct.st_uid; -} \ No newline at end of file +} + +JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_createMemoryFileDescriptor + (JNIEnv *env, jclass clazz, jstring jname) +{ + const char *name = env->GetStringUTFChars(jname, NULL); + + int fd = syscall(SYS_memfd_create, name, MFD_CLOEXEC); + + env->ReleaseStringUTFChars(jname, name); + + return fd; +} diff --git a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h index abf13b35b..12faa685e 100644 --- a/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h +++ b/jni/utils/org_thoughtcrime_securesms_util_FileUtils.h @@ -15,6 +15,14 @@ extern "C" { JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwner (JNIEnv *, jclass, jobject); +/* + * Class: org_thoughtcrime_securesms_util_FileUtils + * Method: createMemoryFileDescriptor + * Signature: (Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_createMemoryFileDescriptor + (JNIEnv *, jclass, jstring); + #ifdef __cplusplus } #endif diff --git a/libs/arm64-v8a/libnative-utils.so b/libs/arm64-v8a/libnative-utils.so index a6af30560e8372d6a1d11b0ed274ef69e7b74952..0d84ac84ee43811faa38bb5cafaf971fb17a5584 100755 GIT binary patch literal 10008 zcmeHNYitzP6~609i&I1FP^hZ{@j|HrZL;k8wd)c!i+RPtF$)Q$il|1jvx9fZdKb-% zT^^QdL4KgBKWr&#sYFsrA{CI@BJ}~5qQ*^Hg`boPgsP~rjaoUhiD=YTY#Mj_otblB zy!Ey~+kbn}^>@$no_prbz01Ax_07Fo>OvtUMN{V$QOO5=PLc)JR&oQ7R0&m&zf08; zPxoVYME}y-|B%lTRGx+Z_u~+qTj9&kt?*+I{7RkA7Zf{UlidW{O|YHdB_4`kz3)an z0HmteZzZl~JwZ>GZK!>1k*`pR$JtI$>{7pXGt@q@&{wF$O>8G9b~|84g6A4WSnPL;_Pu)Qx7@i7 zwtuZ)*-C%DFRnk`*R#1W6pL;p9V?&d*|a$o*Y}xu!*NpE%xBy5IeIbZ^%Q*7&8dXG zubAFvnQp2}-?!H{pQGLm_3fUrU98;KEl-pKkH_b@0QkybjL=_`!YQ zLVz!8AQ?;q_`$XIVt~&fdj2m3_`KwJ{A&R|Z+#wrGQj7p-Q&*$`1CGHxf$To9#C0s z9)9D@D1HV!e0giB>ByJH!gVcITf*8Esh;dAWT$+YNB)Mr&`x`o=w(o{>bG}E9;7t7^uPcBoXnPth+X{COOS+k^GjVzwot;So>_H-e` z^c?MuwvJ(+9yRy0j{S`6n&GQCSvpx^hyC2QLm$(Szd^c>s&V)lpE5&ZuY!h6d$2Fe z(IqoZa-8}l{><*=xWs>>Az9iMN|x4!8vZ!bsK(#bROuH=Jx6+zjmh!Z!+mEk_A$!O zariF3rI3d@JTB(wymBTc4w06*XWiLpO*?yij_-|U;M4a#Pkhse zWloHv8%ooNkzzyKsN@r|YX3e~JiE*9t{S;~LsPB6xIN#9k$l`}@N>9>o(sk^g?=XC z7yXT=z=Q4QZy?XNZ6M!Cx?ZmDaU8F8-~Bl5CNHP&-e2aytoQ-zGmQ zm+3ds%xt+l41DY#Am__+o=pG0r$T4cRD}Lf1$~}rIkWwc z>CgCPxQF!_%b9Lsl=F}^eBLd0y^}1EGwDk#mowzWO!Kvo`ZW8adqXOtaxUzX>L#O{ z*T|W6GxLuzL-HrbvIW*q^tY;-GC>5wke3&Mg-6X4c3R*7^4B(OA4K zZKOL}Ei>NU*&1t0bi^~!M4K7466vUsPIq)B;$6|T_vAaD9~ub9jn2+kBGcJr#jQjv z7PDfR_O6TxG7)QwcgD>Q(};I;8gPAkbiKV_e#Re*GgMfsRK&^RlFK#HDpDxAR;0f$ z7)cN2@|nkRfELLbPF6)S2Mf-@0Uupkc{-eyIfM?GZRHIz;MV@Ut0K63#9zDU-E2jy ztiIPaaPOGSKwl{CUecP&SOwVN8Z)OGwrw2rdN&4gCVDJF;`>6v2T1Y1FZABhE5Wbv zodz>jS9Brkgs|zo^KZ(0j;tSoud$-MNAiA(AM#HzNdL0F2sW{R^vQ?V%X|MA^Tl4) zD?wSGpym}>4}@+weN;Rzq0>VAfRt(Bu%nE%lAE!XR(+51^ zk6}x>6nojPNB!#)wI8xz5qrT45C-jKpHX4oO7z4>CH-@@kBS0geb{d-kO$3tA%iHf Tvx*0eD}CziAQNq|6ZSs=r5Pl8 delta 1932 zcmb7FU2F_d6uvWS>5g4y7O9Bt*6eocr`h`5v?5x~B0^|9h`*>>5yAsYh@Ys62;pHT zr=breNFOSZsCvp z_Tz*MdsKlmq`W*OAS>?c@3R;yl(~8YoL-W?aMUK>Yxm9-VH%4pZ?_Y$@OF$HBj10y-t13dlOCVhiQK9q5rIMA}O^ z4pQ?mUZ@w*++lJ7l*4Z{&e~r57(%+%pAoZ2u z=eEOp1h%;xhEYH1(!Zd7&82@s{eh#O5TYBCvxZ=WZ|ayQW!6aqV2>}UJcV1nDt}|l z!o8MB9gA5H;H__#@i3pin5z`Mxyt(9mnp@%2^HEVr4w4UPjKEpF*Y5e$r611p3f&y zihMr59Yqr5DU?@Gb_#gmH=A#ui`|Z^{AK!O^^zPWX)5v}+Yc4-hKbD zA4B`++xG*9JR7;IxQnRygzQX7tn&x!yi>+GZek@FrjR}m04)?ZYbb5*qce1@(UB=q z3TzC8j4tj78r=nFL*d|b8pX|)^Fm)}wc76mEgVuiyf8C-ykU@>{#WuDF7(@K`9W3L z42R9?Am_)jTY)|^9^~I(iD`c`<3gkqlQNJ7hzirM!>5(fVi%Pts+ zR2zS0pMhISDKtiZU1waN`6?WV#>Idie_Z2?nI}|?UKbqrZ2jXG|5fvG6Jyy)4YUzVCG4JF2+ zjH0pvQ^|qyQWGmQ<`)*4n6cz%g~ph&QlxL@->J>JcWqs@Z_C!KP5TeUui9Kx{z5rg zdwKgdLwa$(scchD>b{?s$)TKLw(rN>g!t!+j79lnML(VYbOFy3BRyU!E6*=yWtC+G z`9>o@E}FJ5TD~Bl1p&Yxcmmu`<_@rR(Uopn#WgbO1QcSgv6wA_2ilwoWV%dgC{qQ7=lY#k9l44#{q-w^d zNwNN7-^u~8@0|h9zS=(R_~ilpD$KC6_Ky0~nmc;OrGZ&nRM^uGevD5l>|ur7FWY8k zL%oWMFBSGRg?&X~UsBjD3cFEZ*DLI^3j2h@u34;bs$q-lodL#b`#L3Qup)&Uw9DtU z!R(VHDKmw?qKy%X`8jQv@FM?AyJ-HlF*l2wRew%$?BEmHc(09Poypilix~3&hGd?g zJD*o|vD9$H=4+@iykT&+XpXrZcelEoX5lJ+eM;=OKE?VZ$2X&*MLeB$|1~cmType` zsx6cwo{?><)>hTlby0O;ZYx4@_f)+P@6bIS^i$Ohay9l-bFZ#NrAZiJ?Rkm(q`oGE zjf5H`hd3fRv&$LcirtdDI4W5kTS){&;uSW>Bl?JJ4&;h<~!A-%$y}f)@EQT-fsuR}mPrRbt z*^ozO_GA!$=ouC%4pD#FAv80Dfv!X|E6@zKEp6d6Xs|C|@4Yqn+8`y~7?d1e48|el zkJ!;4fg50x55MK@&pW(>#F{~SG@LlmG}sc&eS9iH_YR5^IV^kf^g+q77mctmHooE^ z3+-y_$LH~6bn|(iMwNKOZh3<5@+%Ip*{PovyX0uKv-TgBGIkix>OnkTm0*P5_S?Dm zkepM2oLBAD$Pkcm618e@m~Z#54Gxtv!jZAW-m`?U?RZ`v;Mv^?gaup`a3*ZV@z=3X z!;GVIY{v2F7;6tk+FN*hi;?m+8r;m=02LRg0WGM3vS}=(t@sY@HXj$euLN=@B!jh_$d@`{y1oZde06@%dE&au-W;W zE!)oD+jwbNkeUno3GxJ%=FzC?UK0>2KJCyTYakP zD`*3J|L>jg3@F84hf>_;ir<3L^*vCV%~lmUD4FXJs2-Ng_0T6qocY(F)c(RTXC5iq zF|)Os)P->_SN*&fiX<&bXBFuWARv3ZzCG%Nm22>sp zl-g;5(gf^Ko6T&iy6dW7|8_Q%3Z;fqT>cuDZ-O4dg_m7^7qlAwv@4!F>AWrvO7Siz z<*)sZWUhhsL2;&ICmu-V6VM*0H5o=f3>vAI!#UDoD2=opN+UHsbk4XHN&}mG#CL=( z=V{@3qPZ%pZl2kK`}!)FEq-|W(R=bSuoSEWi69+hfo$*|XaXOAHZTFU8m=sq||2$;5CC z&0mbJqm3O9pxUp0~Tr4uAH} zeD-_Jx#!(??t6FdebZIbRZZm&)@2YedP0jMk{~6S;GqEL5x4*h!G0aI75%N~Z$0(e z*6c1bQHT~KGBT2w;-Q-I@WI;9*Cv)ruEcbceJuaE?2J#7)WUa5u3VY8`=L{KC5Q52 z6j3KGji+e(4K&o^VF>W7k!U^ChW_`Ve}lg|=#M~i;jak#^KiRRBXqX43HVF`S}`Y@mSH_h^&oaJwyAm z1$5|>C_-E`?3NltT4bt`S5-3lo)X(44=67Oy)!{=32NKJfvonRo)0SJ2jX&2TM9?HwZ9D5oEBR+8s=J~sZLbg53UglSR z>ac`$EC{pX!tAIpo5F0r@27FRVYWNW9tg9qhiogV>kT=uufmq;)ghwBfzMQRq-%|6 zFs?B5z#LFj)wxDoHfBmk#JJHa?H948)HxL%eUtv>=yR&CQskPl<4Qd9*WhoKLbM#5 zUM+rUdehZ6=e*8vcAUD)bH&LRr}^n|OM&X!F|PV{A$ayaqFY#OMOYS5Sfp=@wXuzh zWG%;l9MiZrn`kXE?trI(nj~(<4obfkAIBA@?-=93c8#gNiZSYLLF!-aL>{d2eyrR$ za6#nG`*Qw`(Ybd{m6^`j3MietvR`Kzz9rkmqWBi6S#-zW8(B3vcYV=hintwTUA$vd z8*~(dj-u==42m7kgN1MtFFOuQ#EaDlo6?qQiOZ0$iyT<~YB>RYHKS&+2YP2D!6?ZqyUhm?J| zWwN+%PoX^M>B|thl*;5@Pi;rUD|;-i#wxym!R)HRW^UXpsmnuY7uy*XhinDM;cb#Ux3|UC*Z?vJ9q|c16#osPy{xD4Myy!R`I0ueqEVc z{K0Bd@Jbclggc?lFe-;tC93|9bq&xa_@6*I?uT-o7uw$5UY9qbDkl&)Go~s|X#RL0 z|63^Muk-}+Sb3b1pkE5*{K-IEA^CP75r>u5Q3mA;oKPO9GuYn+<-&TP2$L6D2gClI zs?3%5P#jEbfL7ze{-E!MR>4o448*IU z`~+&D9CrufC3jS%2EHF!3vK&BRrq}NLb=dsC>OfS7bvt2>a-xx1%oTCp9;*l7s@w? zp218-+Tv8#ne;Z3>x1PEo$>@$a2^P?G;C02U};bY69tJs@9%3QK`GcLj!rt_#-`%u#d&hZAs z5YoO7SljS}|30Mx?B-M=mSkD(LGrY!9_WMKg>vEy)T+aKw(>XSS z7beQS43i&`-5_q}*o^%2m|8K*;!x(%JP_6Yz-DdcoCNm?~X zFike&8k(#8@RcQ)wxKm4p+AYyfMS$rjJ5FxNo#EEl44v9nj(+!JHsxb(L4F(o9~`` z&pG$pJ2T5imXC~gJ=waCF@lky*%&i`7WM;KJE%kl!4$I#;3e3XU|+g4S(vL(Jl}_@gp>m{}oX46S+!z;N z(jLI}5Mx(#ZDp6iJ~Lx=aBXK+wn}!1u~Bd?v;+NN=>C|lhW^;XSWQfyAq{^qdl8#J z$I-;y4z>s_W@9v(js2>9^)LM`pA?$*b^8My0p>sN@9gO7|z)moZ07F&Kgx$-4S-Tf+5UWS%WYDb7W6ZL5^!F}!g!^S;gt|kFFn5XJ zHbRVvtHcNyB}Qs%#0dR>7-?=2Q#!Gx%~I~#6vag2zldiWwW)kO_LSka;?vNk9wS{z z`UPDdC0$S24Siy4)e?bVZKLz@!;FQ-iXnoc5_yWhWULbAo%|zXR>_o;v9VQg#~Mm+Oocox)+l;~;4wS+vXU8WQk=mS#SsiB zwn)b2-_aLEr3>2=pF@X&=0wapV-&U4LUCI3JmE!z7mIp;hWx&vX;^>~@~%DK-gbIE z_(y2&5S5iOOx%>ga2kZEG%sDEGgF*MIxg- zN0ND~WHYvIXZj{e`?ws`fgx_5wM6HtmuCeTnrWw*8{!*ehsY5{vi1c1yQ_8GXT9| zOE|amcDaEx;402Omdui47&r3m{RQPwjG>n-ihL;lJGjvv$N=$e+=%btMz3%NkPa+x zZ;~s$7*3JWU<(+j4AMxXJLog=_miF|qOWqir6R9KfD)h%cn;_WhJlN~1aR3jyPY&G z2+Y8z?O77#rh|WDcS)2Pp@pwQQDj0k&yigc#V2I)v$D)Rveo2+LKXvZu4x>PSnY(I zD^U>$ZvF{o>0cg%o#898tAGkeh*BA14SyiLskqbHihwXa4&g*z;>azW!6U&#wpfeL zeh-?uX%BXPNPN^T^LHIqqy9hRA3E}k`k#w`M^D(XCBEgzlk}$~EKna`krMt3k$2aC delta 1210 zcmZ8hTSyd97(O%ZWhTel&YEkQtLrLW!mPyBZYYU8B}&pwA55YrYc+*>OW6oS4dt)s zp}?RgAqX~n^1-kkqK7Oj_Q8iBgOQA^B=UOt{@FFGGko)%fBwt)&;Oq}bLDg8BNvZa zx)2Pfz(^ND0nMZWd2v+X1H+ZjtKb;)G3aBXj}B%$gYy=upsX5Q*qgCELM6N_sZ@!2 zbB)onudc7IOsJ8(Puc0R=Ft5{T2g9nG{P?*D>T5=BMfl|@|0VMesBQtB)CZv;&MX% z0`7voJ|TPHw>3PN_#iv-<6qLncj^C64*o0CH}x1DPO7DsNq3!>dU`MP(uAYT3;!g* z0I)}~14yQ3#oWg=;1yZx0Jt{D&aNS{!cgr;ffYY0opc^=<-c7;prwtMuAcr}}qM zsynY*W(yHnbi}p8gR#$2Hp$+gui0`*DI+g@tH)e6)zKKh_F zs%{?@>$lZwSu~-ar?KQpwQ_^rC6}o!o3xo6aP}cwKPiSuAtPO}=($m-+s$8-vX8zR zg{3}Z%x85EzRY6Yxh`cDcydF^cCdj6H^C30%_%)fv@4}0{-Umw*@`iEwgcR_7|O_v z3;|idV|0>Ro{_+igMOq|)I5a^cfO8~40sv7+%DqiSzJpTU$dPyHS<^I^h>3;y@AU=2O+ j8czGJp(ox#r+ur?d!C^^G1~AJ>h>&6h!WfWNaoldWWpUh diff --git a/libs/x86_64/libnative-utils.so b/libs/x86_64/libnative-utils.so index 27d3c63f4f54d6e5a1b4c050160fd452bdd9b044..fc03a67d7df254dd134c8e046356ae47c30543ca 100755 GIT binary patch literal 10280 zcmeHNZERat89qs~Hqf*QAEjX%)Kn{^sCDdo*iHvq`r+)RO}o%G6Pl>*uCH%mcaB~8 z+F4tztfNVpb*YGePWds3B31lhh)Ar{KvZOHwW0|@kjAJ`rABCDvFMK=DhRM)p7-4I z-r8O=(IWmJpX%J_yr1X1@4ffE?m2lTI?z{NS0}iw5D$slln}j2(6*5^h!+u!xVDP* zVkOuO=)Fo@jWfy_W24d|_C0F66r5Y@av4R7OOqYV`P`>m`nj!@c(UH58QG55WH+Vk zrj#9{P^Q-Y^Btv7Ax}XQ&&c$iGMa5B#IWj_HWV<%_6><|FYLd0 z?)0tio&Q6_kH7ktA8qJA@%-)&a!&DlbP0vc%kV?X@S~F7B%WC9I*Yp?zNz?u54nsE znh@=`=o>|onEtRN#Q_(@dy3zp{OnQuEwCf~BT7G__y-h!wo3mQ#rHM2itJ|^{H5Z3 zbs7Fu@LR;6ON~{x=vT||f4>Z0*vD+sNTtL;|Bjs^m5iBwJ42W@a>+4kYd)7u*~HRy z^QfU4IqPULr-HtYwedIyuq%k|}G5 zyy<4vf}g$CXeN7%xVtPH7DsZK?45cVv2v9T9(^ipWrhCuzOcS~UwI#w#) zzbiTz)(@L0!?p+am= zRH%iab|9BcrbmVj_U%jD9Ik;UbLPrCdpqY)=%8JvHew=7}G>YdWy`N~hX~juN?i@9Yn7p`iP{r z5ltyAJ|XE2qAAqkAZYE(7dL}ZJcx1ghF)=b*x$Ti98x^48+JkPCkSryCbfxDuD1o}K&kCUxUMsGb&fuU|3~7b8wP$b6!e3l-8aB0IpcSHvTH)fC$*6Ea zuA*>OBg%1^&IO|8=l(?s6TKukb&)_cPc*~9)rmL=YQ-t?=b1^>GwUuRo!Xf-R{`@s zaGgY;3?~HdcS0YX+fLk-oF&!y6R<8!bYbXRq^ox3LW1~t@J}s@=JBfKP+pzrctI}E zkynJ>EBp<4*PQ5lX;wRR#dpZNKgvGf*%yYOy?0J?_QKCcH0N6jlq`ol z!;cf4gXT4@@M|alhGwssMe2xg`0z@f`at*_EQ1?qHY&dWYL$M6f^QOtS*zJI4V zLx4Pdzo`{2L2!-z%IA&O8=S$U95b!iLHtf0US6MXeXms7{>sC6hjtaN;$cEC`MICc zD>W_VS3_91R2aJMSzyBXd%$&~*O41Hfa?4%vwjWl+1;(ba_yXt!8`Fc8NdCV?niL4 zaXqg)zF*WGZK=DjX;tGSLMQ(H=z(`jrSiH*{4Vr1@F_#wh(Cbt1D}csH{!?9Tfr}{ z!^D3Yy#@SF0=Yea!FSL}U;d`szP9Cb{mwNjSG3h(Y1^Atwn$Z48~x@F;6}*bY^-NP z3w)AaM<0Y-{sv@u{XVHc@^{fEA(y}5Jb9ldzaIh4Ku+ryo?Q3jUx2&?Z`e}y6vtN} z?_EYtE~$~;IP_ZoQ7TOnV{J>{+Sc98tDkB-DIUJ}!OwR-u#wo*>h4?d;2`|bxBu4P zOQkpP-k*nE3cQzqKd> z*wm73LE(CZZL~9rM*A7m6zSod+GOUv56xBc_QzWljsMPPXvp*O-t>%`nD>}jpZ8dL z8(asxcf5eT1Zs*c0?LtMd*=mA-%tal-%$fTSKxjAdtJBQem?D=P;+-i=?ZrHVmL~Q`!Z=?wzIRRE6}sqhoi6bh;Piw z+DWh@{(!&3*OtiTj@aAU+YR^d*gulVjHE0(lh2x#-^`4*=Q5d;nJ|*+cGtcu5Da(5 zj993{GQ(Y=j$mh`I~)&0I?aF;i3N;UtUDA5_XIZIk?&M$Y}6MvLZM(J9_q2eRwNh< zTETc%Puv6<33i4{GWh~?01k78h`f~`MXH=A4BK-vHjP;qkE#tx$r3kM!EM5IXC{U zFn%3^a(n(xGrqtJR}R>Y{hd+vvW{S!QimFhw5ML)`1xTx3n7g;esR0ZA41TK&SCPoI{&>;ta4cWv~!!^W!n$0Xwo nsPS(*&lrQ2&$tx7zbxZCGEIIscg!Q)qU`@#&cvAKW%j=TR}IT6 delta 2056 zcmZ`)U1%It6uvV}wmV~HyAwlp)ufrYKZ!rMTS;RSLc3|uZ7mwGs7UFiMCxin6`BVh zgiS&a3N0j@zE~AD55gh}qY>&uz|e+&V2X# z&iy^-&Q5KtmS4EfXkWbl^b&>VgSbviB z2J0KqP#9u8iSxx(BZzt|F!Y3a?v}sc&UWViOJGLd9rlk_{&r9 zWy=~wBmGI9dk110I&G0MYcI~;xrfe00ZLX2dMi4enf6QNp-A{Ml>WiHk$Wl~I-;~f zQ}SI<6Aq0x^$gTSE(4!ljd40_Tx`+vtSgU3g{$j^r=RNco)D=U?zR##d~P%axZXOp z*`&)N6xeFg!{h1h>T;7Vd!fjcCY@3hT1Bb9ngWzJv0$%xmr_Bd1)mFOL56sl!u72H zE%P_lqfB$vH#Gm9IPJZ@s`;7}4s@9M4D#aH z;dK=1^YC4C)T&^!czUh=1Z1tp&3FJOtw)??T)lYKxqxu_2l&_;Ha>(aR>F7>qV}`K zF(}#FFxj@Zvj2JzP<6$q>I3LQsNYP6QluY7&!bNJ5+nUZ^nTQvi$MAt=pO2roEw(L z);VrSG*=h;kLcqV*Ds;O&8Kt+nrj=q3O_k1 zD~;Rz^jZ+ifD=nS@8cL7W9Lx$22<9t9Ry#XmPY&;aR6}}aTKuw<3|u5M4Uz(Mf?Q! zH--2I;u(r#-wzO%mty_q3v$QhPE#q#e7-SqO>3d75{yy4kCYuUxrTzPt#GOPSY?an zqXUo?xDs?j$ojbgQXnay@H{t!4{;+F4%fw`=H1NY6{duv!DqYo6sC9$%*U%F&NIh+ zXfAVOUNo31*Ac0RVo$NrY_7sjiM5k7w>V^;FvEt_6shv9*h%DIvi z!2%paRdOLX3a3$(-3umRJ(aWN4D+pErUz}=afA zf-~@ukpM4~Y?r$rc*cZ7nTPBZ{#vY9Fqav0H<$_@hV@L&z2)l%RyJo#S*i|0=u`}SFG7`dw8P-^$T D4&!gN diff --git a/res/values/strings.xml b/res/values/strings.xml index 5f4064343..adbf618de 100644 --- a/res/values/strings.xml +++ b/res/values/strings.xml @@ -72,6 +72,7 @@ Uploading media... + Compressing video... Error playing audio! diff --git a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java index 098de8d9d..ee359994c 100644 --- a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java +++ b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java @@ -16,20 +16,22 @@ */ package org.thoughtcrime.securesms.database; -import android.annotation.SuppressLint; import android.content.ContentValues; import android.content.Context; import android.database.Cursor; import android.graphics.Bitmap; +import android.media.MediaDataSource; import android.media.MediaMetadataRetriever; import android.net.Uri; import android.os.Build; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.VisibleForTesting; import android.text.TextUtils; import android.util.Pair; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.annotation.VisibleForTesting; + import com.bumptech.glide.Glide; import net.sqlcipher.database.SQLiteDatabase; @@ -495,13 +497,12 @@ public class AttachmentDatabase extends Database { return insertedAttachments; } - public @NonNull Attachment updateAttachmentData(@NonNull Attachment attachment, - @NonNull MediaStream mediaStream) + public @NonNull DatabaseAttachment updateAttachmentData(@NonNull DatabaseAttachment databaseAttachment, + @NonNull MediaStream mediaStream) throws MmsException { - SQLiteDatabase database = databaseHelper.getWritableDatabase(); - DatabaseAttachment databaseAttachment = (DatabaseAttachment) attachment; - DataInfo dataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA); + SQLiteDatabase database = databaseHelper.getWritableDatabase(); + DataInfo dataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA); if (dataInfo == null) { throw new MmsException("No attachment data found!"); @@ -839,8 +840,9 @@ public class AttachmentDatabase extends Database { Bitmap bitmap = MediaUtil.getVideoThumbnail(context, attachment.getDataUri()); if (bitmap != null) { - ThumbnailData thumbnailData = new ThumbnailData(bitmap); - updateAttachmentThumbnail(attachmentId, thumbnailData.toDataStream(), thumbnailData.getAspectRatio()); + try (ThumbnailData thumbnailData = new ThumbnailData(bitmap)) { + updateAttachmentThumbnail(attachmentId, thumbnailData.toDataStream(), thumbnailData.getAspectRatio()); + } } else { Log.w(TAG, "Retrieving video thumbnail failed, submitting thumbnail generation job..."); thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId)); @@ -912,46 +914,53 @@ public class AttachmentDatabase extends Database { return null; } - ThumbnailData data = null; - if (MediaUtil.isVideoType(attachment.getContentType())) { - data = generateVideoThumbnail(attachmentId); + + try (ThumbnailData data = generateVideoThumbnail(attachmentId)) { + + if (data != null) { + updateAttachmentThumbnail(attachmentId, data.toDataStream(), data.getAspectRatio()); + + return getDataStream(attachmentId, THUMBNAIL, 0); + } + } } - if (data == null) { - return null; - } - - updateAttachmentThumbnail(attachmentId, data.toDataStream(), data.getAspectRatio()); - - return getDataStream(attachmentId, THUMBNAIL, 0); + return null; } - @SuppressLint("NewApi") - private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId) throws IOException { + if (Build.VERSION.SDK_INT < 23) { Log.w(TAG, "Video thumbnails not supported..."); return null; } - DataInfo dataInfo = getAttachmentDataFileInfo(attachmentId, DATA); + try (MediaDataSource dataSource = mediaDataSourceFor(attachmentId)) { + if (dataSource == null) return null; - if (dataInfo == null) { - Log.w(TAG, "No data file found for video thumbnail..."); - return null; + MediaMetadataRetriever retriever = new MediaMetadataRetriever(); + retriever.setDataSource(dataSource); + + Bitmap bitmap = retriever.getFrameAtTime(1000); + + Log.i(TAG, "Generated video thumbnail..."); + return bitmap != null ? new ThumbnailData(bitmap) : null; } - - EncryptedMediaDataSource dataSource = new EncryptedMediaDataSource(attachmentSecret, dataInfo.file, dataInfo.random, dataInfo.length); - MediaMetadataRetriever retriever = new MediaMetadataRetriever(); - retriever.setDataSource(dataSource); - - Bitmap bitmap = retriever.getFrameAtTime(1000); - - Log.i(TAG, "Generated video thumbnail..."); - return new ThumbnailData(bitmap); } } + @RequiresApi(23) + public @Nullable MediaDataSource mediaDataSourceFor(@NonNull AttachmentId attachmentId) { + DataInfo dataInfo = getAttachmentDataFileInfo(attachmentId, DATA); + + if (dataInfo == null) { + Log.w(TAG, "No data file found for video attachment..."); + return null; + } + + return EncryptedMediaDataSource.createFor(attachmentSecret, dataInfo.file, dataInfo.random, dataInfo.length); + } + private static class DataInfo { private final File file; private final long length; diff --git a/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java b/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java index d92306c94..3addf9c4a 100644 --- a/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java +++ b/src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java @@ -18,8 +18,6 @@ import org.thoughtcrime.securesms.jobmanager.Job; import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.MediaConstraints; -import org.thoughtcrime.securesms.mms.MediaStream; -import org.thoughtcrime.securesms.mms.MmsException; import org.thoughtcrime.securesms.mms.PartAuthority; import org.thoughtcrime.securesms.service.GenericForegroundService; import org.thoughtcrime.securesms.service.NotificationController; @@ -48,13 +46,26 @@ public class AttachmentUploadJob extends BaseJob { */ private static final int FOREGROUND_LIMIT = 10 * 1024 * 1024; - private AttachmentId attachmentId; + /** + * The {@link PartProgressEvent} on the {@link EventBus} is shared between transcoding and uploading. + *

+ * This number is the ratio that represents the transcoding effort, after which it will hand + * over to the to complete the progress. + */ + private static final double ENCODING_PROGRESS_RATIO = 0.75; - public AttachmentUploadJob(AttachmentId attachmentId) { + private final AttachmentId attachmentId; + + public static AttachmentUploadJob fromAttachment(DatabaseAttachment databaseAttachment) { + return new AttachmentUploadJob(databaseAttachment.getAttachmentId(), MediaUtil.isVideo(databaseAttachment) && MediaConstraints.isVideoTranscodeAvailable()); + } + + private AttachmentUploadJob(AttachmentId attachmentId, boolean isVideoTranscode) { this(new Job.Parameters.Builder() .addConstraint(NetworkConstraint.KEY) .setLifespan(TimeUnit.DAYS.toMillis(1)) .setMaxAttempts(Parameters.UNLIMITED) + .setQueue(isVideoTranscode ? "VIDEO_TRANSCODE" : null) .build(), attachmentId); } @@ -86,11 +97,13 @@ public class AttachmentUploadJob extends BaseJob { throw new IllegalStateException("Cannot find the specified attachment."); } - MediaConstraints mediaConstraints = MediaConstraints.getPushMediaConstraints(); - Attachment scaledAttachment = scaleAndStripExif(database, mediaConstraints, databaseAttachment); + MediaConstraints mediaConstraints = MediaConstraints.getPushMediaConstraints(); + Attachment scaledAttachment = scaleAndStripExif(database, mediaConstraints, databaseAttachment); + boolean videoTranscodeOccurred = databaseAttachment != scaledAttachment && MediaUtil.isVideo(scaledAttachment); + double progressStartPoint = videoTranscodeOccurred ? ENCODING_PROGRESS_RATIO : 0; try (NotificationController notification = getNotificationForAttachment(scaledAttachment)) { - SignalServiceAttachment localAttachment = getAttachmentFor(scaledAttachment, notification); + SignalServiceAttachment localAttachment = getAttachmentFor(scaledAttachment, notification, progressStartPoint); SignalServiceAttachmentPointer remoteAttachment = messageSender.uploadAttachment(localAttachment.asStream(), databaseAttachment.isSticker()); Attachment attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.getFastPreflightId()).get(); @@ -114,7 +127,12 @@ public class AttachmentUploadJob extends BaseJob { return exception instanceof IOException; } - private SignalServiceAttachment getAttachmentFor(Attachment attachment, @Nullable NotificationController notification) { + /** + * @param progressStartPoint A value from 0..1 that represents any progress already shown. + * The {@link PartProgressEvent} of this task will fit in the remaining + * 1 - progressStartPoint. + */ + private SignalServiceAttachment getAttachmentFor(Attachment attachment, @Nullable NotificationController notification, double progressStartPoint) { try { if (attachment.getDataUri() == null || attachment.getSize() == 0) throw new IOException("Assertion failed, outgoing attachment has no data!"); InputStream is = PartAuthority.getAttachmentStream(context, attachment.getDataUri()); @@ -128,7 +146,8 @@ public class AttachmentUploadJob extends BaseJob { .withHeight(attachment.getHeight()) .withCaption(attachment.getCaption()) .withListener((total, progress) -> { - EventBus.getDefault().postSticky(new PartProgressEvent(attachment, total, progress)); + long cumulativeProgress = (long) ((1.0 - progressStartPoint) * progress + total * progressStartPoint); + EventBus.getDefault().postSticky(new PartProgressEvent(attachment, total, cumulativeProgress)); if (notification != null) { notification.setProgress(total, progress); } @@ -142,26 +161,21 @@ public class AttachmentUploadJob extends BaseJob { private Attachment scaleAndStripExif(@NonNull AttachmentDatabase attachmentDatabase, @NonNull MediaConstraints constraints, - @NonNull Attachment attachment) + @NonNull DatabaseAttachment attachment) throws UndeliverableMessageException { - try { - if (constraints.isSatisfied(context, attachment)) { - if (MediaUtil.isJpeg(attachment)) { - MediaStream stripped = constraints.getResizedMedia(context, attachment); - return attachmentDatabase.updateAttachmentData(attachment, stripped); - } else { - return attachment; - } - } else if (constraints.canResize(attachment)) { - MediaStream resized = constraints.getResizedMedia(context, attachment); - return attachmentDatabase.updateAttachmentData(attachment, resized); - } else { - throw new UndeliverableMessageException("Size constraints could not be met!"); - } - } catch (IOException | MmsException e) { - throw new UndeliverableMessageException(e); - } + MediaResizer mediaResizer = new MediaResizer(context, constraints); + + MediaResizer.ProgressListener progressListener = (progress, total) -> { + PartProgressEvent event = new PartProgressEvent(attachment, + total, + (long) (progress * ENCODING_PROGRESS_RATIO)); + EventBus.getDefault().postSticky(event); + }; + + return mediaResizer.scaleAndStripExifToDatabase(attachmentDatabase, + attachment, + progressListener); } public static final class Factory implements Job.Factory { diff --git a/src/org/thoughtcrime/securesms/jobs/BaseJob.java b/src/org/thoughtcrime/securesms/jobs/BaseJob.java index 73053b047..befe91d65 100644 --- a/src/org/thoughtcrime/securesms/jobs/BaseJob.java +++ b/src/org/thoughtcrime/securesms/jobs/BaseJob.java @@ -1,6 +1,7 @@ package org.thoughtcrime.securesms.jobs; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import org.thoughtcrime.securesms.jobmanager.Job; import org.thoughtcrime.securesms.jobmanager.JobLogger; @@ -33,4 +34,20 @@ public abstract class BaseJob extends Job { protected abstract void onRun() throws Exception; protected abstract boolean onShouldRetry(@NonNull Exception e); + + protected void log(@NonNull String tag, @NonNull String message) { + Log.i(tag, JobLogger.format(this, message)); + } + + protected void warn(@NonNull String tag, @NonNull String message) { + warn(tag, message, null); + } + + protected void warn(@NonNull String tag, @Nullable Throwable t) { + warn(tag, "", t); + } + + protected void warn(@NonNull String tag, @NonNull String message, @Nullable Throwable t) { + Log.w(tag, JobLogger.format(this, message), t); + } } diff --git a/src/org/thoughtcrime/securesms/jobs/MediaResizer.java b/src/org/thoughtcrime/securesms/jobs/MediaResizer.java new file mode 100644 index 000000000..860187ed1 --- /dev/null +++ b/src/org/thoughtcrime/securesms/jobs/MediaResizer.java @@ -0,0 +1,142 @@ +package org.thoughtcrime.securesms.jobs; + +import android.content.Context; +import android.media.MediaDataSource; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.R; +import org.thoughtcrime.securesms.attachments.Attachment; +import org.thoughtcrime.securesms.attachments.DatabaseAttachment; +import org.thoughtcrime.securesms.database.AttachmentDatabase; +import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader; +import org.thoughtcrime.securesms.mms.MediaConstraints; +import org.thoughtcrime.securesms.mms.MediaStream; +import org.thoughtcrime.securesms.mms.MmsException; +import org.thoughtcrime.securesms.service.GenericForegroundService; +import org.thoughtcrime.securesms.service.NotificationController; +import org.thoughtcrime.securesms.transport.UndeliverableMessageException; +import org.thoughtcrime.securesms.util.BitmapDecodingException; +import org.thoughtcrime.securesms.util.BitmapUtil; +import org.thoughtcrime.securesms.util.MediaUtil; +import org.thoughtcrime.securesms.video.InMemoryTranscoder; +import org.thoughtcrime.securesms.video.videoconverter.BadVideoException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +final class MediaResizer { + + @NonNull private final Context context; + @NonNull private final MediaConstraints constraints; + + MediaResizer(@NonNull Context context, + @NonNull MediaConstraints constraints) + { + this.context = context; + this.constraints = constraints; + } + + List scaleAndStripExifToDatabase(@NonNull AttachmentDatabase attachmentDatabase, + @NonNull List attachments) + throws UndeliverableMessageException + { + List results = new ArrayList<>(attachments.size()); + + for (Attachment attachment : attachments) { + results.add(scaleAndStripExifToDatabase(attachmentDatabase, (DatabaseAttachment) attachment, null)); + } + + return results; + } + + DatabaseAttachment scaleAndStripExifToDatabase(@NonNull AttachmentDatabase attachmentDatabase, + @NonNull DatabaseAttachment attachment, + @Nullable ProgressListener transcodeProgressListener) + throws UndeliverableMessageException + { + try { + if (MediaUtil.isVideo(attachment) && MediaConstraints.isVideoTranscodeAvailable()) { + return transcodeVideoIfNeededToDatabase(attachmentDatabase, attachment, transcodeProgressListener); + } else if (constraints.isSatisfied(context, attachment)) { + if (MediaUtil.isJpeg(attachment)) { + MediaStream stripped = getResizedMedia(context, attachment); + return attachmentDatabase.updateAttachmentData(attachment, stripped); + } else { + return attachment; + } + } else if (constraints.canResize(attachment)) { + MediaStream resized = getResizedMedia(context, attachment); + return attachmentDatabase.updateAttachmentData(attachment, resized); + } else { + throw new UndeliverableMessageException("Size constraints could not be met!"); + } + } catch (IOException | MmsException e) { + throw new UndeliverableMessageException(e); + } + } + + @RequiresApi(26) + private @NonNull DatabaseAttachment transcodeVideoIfNeededToDatabase(@NonNull AttachmentDatabase attachmentDatabase, + @NonNull DatabaseAttachment attachment, + @Nullable ProgressListener progressListener) + throws UndeliverableMessageException + { + try (NotificationController notification = GenericForegroundService.startForegroundTask(context, context.getString(R.string.AttachmentUploadJob_compressing_video_start))) { + + notification.setIndeterminateProgress(); + + try (MediaDataSource dataSource = attachmentDatabase.mediaDataSourceFor(attachment.getAttachmentId())) { + + if (dataSource == null) { + throw new UndeliverableMessageException("Cannot get media data source for attachment."); + } + + try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, constraints.getCompressedVideoMaxSize(context))) { + + if (transcoder.isTranscodeRequired()) { + + MediaStream mediaStream = transcoder.transcode(percent -> { + notification.setProgress(100, percent); + + if (progressListener != null) { + progressListener.onProgress(percent, 100); + } + }); + + return attachmentDatabase.updateAttachmentData(attachment, mediaStream); + } else { + return attachment; + } + } + } + } catch (IOException | MmsException | BadVideoException e) { + throw new UndeliverableMessageException("Failed to transcode", e); + } + } + + private MediaStream getResizedMedia(@NonNull Context context, @NonNull Attachment attachment) + throws IOException + { + if (!constraints.canResize(attachment)) { + throw new UnsupportedOperationException("Cannot resize this content type"); + } + + try { + // XXX - This is loading everything into memory! We want the send path to be stream-like. + BitmapUtil.ScaleResult scaleResult = BitmapUtil.createScaledBytes(context, new DecryptableStreamUriLoader.DecryptableUri(attachment.getDataUri()), constraints); + return new MediaStream(new ByteArrayInputStream(scaleResult.getBitmap()), MediaUtil.IMAGE_JPEG, scaleResult.getWidth(), scaleResult.getHeight()); + } catch (BitmapDecodingException e) { + throw new IOException(e); + } + } + + public interface ProgressListener { + + void onProgress(long progress, long total); + } +} diff --git a/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java b/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java index 2a75ad07d..6077c4e06 100644 --- a/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java +++ b/src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java @@ -94,7 +94,7 @@ public class PushGroupSendJob extends PushSendJob { attachments.addAll(Stream.of(message.getLinkPreviews()).filter(p -> p.getThumbnail().isPresent()).map(p -> p.getThumbnail().get()).toList()); attachments.addAll(Stream.of(message.getSharedContacts()).filter(c -> c.getAvatar() != null).map(c -> c.getAvatar().getAttachment()).withoutNulls().toList()); - List attachmentJobs = Stream.of(attachments).map(a -> new AttachmentUploadJob(((DatabaseAttachment) a).getAttachmentId())).toList(); + List attachmentJobs = Stream.of(attachments).map(a -> AttachmentUploadJob.fromAttachment((DatabaseAttachment) a)).toList(); if (attachmentJobs.isEmpty()) { jobManager.add(new PushGroupSendJob(messageId, destination, filterAddress)); diff --git a/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java b/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java index adc627a76..fc2b73ddb 100644 --- a/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java +++ b/src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java @@ -81,7 +81,7 @@ public class PushMediaSendJob extends PushSendJob { attachments.addAll(Stream.of(message.getLinkPreviews()).filter(p -> p.getThumbnail().isPresent()).map(p -> p.getThumbnail().get()).toList()); attachments.addAll(Stream.of(message.getSharedContacts()).filter(c -> c.getAvatar() != null).map(c -> c.getAvatar().getAttachment()).withoutNulls().toList()); - List attachmentJobs = Stream.of(attachments).map(a -> new AttachmentUploadJob(((DatabaseAttachment) a).getAttachmentId())).toList(); + List attachmentJobs = Stream.of(attachments).map(a -> AttachmentUploadJob.fromAttachment((DatabaseAttachment) a)).toList(); if (attachmentJobs.isEmpty()) { jobManager.add(new PushMediaSendJob(messageId, destination)); diff --git a/src/org/thoughtcrime/securesms/jobs/SendJob.java b/src/org/thoughtcrime/securesms/jobs/SendJob.java index 525d41229..1775da3a4 100644 --- a/src/org/thoughtcrime/securesms/jobs/SendJob.java +++ b/src/org/thoughtcrime/securesms/jobs/SendJob.java @@ -1,7 +1,6 @@ package org.thoughtcrime.securesms.jobs; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import org.thoughtcrime.securesms.BuildConfig; import org.thoughtcrime.securesms.TextSecureExpiredException; @@ -9,17 +8,11 @@ import org.thoughtcrime.securesms.attachments.Attachment; import org.thoughtcrime.securesms.database.AttachmentDatabase; import org.thoughtcrime.securesms.database.DatabaseFactory; import org.thoughtcrime.securesms.jobmanager.Job; -import org.thoughtcrime.securesms.jobmanager.JobLogger; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.MediaConstraints; -import org.thoughtcrime.securesms.mms.MediaStream; -import org.thoughtcrime.securesms.mms.MmsException; import org.thoughtcrime.securesms.transport.UndeliverableMessageException; -import org.thoughtcrime.securesms.util.MediaUtil; import org.thoughtcrime.securesms.util.Util; -import java.io.IOException; -import java.util.LinkedList; import java.util.List; public abstract class SendJob extends BaseJob { @@ -58,45 +51,9 @@ public abstract class SendJob extends BaseJob { @NonNull List attachments) throws UndeliverableMessageException { + MediaResizer mediaResizer = new MediaResizer(context, constraints); AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context); - List results = new LinkedList<>(); - for (Attachment attachment : attachments) { - try { - if (constraints.isSatisfied(context, attachment)) { - if (MediaUtil.isJpeg(attachment)) { - MediaStream stripped = constraints.getResizedMedia(context, attachment); - results.add(attachmentDatabase.updateAttachmentData(attachment, stripped)); - } else { - results.add(attachment); - } - } else if (constraints.canResize(attachment)) { - MediaStream resized = constraints.getResizedMedia(context, attachment); - results.add(attachmentDatabase.updateAttachmentData(attachment, resized)); - } else { - throw new UndeliverableMessageException("Size constraints could not be met!"); - } - } catch (IOException | MmsException e) { - throw new UndeliverableMessageException(e); - } - } - - return results; - } - - protected void log(@NonNull String tag, @NonNull String message) { - Log.i(tag, JobLogger.format(this, message)); - } - - protected void warn(@NonNull String tag, @NonNull String message) { - warn(tag, message, null); - } - - protected void warn(@NonNull String tag, @Nullable Throwable t) { - warn(tag, "", t); - } - - protected void warn(@NonNull String tag, @NonNull String message, @Nullable Throwable t) { - Log.w(tag, JobLogger.format(this, message), t); + return mediaResizer.scaleAndStripExifToDatabase(attachmentDatabase, attachments); } } diff --git a/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java b/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java index 1834a197e..6d642c0c9 100644 --- a/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java +++ b/src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java @@ -466,7 +466,7 @@ class MediaSendViewModel extends ViewModel { .filter(m -> { return (MediaUtil.isImageType(m.getMimeType()) && !MediaUtil.isGif(m.getMimeType())) || (MediaUtil.isGif(m.getMimeType()) && m.getSize() < mediaConstraints.getGifMaxSize(context)) || - (MediaUtil.isVideoType(m.getMimeType()) && m.getSize() < mediaConstraints.getVideoMaxSize(context)); + (MediaUtil.isVideoType(m.getMimeType()) && m.getSize() < mediaConstraints.getUncompressedVideoMaxSize(context)); }).toList(); } diff --git a/src/org/thoughtcrime/securesms/mms/MediaConstraints.java b/src/org/thoughtcrime/securesms/mms/MediaConstraints.java index 37ffb87e8..dab8c05fa 100644 --- a/src/org/thoughtcrime/securesms/mms/MediaConstraints.java +++ b/src/org/thoughtcrime/securesms/mms/MediaConstraints.java @@ -2,18 +2,18 @@ package org.thoughtcrime.securesms.mms; import android.content.Context; import android.net.Uri; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import org.thoughtcrime.securesms.logging.Log; +import android.os.Build; import android.util.Pair; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + import org.thoughtcrime.securesms.attachments.Attachment; -import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader.DecryptableUri; +import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.util.BitmapDecodingException; import org.thoughtcrime.securesms.util.BitmapUtil; import org.thoughtcrime.securesms.util.MediaUtil; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -34,6 +34,15 @@ public abstract class MediaConstraints { public abstract int getGifMaxSize(Context context); public abstract int getVideoMaxSize(Context context); + + public int getUncompressedVideoMaxSize(Context context) { + return getVideoMaxSize(context); + } + + public int getCompressedVideoMaxSize(Context context) { + return getVideoMaxSize(context); + } + public abstract int getAudioMaxSize(Context context); public abstract int getDocumentMaxSize(Context context); @@ -61,23 +70,12 @@ public abstract class MediaConstraints { } } - public boolean canResize(@Nullable Attachment attachment) { - return attachment != null && MediaUtil.isImage(attachment) && !MediaUtil.isGif(attachment); + public boolean canResize(@NonNull Attachment attachment) { + return MediaUtil.isImage(attachment) && !MediaUtil.isGif(attachment) || + MediaUtil.isVideo(attachment) && isVideoTranscodeAvailable(); } - public MediaStream getResizedMedia(@NonNull Context context, @NonNull Attachment attachment) - throws IOException - { - if (!canResize(attachment)) { - throw new UnsupportedOperationException("Cannot resize this content type"); - } - - try { - // XXX - This is loading everything into memory! We want the send path to be stream-like. - BitmapUtil.ScaleResult scaleResult = BitmapUtil.createScaledBytes(context, new DecryptableUri(attachment.getDataUri()), this); - return new MediaStream(new ByteArrayInputStream(scaleResult.getBitmap()), MediaUtil.IMAGE_JPEG, scaleResult.getWidth(), scaleResult.getHeight()); - } catch (BitmapDecodingException e) { - throw new IOException(e); - } + public static boolean isVideoTranscodeAvailable() { + return Build.VERSION.SDK_INT >= 26; } } diff --git a/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java b/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java index d7b6c9666..892c610b0 100644 --- a/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java +++ b/src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java @@ -39,6 +39,11 @@ final class MmsMediaConstraints extends MediaConstraints { return getMaxMessageSize(context); } + @Override + public int getUncompressedVideoMaxSize(Context context) { + return Math.max(getVideoMaxSize(context), 15 * 1024 * 1024); + } + @Override public int getAudioMaxSize(Context context) { return getMaxMessageSize(context); diff --git a/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java b/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java index 30a6b1ea1..ef936dfeb 100644 --- a/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java +++ b/src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java @@ -36,6 +36,18 @@ public class PushMediaConstraints extends MediaConstraints { return 100 * MB; } + @Override + public int getUncompressedVideoMaxSize(Context context) { + return isVideoTranscodeAvailable() ? 200 * MB + : getVideoMaxSize(context); + } + + @Override + public int getCompressedVideoMaxSize(Context context) { + return Util.isLowMemory(context) ? 30 * MB + : 50 * MB; + } + @Override public int getAudioMaxSize(Context context) { return 100 * MB; diff --git a/src/org/thoughtcrime/securesms/mms/Slide.java b/src/org/thoughtcrime/securesms/mms/Slide.java index 2c5dc1cb1..a7743f972 100644 --- a/src/org/thoughtcrime/securesms/mms/Slide.java +++ b/src/org/thoughtcrime/securesms/mms/Slide.java @@ -105,7 +105,7 @@ public abstract class Slide { public @NonNull String getContentDescription() { return ""; } - public Attachment asAttachment() { + public @NonNull Attachment asAttachment() { return attachment; } diff --git a/src/org/thoughtcrime/securesms/sms/MessageSender.java b/src/org/thoughtcrime/securesms/sms/MessageSender.java index 5b4b4da09..97f865c55 100644 --- a/src/org/thoughtcrime/securesms/sms/MessageSender.java +++ b/src/org/thoughtcrime/securesms/sms/MessageSender.java @@ -20,6 +20,8 @@ import android.app.Application; import android.content.Context; import androidx.annotation.NonNull; +import com.annimon.stream.Stream; + import org.thoughtcrime.securesms.attachments.AttachmentId; import org.thoughtcrime.securesms.attachments.DatabaseAttachment; import org.thoughtcrime.securesms.database.MessagingDatabase.SyncMessageId; @@ -133,14 +135,14 @@ public class MessageSender { return; } - ThreadDatabase threadDatabase = DatabaseFactory.getThreadDatabase(context); - MmsDatabase mmsDatabase = DatabaseFactory.getMmsDatabase(context); - AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context); - List> attachmentIds = new ArrayList<>(messages.get(0).getAttachments().size()); - List messageIds = new ArrayList<>(messages.size()); + ThreadDatabase threadDatabase = DatabaseFactory.getThreadDatabase(context); + MmsDatabase mmsDatabase = DatabaseFactory.getMmsDatabase(context); + AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context); + List> databaseAttachments = new ArrayList<>(messages.get(0).getAttachments().size()); + List messageIds = new ArrayList<>(messages.size()); for (int i = 0; i < messages.get(0).getAttachments().size(); i++) { - attachmentIds.add(new ArrayList<>(messages.size())); + databaseAttachments.add(new ArrayList<>(messages.size())); } try { @@ -152,13 +154,13 @@ public class MessageSender { long messageId = mmsDatabase.insertMessageOutbox(message, allocatedThreadId, false, null); List attachments = attachmentDatabase.getAttachmentsForMessage(messageId); - if (attachments.size() != attachmentIds.size()) { - Log.w(TAG, "Got back an attachment list that was a different size than expected. Expected: " + attachmentIds.size() + " Actual: "+ attachments.size()); + if (attachments.size() != databaseAttachments.size()) { + Log.w(TAG, "Got back an attachment list that was a different size than expected. Expected: " + databaseAttachments.size() + " Actual: "+ attachments.size()); return; } for (int i = 0; i < attachments.size(); i++) { - attachmentIds.get(i).add(attachments.get(i).getAttachmentId()); + databaseAttachments.get(i).add(attachments.get(i)); } messageIds.add(messageId); @@ -169,16 +171,20 @@ public class MessageSender { mmsDatabase.endTransaction(); } - List uploadJobs = new ArrayList<>(attachmentIds.size()); - List copyJobs = new ArrayList<>(attachmentIds.size()); - List messageJobs = new ArrayList<>(attachmentIds.get(0).size()); + List uploadJobs = new ArrayList<>(databaseAttachments.size()); + List copyJobs = new ArrayList<>(databaseAttachments.size()); + List messageJobs = new ArrayList<>(databaseAttachments.get(0).size()); - for (List idList : attachmentIds) { - uploadJobs.add(new AttachmentUploadJob(idList.get(0))); + for (List attachmentList : databaseAttachments) { + DatabaseAttachment source = attachmentList.get(0); - if (idList.size() > 1) { - AttachmentId sourceId = idList.get(0); - List destinationIds = idList.subList(1, idList.size()); + uploadJobs.add(AttachmentUploadJob.fromAttachment(source)); + + if (attachmentList.size() > 1) { + AttachmentId sourceId = source.getAttachmentId(); + List destinationIds = Stream.of(attachmentList.subList(1, attachmentList.size())) + .map(DatabaseAttachment::getAttachmentId) + .toList(); copyJobs.add(new AttachmentCopyJob(sourceId, destinationIds)); } diff --git a/src/org/thoughtcrime/securesms/util/FileUtils.java b/src/org/thoughtcrime/securesms/util/FileUtils.java index e437edd84..7b516a9f9 100644 --- a/src/org/thoughtcrime/securesms/util/FileUtils.java +++ b/src/org/thoughtcrime/securesms/util/FileUtils.java @@ -7,7 +7,7 @@ import java.io.IOException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -public class FileUtils { +public final class FileUtils { static { System.loadLibrary("native-utils"); @@ -15,6 +15,8 @@ public class FileUtils { public static native int getFileDescriptorOwner(FileDescriptor fileDescriptor); + static native int createMemoryFileDescriptor(String name); + public static byte[] getFileDigest(FileInputStream fin) throws IOException { try { MessageDigest digest = MessageDigest.getInstance("SHA256"); diff --git a/src/org/thoughtcrime/securesms/util/MediaUtil.java b/src/org/thoughtcrime/securesms/util/MediaUtil.java index 625337f3d..7d510b8fc 100644 --- a/src/org/thoughtcrime/securesms/util/MediaUtil.java +++ b/src/org/thoughtcrime/securesms/util/MediaUtil.java @@ -280,16 +280,17 @@ public class MediaUtil { return sections.length > 1 ? sections[0] : null; } - public static class ThumbnailData { - Bitmap bitmap; - float aspectRatio; + public static class ThumbnailData implements AutoCloseable { - public ThumbnailData(Bitmap bitmap) { + @NonNull private final Bitmap bitmap; + private final float aspectRatio; + + public ThumbnailData(@NonNull Bitmap bitmap) { this.bitmap = bitmap; this.aspectRatio = (float) bitmap.getWidth() / (float) bitmap.getHeight(); } - public Bitmap getBitmap() { + public @NonNull Bitmap getBitmap() { return bitmap; } @@ -300,5 +301,10 @@ public class MediaUtil { public InputStream toDataStream() { return BitmapUtil.toCompressedJpeg(bitmap); } + + @Override + public void close() { + bitmap.recycle(); + } } } diff --git a/src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java b/src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java new file mode 100644 index 000000000..dcc6ea278 --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java @@ -0,0 +1,158 @@ +package org.thoughtcrime.securesms.util; + +import android.app.ActivityManager; +import android.content.Context; +import android.os.ParcelFileDescriptor; + +import androidx.annotation.NonNull; + +import org.thoughtcrime.securesms.logging.Log; + +import java.io.Closeable; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.text.NumberFormat; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicLong; + +public final class MemoryFileDescriptor implements Closeable { + + private static final String TAG = Log.tag(MemoryFileDescriptor.class); + + private final ParcelFileDescriptor parcelFileDescriptor; + private final AtomicLong sizeEstimate; + + /** + * memfd files do not show on the available RAM, so we must track our allocations in addition. + */ + private static long sizeOfAllMemoryFileDescriptors; + + private MemoryFileDescriptor(@NonNull ParcelFileDescriptor parcelFileDescriptor, long sizeEstimate) { + this.parcelFileDescriptor = parcelFileDescriptor; + this.sizeEstimate = new AtomicLong(sizeEstimate); + } + + /** + * @param debugName The name supplied in name is used as a filename and will be displayed + * as the target of the corresponding symbolic link in the directory + * /proc/self/fd/. The displayed name is always prefixed with memfd: + * and serves only for debugging purposes. Names do not affect the + * behavior of the file descriptor, and as such multiple files can have + * the same name without any side effects. + * @param sizeEstimate An estimated upper bound on this file. This is used to check there will be + * enough RAM available and to register with a global counter of reservations. + * Use zero to avoid RAM check. + * @return MemoryFileDescriptor + * @throws MemoryLimitException If there is not enough available RAM to comfortably fit this file. + * @throws IOException If fails to create a memory file descriptor. + */ + public static MemoryFileDescriptor newMemoryFileDescriptor(@NonNull Context context, + @NonNull String debugName, + long sizeEstimate) + throws MemoryLimitException, IOException + { + if (sizeEstimate < 0) throw new IllegalArgumentException(); + + if (sizeEstimate > 0) { + ActivityManager activityManager = ServiceUtil.getActivityManager(context); + ActivityManager.MemoryInfo memoryInfo = new ActivityManager.MemoryInfo(); + + synchronized (MemoryFileDescriptor.class) { + activityManager.getMemoryInfo(memoryInfo); + + long remainingRam = memoryInfo.availMem - memoryInfo.threshold - sizeEstimate - sizeOfAllMemoryFileDescriptors; + + if (remainingRam <= 0) { + NumberFormat numberFormat = NumberFormat.getInstance(Locale.US); + Log.w(TAG, String.format("Not enough RAM available without taking the system into a low memory state.%n" + + "Available: %s%n" + + "Low memory threshold: %s%n" + + "Requested: %s%n" + + "Total MemoryFileDescriptor limit: %s%n" + + "Shortfall: %s", + numberFormat.format(memoryInfo.availMem), + numberFormat.format(memoryInfo.threshold), + numberFormat.format(sizeEstimate), + numberFormat.format(sizeOfAllMemoryFileDescriptors), + numberFormat.format(remainingRam) + )); + throw new MemoryLimitException(); + } + + sizeOfAllMemoryFileDescriptors += sizeEstimate; + } + } + + int fileDescriptor = FileUtils.createMemoryFileDescriptor(debugName); + + if (fileDescriptor < 0) { + throw new IOException("Failed to create a memory file descriptor " + fileDescriptor); + } + + return new MemoryFileDescriptor(ParcelFileDescriptor.adoptFd(fileDescriptor), sizeEstimate); + } + + @Override + public void close() throws IOException { + try { + clearAndRemoveAllocation(); + } catch (Exception e) { + Log.w(TAG, "Failed to clear data in MemoryFileDescriptor", e); + } finally { + parcelFileDescriptor.close(); + } + } + + private void clearAndRemoveAllocation() throws IOException { + clear(); + + long oldEstimate = sizeEstimate.getAndSet(0); + + synchronized (MemoryFileDescriptor.class) { + sizeOfAllMemoryFileDescriptors -= oldEstimate; + } + } + + /** Rewinds and clears all bytes. */ + private void clear() throws IOException { + long size; + try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) { + FileChannel channel = fileInputStream.getChannel(); + size = channel.size(); + + if (size == 0) return; + + channel.position(0); + } + byte[] zeros = new byte[16 * 1024]; + + try (FileOutputStream output = new FileOutputStream(getFileDescriptor())) { + while (size > 0) { + int limit = (int) Math.min(size, zeros.length); + + output.write(zeros, 0, limit); + + size -= limit; + } + } + } + + public FileDescriptor getFileDescriptor() { + return parcelFileDescriptor.getFileDescriptor(); + } + + public void seek(long position) throws IOException { + try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) { + fileInputStream.getChannel().position(position); + } + } + + public long size() throws IOException { + try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) { + return fileInputStream.getChannel().size(); + } + } +} diff --git a/src/org/thoughtcrime/securesms/util/MemoryLimitException.java b/src/org/thoughtcrime/securesms/util/MemoryLimitException.java new file mode 100644 index 000000000..2dfe9d0d8 --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/MemoryLimitException.java @@ -0,0 +1,6 @@ +package org.thoughtcrime.securesms.util; + +import java.io.IOException; + +public final class MemoryLimitException extends IOException { +} diff --git a/src/org/thoughtcrime/securesms/util/ServiceUtil.java b/src/org/thoughtcrime/securesms/util/ServiceUtil.java index 76a413b7f..a86d06597 100644 --- a/src/org/thoughtcrime/securesms/util/ServiceUtil.java +++ b/src/org/thoughtcrime/securesms/util/ServiceUtil.java @@ -1,6 +1,7 @@ package org.thoughtcrime.securesms.util; import android.app.Activity; +import android.app.ActivityManager; import android.app.AlarmManager; import android.app.NotificationManager; import android.app.job.JobScheduler; @@ -69,4 +70,8 @@ public class ServiceUtil { public static @Nullable SubscriptionManager getSubscriptionManager(@NonNull Context context) { return (SubscriptionManager) context.getSystemService(Context.TELEPHONY_SUBSCRIPTION_SERVICE); } + + public static ActivityManager getActivityManager(@NonNull Context context) { + return (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + } } diff --git a/src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java b/src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java new file mode 100644 index 000000000..f4a019978 --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java @@ -0,0 +1,54 @@ +package org.thoughtcrime.securesms.video; + +import android.media.MediaDataSource; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.crypto.AttachmentSecret; +import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream; +import org.thoughtcrime.securesms.util.Util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +@RequiresApi(23) +final class ClassicEncryptedMediaDataSource extends MediaDataSource { + + private final AttachmentSecret attachmentSecret; + private final File mediaFile; + private final long length; + + ClassicEncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, long length) { + this.attachmentSecret = attachmentSecret; + this.mediaFile = mediaFile; + this.length = length; + } + + @Override + public int readAt(long position, byte[] bytes, int offset, int length) throws IOException { + try (InputStream inputStream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, mediaFile)) { + byte[] buffer = new byte[4096]; + long headerRemaining = position; + + while (headerRemaining > 0) { + int read = inputStream.read(buffer, 0, Util.toIntExact(Math.min((long)buffer.length, headerRemaining))); + + if (read == -1) return -1; + + headerRemaining -= read; + } + + return inputStream.read(bytes, offset, length); + } + } + + @Override + public long getSize() { + return length; + } + + @Override + public void close() {} +} diff --git a/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java b/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java index f2e036ed8..d4af426dc 100644 --- a/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java +++ b/src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java @@ -1,78 +1,23 @@ package org.thoughtcrime.securesms.video; - -import android.annotation.TargetApi; import android.media.MediaDataSource; -import android.os.Build; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; import org.thoughtcrime.securesms.crypto.AttachmentSecret; -import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream; -import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream; -import org.thoughtcrime.securesms.util.Util; import java.io.File; -import java.io.IOException; -import java.io.InputStream; -@TargetApi(Build.VERSION_CODES.M) -public class EncryptedMediaDataSource extends MediaDataSource { +@RequiresApi(23) +public final class EncryptedMediaDataSource { - private final AttachmentSecret attachmentSecret; - private final File mediaFile; - private final byte[] random; - private final long length; - - public EncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @Nullable byte[] random, long length) { - this.attachmentSecret = attachmentSecret; - this.mediaFile = mediaFile; - this.random = random; - this.length = length; - } - - @Override - public int readAt(long position, byte[] bytes, int offset, int length) throws IOException { - if (random == null) return readAtClassic(position, bytes, offset, length); - else return readAtModern(position, bytes, offset, length); - } - - private int readAtClassic(long position, byte[] bytes, int offset, int length) throws IOException { - InputStream inputStream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, mediaFile); - byte[] buffer = new byte[4096]; - long headerRemaining = position; - - while (headerRemaining > 0) { - int read = inputStream.read(buffer, 0, Util.toIntExact(Math.min((long)buffer.length, headerRemaining))); - - if (read == -1) return -1; - - headerRemaining -= read; + public static MediaDataSource createFor(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @Nullable byte[] random, long length) { + if (random == null) { + return new ClassicEncryptedMediaDataSource(attachmentSecret, mediaFile, length); + } else { + return new ModernEncryptedMediaDataSource(attachmentSecret, mediaFile, random, length); } - - int returnValue = inputStream.read(bytes, offset, length); - inputStream.close(); - return returnValue; - } - - private int readAtModern(long position, byte[] bytes, int offset, int length) throws IOException { - assert(random != null); - - InputStream inputStream = ModernDecryptingPartInputStream.createFor(attachmentSecret, random, mediaFile, position); - int returnValue = inputStream.read(bytes, offset, length); - - inputStream.close(); - - return returnValue; - } - - @Override - public long getSize() throws IOException { - return length; - } - - @Override - public void close() throws IOException { - } } diff --git a/src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java b/src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java new file mode 100644 index 000000000..4863e6ffe --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java @@ -0,0 +1,201 @@ +package org.thoughtcrime.securesms.video; + +import android.content.Context; +import android.media.MediaDataSource; +import android.media.MediaMetadataRetriever; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import com.google.android.exoplayer2.util.MimeTypes; + +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.mms.MediaStream; +import org.thoughtcrime.securesms.transport.UndeliverableMessageException; +import org.thoughtcrime.securesms.util.MemoryFileDescriptor; +import org.thoughtcrime.securesms.video.videoconverter.BadVideoException; +import org.thoughtcrime.securesms.video.videoconverter.MediaConverter; + +import java.io.Closeable; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.IOException; +import java.text.NumberFormat; +import java.util.Locale; + +@RequiresApi(26) +public final class InMemoryTranscoder implements Closeable { + + private static final String TAG = Log.tag(InMemoryTranscoder.class); + + private static final int MAXIMUM_TARGET_VIDEO_BITRATE = 2_000_000; + private static final int LOW_RES_TARGET_VIDEO_BITRATE = 1_750_000; + private static final int MINIMUM_TARGET_VIDEO_BITRATE = 500_000; + private static final int AUDIO_BITRATE = 192_000; + private static final int OUTPUT_FORMAT = 720; + private static final int LOW_RES_OUTPUT_FORMAT = 480; + + private final Context context; + private final MediaDataSource dataSource; + private final long upperSizeLimit; + private final long inSize; + private final long duration; + private final int inputBitRate; + private final int targetVideoBitRate; + private final long memoryFileEstimate; + private final boolean transcodeRequired; + private final long fileSizeEstimate; + private final int outputFormat; + + private @Nullable MemoryFileDescriptor memoryFile; + + /** + * @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller. + */ + public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, long upperSizeLimit) throws IOException { + this.context = context; + this.dataSource = dataSource; + + final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever(); + mediaMetadataRetriever.setDataSource(dataSource); + + long upperSizeLimitWithMargin = (long) (upperSizeLimit / 1.1); + + this.inSize = dataSource.getSize(); + this.duration = getDuration(mediaMetadataRetriever); + this.inputBitRate = bitRate(inSize, duration); + this.targetVideoBitRate = getTargetVideoBitRate(upperSizeLimitWithMargin, duration); + this.upperSizeLimit = upperSizeLimit; + + this.transcodeRequired = inputBitRate >= targetVideoBitRate * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever); + if (!transcodeRequired) { + Log.i(TAG, "Video is within 20% of target bitrate, below the size limit and contained no location metadata."); + } + + this.fileSizeEstimate = (targetVideoBitRate + AUDIO_BITRATE) * duration / 8000; + this.memoryFileEstimate = (long) (fileSizeEstimate * 1.1); + this.outputFormat = targetVideoBitRate < LOW_RES_TARGET_VIDEO_BITRATE + ? LOW_RES_OUTPUT_FORMAT + : OUTPUT_FORMAT; + } + + public @NonNull MediaStream transcode(@NonNull Progress progress) throws IOException, UndeliverableMessageException, BadVideoException { + if (memoryFile != null) throw new AssertionError("Not expecting to reuse transcoder"); + + float durationSec = duration / 1000f; + + NumberFormat numberFormat = NumberFormat.getInstance(Locale.US); + + Log.i(TAG, String.format(Locale.US, + "Transcoding:\n" + + "Target bitrate : %s + %s = %s\n" + + "Target format : %dp\n" + + "Video duration : %.1fs\n" + + "Size limit : %s kB\n" + + "Estimate : %s kB\n" + + "Input size : %s kB\n" + + "Input bitrate : %s bps", + numberFormat.format(targetVideoBitRate), + numberFormat.format(AUDIO_BITRATE), + numberFormat.format(targetVideoBitRate + AUDIO_BITRATE), + outputFormat, + durationSec, + numberFormat.format(upperSizeLimit / 1024), + numberFormat.format(fileSizeEstimate / 1024), + numberFormat.format(inSize / 1024), + numberFormat.format(inputBitRate))); + + if (fileSizeEstimate > upperSizeLimit) { + throw new UndeliverableMessageException("Size constraints could not be met!"); + } + + memoryFile = MemoryFileDescriptor.newMemoryFileDescriptor(context, + "TRANSCODE", + memoryFileEstimate); + final long startTime = System.currentTimeMillis(); + + final FileDescriptor memoryFileFileDescriptor = memoryFile.getFileDescriptor(); + + final MediaConverter converter = new MediaConverter(); + + converter.setInput(dataSource); + converter.setOutput(memoryFileFileDescriptor); + converter.setVideoResolution(outputFormat); + converter.setVideoBitrate(targetVideoBitRate); + converter.setAudioBitrate(AUDIO_BITRATE); + + converter.setListener(percent -> { + progress.onProgress(percent); + return false; + }); + + converter.convert(); + + // output details of the transcoding + long outSize = memoryFile.size(); + float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f; + + Log.i(TAG, String.format(Locale.US, + "Transcoding complete:\n" + + "Transcode time : %.1fs (%.1fx)\n" + + "Output size : %s kB\n" + + " of Original : %.1f%%\n" + + " of Estimate : %.1f%%\n" + + " of Memory : %.1f%%\n" + + "Output bitrate : %s bps", + encodeDurationSec, + durationSec / encodeDurationSec, + numberFormat.format(outSize / 1024), + (outSize * 100d) / inSize, + (outSize * 100d) / fileSizeEstimate, + (outSize * 100d) / memoryFileEstimate, + numberFormat.format(bitRate(outSize, duration)))); + + if (outSize > upperSizeLimit) { + throw new UndeliverableMessageException("Size constraints could not be met!"); + } + + memoryFile.seek(0); + + return new MediaStream(new FileInputStream(memoryFileFileDescriptor), MimeTypes.VIDEO_MP4, 0, 0); + } + + public boolean isTranscodeRequired() { + return transcodeRequired; + } + + @Override + public void close() throws IOException { + if (memoryFile != null) { + memoryFile.close(); + } + } + + private static int bitRate(long bytes, long duration) { + return (int) (bytes * 8 / (duration / 1000f)); + } + + private static int getTargetVideoBitRate(long sizeGuideBytes, long duration) { + sizeGuideBytes -= (duration / 1000d) * AUDIO_BITRATE / 8; + + double targetAttachmentSizeBits = sizeGuideBytes * 8L; + + double bitRateToFixTarget = targetAttachmentSizeBits / (duration / 1000d); + return Math.max(MINIMUM_TARGET_VIDEO_BITRATE, Math.min(MAXIMUM_TARGET_VIDEO_BITRATE, (int) bitRateToFixTarget)); + } + + private static long getDuration(MediaMetadataRetriever mediaMetadataRetriever) { + return Long.parseLong(mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)); + } + + private static boolean containsLocation(MediaMetadataRetriever mediaMetadataRetriever) { + String locationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION); + return locationString != null; + } + + public interface Progress { + + void onProgress(int percent); + } +} diff --git a/src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java b/src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java new file mode 100644 index 000000000..738377a96 --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java @@ -0,0 +1,71 @@ +package org.thoughtcrime.securesms.video; + +import android.media.MediaDataSource; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.crypto.AttachmentSecret; +import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +/** + * Create via {@link EncryptedMediaDataSource}. + *

+ * A {@link MediaDataSource} that points to an encrypted file. + *

+ * It is "modern" compared to the {@link ClassicEncryptedMediaDataSource}. And "modern" refers to + * the presence of a random part of the key supplied in the constructor. + */ +@RequiresApi(23) +final class ModernEncryptedMediaDataSource extends MediaDataSource { + + private final AttachmentSecret attachmentSecret; + private final File mediaFile; + private final byte[] random; + private final long length; + + ModernEncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @NonNull byte[] random, long length) { + this.attachmentSecret = attachmentSecret; + this.mediaFile = mediaFile; + this.random = random; + this.length = length; + } + + @Override + public int readAt(long position, byte[] bytes, int offset, int length) throws IOException { + try (InputStream inputStream = ModernDecryptingPartInputStream.createFor(attachmentSecret, random, mediaFile, position)) { + int totalRead = 0; + + while (length > 0) { + int read = inputStream.read(bytes, offset, length); + + if (read == -1) { + if (totalRead == 0) { + return -1; + } else { + return totalRead; + } + } + + length -= read; + offset += read; + totalRead += read; + } + + return totalRead; + } + } + + @Override + public long getSize() { + return length; + } + + @Override + public void close() { + } +}