2019-09-26 16:12:51 +02:00
package org.thoughtcrime.securesms.jobs ;
import android.content.Context ;
import androidx.annotation.NonNull ;
import com.annimon.stream.Stream ;
import org.thoughtcrime.securesms.database.DatabaseFactory ;
import org.thoughtcrime.securesms.database.RecipientDatabase ;
import org.thoughtcrime.securesms.database.RecipientDatabase.RecipientSettings ;
import org.thoughtcrime.securesms.database.StorageKeyDatabase ;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies ;
import org.thoughtcrime.securesms.jobmanager.Data ;
import org.thoughtcrime.securesms.jobmanager.Job ;
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint ;
2020-01-17 17:14:54 +01:00
import org.thoughtcrime.securesms.keyvalue.SignalStore ;
2019-09-26 16:12:51 +02:00
import org.thoughtcrime.securesms.logging.Log ;
2020-05-19 23:48:26 +02:00
import org.thoughtcrime.securesms.recipients.Recipient ;
2019-09-26 16:12:51 +02:00
import org.thoughtcrime.securesms.recipients.RecipientId ;
2020-05-19 23:48:26 +02:00
import org.thoughtcrime.securesms.storage.StorageSyncHelper ;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.KeyDifferenceResult ;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.LocalWriteResult ;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.MergeResult ;
import org.thoughtcrime.securesms.storage.StorageSyncHelper.WriteOperationResult ;
import org.thoughtcrime.securesms.storage.StorageSyncModels ;
2020-03-18 21:31:45 +01:00
import org.thoughtcrime.securesms.storage.StorageSyncValidations ;
2019-09-26 16:12:51 +02:00
import org.thoughtcrime.securesms.transport.RetryLaterException ;
import org.thoughtcrime.securesms.util.FeatureFlags ;
import org.thoughtcrime.securesms.util.TextSecurePreferences ;
import org.thoughtcrime.securesms.util.Util ;
import org.whispersystems.libsignal.InvalidKeyException ;
import org.whispersystems.libsignal.util.guava.Optional ;
import org.whispersystems.signalservice.api.SignalServiceAccountManager ;
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException ;
2020-05-19 23:48:26 +02:00
import org.whispersystems.signalservice.api.storage.SignalAccountRecord ;
2019-09-26 16:12:51 +02:00
import org.whispersystems.signalservice.api.storage.SignalStorageManifest ;
import org.whispersystems.signalservice.api.storage.SignalStorageRecord ;
2020-05-19 23:48:26 +02:00
import org.whispersystems.signalservice.api.storage.StorageId ;
import org.whispersystems.signalservice.api.storage.StorageKey ;
2020-03-18 21:31:45 +01:00
import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord ;
2019-09-26 16:12:51 +02:00
import java.io.IOException ;
import java.util.ArrayList ;
2020-04-27 19:05:22 +02:00
import java.util.Arrays ;
2020-03-18 21:31:45 +01:00
import java.util.Collections ;
2019-09-26 16:12:51 +02:00
import java.util.List ;
2020-02-10 19:42:43 +01:00
import java.util.Locale ;
2020-03-18 21:31:45 +01:00
import java.util.Set ;
2019-09-26 16:12:51 +02:00
import java.util.concurrent.TimeUnit ;
/ * *
* Does a full sync of our local storage state with the remote storage state . Will write any pending
* local changes and resolve any conflicts with remote storage .
*
* This should be performed whenever a change is made locally , or whenever we want to retrieve
* changes that have been made remotely .
* /
public class StorageSyncJob extends BaseJob {
public static final String KEY = "StorageSyncJob" ;
public static final String QUEUE_KEY = "StorageSyncingJobs" ;
private static final String TAG = Log . tag ( StorageSyncJob . class ) ;
public StorageSyncJob ( ) {
this ( new Job . Parameters . Builder ( ) . addConstraint ( NetworkConstraint . KEY )
. setQueue ( QUEUE_KEY )
2020-03-18 21:31:45 +01:00
. setMaxInstances ( 2 )
2019-09-26 16:12:51 +02:00
. setLifespan ( TimeUnit . DAYS . toMillis ( 1 ) )
. build ( ) ) ;
}
private StorageSyncJob ( @NonNull Parameters parameters ) {
super ( parameters ) ;
}
@Override
public @NonNull Data serialize ( ) {
return Data . EMPTY ;
}
@Override
public @NonNull String getFactoryKey ( ) {
return KEY ;
}
@Override
protected void onRun ( ) throws IOException , RetryLaterException {
2020-07-14 00:52:30 +02:00
if ( ! SignalStore . kbsValues ( ) . hasPin ( ) & & ! SignalStore . kbsValues ( ) . hasOptedOut ( ) ) {
2020-04-02 23:09:25 +02:00
Log . i ( TAG , "Doesn't have a PIN. Skipping." ) ;
return ;
}
2020-03-18 21:31:45 +01:00
if ( ! TextSecurePreferences . isPushRegistered ( context ) ) {
Log . i ( TAG , "Not registered. Skipping." ) ;
return ;
}
2019-09-26 16:12:51 +02:00
try {
boolean needsMultiDeviceSync = performSync ( ) ;
if ( TextSecurePreferences . isMultiDevice ( context ) & & needsMultiDeviceSync ) {
ApplicationDependencies . getJobManager ( ) . add ( new MultiDeviceStorageSyncRequestJob ( ) ) ;
}
2020-02-10 19:42:43 +01:00
SignalStore . storageServiceValues ( ) . onSyncCompleted ( ) ;
2019-09-26 16:12:51 +02:00
} catch ( InvalidKeyException e ) {
Log . w ( TAG , "Failed to decrypt remote storage! Force-pushing and syncing the storage key to linked devices." , e ) ;
ApplicationDependencies . getJobManager ( ) . startChain ( new MultiDeviceKeysUpdateJob ( ) )
. then ( new StorageForcePushJob ( ) )
. then ( new MultiDeviceStorageSyncRequestJob ( ) )
. enqueue ( ) ;
}
}
@Override
protected boolean onShouldRetry ( @NonNull Exception e ) {
return e instanceof PushNetworkException | | e instanceof RetryLaterException ;
}
@Override
2020-01-03 20:10:16 +01:00
public void onFailure ( ) {
2019-09-26 16:12:51 +02:00
}
private boolean performSync ( ) throws IOException , RetryLaterException , InvalidKeyException {
SignalServiceAccountManager accountManager = ApplicationDependencies . getSignalServiceAccountManager ( ) ;
RecipientDatabase recipientDatabase = DatabaseFactory . getRecipientDatabase ( context ) ;
StorageKeyDatabase storageKeyDatabase = DatabaseFactory . getStorageKeyDatabase ( context ) ;
2020-04-02 23:09:25 +02:00
StorageKey storageServiceKey = SignalStore . storageServiceValues ( ) . getOrCreateStorageKey ( ) ;
2019-09-26 16:12:51 +02:00
2020-02-10 19:42:43 +01:00
boolean needsMultiDeviceSync = false ;
2020-09-10 20:01:41 +02:00
boolean needsForcePush = false ;
2020-02-10 19:42:43 +01:00
long localManifestVersion = TextSecurePreferences . getStorageManifestVersion ( context ) ;
Optional < SignalStorageManifest > remoteManifest = accountManager . getStorageManifestIfDifferentVersion ( storageServiceKey , localManifestVersion ) ;
long remoteManifestVersion = remoteManifest . transform ( SignalStorageManifest : : getVersion ) . or ( localManifestVersion ) ;
2019-09-26 16:12:51 +02:00
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "Our version: " + localManifestVersion + ", their version: " + remoteManifestVersion ) ;
2019-09-26 16:12:51 +02:00
2020-02-10 19:42:43 +01:00
if ( remoteManifest . isPresent ( ) & & remoteManifestVersion > localManifestVersion ) {
Log . i ( TAG , "[Remote Newer] Newer manifest version found!" ) ;
2019-09-26 16:12:51 +02:00
2020-04-27 19:05:22 +02:00
List < StorageId > allLocalStorageKeys = getAllLocalStorageIds ( context , Recipient . self ( ) . fresh ( ) ) ;
2020-03-18 21:31:45 +01:00
KeyDifferenceResult keyDifference = StorageSyncHelper . findKeyDifference ( remoteManifest . get ( ) . getStorageIds ( ) , allLocalStorageKeys ) ;
2019-09-26 16:12:51 +02:00
2020-09-10 20:01:41 +02:00
if ( keyDifference . hasTypeMismatches ( ) ) {
Log . w ( TAG , "Found type mismatches in the key sets! Scheduling a force push after this sync completes." ) ;
needsForcePush = true ;
}
2019-09-26 16:12:51 +02:00
if ( ! keyDifference . isEmpty ( ) ) {
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "[Remote Newer] There's a difference in keys. Local-only: " + keyDifference . getLocalOnlyKeys ( ) . size ( ) + ", Remote-only: " + keyDifference . getRemoteOnlyKeys ( ) . size ( ) ) ;
2020-10-06 16:24:14 +02:00
List < SignalStorageRecord > localOnly = buildLocalStorageRecords ( context , keyDifference . getLocalOnlyKeys ( ) ) ;
2019-09-26 16:12:51 +02:00
List < SignalStorageRecord > remoteOnly = accountManager . readStorageRecords ( storageServiceKey , keyDifference . getRemoteOnlyKeys ( ) ) ;
MergeResult mergeResult = StorageSyncHelper . resolveConflict ( remoteOnly , localOnly ) ;
2020-02-10 19:42:43 +01:00
WriteOperationResult writeOperationResult = StorageSyncHelper . createWriteOperation ( remoteManifest . get ( ) . getVersion ( ) , allLocalStorageKeys , mergeResult ) ;
2019-09-26 16:12:51 +02:00
2020-09-10 20:01:41 +02:00
if ( remoteOnly . size ( ) ! = keyDifference . getRemoteOnlyKeys ( ) . size ( ) ) {
Log . w ( TAG , "Could not find all remote-only records! Requested: " + keyDifference . getRemoteOnlyKeys ( ) . size ( ) + ", Found: " + remoteOnly . size ( ) + ". Scheduling a force push after this sync completes." ) ;
needsForcePush = true ;
}
2020-03-18 21:31:45 +01:00
StorageSyncValidations . validate ( writeOperationResult ) ;
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "[Remote Newer] MergeResult :: " + mergeResult ) ;
2019-09-26 16:12:51 +02:00
2020-02-10 19:42:43 +01:00
if ( ! writeOperationResult . isEmpty ( ) ) {
Log . i ( TAG , "[Remote Newer] WriteOperationResult :: " + writeOperationResult ) ;
Log . i ( TAG , "[Remote Newer] We have something to write remotely." ) ;
2020-02-28 19:03:06 +01:00
if ( writeOperationResult . getManifest ( ) . getStorageIds ( ) . size ( ) ! = remoteManifest . get ( ) . getStorageIds ( ) . size ( ) + writeOperationResult . getInserts ( ) . size ( ) - writeOperationResult . getDeletes ( ) . size ( ) ) {
2020-02-10 19:42:43 +01:00
Log . w ( TAG , String . format ( Locale . ENGLISH , "Bad storage key management! originalRemoteKeys: %d, newRemoteKeys: %d, insertedKeys: %d, deletedKeys: %d" ,
2020-02-28 19:03:06 +01:00
remoteManifest . get ( ) . getStorageIds ( ) . size ( ) , writeOperationResult . getManifest ( ) . getStorageIds ( ) . size ( ) , writeOperationResult . getInserts ( ) . size ( ) , writeOperationResult . getDeletes ( ) . size ( ) ) ) ;
2020-02-10 19:42:43 +01:00
}
Optional < SignalStorageManifest > conflict = accountManager . writeStorageRecords ( storageServiceKey , writeOperationResult . getManifest ( ) , writeOperationResult . getInserts ( ) , writeOperationResult . getDeletes ( ) ) ;
if ( conflict . isPresent ( ) ) {
Log . w ( TAG , "[Remote Newer] Hit a conflict when trying to resolve the conflict! Retrying." ) ;
throw new RetryLaterException ( ) ;
}
remoteManifestVersion = writeOperationResult . getManifest ( ) . getVersion ( ) ;
2020-10-07 19:32:59 +02:00
needsMultiDeviceSync = true ;
2020-02-10 19:42:43 +01:00
} else {
Log . i ( TAG , "[Remote Newer] After resolving the conflict, all changes are local. No remote writes needed." ) ;
2019-09-26 16:12:51 +02:00
}
2020-05-19 23:48:26 +02:00
recipientDatabase . applyStorageSyncUpdates ( mergeResult . getLocalContactInserts ( ) , mergeResult . getLocalContactUpdates ( ) , mergeResult . getLocalGroupV1Inserts ( ) , mergeResult . getLocalGroupV1Updates ( ) , mergeResult . getLocalGroupV2Inserts ( ) , mergeResult . getLocalGroupV2Updates ( ) ) ;
2019-09-26 16:12:51 +02:00
storageKeyDatabase . applyStorageSyncUpdates ( mergeResult . getLocalUnknownInserts ( ) , mergeResult . getLocalUnknownDeletes ( ) ) ;
2020-03-18 21:31:45 +01:00
StorageSyncHelper . applyAccountStorageSyncUpdates ( context , mergeResult . getLocalAccountUpdate ( ) ) ;
2019-09-26 16:12:51 +02:00
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "[Remote Newer] Updating local manifest version to: " + remoteManifestVersion ) ;
TextSecurePreferences . setStorageManifestVersion ( context , remoteManifestVersion ) ;
2019-09-26 16:12:51 +02:00
} else {
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "[Remote Newer] Remote version was newer, but our local data matched." ) ;
Log . i ( TAG , "[Remote Newer] Updating local manifest version to: " + remoteManifest . get ( ) . getVersion ( ) ) ;
TextSecurePreferences . setStorageManifestVersion ( context , remoteManifest . get ( ) . getVersion ( ) ) ;
2019-09-26 16:12:51 +02:00
}
}
localManifestVersion = TextSecurePreferences . getStorageManifestVersion ( context ) ;
2020-04-27 19:05:22 +02:00
Recipient self = Recipient . self ( ) . fresh ( ) ;
List < StorageId > allLocalStorageKeys = getAllLocalStorageIds ( context , self ) ;
2020-03-18 21:31:45 +01:00
List < RecipientSettings > pendingUpdates = recipientDatabase . getPendingRecipientSyncUpdates ( ) ;
List < RecipientSettings > pendingInsertions = recipientDatabase . getPendingRecipientSyncInsertions ( ) ;
List < RecipientSettings > pendingDeletions = recipientDatabase . getPendingRecipientSyncDeletions ( ) ;
2020-04-27 19:05:22 +02:00
Optional < SignalAccountRecord > pendingAccountInsert = StorageSyncHelper . getPendingAccountSyncInsert ( context , self ) ;
Optional < SignalAccountRecord > pendingAccountUpdate = StorageSyncHelper . getPendingAccountSyncUpdate ( context , self ) ;
2020-03-18 21:31:45 +01:00
Optional < LocalWriteResult > localWriteResult = StorageSyncHelper . buildStorageUpdatesForLocal ( localManifestVersion ,
allLocalStorageKeys ,
pendingUpdates ,
pendingInsertions ,
pendingDeletions ,
pendingAccountUpdate ,
2020-10-06 16:24:14 +02:00
pendingAccountInsert ) ;
2019-09-26 16:12:51 +02:00
if ( localWriteResult . isPresent ( ) ) {
2020-04-23 18:03:23 +02:00
Log . i ( TAG , String . format ( Locale . ENGLISH , "[Local Changes] Local changes present. %d updates, %d inserts, %d deletes, account update: %b, account insert: %b." , pendingUpdates . size ( ) , pendingInsertions . size ( ) , pendingDeletions . size ( ) , pendingAccountUpdate . isPresent ( ) , pendingAccountInsert . isPresent ( ) ) ) ;
2020-02-10 19:42:43 +01:00
WriteOperationResult localWrite = localWriteResult . get ( ) . getWriteResult ( ) ;
2020-03-18 21:31:45 +01:00
StorageSyncValidations . validate ( localWrite ) ;
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "[Local Changes] WriteOperationResult :: " + localWrite ) ;
if ( localWrite . isEmpty ( ) ) {
throw new AssertionError ( "Decided there were local writes, but our write result was empty!" ) ;
}
2020-03-18 21:31:45 +01:00
Optional < SignalStorageManifest > conflict = accountManager . writeStorageRecords ( storageServiceKey , localWrite . getManifest ( ) , localWrite . getInserts ( ) , localWrite . getDeletes ( ) ) ;
2019-09-26 16:12:51 +02:00
if ( conflict . isPresent ( ) ) {
2020-02-10 19:42:43 +01:00
Log . w ( TAG , "[Local Changes] Hit a conflict when trying to upload our local writes! Retrying." ) ;
2019-09-26 16:12:51 +02:00
throw new RetryLaterException ( ) ;
}
2020-03-18 21:31:45 +01:00
List < RecipientId > clearIds = new ArrayList < > ( pendingUpdates . size ( ) + pendingInsertions . size ( ) + pendingDeletions . size ( ) + 1 ) ;
2019-09-26 16:12:51 +02:00
clearIds . addAll ( Stream . of ( pendingUpdates ) . map ( RecipientSettings : : getId ) . toList ( ) ) ;
clearIds . addAll ( Stream . of ( pendingInsertions ) . map ( RecipientSettings : : getId ) . toList ( ) ) ;
clearIds . addAll ( Stream . of ( pendingDeletions ) . map ( RecipientSettings : : getId ) . toList ( ) ) ;
2020-03-18 21:31:45 +01:00
clearIds . add ( Recipient . self ( ) . getId ( ) ) ;
2019-09-26 16:12:51 +02:00
recipientDatabase . clearDirtyState ( clearIds ) ;
recipientDatabase . updateStorageKeys ( localWriteResult . get ( ) . getStorageKeyUpdates ( ) ) ;
needsMultiDeviceSync = true ;
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "[Local Changes] Updating local manifest version to: " + localWriteResult . get ( ) . getWriteResult ( ) . getManifest ( ) . getVersion ( ) ) ;
2019-09-26 16:12:51 +02:00
TextSecurePreferences . setStorageManifestVersion ( context , localWriteResult . get ( ) . getWriteResult ( ) . getManifest ( ) . getVersion ( ) ) ;
} else {
2020-02-10 19:42:43 +01:00
Log . i ( TAG , "[Local Changes] No local changes." ) ;
2019-09-26 16:12:51 +02:00
}
2020-09-10 20:01:41 +02:00
if ( needsForcePush ) {
Log . w ( TAG , "Scheduling a force push." ) ;
ApplicationDependencies . getJobManager ( ) . add ( new StorageForcePushJob ( ) ) ;
}
2019-09-26 16:12:51 +02:00
return needsMultiDeviceSync ;
}
2020-04-27 19:05:22 +02:00
private static @NonNull List < StorageId > getAllLocalStorageIds ( @NonNull Context context , @NonNull Recipient self ) {
2020-03-18 21:31:45 +01:00
return Util . concatenatedList ( DatabaseFactory . getRecipientDatabase ( context ) . getContactStorageSyncIds ( ) ,
Collections . singletonList ( StorageId . forAccount ( self . getStorageServiceId ( ) ) ) ,
2019-09-26 16:12:51 +02:00
DatabaseFactory . getStorageKeyDatabase ( context ) . getAllKeys ( ) ) ;
}
2020-10-06 16:24:14 +02:00
private static @NonNull List < SignalStorageRecord > buildLocalStorageRecords ( @NonNull Context context , @NonNull List < StorageId > ids ) {
2020-04-27 19:05:22 +02:00
Recipient self = Recipient . self ( ) . fresh ( ) ;
2019-09-26 16:12:51 +02:00
RecipientDatabase recipientDatabase = DatabaseFactory . getRecipientDatabase ( context ) ;
StorageKeyDatabase storageKeyDatabase = DatabaseFactory . getStorageKeyDatabase ( context ) ;
2020-02-28 19:03:06 +01:00
List < SignalStorageRecord > records = new ArrayList < > ( ids . size ( ) ) ;
2019-09-26 16:12:51 +02:00
2020-02-28 19:03:06 +01:00
for ( StorageId id : ids ) {
2020-03-18 21:31:45 +01:00
switch ( id . getType ( ) ) {
case ManifestRecord . Identifier . Type . CONTACT_VALUE :
case ManifestRecord . Identifier . Type . GROUPV1_VALUE :
case ManifestRecord . Identifier . Type . GROUPV2_VALUE :
RecipientSettings settings = recipientDatabase . getByStorageId ( id . getRaw ( ) ) ;
if ( settings ! = null ) {
2020-10-06 16:24:14 +02:00
if ( settings . getGroupType ( ) = = RecipientDatabase . GroupType . SIGNAL_V2 & & settings . getSyncExtras ( ) . getGroupMasterKey ( ) = = null ) {
2020-05-19 23:48:26 +02:00
Log . w ( TAG , "Missing master key on gv2 recipient" ) ;
} else {
2020-10-06 16:24:14 +02:00
records . add ( StorageSyncModels . localToRemoteRecord ( settings ) ) ;
2020-05-19 23:48:26 +02:00
}
2020-03-18 21:31:45 +01:00
} else {
Log . w ( TAG , "Missing local recipient model! Type: " + id . getType ( ) ) ;
}
break ;
case ManifestRecord . Identifier . Type . ACCOUNT_VALUE :
2020-04-27 19:05:22 +02:00
if ( ! Arrays . equals ( self . getStorageServiceId ( ) , id . getRaw ( ) ) ) {
throw new AssertionError ( "Local storage ID doesn't match self!" ) ;
}
records . add ( StorageSyncHelper . buildAccountRecord ( context , self ) ) ;
2020-03-18 21:31:45 +01:00
break ;
default :
SignalStorageRecord unknown = storageKeyDatabase . getById ( id . getRaw ( ) ) ;
if ( unknown ! = null ) {
records . add ( unknown ) ;
} else {
Log . w ( TAG , "Missing local unknown model! Type: " + id . getType ( ) ) ;
}
break ;
}
2019-09-26 16:12:51 +02:00
}
return records ;
}
public static final class Factory implements Job . Factory < StorageSyncJob > {
@Override
public @NonNull StorageSyncJob create ( @NonNull Parameters parameters , @NonNull Data data ) {
return new StorageSyncJob ( parameters ) ;
}
}
}