diff --git a/app/src/main/java/to/bitkit/data/backup/UsdtRecoveryBackupStore.kt b/app/src/main/java/to/bitkit/data/backup/UsdtRecoveryBackupStore.kt index 335429403a..4364ebe12c 100644 --- a/app/src/main/java/to/bitkit/data/backup/UsdtRecoveryBackupStore.kt +++ b/app/src/main/java/to/bitkit/data/backup/UsdtRecoveryBackupStore.kt @@ -28,7 +28,7 @@ class UsdtRecoveryBackupStore( ) { constructor(client: VssBackupClient) : this( put = { key, data -> - client.setup().getOrThrow() + client.ensureSetupForUpload() client.putObject(key, data).getOrThrow() }, keys = { diff --git a/app/src/main/java/to/bitkit/data/backup/VssBackupClient.kt b/app/src/main/java/to/bitkit/data/backup/VssBackupClient.kt index 728b112e41..aaa1677680 100644 --- a/app/src/main/java/to/bitkit/data/backup/VssBackupClient.kt +++ b/app/src/main/java/to/bitkit/data/backup/VssBackupClient.kt @@ -19,6 +19,7 @@ import to.bitkit.data.keychain.Keychain import to.bitkit.di.IoDispatcher import to.bitkit.env.Env import to.bitkit.ext.runSuspendCatching +import to.bitkit.utils.AppError import to.bitkit.utils.Logger import to.bitkit.utils.ServiceError import javax.inject.Inject @@ -28,6 +29,17 @@ import kotlin.time.Duration.Companion.seconds class MnemonicNotAvailableException : Exception("Mnemonic not available") +internal class BackupClientSetupTimeout(cause: TimeoutCancellationException) : + AppError("Backup client setup timed out", cause) + +internal suspend fun VssBackupClient.ensureSetupForUpload() { + val result = setup() + currentCoroutineContext().ensureActive() + val error = result.exceptionOrNull() + if (error is TimeoutCancellationException) throw BackupClientSetupTimeout(error) + result.getOrThrow() +} + @Singleton class VssBackupClient @Inject constructor( @IoDispatcher private val ioDispatcher: CoroutineDispatcher, diff --git a/app/src/main/java/to/bitkit/repositories/BackupRepo.kt b/app/src/main/java/to/bitkit/repositories/BackupRepo.kt index 48ce290f12..ba835fee4e 100644 --- a/app/src/main/java/to/bitkit/repositories/BackupRepo.kt +++ b/app/src/main/java/to/bitkit/repositories/BackupRepo.kt @@ -4,6 +4,7 @@ import android.content.Context import com.synonym.bitkitcore.UsdtException import com.synonym.vssclient.VssException import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineStart @@ -11,10 +12,14 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.Job import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -29,7 +34,9 @@ import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.transformLatest import kotlinx.coroutines.flow.update import kotlinx.coroutines.isActive +import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch +import kotlinx.coroutines.supervisorScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext @@ -50,11 +57,13 @@ import to.bitkit.data.SettingsStore import to.bitkit.data.WalletConnectionRecovery import to.bitkit.data.WatchOnlyAccountStore import to.bitkit.data.WidgetsStore +import to.bitkit.data.backup.BackupClientSetupTimeout import to.bitkit.data.backup.UsdtRecoveryBackupStore import to.bitkit.data.backup.VssBackupClient import to.bitkit.data.backup.VssBackupClientLdk import to.bitkit.data.backup.VssClientCoordinator import to.bitkit.data.backup.VssStoreIdProvider +import to.bitkit.data.backup.ensureSetupForUpload import to.bitkit.data.entities.TransferEntity import to.bitkit.data.hwWalletNames import to.bitkit.data.keychain.Keychain @@ -94,11 +103,13 @@ import to.bitkit.utils.Logger import to.bitkit.utils.NetworkValidationHelper import to.bitkit.utils.jsonLogOf import java.io.File +import java.net.URI import java.security.MessageDigest import java.util.concurrent.ConcurrentHashMap import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton +import kotlin.coroutines.cancellation.CancellationException import kotlin.time.Clock import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.minutes @@ -126,7 +137,6 @@ class BackupRepo @Inject constructor( @IoDispatcher private val ioDispatcher: CoroutineDispatcher, private val cacheStore: CacheStore, private val vssBackupClient: VssBackupClient, - private val vssClients: VssClientCoordinator, private val vssBackupClientLdk: VssBackupClientLdk, private val settingsStore: SettingsStore, private val widgetsStore: WidgetsStore, @@ -157,47 +167,56 @@ class BackupRepo @Inject constructor( private val clock: Clock, private val db: AppDb, private val connectionRecovery: WalletConnectionRecovery, + private val vssClients: VssClientCoordinator, ) { private val walletBackupMutex = Mutex() private val usdtBackupMutex = Mutex() - private val activityBackupMutex = Mutex() private val usdtRecoveryBackup = UsdtRecoveryBackupStore(vssBackupClient) private val scope = appScope(ioDispatcher, TAG) - private val backupJobs = ConcurrentHashMap.newKeySet() - private val backupJobLock = Any() - private val pendingBackups = ConcurrentHashMap() - private class PendingBackup( - val owner: VssClientCoordinator.Owner, - @Volatile var skipUnchangedActivity: Boolean, - ) { - @Volatile - var admitted = false + private val lifecycleLock = Any() + private val resetMutex = Mutex() + private var generation = 0L + private var resetting = false + + // Reset owns all pending work; stopping observation only cancels its child scope. + // Retiring observation parents remain attached until their cancellation cleanup finishes. + private var pendingScope = newBackupScope(scope) + private var observation: Observation? = null + private var observationController: CoroutineScope? = null + private var nativeGeneration: Long? = null + private val retiringObservations = mutableSetOf() + private val backupJobs = mutableMapOf() + private val explicitBackups = mutableMapOf() + private val categoryMutexes = BackupCategory.entries.associateWith { Mutex() } + + private class ActivityUploadCache { + var ownerGeneration: Long? = null + var digest: ByteArray? = null } - private val statusObserverJobs = mutableListOf() - private val dataListenerJobs = mutableListOf() - private var periodicCheckJob: Job? = null - private var setupJob: Job? = null - private var setupGeneration: Long? = null + // ACTIVITY's category mutex serializes uploads. Swapping the cache detaches any outstanding ACK. + @Volatile + private var activityUploadCache = ActivityUploadCache() private val runningBackups = ConcurrentHashMap.newKeySet() // Tracks active jobs since app start private val failedBackupRequired = ConcurrentHashMap() private val temporaryFailureRetries = ConcurrentHashMap() private val temporaryFailureSince = ConcurrentHashMap() - - @Volatile - private var isObserving = false private var lastNotificationTime = 0L - private class ActivityUploadCache { - var ownerGeneration: Long? = null - var digest: ByteArray? = null + private class BackupOwner(val generation: Long, val native: VssClientCoordinator.Owner) + + private class PendingBackup(val owner: BackupOwner, val job: Job) { + var admitted = false } - // Uploads are serialized by activityBackupMutex. Reset swaps the cache so an old ACK cannot populate it. - @Volatile - private var activityUploadCache = ActivityUploadCache() + private class Observation(val scope: CoroutineScope, val generation: BackupOwner) { + val ready = CompletableDeferred() + } + + private fun newBackupScope(parent: CoroutineScope) = + CoroutineScope(parent.coroutineContext + SupervisorJob(parent.coroutineContext[Job])) /** Spark's current connection was backed up once ready, so a restart of the observers doesn't upload it again. */ @Volatile @@ -231,9 +250,35 @@ class BackupRepo @Inject constructor( } .distinctUntilChanged() - fun reset() { - stopObservingBackups() - clearUploadedActivityDigest() + suspend fun reset() = withContext(ioDispatcher) { + resetMutex.withLock { + val pending = synchronized(lifecycleLock) { + resetting = true + activityUploadCache = ActivityUploadCache() + observation = null + observationController = null + pendingScope.coroutineContext[Job] + } + withContext(NonCancellable) { + try { + pending?.cancel() + pending?.join() + synchronized(lifecycleLock) { generation += 1 } + synchronized(lifecycleLock) { + backupJobs.clear() + explicitBackups.clear() + failedBackupRequired.clear() + temporaryFailureRetries.clear() + temporaryFailureSince.clear() + } + } finally { + synchronized(lifecycleLock) { + pendingScope = newBackupScope(scope) + resetting = false + } + } + } + } } fun setWiping(isWiping: Boolean) { @@ -281,53 +326,99 @@ class BackupRepo @Inject constructor( failedBackupRequired[category] != this.required fun startObservingBackups() { - if (isObserving) return - if (_isWiping.value) { - Logger.debug("Skipped observing backups while wiping", context = TAG) - return + val startup = synchronized(lifecycleLock) { + if (observationController != null || resetting || _isWiping.value) return + val previous = retiringObservations.toList() + val controller = newBackupScope(pendingScope) + observationController = controller + val parent = checkNotNull(controller.coroutineContext[Job]) + retiringObservations.add(parent) + parent.invokeOnCompletion { synchronized(lifecycleLock) { retiringObservations.remove(parent) } } + controller.launch(start = CoroutineStart.LAZY) { + combine(vssClients.owners, isWiping) { native, wiping -> native.takeUnless { wiping } } + .collectLatest { native -> + if (native == null) return@collectLatest + try { + vssClients.withWallet(native) { + supervisorScope { + val original = synchronized(lifecycleLock) { BackupOwner(generation, native) } + adoptOwner(original) + val owner = Observation(this, original) + synchronized(lifecycleLock) { + if (observationController !== controller) { + throw CancellationException( + "Backup observation stopped" + ) + } + observation = owner + } + try { + // Record consumed changes during ordinary stop cleanup, before uploads resume. + startDataStoreListeners(owner) + previous.joinAll() + currentCoroutineContext().ensureActive() + owner.ready.complete(Unit) + startObservation(owner) + // Keep every listener and its non-cancellable tail inside the native drain. + awaitCancellation() + } finally { + synchronized(lifecycleLock) { + if (observation === owner) observation = null + } + } + } + } + } catch (_: CancellationException) { + currentCoroutineContext().ensureActive() + // Native retirement cancels only this generation; the controller awaits reopening. + } + } + } } + startup.start() + } - isObserving = true - Logger.debug("Start observing backup statuses and data store changes", context = TAG) + private fun adoptOwner(owner: BackupOwner) = synchronized(lifecycleLock) { + check(generation == owner.generation && !resetting) + if (nativeGeneration != owner.native.generation) { + // Admission guarantees the previous owner's native work and local tails have drained. + nativeGeneration = owner.native.generation + explicitBackups.entries.removeAll { it.value != owner.native.generation } + failedBackupRequired.clear() + temporaryFailureRetries.clear() + temporaryFailureSince.clear() + } + } - setupJob = scope.launch { - vssClients.owners.collectLatest { owner -> - if (owner == null) return@collectLatest - vssClients.withWallet(owner) { - if (setupGeneration != null && setupGeneration != owner.generation) { - failedBackupRequired.clear() - temporaryFailureRetries.clear() - temporaryFailureSince.clear() - } - setupGeneration = owner.generation - vssBackupClient.setupWithRetry { - onSuccess = { attempt -> - Logger.debug("VSS client setup succeeded on attempt $attempt", context = TAG) - } - onRetry = { attempt, maxAttempts, delayMs -> - Logger.debug( - "VSS client setup deferred, retrying in ${delayMs}ms (attempt $attempt/$maxAttempts)", - context = TAG, - ) - } - onExhausted = { maxAttempts -> - Logger.warn("VSS client setup failed after $maxAttempts attempts", context = TAG) - } - }.onSuccess { - vssBackupClientLdk.setup() - _isWiping.first { !it } - cacheStore.backupStatuses.first().forEach { (category, status) -> - if (status.shouldBackup(category)) scheduleBackup(category) - } - } + private fun ownsState(owner: BackupOwner) = + generation == owner.generation && nativeGeneration == owner.native.generation + + private fun startObservation(owner: Observation) { + Logger.debug("Start observing backup statuses and data store changes", context = TAG) + owner.scope.launch { + vssBackupClient.setupWithRetry { + onSuccess = { attempt -> + Logger.debug("VSS client setup succeeded on attempt $attempt", context = TAG) + } + onRetry = { attempt, maxAttempts, delayMs -> + Logger.debug( + "VSS client setup deferred, retrying in ${delayMs}ms (attempt $attempt/$maxAttempts)", + context = TAG, + ) + } + onExhausted = { maxAttempts -> + Logger.warn("VSS client setup failed after $maxAttempts attempts", context = TAG) } + }.onSuccess { + currentCoroutineContext().ensureActive() + vssBackupClientLdk.setup() } } - launchForWallet { + owner.scope.launch { BackupCategory.entries.forEach { category -> - cacheStore.updateBackupStatus(category) { status -> - if (status.running && category !in runningBackups && pendingBackups[category]?.admitted != true) { + updateBackupStatus(category, owner.generation) { status -> + if (status.running && category !in runningBackups && backupJobs[category]?.admitted != true) { Logger.debug("Clearing stale running flag for: '$category'", context = TAG) status.copy(running = false) } else { @@ -337,46 +428,25 @@ class BackupRepo @Inject constructor( } } - startBackupStatusObservers() - startDataStoreListeners() - startPeriodicBackupFailureCheck() + startBackupStatusObservers(owner) + startPeriodicBackupFailureCheck(owner) } fun stopObservingBackups() { - if (!isObserving) return - - val pendingJobs = synchronized(backupJobLock) { - isObserving = false - backupJobs.toList().also { - backupJobs.clear() - pendingBackups.clear() + val parent = synchronized(lifecycleLock) { + observationController?.coroutineContext?.get(Job).also { + observationController = null + observation = null } } - setupJob?.cancel() - setupJob = null - - // Cancel all backup jobs - pendingJobs.forEach { it.cancel() } - - // Cancel backup status observer jobs - statusObserverJobs.forEach { it.cancel() } - statusObserverJobs.clear() - - // Cancel data store listener jobs - dataListenerJobs.forEach { it.cancel() } - dataListenerJobs.clear() - - // Cancel periodic check job - periodicCheckJob?.cancel() - periodicCheckJob = null - + parent?.cancel() Logger.debug("Stopped observing backup statuses and data store changes", context = TAG) } - private fun startBackupStatusObservers() { + private fun startBackupStatusObservers(owner: Observation) { // Observe backup status changes for each category BackupCategory.entries.forEach { category -> - val job = scope.launch { + owner.scope.launch { val statuses = cacheStore.backupStatuses .map { statuses -> statuses[category] ?: BackupItemStatus() } .distinctUntilChanged { old, new -> @@ -391,184 +461,152 @@ class BackupRepo @Inject constructor( } resumableStatuses.collect { status -> if (status.shouldBackup(category)) { - scheduleBackup(category) + scheduleBackup(category, owner) } } } - statusObserverJobs.add(job) } - - Logger.debug("Started ${statusObserverJobs.size} backup status observers", context = TAG) } - private fun startDataStoreListeners() { - val settingsJob = scope.launch { + private fun startDataStoreListeners(owner: Observation) { + owner.scope.launch { settingsStore.data .distinctUntilChanged() .drop(1) .collect { if (shouldSkipBackup()) return@collect - markBackupRequired(BackupCategory.SETTINGS) + markBackupRequired(owner, BackupCategory.SETTINGS) } } - dataListenerJobs.add(settingsJob) - val widgetsJob = scope.launch { + owner.scope.launch { widgetsStore.data .distinctUntilChanged() .drop(1) .collect { if (shouldSkipBackup()) return@collect - markBackupRequired(BackupCategory.WIDGETS) + markBackupRequired(owner, BackupCategory.WIDGETS) } } - dataListenerJobs.add(widgetsJob) // WALLET - Observe transfers - val transfersJob = scope.launch { + owner.scope.launch { db.transferDao().observeAll() .distinctUntilChanged() .drop(1) .collect { if (shouldSkipBackup()) return@collect - markBackupRequired(BackupCategory.WALLET) + markBackupRequired(owner, BackupCategory.WALLET) } } - dataListenerJobs.add(transfersJob) - val watchOnlyAccountsJob = scope.launch { + owner.scope.launch { watchOnlyAccountStore.data .distinctUntilChanged() .drop(1) .collect { if (shouldSkipBackup()) return@collect - markBackupRequired(BackupCategory.WALLET) + markBackupRequired(owner, BackupCategory.WALLET) } } - dataListenerJobs.add(watchOnlyAccountsJob) // METADATA - Observe entire CacheStore excluding backup statuses - val cacheMetadataJob = scope.launch { + owner.scope.launch { cacheStore.data .map { it.copy(backupStatuses = mapOf()) } .distinctUntilChanged() .drop(1) .collect { if (shouldSkipBackup()) return@collect - markBackupRequired(BackupCategory.METADATA) + markBackupRequired(owner, BackupCategory.METADATA) } } - dataListenerJobs.add(cacheMetadataJob) // METADATA - Observe pre-activity metadata changes - val preActivityMetadataJob = scope.launch { + owner.scope.launch { preActivityMetadataRepo.preActivityMetadataChanged .drop(1) .collect { if (shouldSkipBackup()) return@collect - markBackupRequired(BackupCategory.METADATA) + markBackupRequired(owner, BackupCategory.METADATA) } } - dataListenerJobs.add(preActivityMetadataJob) // METADATA - Observe hardware wallet names only: the store is also rewritten by every connect, // and reconnect traffic must not re-upload the whole metadata envelope. - val hwWalletNamesJob = scope.launch { + owner.scope.launch { hwWalletStore.data .map { it.hwWalletNames() } .distinctUntilChanged() .drop(1) .collect { if (shouldSkipBackup()) return@collect - markBackupRequired(BackupCategory.METADATA) + markBackupRequired(owner, BackupCategory.METADATA) } } - dataListenerJobs.add(hwWalletNamesJob) - dataListenerJobs.add(observeBackupChanges(pubkyRepo.backupStateVersion, BackupCategory.METADATA)) - dataListenerJobs.add(observeBackupChanges(privatePaykitRepo.get().backupStateVersion, BackupCategory.WALLET)) - dataListenerJobs.add(observeBackupChanges(paykitSdkService.backupStateVersion, BackupCategory.WALLET)) - dataListenerJobs.add(observeBackupChanges(paykitPaymentProofStore.backupStateVersion, BackupCategory.WALLET)) - dataListenerJobs.add(observeBackupChanges(onchainSendAttemptStore.backupStateVersion, BackupCategory.WALLET)) - dataListenerJobs.add( - observeBackupChanges(paykitUsdtPaymentRepo.get().backupStateVersion, BackupCategory.WALLET) - ) - dataListenerJobs.add( - observeBackupChanges(paykitUsdtPaymentRepo.get().backupStateVersion, BackupCategory.USDT_RECOVERY) + observeBackupChanges(owner, pubkyRepo.backupStateVersion, BackupCategory.METADATA) + observeBackupChanges(owner, privatePaykitRepo.get().backupStateVersion, BackupCategory.WALLET) + observeBackupChanges(owner, paykitSdkService.backupStateVersion, BackupCategory.WALLET) + observeBackupChanges(owner, paykitPaymentProofStore.backupStateVersion, BackupCategory.WALLET) + observeBackupChanges(owner, onchainSendAttemptStore.backupStateVersion, BackupCategory.WALLET) + observeBackupChanges(owner, paykitUsdtPaymentRepo.get().backupStateVersion, BackupCategory.WALLET) + observeBackupChanges(owner, paykitUsdtPaymentRepo.get().backupStateVersion, BackupCategory.USDT_RECOVERY) + observeBackupChanges( + owner, + usdtRepo.get().state.map { it.transfers }.distinctUntilChanged(), + BackupCategory.USDT_RECOVERY, ) - dataListenerJobs.add( - observeBackupChanges( - usdtRepo.get().state.map { it.transfers }.distinctUntilChanged(), - BackupCategory.USDT_RECOVERY, - ) - ) - dataListenerJobs.add( - observeBackupChanges( - usdtRepo.get().state.map { it.transfers }.distinctUntilChanged(), - BackupCategory.WALLET, - ) + observeBackupChanges( + owner, + usdtRepo.get().state.map { it.transfers }.distinctUntilChanged(), + BackupCategory.WALLET, ) - dataListenerJobs.add(observeBackupChanges(paykitPresentationStore.backupStateVersion, BackupCategory.WALLET)) - dataListenerJobs.add(observeBackupChanges(paykitAllowanceStore.backupStateVersion, BackupCategory.WALLET)) - dataListenerJobs.add( - observeBackupChanges( - privatePaykitAddressReservationRepo.get().backupStateVersion, - BackupCategory.WALLET, - ) + observeBackupChanges(owner, paykitPresentationStore.backupStateVersion, BackupCategory.WALLET) + observeBackupChanges(owner, paykitAllowanceStore.backupStateVersion, BackupCategory.WALLET) + observeBackupChanges( + owner, + privatePaykitAddressReservationRepo.get().backupStateVersion, + BackupCategory.WALLET, ) // BLOCKTANK - Observe blocktank state changes (orders, cjitEntries, info) - dataListenerJobs.add(observeBackupChanges(blocktankRepo.blocktankState, BackupCategory.BLOCKTANK)) + observeBackupChanges(owner, blocktankRepo.blocktankState, BackupCategory.BLOCKTANK) // ACTIVITY - Observe activity changes - dataListenerJobs.add(observeBackupChanges(activityRepo.activitiesChanged, BackupCategory.ACTIVITY)) + observeBackupChanges(owner, activityRepo.activitiesChanged, BackupCategory.ACTIVITY) // Hardware tags are carried by the metadata backup. Observe the narrower tag signal so ordinary // payment and sync traffic does not re-upload the whole metadata envelope. - dataListenerJobs.add(observeBackupChanges(activityRepo.activityTagsChanged, BackupCategory.METADATA)) - dataListenerJobs.add(observeBackupChanges(SparkExitEarlierAddressStore.writes, BackupCategory.METADATA)) - - // Replay unrecorded Spark changes when wallet admission or the restore gate reopens. - dataListenerJobs.add( - scope.launch { - vssClients.owners.collectLatest { owner -> - if (owner == null) return@collectLatest - vssClients.withWallet(owner) { - combine( - sparkRepo.state, - sparkRepo.exitStateGeneration, - restoreInProgress - ) { state, generation, _ -> - state to generation - }.collect { (state, generation) -> - val ready = state.isReady && state.hasSynced - if (!ready) hasBackedUpSparkConnection = false - val connectionNeedsBackup = ready && !hasBackedUpSparkConnection - if ((!connectionNeedsBackup && generation <= markedSparkExitGeneration) || - shouldSkipBackup() - ) { - return@collect - } - runSuspendCatching { updateBackupRequired(BackupCategory.SPARK_EXIT) } - .onSuccess { - hasBackedUpSparkConnection = ready - markedSparkExitGeneration = maxOf(markedSparkExitGeneration, generation) - } - .onFailure { - Logger.error( - "Failed to mark Spark exit backup required", - it, - context = TAG - ) - } - } - } + observeBackupChanges(owner, activityRepo.activityTagsChanged, BackupCategory.METADATA) + observeBackupChanges(owner, SparkExitEarlierAddressStore.writes, BackupCategory.METADATA) + + // Replay unrecorded Spark changes when the admitted observation or restore gate reopens. + owner.scope.launch { + combine( + sparkRepo.state, + sparkRepo.exitStateGeneration, + restoreInProgress + ) { state, generation, _ -> + state to generation + }.collect { (state, generation) -> + val ready = state.isReady && state.hasSynced + if (!ready) hasBackedUpSparkConnection = false + val connectionNeedsBackup = ready && !hasBackedUpSparkConnection + if ((!connectionNeedsBackup && generation <= markedSparkExitGeneration) || shouldSkipBackup()) { + return@collect } + runSuspendCatching { updateBackupRequired(BackupCategory.SPARK_EXIT, owner.generation) } + .onSuccess { + hasBackedUpSparkConnection = ready + markedSparkExitGeneration = maxOf(markedSparkExitGeneration, generation) + } + .onFailure { Logger.error("Failed to mark Spark exit backup required", it, context = TAG) } } - ) + } // LIGHTNING_CONNECTIONS - Only display sync timestamp, ldk-node manages its own backups @OptIn(FlowPreview::class) - val lightningConnectionsJob = scope.launch { + owner.scope.launch { lightningService.syncStatusChanged .debounce(SYNC_STATUS_DEBOUNCE) .collect { @@ -576,136 +614,217 @@ class BackupRepo @Inject constructor( ?.let { it * 1000 } // Convert seconds to millis ?: return@collect if (shouldSkipBackup()) return@collect - cacheStore.updateBackupStatus(BackupCategory.LIGHTNING_CONNECTIONS) { + updateBackupStatus(BackupCategory.LIGHTNING_CONNECTIONS, owner.generation) { it.copy(required = lastSync, synced = lastSync, running = false) } } } - dataListenerJobs.add(lightningConnectionsJob) - - Logger.debug("Started ${dataListenerJobs.size} data store listeners", context = TAG) } - private fun observeBackupChanges(flow: Flow<*>, category: BackupCategory): Job = - scope.launch { + private fun observeBackupChanges(owner: Observation, flow: Flow<*>, category: BackupCategory): Job = + owner.scope.launch { flow.drop(1).collect { if (shouldSkipBackup(category)) return@collect - markBackupRequired(category) + markBackupRequired(owner, category) } } - private fun startPeriodicBackupFailureCheck() { - periodicCheckJob = scope.launch { + private fun startPeriodicBackupFailureCheck(owner: Observation) { + owner.scope.launch { while (currentCoroutineContext().isActive) { delay(BACKUP_CHECK_INTERVAL) - checkForFailedBackups() + checkForFailedBackups(owner) } } } - private fun launchForWallet(block: suspend CoroutineScope.() -> Unit): Job { - val owner = runCatching { vssClients.captureOwner() }.getOrNull() ?: return Job().apply { cancel() } - return scope.launch { vssClients.withWallet(owner) { block() } } - } + private suspend fun markBackupRequired(owner: Observation, category: BackupCategory) = + markBackupRequired(category, owner.generation) - private fun markBackupRequired(category: BackupCategory) { - launchForWallet { updateBackupRequired(category) } - } + private suspend fun markBackupRequired(category: BackupCategory, expectedGeneration: BackupOwner) = + withContext(NonCancellable) { + // A consumed change must remain pending even if observation stops before the cache write. + updateBackupRequired(category, expectedGeneration) + } - private suspend fun updateBackupRequired(category: BackupCategory) { - failedBackupRequired -= category - cacheStore.updateBackupStatus(category) { + private suspend fun updateBackupRequired(category: BackupCategory, expectedGeneration: BackupOwner) { + updateBackupStatus(category, expectedGeneration) { + failedBackupRequired -= category it.copy(required = maxOf(currentTimeMillis(), it.required + 1, it.synced + 1)) } Logger.verbose("Marked backup required for: '$category'", context = TAG) } + private suspend fun updateBackupStatus( + category: BackupCategory, + expectedGeneration: BackupOwner, + transform: (BackupItemStatus) -> BackupItemStatus, + ) { + cacheStore.update { data -> + synchronized(lifecycleLock) { + // Reset drains admitted writers before advancing the generation or clearing clients. + if (!ownsState(expectedGeneration)) return@update data + val status = data.backupStatuses[category] ?: BackupItemStatus() + data.copy(backupStatuses = data.backupStatuses + (category to transform(status))) + } + } + } + private fun scheduleBackup( category: BackupCategory, - expectedOwner: VssClientCoordinator.Owner? = null, - requireObservation: Boolean = true, - skipUnchangedActivity: Boolean = true, + owner: Observation? = null, + requestedGeneration: BackupOwner? = null, ) { - val owner = expectedOwner ?: runCatching { vssClients.captureOwner() }.getOrNull() ?: return - val job = synchronized(backupJobLock) { - if (requireObservation && !isObserving) return - val pending = PendingBackup(owner, skipUnchangedActivity) - var reserved = false - pendingBackups.compute(category) { _, previous -> - if (previous == null || previous.owner.generation < owner.generation) { - reserved = true - pending - } else { - if (previous.owner.generation == owner.generation && !skipUnchangedActivity) { - previous.skipUnchangedActivity = false - } - previous - } + val expectedGeneration = owner?.generation ?: requestedGeneration ?: return + val job = synchronized(lifecycleLock) { + if (expectedGeneration.native.generation != vssClients.owners.value?.generation || + resetting || _isWiping.value || (owner != null && observation !== owner) || + (requestedGeneration != null && requestedGeneration.generation != generation) + ) { + return } - if (!reserved) return - Logger.debug("Scheduling backup for: '$category'", context = TAG) - scope.launch(start = CoroutineStart.LAZY) { - try { - vssClients.withWallet(owner) { - pending.admitted = true - try { - cacheStore.updateBackupStatus(category) { it.copy(running = true) } - val debounce = if (category == BackupCategory.SPARK_EXIT) { - SPARK_EXIT_BACKUP_DEBOUNCE - } else { - BACKUP_DEBOUNCE - } - delay(debounce) - if (category == BackupCategory.WALLET) { - paykitSdkService.isPaymentSubmissionActive.first { !it } - } - runPendingBackup(category, pending) - } finally { - withContext(NonCancellable) { - pendingBackups.remove(category, pending) - cacheStore.updateBackupStatus(category) { - it.copy( - running = category in runningBackups || - pendingBackups[category]?.admitted == true - ) + if (owner == null) explicitBackups[category] = expectedGeneration.native.generation + val existing = backupJobs[category] + if (existing?.owner?.native?.generation == expectedGeneration.native.generation && + !existing.job.isCompleted && !existing.job.isCancelled + ) { + return + } + val jobScope = owner?.scope ?: pendingScope + jobScope.launch(start = CoroutineStart.LAZY) { + val scheduledJob = currentCoroutineContext()[Job] + vssClients.withWallet(expectedGeneration.native) { + adoptOwner(expectedGeneration) + if (owner == null) { + synchronized( + lifecycleLock + ) { explicitBackups[category] = expectedGeneration.native.generation } + } + try { + synchronized(lifecycleLock) { + backupJobs[category]?.takeIf { it.job === scheduledJob }?.admitted = true + } + updateBackupStatus(category, expectedGeneration) { it.copy(running = true) } + owner?.ready?.await() + // Coalescing pending work must not delay an explicit recovery acknowledgement. + val debounce = if (category == BackupCategory.SPARK_EXIT) { + SPARK_EXIT_BACKUP_DEBOUNCE + } else { + BACKUP_DEBOUNCE + } + delay(debounce) + categoryMutexes.getValue(category).withLock { + runningBackups += category + try { + updateBackupStatus(category, expectedGeneration) { it.copy(running = true) } + if (category == BackupCategory.WALLET) { + paykitSdkService.isPaymentSubmissionActive.first { !it } } + val status = cacheStore.backupStatuses.first()[category] ?: BackupItemStatus() + if (status.isRequired && !shouldSkipBackup(category) && + failedBackupRequired[category] != status.required + ) { + if (performBackup(category, expectedGeneration).getOrNull() == false) { + finishUnchangedActivity(expectedGeneration, scheduledJob) + } + } + } finally { + finishBackup(category, expectedGeneration) } } + } finally { + synchronized(lifecycleLock) { + if (backupJobs[category]?.job === scheduledJob) backupJobs.remove(category) + } + withContext(NonCancellable) { + updateBackupStatus(category, expectedGeneration) { + it.copy(running = category in runningBackups || backupJobs[category]?.admitted == true) + } + schedulePendingBackup(category, expectedGeneration, owner) + } } - } finally { - pendingBackups.remove(category, pending) } - }.also { scheduled -> - backupJobs.add(scheduled) - scheduled.invokeOnCompletion { backupJobs.remove(scheduled) } - } + }.also { backupJobs[category] = PendingBackup(expectedGeneration, it) } } + job.invokeOnCompletion { resumeUnstartedBackup(category, expectedGeneration, job) } job.start() } - private suspend fun runPendingBackup(category: BackupCategory, pending: PendingBackup) { - // A full backup can still upgrade an observed ACTIVITY request while its payload is being read. - if (category != BackupCategory.ACTIVITY) pendingBackups.remove(category, pending) - val status = cacheStore.backupStatuses.first()[category] ?: BackupItemStatus() - if (!status.copy(running = category in runningBackups).shouldBackup(category)) return + private fun resumeUnstartedBackup(category: BackupCategory, original: BackupOwner, completed: Job) { + val followup = synchronized(lifecycleLock) { + // Entered jobs remove their reservation before their normal pending handoff. + if (backupJobs[category]?.job !== completed) return + backupJobs.remove(category) + if (resetting || _isWiping.value || !ownsState(original) || + vssClients.owners.value?.generation != original.native.generation || + explicitBackups[category] != original.native.generation + ) { + return + } + pendingScope.launch(start = CoroutineStart.LAZY) { + vssClients.withWallet(original.native) { + schedulePendingBackup(category, original) + } + } + } + followup.start() + } - val uploaded = triggerBackup(category, pending.skipUnchangedActivity) - if (category == BackupCategory.ACTIVITY && pendingBackups.remove(category, pending)) { - if (uploaded.getOrNull() == false && !pending.skipUnchangedActivity && !shouldSkipBackup(category)) { - triggerBackup(category, skipUnchangedActivity = false) + private suspend fun finishUnchangedActivity(owner: BackupOwner, job: Job?) = withContext(NonCancellable) { + val category = BackupCategory.ACTIVITY + // The skip has been published. Retire its reservation atomically with checking for a joined full request. + val force = synchronized(lifecycleLock) { + if (backupJobs[category]?.job === job) backupJobs.remove(category) + explicitBackups[category] == owner.native.generation + } + if (force) { + updateBackupStatus(category, owner) { + explicitBackups[category] = owner.native.generation + if (it.isRequired) it else it.copy(required = maxOf(currentTimeMillis(), it.synced + 1)) } - // An observer may have joined this reservation after the upload captured its revision. - if (cacheStore.backupStatuses.first()[category]?.shouldBackup(category) == true) { - scheduleBackup(category, expectedOwner = pending.owner) + } + } + + private suspend fun schedulePendingBackup( + category: BackupCategory, + expectedGeneration: BackupOwner, + previousOwner: Observation? = null, + ) { + var status = BackupItemStatus() + updateBackupStatus(category, expectedGeneration) { + status = it + if (!it.isRequired) explicitBackups -= category + it + } + val nextOwner = synchronized(lifecycleLock) { + if (resetting || !ownsState(expectedGeneration)) return + if (!status.shouldBackup(category)) return + if (category in explicitBackups) { + null + } else { + val owner = observation?.takeIf { previousOwner == null || previousOwner === it } ?: return + if (!owner.scope.isActive) return + owner } } + scheduleBackup(category, nextOwner, expectedGeneration) + } + + private suspend fun finishBackup(category: BackupCategory, expectedGeneration: BackupOwner) = withContext( + NonCancellable + ) { + try { + updateBackupStatus(category, expectedGeneration) { it.copy(running = false) } + } finally { + runningBackups -= category + } } - private fun checkForFailedBackups() { + private fun checkForFailedBackups(owner: Observation) { val currentTime = currentTimeMillis() // find if there are any backup categories that have been failing for more than 30 minutes - launchForWallet { + owner.scope.launch { val backupStatuses = cacheStore.backupStatuses.first() val hasFailedBackups = BackupCategory.entries.any { category -> val status = backupStatuses[category] ?: BackupItemStatus() @@ -717,18 +836,18 @@ class BackupRepo @Inject constructor( } if (hasFailedBackups) { - showBackupFailureNotification(currentTime) + showBackupFailureNotification(owner, currentTime) } } } - private fun showBackupFailureNotification(currentTime: Long) { + private fun showBackupFailureNotification(owner: Observation, currentTime: Long) { // Throttle notifications to avoid spam if (currentTime - lastNotificationTime < FAILED_BACKUP_NOTIFICATION_INTERVAL) return lastNotificationTime = currentTime - launchForWallet { + owner.scope.launch { ToastEventBus.send( type = Toast.ToastType.ERROR, title = context.getString(R.string.settings__backup__failed_title), @@ -739,78 +858,129 @@ class BackupRepo @Inject constructor( } } - suspend fun triggerBackup(category: BackupCategory): Result = - triggerBackup(category, skipUnchangedActivity = false).map {} + suspend fun triggerBackup(category: BackupCategory): Result = performExplicitBackup(category) - private suspend fun triggerBackup( + private suspend fun performExplicitBackup( category: BackupCategory, - skipUnchangedActivity: Boolean, - ): Result = withContext(ioDispatcher) { - vssClients.withWallet { - if (category == BackupCategory.WALLET || category == BackupCategory.METADATA) { - walletBackupMutex.withLock { performBackup(category, skipUnchangedActivity) } - } else { - performBackup(category, skipUnchangedActivity) + usdtSnapshot: String? = null, + ): Result = withBackupOperation { expectedGeneration -> + var reservedRequired: Long? = null + if (usdtSnapshot != null) { + withContext(NonCancellable) { + updateBackupStatus(category, expectedGeneration) { + val required = maxOf(currentTimeMillis(), it.required + 1, it.synced + 1) + reservedRequired = required + it.copy(required = required) + } + } + } + categoryMutexes.getValue(category).withLock { + if (synchronized(lifecycleLock) { resetting || !ownsState(expectedGeneration) }) { + return@withLock Result.failure(UsdtException.BackupUnavailable()) + } + runningBackups += category + try { + performBackup( + category, + expectedGeneration, + usdtSnapshot, + explicitAttempt = true, + reservedRequired = reservedRequired, + ).map { } + } finally { + finishBackup(category, expectedGeneration) + withContext(NonCancellable) { + schedulePendingBackup(category, expectedGeneration) + } } } } - private suspend fun performBackup(category: BackupCategory, skipUnchangedActivity: Boolean): Result { - Logger.debug("Backup starting for: '$category'", context = TAG) - var backupRequired = currentTimeMillis() - runningBackups += category - failedBackupRequired -= category - try { - cacheStore.updateBackupStatus(category) { - backupRequired = maxOf(backupRequired, it.required, it.synced + 1) - it.copy(running = true, required = backupRequired) + private suspend fun performBackup( + category: BackupCategory, + expectedGeneration: BackupOwner, + usdtSnapshot: String? = null, + explicitAttempt: Boolean = false, + reservedRequired: Long? = null, + ): Result = + if (category == BackupCategory.WALLET || category == BackupCategory.METADATA) { + walletBackupMutex.withLock { + performSerializedBackup(category, expectedGeneration, usdtSnapshot, explicitAttempt, reservedRequired) } + } else { + performSerializedBackup(category, expectedGeneration, usdtSnapshot, explicitAttempt, reservedRequired) + } - val upload = when (category) { - BackupCategory.WALLET -> uploadWalletBackup().map { true } - BackupCategory.USDT_RECOVERY -> uploadUsdtRecoveryBackup().map { true } - BackupCategory.SPARK_EXIT -> uploadSparkExitBackup().map { true } - BackupCategory.ACTIVITY -> uploadActivityBackup(skipUnchangedActivity) - else -> runSuspendCatching { - vssBackupClient.putObject(key = category.name, data = getBackupDataBytes(category)).getOrThrow() - true - } + private suspend fun performSerializedBackup( + category: BackupCategory, + expectedGeneration: BackupOwner, + usdtSnapshot: String?, + explicitAttempt: Boolean, + reservedRequired: Long?, + ): Result { + Logger.debug("Backup starting for: '$category'", context = TAG) + var backupRequired = reservedRequired ?: currentTimeMillis() + updateBackupStatus(category, expectedGeneration) { + if (reservedRequired == null) { + val required = if (explicitAttempt) it.required + 1 else it.required + backupRequired = maxOf(backupRequired, required, it.synced + 1) + } + if ((failedBackupRequired[category] ?: 0) <= backupRequired) failedBackupRequired -= category + it.copy(running = true, required = maxOf(it.required, backupRequired)) + } + if (synchronized(lifecycleLock) { resetting || !ownsState(expectedGeneration) }) { + return Result.failure(UsdtException.BackupUnavailable()) + } + val upload = when (category) { + BackupCategory.WALLET -> uploadWalletBackup().map { true } + BackupCategory.USDT_RECOVERY -> uploadUsdtRecoveryBackup(usdtSnapshot).map { true } + BackupCategory.SPARK_EXIT -> uploadSparkExitBackup().map { true } + BackupCategory.ACTIVITY -> uploadActivityBackup(expectedGeneration, explicitAttempt) + else -> runSuspendCatching { + val data = getBackupDataBytes(category) + vssBackupClient.ensureSetupForUpload() + vssBackupClient.putObject(key = category.name, data = data).getOrThrow() + true } - return upload - .onSuccess { uploaded -> - withContext(NonCancellable) { - cacheStore.updateBackupStatus(category) { - it.copy(running = false, synced = backupRequired) + } + return upload + .onSuccess { uploaded -> + withContext(NonCancellable) { + updateBackupStatus(category, expectedGeneration) { + // A known full request must remain durably pending until the upload succeeds. + if (!uploaded && explicitBackups[category] == expectedGeneration.native.generation) { + return@updateBackupStatus it.copy(running = false) } - if (category == BackupCategory.WALLET) { + if ((failedBackupRequired[category] ?: 0) <= backupRequired) { + failedBackupRequired -= category + temporaryFailureRetries -= category + temporaryFailureSince -= category + } + if (uploaded && it.required <= backupRequired) { + explicitBackups -= category + } else if (explicitAttempt) { + explicitBackups[category] = expectedGeneration.native.generation + } + it.copy(running = false, synced = maxOf(it.synced, backupRequired)) + } + if (category == BackupCategory.WALLET) { + updateBackupStatus(BackupCategory.METADATA, expectedGeneration) { failedBackupRequired -= BackupCategory.METADATA - cacheStore.updateBackupStatus(BackupCategory.METADATA) { - if (it.isRequired) { - it.copy(required = maxOf(currentTimeMillis(), it.required + 1, it.synced + 1)) - } else { - it - } + if (it.isRequired) { + it.copy(required = maxOf(currentTimeMillis(), it.required + 1, it.synced + 1)) + } else { + it } } - runningBackups -= category - failedBackupRequired -= category - temporaryFailureRetries -= category - temporaryFailureSince -= category - } - if (uploaded) { - Logger.info("Backup succeeded for: '$category'", context = TAG) - } else { - Logger.debug("Skipped backup for: '$category', payload unchanged", context = TAG) } } - .onFailure { markBackupFailed(category, backupRequired, it) } - } finally { - if (runningBackups.remove(category)) { - withContext(NonCancellable) { - cacheStore.updateBackupStatus(category) { it.copy(running = false) } + if (uploaded) { + Logger.info("Backup succeeded for: '$category'", context = TAG) + } else { + Logger.debug("Skipped backup for: '$category', payload unchanged", context = TAG) } } - } + .onFailure { markBackupFailed(category, backupRequired, expectedGeneration, it, explicitAttempt) } } /** @@ -821,9 +991,12 @@ class BackupRepo @Inject constructor( private suspend fun uploadSparkExitBackup(): Result = runSuspendCatching { if (!sparkRepo.hasMergedRemoteExitState) { sparkRepo.exitBackup().getOrThrow() ?: throw SparkExitBackupDeferred() + vssBackupClient.ensureSetupForUpload() val remote = vssBackupClient.getObject(BackupCategory.SPARK_EXIT.name).getOrThrow()?.value ?.let { json.decodeFromString(String(it)) } sparkRepo.mergeRemoteExitBackup(remote).getOrThrow() + } else { + vssBackupClient.ensureSetupForUpload() } val backup = sparkRepo.exitBackup().getOrThrow() ?: throw SparkExitBackupDeferred() vssBackupClient.putObject( @@ -832,21 +1005,44 @@ class BackupRepo @Inject constructor( ).getOrThrow() } - suspend fun persistWalletBackup(): Result = - withContext(ioDispatcher) { - vssClients.withWallet { - walletBackupMutex.withLock { uploadWalletBackup() } - .onFailure { markBackupRequired(BackupCategory.WALLET) } - } - } + suspend fun persistWalletBackup(): Result = withBackupOperation { expectedGeneration -> + walletBackupMutex.withLock { uploadWalletBackup() } + .onFailure { markBackupRequired(BackupCategory.WALLET, expectedGeneration) } + } - suspend fun persistUsdtRecoveryBackup(snapshot: String): Result = vssClients.withWallet { - runSuspendCatching { - cacheStore.updateBackupStatus(BackupCategory.USDT_RECOVERY) { - it.copy(required = maxOf(currentTimeMillis(), it.required + 1, it.synced + 1)) + suspend fun persistUsdtRecoveryBackup(snapshot: String): Result = + performExplicitBackup(BackupCategory.USDT_RECOVERY, snapshot) + + private suspend fun withBackupOperation( + operation: suspend (BackupOwner) -> Result, + ): Result { + val native = try { + vssClients.captureOwner() + } catch (e: CancellationException) { + currentCoroutineContext().ensureActive() + return Result.failure(UsdtException.BackupUnavailable().apply { initCause(e) }) + } + return withContext(ioDispatcher) { + val callerContext = currentCoroutineContext().minusKey(Job) + val task = synchronized(lifecycleLock) { + if (resetting || _isWiping.value) return@withContext Result.failure(UsdtException.BackupUnavailable()) + val original = BackupOwner(generation, native) + pendingScope.async(context = callerContext, start = CoroutineStart.LAZY) { + vssClients.withWallet(native) { + adoptOwner(original) + operation(original) + } + } + } + try { + task.await() + } catch (e: CancellationException) { + currentCoroutineContext().ensureActive() + Result.failure(UsdtException.BackupUnavailable().apply { initCause(e) }) + } finally { + withContext(NonCancellable) { task.cancelAndJoin() } } - uploadUsdtRecoveryBackup(snapshot).getOrThrow() - }.onFailure { markBackupRequired(BackupCategory.USDT_RECOVERY) } + } } private suspend fun uploadUsdtRecoveryBackup(snapshot: String? = null): Result = withContext(ioDispatcher) { @@ -876,47 +1072,79 @@ class BackupRepo @Inject constructor( private suspend fun uploadWalletBackup(): Result = withContext(ioDispatcher) { runSuspendCatching { if (shouldSkipBackup()) throw UsdtException.BackupUnavailable() - vssBackupClient.setup().getOrThrow() + vssBackupClient.ensureSetupForUpload() val data = getWalletBackupDataBytes() if (shouldSkipBackup()) throw UsdtException.BackupUnavailable() vssBackupClient.putObject(key = BackupCategory.WALLET.name, data = data).getOrThrow() }.map { } } - private suspend fun markBackupFailed(category: BackupCategory, backupRequired: Long, e: Throwable) { + private suspend fun markBackupFailed( + category: BackupCategory, + backupRequired: Long, + expectedGeneration: BackupOwner, + e: Throwable, + explicitAttempt: Boolean, + ) { + val temporary = e.isTemporaryBackupFailure() withContext(NonCancellable) { - cacheStore.updateBackupStatus(category) { + updateBackupStatus(category, expectedGeneration) { + if (temporary && explicitAttempt) explicitBackups[category] = expectedGeneration.native.generation + if (!temporary && e !is WalletBackupSnapshotChanged && it.required == backupRequired) { + explicitBackups -= category + } if (it.required == backupRequired && e !is WalletBackupSnapshotChanged) { failedBackupRequired[category] = backupRequired - } else { + } else if ((failedBackupRequired[category] ?: 0) <= backupRequired) { failedBackupRequired -= category } it.copy(running = false) } - runningBackups -= category } when { e is WalletBackupSnapshotChanged -> Logger.debug("Deferred changing backup snapshot for: '$category'", context = TAG) e is SparkExitBackupDeferred -> Logger.debug("Deferred backup until Spark syncs for: '$category'", context = TAG) - e.isTemporaryBackupFailure() -> retryAfterTemporaryFailure(category, backupRequired, e) + temporary -> retryAfterTemporaryFailure(category, backupRequired, expectedGeneration, e) else -> Logger.error("Backup failed for: '$category'", e, context = TAG) } } - private fun retryAfterTemporaryFailure(category: BackupCategory, backupRequired: Long, e: Throwable) { - val attempt = temporaryFailureRetries.merge(category, 1, Int::plus) ?: 1 - temporaryFailureSince.putIfAbsent(category, backupRequired) - val retryDelay = (TEMPORARY_FAILURE_RETRY_DELAY * (1 shl (attempt - 1).coerceAtMost(MAX_RETRY_SHIFT))) - .coerceAtMost(TEMPORARY_FAILURE_RETRY_MAX_DELAY) - Logger.warn("Retrying backup for: '$category' in '$retryDelay'", e, context = TAG) - launchForWallet { - delay(retryDelay) - if (!failedBackupRequired.remove(category, backupRequired) || !isObserving) return@launchForWallet - val status = cacheStore.backupStatuses.first()[category] ?: return@launchForWallet - if (status.shouldBackup(category)) scheduleBackup(category) + private fun retryAfterTemporaryFailure( + category: BackupCategory, + backupRequired: Long, + expectedGeneration: BackupOwner, + e: Throwable, + ) { + val job = synchronized(lifecycleLock) { + if (resetting || !ownsState(expectedGeneration) || + (failedBackupRequired[category] ?: 0) > backupRequired + ) { + return + } + val attempt = temporaryFailureRetries.merge(category, 1, Int::plus) ?: 1 + temporaryFailureSince.putIfAbsent(category, backupRequired) + val retryDelay = (TEMPORARY_FAILURE_RETRY_DELAY * (1 shl (attempt - 1).coerceAtMost(MAX_RETRY_SHIFT))) + .coerceAtMost(TEMPORARY_FAILURE_RETRY_MAX_DELAY) + Logger.warn("Retrying backup for: '$category' in '$retryDelay'", e, context = TAG) + pendingScope.launch(start = CoroutineStart.LAZY) { + delay(retryDelay) + vssClients.withWallet(expectedGeneration.native) { + synchronized(lifecycleLock) { + if (resetting || !ownsState(expectedGeneration)) return@withWallet + val failed = failedBackupRequired[category] + if (failed == backupRequired) { + failedBackupRequired.remove(category, backupRequired) + } else if (failed != null || category !in explicitBackups) { + return@withWallet + } + } + schedulePendingBackup(category, expectedGeneration) + } + } } + job.start() } private suspend fun getBackupDataBytes(category: BackupCategory): ByteArray = when (category) { @@ -943,13 +1171,11 @@ class BackupRepo @Inject constructor( BackupCategory.METADATA -> getMetadataBackupDataBytes() BackupCategory.BLOCKTANK -> { - val blocktankState = blocktankRepo.blocktankState.first() - val payload = BlocktankBackupV1( createdAt = currentTimeMillis(), - orders = blocktankState.orders, - cjitEntries = blocktankState.cjitEntries, - info = blocktankState.info, + orders = coreService.blocktank.orders(refresh = false), + cjitEntries = coreService.blocktank.cjitEntries(refresh = false), + info = coreService.blocktank.info(refresh = false), ) json.encodeToString(payload).toByteArray() @@ -965,41 +1191,34 @@ class BackupRepo @Inject constructor( } } - private suspend fun getActivityBackupPayload(): ActivityBackupV1 = ActivityBackupV1( + private suspend fun getActivityBackupPayload() = ActivityBackupV1( createdAt = currentTimeMillis(), - activities = activityRepo.getActivities().getOrDefault(emptyList()), - activityTags = activityRepo.getAllActivitiesTags().getOrDefault(emptyList()), - closedChannels = activityRepo.getClosedChannels().getOrDefault(emptyList()), + activities = activityRepo.getActivities().getOrThrow(), + activityTags = activityRepo.getAllActivitiesTags().getOrThrow(), + closedChannels = activityRepo.getClosedChannels().getOrThrow(), ) - private suspend fun uploadActivityBackup(skipIfUnchanged: Boolean): Result = runSuspendCatching { - activityBackupMutex.withLock { uploadActivityBackupLocked(skipIfUnchanged) } - } - - private suspend fun uploadActivityBackupLocked(skipIfUnchanged: Boolean): Boolean { + private suspend fun uploadActivityBackup(owner: BackupOwner, explicitAttempt: Boolean) = runSuspendCatching { + val category = BackupCategory.ACTIVITY val cache = activityUploadCache - val ownerGeneration = vssClients.captureOwner().generation val payload = getActivityBackupPayload() val digest = MessageDigest.getInstance("SHA-256") .digest(json.encodeToString(payload.copy(createdAt = 0)).toByteArray()) - if (skipIfUnchanged && cache === activityUploadCache && - cache.ownerGeneration == ownerGeneration && digest.contentEquals(cache.digest) + val force = explicitAttempt || synchronized(lifecycleLock) { + explicitBackups[category] == owner.native.generation + } + if (!force && cache === activityUploadCache && cache.ownerGeneration == owner.native.generation && + digest.contentEquals(cache.digest) ) { - return false + return@runSuspendCatching false } - cache.digest = null - vssBackupClient.putObject( - key = BackupCategory.ACTIVITY.name, - data = json.encodeToString(payload).toByteArray(), - ).getOrThrow() - cache.ownerGeneration = ownerGeneration + vssBackupClient.ensureSetupForUpload() + cache.digest = null // A failed PUT may already have changed the remote value. + vssBackupClient.putObject(key = category.name, data = json.encodeToString(payload).toByteArray()).getOrThrow() + cache.ownerGeneration = owner.native.generation cache.digest = digest - return true - } - - private fun clearUploadedActivityDigest() { - activityUploadCache = ActivityUploadCache() + true } private suspend fun getMetadataBackupDataBytes(): ByteArray = withContext(ioDispatcher) { @@ -1177,7 +1396,7 @@ class BackupRepo @Inject constructor( Logger.debug("Full restore starting", context = TAG) _isRestoring.update { true } - clearUploadedActivityDigest() + activityUploadCache = ActivityUploadCache() // Mutated only by the sequential restore steps below, inside this single coroutine. val categoriesNeedingRewrite = mutableSetOf() @@ -1525,22 +1744,25 @@ class BackupRepo @Inject constructor( wire.validateProofs(proofs, WalletScope.default) } - suspend fun getLatestBackupTime(): ULong? = withContext(ioDispatcher) { - vssClients.withWallet { - runCatching { - withTimeout(VSS_TIMESTAMP_TIMEOUT) { - vssBackupClient.setup().getOrThrow() - coroutineScope { - LATEST_BACKUP_TIME_CATEGORIES - .map { category -> async { getRemoteBackupTimestamp(category) } } - .mapNotNull { it.await() } - .filter { it > 0uL } - .maxOrNull() + suspend fun getLatestBackupTime(): ULong? { + val original = vssClients.captureOwner() + return withContext(ioDispatcher) { + vssClients.withWallet(original) { + runCatching { + withTimeout(VSS_TIMESTAMP_TIMEOUT) { + vssBackupClient.setup().getOrThrow() + coroutineScope { + LATEST_BACKUP_TIME_CATEGORIES + .map { category -> async { getRemoteBackupTimestamp(category) } } + .mapNotNull { it.await() } + .filter { it > 0uL } + .maxOrNull() + } } - } - }.onFailure { e -> - Logger.warn("Failed to get VSS backup timestamp: $e", context = TAG) - }.getOrNull() + }.onFailure { e -> + Logger.warn("Failed to get VSS backup timestamp: $e", context = TAG) + }.getOrNull() + } } } @@ -1558,12 +1780,17 @@ class BackupRepo @Inject constructor( } fun scheduleFullBackup() { - val owner = runCatching { vssClients.captureOwner() }.getOrNull() ?: return Logger.debug("Scheduling backups for all categories", context = TAG) + val native = try { + vssClients.captureOwner() + } catch (_: CancellationException) { + return + } + val requestedGeneration = synchronized(lifecycleLock) { BackupOwner(generation, native) } BackupCategory.entries .filter { it != BackupCategory.LIGHTNING_CONNECTIONS } .forEach { - scheduleBackup(it, expectedOwner = owner, requireObservation = false, skipUnchangedActivity = false) + scheduleBackup(it, requestedGeneration = requestedGeneration) } } @@ -1738,10 +1965,23 @@ private data class RestoredCoreBackup( ) internal fun Throwable.isTemporaryBackupFailure(): Boolean = isPaykitTemporarilyUnavailable() || when (this) { + is BackupClientSetupTimeout, is VssException.ConnectionException, is VssException.NetworkException, is VssException.StoreException, is VssException.PutException, -> true + is VssException.UnknownException -> isVssRequestSendFailure(errorDetails, "/putObjects") + is VssException.AuthException -> isVssRequestSendFailure( + errorDetails.removePrefix("error performing external request: "), + ) else -> false } + +private fun isVssRequestSendFailure(details: String, pathSuffix: String? = null): Boolean { + val prefix = "error sending request for url (" + if (!details.startsWith(prefix) || !details.endsWith(")")) return false + val url = runCatching { URI(details.removePrefix(prefix).dropLast(1)) }.getOrNull() ?: return false + return url.scheme in setOf("http", "https") && !url.host.isNullOrEmpty() && + (pathSuffix == null || url.path.endsWith(pathSuffix)) +} diff --git a/app/src/main/java/to/bitkit/repositories/PubkyRepo.kt b/app/src/main/java/to/bitkit/repositories/PubkyRepo.kt index c6b34945f3..92ba7bafe1 100644 --- a/app/src/main/java/to/bitkit/repositories/PubkyRepo.kt +++ b/app/src/main/java/to/bitkit/repositories/PubkyRepo.kt @@ -1257,16 +1257,6 @@ class PubkyRepo @Inject constructor( } } - suspend fun deleteProfileWithSessionRetry(): Result = withContext(ioDispatcher) { - val initialResult = deleteProfile() - if (initialResult.isSuccess) return@withContext initialResult - - val refreshedSession = refreshSessionIfPossible().getOrDefault(false) - if (!refreshedSession) return@withContext initialResult - - deleteProfile() - } - suspend fun deleteProfile(): Result = runSuspendCatching { withContext(ioDispatcher) { requireNotNull(keychain.loadString(Keychain.Key.PAYKIT_SESSION.name)) { @@ -1300,10 +1290,10 @@ class PubkyRepo @Inject constructor( val publicKey = _publicKey.value ?: return val known = ownProfileRevision val revision = known ?: pubkyService.fetchPubkyProfileWithRevision(publicKey)?.revision ?: return - runSuspendCatching { withPubkyAppScope { pubkyService.deletePubkyProfile(revision, publicKey) } }.getOrElse { + runSuspendCatching { pubkyService.deletePubkyProfile(revision, publicKey) }.getOrElse { if (known == null || !it.isConcurrentUpdate()) throw it val current = pubkyService.fetchPubkyProfileWithRevision(publicKey)?.revision - if (current != null) withPubkyAppScope { pubkyService.deletePubkyProfile(current, publicKey) } + if (current != null) pubkyService.deletePubkyProfile(current, publicKey) } ownProfileRevision = null } @@ -2344,7 +2334,7 @@ class PubkyRepo @Inject constructor( } } - suspend fun refreshSessionIfPossible(): Result = runSuspendCatching { + private suspend fun refreshSessionIfPossible(): Result = runSuspendCatching { withContext(ioDispatcher) { val storedSecretKeyHex = activeSecretKeyHex() ?: return@withContext false diff --git a/app/src/main/java/to/bitkit/ui/screens/profile/EditProfileViewModel.kt b/app/src/main/java/to/bitkit/ui/screens/profile/EditProfileViewModel.kt index ad2302f4a4..b03f5e61e6 100644 --- a/app/src/main/java/to/bitkit/ui/screens/profile/EditProfileViewModel.kt +++ b/app/src/main/java/to/bitkit/ui/screens/profile/EditProfileViewModel.kt @@ -294,7 +294,7 @@ class EditProfileViewModel @Inject constructor( try { privatePaykitRepo.beginProfileDeletion() privatePaykitRepo.removePublishedEndpointsForCleanup(TAG, deferAppSync = true) - val result = pubkyRepo.deleteProfileWithSessionRetry() + val result = pubkyRepo.deleteProfile() if (result.isSuccess) { privatePaykitRepo.closeAndClear() ToastEventBus.send( diff --git a/app/src/test/java/to/bitkit/repositories/BackupRepoTest.kt b/app/src/test/java/to/bitkit/repositories/BackupRepoTest.kt index ba9f610a93..c1fba60d78 100644 --- a/app/src/test/java/to/bitkit/repositories/BackupRepoTest.kt +++ b/app/src/test/java/to/bitkit/repositories/BackupRepoTest.kt @@ -1,28 +1,38 @@ package to.bitkit.repositories import android.content.Context +import androidx.datastore.core.DataStore +import androidx.datastore.core.DataStoreFactory import com.synonym.bitkitcore.ActivityTags import com.synonym.bitkitcore.PreActivityMetadata import com.synonym.paykit.PaykitException +import com.synonym.vssclient.VssException import com.synonym.vssclient.VssItem import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Delay import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.InternalCoroutinesApi +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.awaitCancellation import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.test.StandardTestDispatcher import kotlinx.coroutines.test.TestScope import kotlinx.coroutines.test.advanceTimeBy @@ -30,6 +40,7 @@ import kotlinx.coroutines.test.currentTime import kotlinx.coroutines.test.runCurrent import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.yield import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive @@ -77,12 +88,14 @@ import to.bitkit.data.backup.VssClientCoordinator import to.bitkit.data.dao.TransferDao import to.bitkit.data.entities.TransferEntity import to.bitkit.data.keychain.Keychain +import to.bitkit.data.serializers.AppCacheSerializer import to.bitkit.data.withPublicSharingDisabledForRestore import to.bitkit.di.json import to.bitkit.env.Env import to.bitkit.models.ActivityBackupV1 import to.bitkit.models.BackupCategory import to.bitkit.models.BackupItemStatus +import to.bitkit.models.BlocktankBackupV1 import to.bitkit.models.KnownDevice import to.bitkit.models.MetadataBackupV1 import to.bitkit.models.PaykitAllowanceStateBackup @@ -100,16 +113,21 @@ import to.bitkit.models.WatchOnlyAccountRecord import to.bitkit.models.WatchOnlyAccountSetupState import to.bitkit.models.WidgetsBackupV1 import to.bitkit.services.AddressDerivationInfo +import to.bitkit.services.BlocktankService import to.bitkit.services.LightningService +import to.bitkit.services.PaykitSdkOperationLock import to.bitkit.services.PaykitSdkService +import to.bitkit.services.UsdtService import to.bitkit.test.BaseUnitTest import to.bitkit.test.connectionRecoveryFixture import to.bitkit.utils.AppError import java.io.File import java.io.IOException +import java.nio.file.Files import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger import javax.inject.Provider import kotlin.coroutines.CoroutineContext import kotlin.coroutines.cancellation.CancellationException @@ -145,6 +163,7 @@ class BackupRepoTest : BaseUnitTest() { private val blocktankRepo = mock() private val activityRepo = mock() private val coreService = mock() + private val blocktankService = mock() private val lightningRepo = mock() private val pubkyRepo = mock() private val paykitSdkService = mock() @@ -178,6 +197,10 @@ class BackupRepoTest : BaseUnitTest() { @Before fun setUp() = test { + whenever(coreService.blocktank).thenReturn(blocktankService) + whenever { blocktankService.orders(refresh = false) }.thenReturn(emptyList()) + whenever { blocktankService.cjitEntries(refresh = false) }.thenReturn(emptyList()) + whenever { blocktankService.info(refresh = false) }.thenReturn(null) whenever(coreService.validateBitcoinAddress(any())).thenAnswer { com.synonym.bitkitcore.ValidationResult( address = it.getArgument(0), @@ -198,7 +221,10 @@ class BackupRepoTest : BaseUnitTest() { } whenever { transferDao.upsert(any>()) }.thenReturn(Unit) whenever { cacheStore.updateBackupStatus(any(), any()) }.thenReturn(Unit) - whenever { cacheStore.update(any()) }.thenReturn(Unit) + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + cacheData.update(transform) + } whenever(cacheStore.data).thenReturn(cacheData) whenever(cacheStore.backupStatuses).thenReturn(cacheData.map { it.backupStatuses }) whenever(settingsStore.data).thenReturn(settingsData) @@ -293,6 +319,83 @@ class BackupRepoTest : BaseUnitTest() { } } + @Test + fun `isolated cache failure preserves other listeners and owner rearm`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000, synced = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var failNext = true + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + if (failNext) { + failNext = false + throw java.io.IOException("isolated cache read failure") + } + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } + } + try { + sut.startObservingBackups() + runCurrent() + settingsData.update { it.copy(selectedCurrency = "EUR") } + advanceTimeBy(5_000) + runCurrent() + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + connectionRecoveryFixture(settingsStore, keychain, vssClients).withIdentityChange { } + runCurrent() + verify(vssBackupClient, times(2)).setupWithRetry(any(), any(), any()) + } finally { + sut.reset() + } + } + + @Test + fun `explicit coalesced request survives observer cancellation before dispatch`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var beforeDispatch: (() -> Unit)? = null + var intercepted = false + val delegate = StandardTestDispatcher(testScheduler) + + @OptIn(InternalCoroutinesApi::class) + val dispatcher = object : CoroutineDispatcher(), Delay by (delegate as Delay) { + override fun dispatch(context: CoroutineContext, block: Runnable) { + val hook = beforeDispatch + beforeDispatch = null + hook?.invoke() + delegate.dispatch(context, block) + } + } + sut = createSut(dispatcher = dispatcher) + whenever(cacheStore.backupStatuses).thenReturn( + flow { + if (!intercepted) { + beforeDispatch = { + intercepted = true + sut.scheduleFullBackup() + sut.stopObservingBackups() + } + } + emit(statuses.value) + beforeDispatch = null + awaitCancellation() + } + ) + try { + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + assertTrue(intercepted) + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + beforeDispatch = null + sut.reset() + } + } + @Test fun `retired replacement reservation cannot leave a cancelled backup running`() = test { for (retireBeforeCleanup in listOf(true, false)) { @@ -313,15 +416,14 @@ class BackupRepoTest : BaseUnitTest() { if (holdDispatch) queued += context to block else delegate.dispatch(context, block) } } - sut = createSut(ioDispatcher = dispatcher) - whenever(cacheStore.updateBackupStatus(any(), any())).doSuspendableAnswer { - val target = it.getArgument(0) - val transform = it.getArgument<(BackupItemStatus) -> BackupItemStatus>(1) - if (target == category && holdCleanup && !cleanupEntered) { + sut = createSut(dispatcher = dispatcher) + whenever(cacheStore.update(any())).doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + if (holdCleanup && !cleanupEntered) { cleanupEntered = true finishCleanup.await() } - statuses.update { it + (target to transform(it[target] ?: BackupItemStatus())) } + statuses.update { transform(AppCacheData(backupStatuses = it)).backupStatuses } } try { sut.startObservingBackups() @@ -370,26 +472,334 @@ class BackupRepoTest : BaseUnitTest() { } @Test - fun `backup failure status drains before replacement credentials`() = test { + fun `retired queued full request cannot hide a new owner request`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + sut.scheduleFullBackup() + connectionRecoveryFixture(settingsStore, keychain, vssClients).withIdentityChange { } + sut.scheduleFullBackup() + advanceTimeBy(5_000) + runCurrent() + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + assertFalse(statuses.value.getValue(category).isRequired) + sut.reset() + } + + @Test + fun `retired detached retry cannot upload for a replacement wallet`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever { vssBackupClient.putObject(eq(category.name), any()) }.thenReturn( + Result.failure(java.io.IOException("temporary transport failure")), + ) + assertTrue(sut.triggerBackup(category).isFailure) + connectionRecoveryFixture(settingsStore, keychain, vssClients).withIdentityChange { } + whenever { vssBackupClient.putObject(eq(category.name), any()) }.thenReturn( + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)), + ) + advanceTimeBy(120_000) + runCurrent() + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + assertTrue(statuses.value.getValue(category).isRequired) + sut.triggerBackup(category).getOrThrow() + verify(vssBackupClient, times(2)).putObject(eq(category.name), any()) + assertFalse(statuses.value.getValue(category).isRequired) + sut.reset() + } + + @Test + fun `replacement waits for consumed settings write before restarting observation`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000, synced = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + sut.startObservingBackups() + runCurrent() val entered = CompletableDeferred() val release = CompletableDeferred() var replaced = false var crossed = false - whenever(cacheStore.updateBackupStatus(eq(BackupCategory.WALLET), any())).doSuspendableAnswer { + var holdNext = true + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + if (holdNext) { + holdNext = false + entered.complete(Unit) + release.await() + crossed = replaced + } + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } + } + settingsData.update { it.copy(selectedCurrency = "EUR") } + entered.await() + val transition = async { + connectionRecoveryFixture(settingsStore, keychain, vssClients).withIdentityChange { replaced = true } + } + try { + runCurrent() + assertFalse(transition.isCompleted) + assertFalse(replaced) + release.complete(Unit) + transition.await() + runCurrent() + assertFalse(crossed) + assertTrue(statuses.value.getValue(category).isRequired) + advanceTimeBy(5_000) + runCurrent() + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + verify(vssBackupClient, times(2)).setupWithRetry(any(), any(), any()) + } finally { + release.complete(Unit) + transition.join() + sut.reset() + } + } + + @Test + fun `retiring explicit payload drains before credential change without upload`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + var replaced = false + whenever { activityRepo.getActivities() }.doSuspendableAnswer { entered.complete(Unit) - release.await() - crossed = replaced - Unit + withContext(NonCancellable) { release.await() } + Result.success(emptyList()) } - whenever(pubkyRepo.hasIdentityForBackup()).thenReturn(true) - assertTrue(sut.persistWalletBackup().isFailure) + val manual = async { sut.triggerBackup(BackupCategory.ACTIVITY) } entered.await() - val recovery = connectionRecoveryFixture(settingsStore, keychain, vssClients) - recovery.withIdentityChange { replaced = true } - release.complete(Unit) - runCurrent() - assertTrue(replaced) - assertFalse(crossed) + val transition = async { + connectionRecoveryFixture(settingsStore, keychain, vssClients).withIdentityChange { replaced = true } + } + try { + runCurrent() + assertFalse(transition.isCompleted) + assertFalse(replaced) + release.complete(Unit) + transition.await() + assertTrue(manual.await().isFailure) + verify(vssBackupClient, never()).putObject(eq(BackupCategory.ACTIVITY.name), any()) + } finally { + release.complete(Unit) + transition.join() + sut.reset() + } + } + + @Test + fun `cold explicit settings backup initializes its client without observation`() = test { + val ready = CompletableDeferred() + whenever { vssBackupClient.setup(any()) }.doSuspendableAnswer { + ready.complete(Unit) + Result.success(Unit) + } + whenever { vssBackupClient.putObject(eq(BackupCategory.SETTINGS.name), any()) }.doSuspendableAnswer { + ready.await() + Result.success(VssItem(BackupCategory.SETTINGS.name, byteArrayOf(), 1)) + } + val backup = async { sut.triggerBackup(BackupCategory.SETTINGS) } + try { + runCurrent() + assertTrue(backup.isCompleted) + backup.await().getOrThrow() + verify(vssBackupClient, never()).setupWithRetry(any(), any(), any()) + verify(vssBackupClientLdk, never()).setup() + } finally { + backup.cancelAndJoin() + sut.reset() + } + } + + @Test + fun `cold explicit settings setup failure keeps its retry without observation`() = test { + val error = VssException.ConnectionException("unavailable") + val statuses = MutableStateFlow(mapOf(BackupCategory.SETTINGS to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever { vssBackupClient.setup(any()) } + .thenReturn(Result.failure(error)) + .thenReturn(Result.success(Unit)) + try { + assertEquals(error, sut.triggerBackup(BackupCategory.SETTINGS).exceptionOrNull()) + verify(vssBackupClient, never()).putObject(any(), any()) + assertTrue(statuses.value.getValue(BackupCategory.SETTINGS).isRequired) + advanceTimeBy(15_000) + runCurrent() + verify(vssBackupClient, times(2)).setup(any()) + verify(vssBackupClient).putObject(eq(BackupCategory.SETTINGS.name), any()) + assertFalse(statuses.value.getValue(BackupCategory.SETTINGS).isRequired) + } finally { + sut.reset() + } + } + + @Test + fun `wallet upload retries after a client setup timeout`() = test { + assertSetupTimeoutRetries(BackupCategory.WALLET) { sut.triggerBackup(BackupCategory.WALLET) } + verify(vssBackupClient).putObject(eq(BackupCategory.WALLET.name), any()) + } + + @Test + fun `USDT recovery upload retries the pending snapshot after a client setup timeout`() = test { + whenever { usdtRepo.backupSnapshot() }.thenReturn("pending-snapshot") + assertSetupTimeoutRetries(BackupCategory.USDT_RECOVERY) { sut.persistUsdtRecoveryBackup("pending-snapshot") } + val payload = argumentCaptor() + verify(vssBackupClient).putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, payload.capture()) + val backup = json.decodeFromString(payload.firstValue.decodeToString()) + assertEquals("pending-snapshot", backup.usdtWallet) + } + + @Test + fun `cold explicit setup timeout keeps its retry without observation`() = test { + assertSetupTimeoutRetries(BackupCategory.SETTINGS) { sut.triggerBackup(BackupCategory.SETTINGS) } + verify(vssBackupClient).putObject(eq(BackupCategory.SETTINGS.name), any()) + } + + private suspend fun TestScope.assertSetupTimeoutRetries( + category: BackupCategory, + upload: suspend () -> Result, + ) { + sut = createSut(dispatcher = StandardTestDispatcher(testScheduler)) + var setupAttempts = 0 + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever { vssBackupClient.setup(any()) }.doSuspendableAnswer { + setupAttempts += 1 + if (setupAttempts == 1) runCatching { withTimeout(1) { awaitCancellation() } } else Result.success(Unit) + } + try { + val result = upload() + assertTrue(result.exceptionOrNull()?.isTemporaryBackupFailure() == true, result.toString()) + verify(vssBackupClient, never()).putObject(any(), any()) + assertTrue(statuses.value.getValue(category).isRequired) + runCurrent() + advanceTimeBy(10_000) + runCurrent() + verify(vssBackupClient, never()).putObject(any(), any()) + verify(vssBackupClient, times(1)).setup(any()) + advanceTimeBy(5_000) + runCurrent() + verify(vssBackupClient, times(2)).setup(any()) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + sut.reset() + } + } + + @Test + fun `reset drains cold explicit setup before resetting clients`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + whenever { vssBackupClient.setup(any()) }.doSuspendableAnswer { + entered.complete(Unit) + withContext(NonCancellable) { release.await() } + Result.success(Unit) + } + val backup = async { sut.triggerBackup(BackupCategory.SETTINGS) } + try { + runCurrent() + assertTrue(entered.isCompleted) + val reset = async { sut.reset() } + runCurrent() + assertFalse(reset.isCompleted) + release.complete(Unit) + reset.await() + assertTrue(backup.await().isFailure) + verify(vssBackupClient, never()).putObject(any(), any()) + } finally { + release.complete(Unit) + backup.cancelAndJoin() + sut.reset() + } + } + + @Test + fun `caller cancellation drains cold explicit setup without upload`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + whenever { vssBackupClient.setup(any()) }.doSuspendableAnswer { + entered.complete(Unit) + withContext(NonCancellable) { release.await() } + Result.success(Unit) + } + val backup = async { sut.triggerBackup(BackupCategory.SETTINGS) } + try { + runCurrent() + assertTrue(entered.isCompleted) + backup.cancel() + runCurrent() + assertFalse(backup.isCompleted) + release.complete(Unit) + backup.join() + assertTrue(backup.isCancelled) + verify(vssBackupClient, never()).putObject(any(), any()) + } finally { + release.complete(Unit) + backup.cancelAndJoin() + sut.reset() + } + } + + @Test + fun `cold blocktank backup captures persisted orders before state refresh`() = test { + val order = previewBtOrder() + whenever(blocktankRepo.blocktankState).thenReturn(MutableStateFlow(BlocktankState())) + whenever { blocktankService.orders(refresh = false) }.thenReturn(listOf(order)) + + sut.triggerBackup(BackupCategory.BLOCKTANK).getOrThrow() + + val payload = argumentCaptor() + verify(vssBackupClient).putObject(eq(BackupCategory.BLOCKTANK.name), payload.capture()) + val backup = json.decodeFromString(payload.firstValue.decodeToString()) + assertEquals(listOf(order), backup.orders) + assertTrue(backup.cjitEntries.isEmpty()) + assertNull(backup.info) + verify(blocktankService).orders(refresh = false) + verify(blocktankService).cjitEntries(refresh = false) + verify(blocktankService).info(refresh = false) + } + + @Test + fun `cold blocktank backup refuses every unreadable cache before setup`() = test { + val error = AppError("Core cache is unavailable") + var unreadable = -1 + whenever { blocktankService.orders(refresh = false) }.doSuspendableAnswer { + if (unreadable == 0) throw error + emptyList() + } + whenever { blocktankService.cjitEntries(refresh = false) }.doSuspendableAnswer { + if (unreadable == 1) throw error + emptyList() + } + whenever { blocktankService.info(refresh = false) }.doSuspendableAnswer { + if (unreadable == 2) throw error + null + } + repeat(3) { unavailable -> + unreadable = unavailable + assertEquals(error, sut.triggerBackup(BackupCategory.BLOCKTANK).exceptionOrNull()) + } + verify(vssBackupClient, never()).setup(any()) + verify(vssBackupClient, never()).putObject(any(), any()) + } + + @Test + fun `cold activity backup refuses every unavailable component before setup`() = test { + val error = AppError("Core is unavailable") + repeat(3) { unavailable -> + whenever { activityRepo.getActivities() }.thenReturn( + if (unavailable == 0) Result.failure(error) else Result.success(emptyList()) + ) + whenever { activityRepo.getClosedChannels() }.thenReturn( + if (unavailable == 1) Result.failure(error) else Result.success(emptyList()) + ) + whenever { activityRepo.getAllActivitiesTags() }.thenReturn( + if (unavailable == 2) Result.failure(error) else Result.success(emptyList()) + ) + assertEquals(error, sut.triggerBackup(BackupCategory.ACTIVITY).exceptionOrNull()) + } + verify(vssBackupClient, never()).setup(any()) + verify(vssBackupClient, never()).putObject(any(), any()) } @Test @@ -460,7 +870,7 @@ class BackupRepoTest : BaseUnitTest() { false } Executors.newSingleThreadExecutor().asCoroutineDispatcher().use { ioDispatcher -> - sut = createSut(ioDispatcher = ioDispatcher) + sut = createSut(dispatcher = ioDispatcher) try { val read = async(Dispatchers.Main) { sut.hasPendingWalletRestore() } val storageThread = withContext(Dispatchers.Default) { @@ -679,16 +1089,16 @@ class BackupRepoTest : BaseUnitTest() { Result.success(SparkExitBackupV1(createdAt = 1_000, network = "regtest", exitState = "state")) ) var failRequiredWrite = true - whenever(cacheStore.updateBackupStatus(any(), any())).doSuspendableAnswer { - val target = it.getArgument(0) - val transform = it.getArgument<(BackupItemStatus) -> BackupItemStatus>(1) - val prior = statuses.value[target] ?: BackupItemStatus() - val next = transform(prior) - if (target == category && failRequiredWrite && next.required > prior.required) { + whenever(cacheStore.update(any())).doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + val prior = statuses.value[category] ?: BackupItemStatus() + val updated = transform(AppCacheData(backupStatuses = statuses.value)) + val next = updated.backupStatuses[category] ?: BackupItemStatus() + if (failRequiredWrite && next.required > prior.required) { failRequiredWrite = false throw IOException("status storage unavailable") } - statuses.update { it + (target to next) } + statuses.value = updated.backupStatuses } try { sut.startObservingBackups() @@ -720,16 +1130,16 @@ class BackupRepoTest : BaseUnitTest() { ) val requiredWrite = CompletableDeferred() var holdRequiredWrite = false - whenever(cacheStore.updateBackupStatus(any(), any())).doSuspendableAnswer { - val target = it.getArgument(0) - val transform = it.getArgument<(BackupItemStatus) -> BackupItemStatus>(1) - val prior = statuses.value[target] ?: BackupItemStatus() - val next = transform(prior) - if (target == category && holdRequiredWrite && next.required > prior.required) { + whenever(cacheStore.update(any())).doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + val prior = statuses.value[category] ?: BackupItemStatus() + val updated = transform(AppCacheData(backupStatuses = statuses.value)) + val next = updated.backupStatuses[category] ?: BackupItemStatus() + if (holdRequiredWrite && next.required > prior.required) { requiredWrite.complete(Unit) awaitCancellation() } - statuses.update { it + (target to next) } + statuses.value = updated.backupStatuses } try { sut.startObservingBackups() @@ -879,14 +1289,14 @@ class BackupRepoTest : BaseUnitTest() { val statusWriteStarted = CompletableDeferred() val finishStatusWrite = CompletableDeferred() whenever(cacheStore.backupStatuses).thenReturn(statuses) - whenever(cacheStore.updateBackupStatus(eq(category), any())).doSuspendableAnswer { - val transform = it.getArgument<(BackupItemStatus) -> BackupItemStatus>(1) - val updated = transform(statuses.value.getValue(category)) - if (!updated.running) { + whenever(cacheStore.update(any())).doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + val updated = transform(AppCacheData(backupStatuses = statuses.value)).backupStatuses + if (!updated.getValue(category).running) { statusWriteStarted.complete(Unit) finishStatusWrite.await() } - statuses.update { current -> current + (category to updated) } + statuses.value = updated } val uploadResult = if (uploadSucceeds) { Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) @@ -1638,16 +2048,20 @@ class BackupRepoTest : BaseUnitTest() { whenever(privatePaykitRepo.backupSnapshot()).thenReturn( Result.failure(PaykitException.SharedStateBusy("shared_state_busy", "lock unavailable")), ) - assertTrue(sut.triggerBackup(BackupCategory.WALLET).isFailure) - sut.persistUsdtRecoveryBackup("same-signed-operation").getOrThrow() - val payload = argumentCaptor() - verify(vssBackupClient).putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, payload.capture()) - assertEquals( - "same-signed-operation", - json.decodeFromString(payload.firstValue.decodeToString()).usdtWallet, - ) - verify(privatePaykitRepo, times(1)).backupSnapshot() - verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + try { + assertTrue(sut.triggerBackup(BackupCategory.WALLET).isFailure) + sut.persistUsdtRecoveryBackup("same-signed-operation").getOrThrow() + val payload = argumentCaptor() + verify(vssBackupClient).putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, payload.capture()) + assertEquals( + "same-signed-operation", + json.decodeFromString(payload.firstValue.decodeToString()).usdtWallet, + ) + verify(privatePaykitRepo, times(1)).backupSnapshot() + verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + } finally { + sut.reset() + } } @Test @@ -1914,7 +2328,7 @@ class BackupRepoTest : BaseUnitTest() { verify(settingsStore, never()).update(any()) verify(keychain, never()).delete(Keychain.Key.PAYKIT_PENDING_BACKUP_RESTORE.name) - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) assertTrue(sut.hasPendingWalletRestore()) assertTrue(keychain.exists(Keychain.Key.USDT_PENDING_BACKUP_RESTORE.name)) assertTrue(sut.triggerBackup(BackupCategory.WALLET).isFailure) @@ -1937,7 +2351,7 @@ class BackupRepoTest : BaseUnitTest() { val restartedReservations = createReservationRepo(reservationStore) assertTrue(restartedReservations.isUnavailableForReusableReceive(publicAddress)) - sut = createSut(restartedReservations) + sut = createSut(reservationRepo = restartedReservations) sut.triggerBackup(BackupCategory.WALLET).getOrThrow() val uploaded = argumentCaptor() @@ -1957,7 +2371,7 @@ class BackupRepoTest : BaseUnitTest() { assertTrue(sut.hasPendingWalletRestore()) assertFalse(settingsData.value.backupVerified) - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) whenever(vssBackupClient.getObject(BackupCategory.METADATA.name)) .thenReturn(Result.failure(BackupRepoTestError("metadata download unavailable"))) @@ -1970,7 +2384,7 @@ class BackupRepoTest : BaseUnitTest() { assertFalse(sut.hasPendingWalletRestore()) val restartedReservations = createReservationRepo(reservationStore) assertTrue(restartedReservations.isUnavailableForReusableReceive(publicAddress)) - sut = createSut(restartedReservations) + sut = createSut(reservationRepo = restartedReservations) sut.triggerBackup(BackupCategory.WALLET).getOrThrow() val uploaded = argumentCaptor() @@ -2058,7 +2472,7 @@ class BackupRepoTest : BaseUnitTest() { assertFalse(reservations.preparePublicAddressRestore()) reservations.capturePublicAddressRestoreSource(metadata.toByteArray()) val source = pendingPublicAddressRestoreRecord("source") - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) for (download in listOf(null, "$metadata ")) { whenever(vssBackupClient.getObject(BackupCategory.METADATA.name)).thenReturn( @@ -2108,7 +2522,7 @@ class BackupRepoTest : BaseUnitTest() { assertTrue(sut.triggerBackup(BackupCategory.METADATA).isFailure) verify(vssBackupClient, never()).putObject(any(), any()) - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) stubMetadataRestore(envelope = metadataEnvelope()) sut.performFullRestoreFromLatestBackup().getOrThrow() @@ -2130,7 +2544,7 @@ class BackupRepoTest : BaseUnitTest() { assertTrue(reservationStore.data.first().quarantinedPublicAddresses.isEmpty()) verify(vssBackupClient, never()).getObject(BackupCategory.WALLET.name) - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) val missingMetadata = listOf( Result.failure(BackupRepoTestError("metadata download unavailable")), Result.success(null), @@ -2187,7 +2601,7 @@ class BackupRepoTest : BaseUnitTest() { assertFalse(settingsData.value.backupVerified) pendingPublicAddressRestoreRecord("source") assertTrue(reservationStore.data.first().quarantinedPublicAddresses.isEmpty()) - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) whenever(vssBackupClient.getObject(BackupCategory.METADATA.name)) .thenReturn(Result.failure(BackupRepoTestError("metadata download unavailable"))) @@ -2442,7 +2856,7 @@ class BackupRepoTest : BaseUnitTest() { assertEquals(setOf(publicAddress), reservationStore.data.first().quarantinedPublicAddresses) assertEquals(publicAddress, pendingPublicAddressRestoreRecord("address")["address"]?.jsonPrimitive?.content) - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) whenever(vssBackupClient.getObject(BackupCategory.METADATA.name)) .thenReturn(Result.failure(BackupRepoTestError("metadata download unavailable"))) sut.performFullRestoreFromLatestBackup().getOrThrow() @@ -2525,21 +2939,100 @@ class BackupRepoTest : BaseUnitTest() { } @Test - fun `metadata restore waits for admitted metadata capture and upload to drain`() = test { - stubMetadataBackupReads() - val captureStarted = CompletableDeferred() - val finishCapture = CompletableDeferred() - val uploadStarted = CompletableDeferred() - val finishUpload = CompletableDeferred() - val metadataRead = CompletableDeferred() - whenever(pubkyRepo.snapshotSessionBackupState()).doSuspendableAnswer { - captureStarted.complete(Unit) - finishCapture.await() - Result.success(null) + fun `wallet acknowledgement automatically resumes blocked metadata observation`() = test { + val publicAddress = "bcrt1qpublic" + val reservationStore = stubPublicAddressRestore(publicAddress) + val legacyMetadata = envelopeWithRawField( + base = metadataEnvelope(cache = AppCacheData(onchainAddress = publicAddress)), + field = "tagMetadata", + rawJson = LEGACY_METADATA_JSON, + ) + val remote = mutableMapOf( + BackupCategory.METADATA.name to legacyMetadata.toByteArray(), + BackupCategory.WALLET.name to """{"version":1,"createdAt":123,"transfers":[]}""".toByteArray(), + ) + whenever(vssBackupClient.getObject(any())).doSuspendableAnswer { + val key = it.getArgument(0) + Result.success(remote[key]?.let { value -> VssItem(key = key, value = value, version = 1) }) } - whenever(vssBackupClient.putObject(eq(BackupCategory.METADATA.name), any())).doSuspendableAnswer { - uploadStarted.complete(Unit) - finishUpload.await() + var walletUploadAvailable = false + val uploadedCategories = mutableListOf() + whenever(vssBackupClient.putObject(any(), any())).doSuspendableAnswer { + val key = it.getArgument(0) + val value = it.getArgument(1) + if (key == BackupCategory.WALLET.name && !walletUploadAvailable) { + Result.failure(BackupRepoTestError("wallet upload unavailable")) + } else { + remote[key] = value + uploadedCategories += key + Result.success(VssItem(key = key, value = value, version = 1)) + } + } + + sut.performFullRestoreFromLatestBackup().getOrThrow() + + assertTrue(settingsData.value.backupVerified) + assertFalse(sut.hasPendingWalletRestore()) + assertEquals(setOf(publicAddress), reservationStore.data.first().quarantinedPublicAddresses) + assertEquals(legacyMetadata, remote.getValue(BackupCategory.METADATA.name).decodeToString()) + assertTrue(cacheData.value.backupStatuses.getValue(BackupCategory.WALLET).isRequired) + assertTrue(cacheData.value.backupStatuses.getValue(BackupCategory.METADATA).isRequired) + verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + + sut.triggerBackup(BackupCategory.METADATA) + assertTrue(sut.triggerBackup(BackupCategory.WALLET).isFailure) + sut.triggerBackup(BackupCategory.METADATA) + assertEquals(legacyMetadata, remote.getValue(BackupCategory.METADATA.name).decodeToString()) + assertTrue(cacheData.value.backupStatuses.getValue(BackupCategory.WALLET).isRequired) + assertTrue(cacheData.value.backupStatuses.getValue(BackupCategory.METADATA).isRequired) + verify(vssBackupClient, never()).putObject(eq(BackupCategory.METADATA.name), any()) + + stubBackupObservers() + sut.startObservingBackups() + runCurrent() + advanceTimeBy(5_000) + runCurrent() + assertEquals(legacyMetadata, remote.getValue(BackupCategory.METADATA.name).decodeToString()) + + try { + walletUploadAvailable = true + sut.triggerBackup(BackupCategory.WALLET).getOrThrow() + val wallet = json.decodeFromString( + remote.getValue(BackupCategory.WALLET.name).decodeToString(), + ) + assertEquals(listOf(publicAddress), wallet.publicPaykitLegacyAddresses) + advanceTimeBy(5_000) + runCurrent() + assertEquals(1, uploadedCategories.count { it == BackupCategory.METADATA.name }) + val metadata = json.decodeFromString( + remote.getValue(BackupCategory.METADATA.name).decodeToString(), + ) + assertEquals("", metadata.cache.onchainAddress) + assertEquals(BackupCategory.WALLET.name, uploadedCategories.first()) + assertTrue(uploadedCategories.indexOf(BackupCategory.METADATA.name) > 0) + assertFalse(cacheData.value.backupStatuses.getValue(BackupCategory.WALLET).isRequired) + assertFalse(cacheData.value.backupStatuses.getValue(BackupCategory.METADATA).isRequired) + } finally { + sut.reset() + } + } + + @Test + fun `metadata restore waits for admitted metadata capture and upload to drain`() = test { + stubMetadataBackupReads() + val captureStarted = CompletableDeferred() + val finishCapture = CompletableDeferred() + val uploadStarted = CompletableDeferred() + val finishUpload = CompletableDeferred() + val metadataRead = CompletableDeferred() + whenever(pubkyRepo.snapshotSessionBackupState()).doSuspendableAnswer { + captureStarted.complete(Unit) + finishCapture.await() + Result.success(null) + } + whenever(vssBackupClient.putObject(eq(BackupCategory.METADATA.name), any())).doSuspendableAnswer { + uploadStarted.complete(Unit) + finishUpload.await() Result.success(VssItem(key = BackupCategory.METADATA.name, value = it.getArgument(1), version = 1)) } whenever(vssBackupClient.getObject(BackupCategory.METADATA.name)).doSuspendableAnswer { @@ -2551,100 +3044,1353 @@ class BackupRepoTest : BaseUnitTest() { val restore = async { sut.performFullRestoreFromLatestBackup() } try { runCurrent() - assertFalse(metadataRead.isCompleted) - assertFalse(restore.isCompleted) - finishCapture.complete(Unit) + assertFalse(metadataRead.isCompleted) + assertFalse(restore.isCompleted) + finishCapture.complete(Unit) + runCurrent() + assertTrue(uploadStarted.isCompleted) + assertFalse(metadataRead.isCompleted) + assertFalse(restore.isCompleted) + + finishUpload.complete(Unit) + backup.await().getOrThrow() + restore.await().getOrThrow() + assertTrue(metadataRead.isCompleted) + } finally { + finishCapture.complete(Unit) + finishUpload.complete(Unit) + backup.cancelAndJoin() + restore.cancelAndJoin() + } + } + + @Test + fun `automatic wallet backup marks failure when Paykit snapshot fails`() = test { + whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(3_000)) + whenever { privatePaykitRepo.backupSnapshot() } + .thenReturn(Result.failure(BackupRepoTestError("paykit session missing capabilities"))) + val backupStatuses = MutableStateFlow( + mapOf( + BackupCategory.WALLET to BackupItemStatus( + synced = 1_000, + required = 2_000, + ), + ) + ) + val allowWalletClear = CompletableDeferred().apply { complete(Unit) } + stubBackupStatuses(backupStatuses, allowWalletClear) {} + stubBackupObservers() + + try { + sut.startObservingBackups() + runCurrent() + advanceTimeBy(5_000) + runCurrent() + + verify(privatePaykitRepo).backupSnapshot() + verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + val status = backupStatuses.value.getValue(BackupCategory.WALLET) + assertTrue(status.isRequired) + assertFalse(status.running) + + advanceTimeBy(5_000) + runCurrent() + + // still one attempt: the failed timestamp suppresses a retry until data changes again + verify(privatePaykitRepo, times(1)).backupSnapshot() + verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + } finally { + sut.stopObservingBackups() + } + } + + @Test + fun `released VSS auth transport failure is temporary without retrying authorization failures`() { + for (prefix in listOf("", "error performing external request: ")) { + for (url in listOf("http://127.0.0.1:1/lnurl_auth/auth", "https://auth.example.com/signed?k1=fixture")) { + assertTrue( + VssException.AuthException( + "${prefix}error sending request for url ($url)" + ).isTemporaryBackupFailure() + ) + } + } + listOf( + "authorization was refused: LNURL Auth failed", + "invalid data: invalid JWT", + "HTTP status client error (401 Unauthorized)", + "error performing external request: error decoding response body", + "prefix error sending request for url (https://auth.example.com/auth)", + "error performing external request: prefix error sending request for url (https://auth.example.com/auth)", + "error sending request for url (file://auth.example.com/auth)", + "error sending request for url (https:///auth)", + "error sending request for url (not a URL)", + "error sending request for url (https://auth.example.com/auth) suffix", + ).forEach { assertFalse(VssException.AuthException(it).isTemporaryBackupFailure(), it) } + assertFalse( + VssException.UnknownException( + "error sending request for url (https://auth.example.com/auth)" + ).isTemporaryBackupFailure() + ) + } + + @Test + fun `cold explicit auth transport failure retries without observation or another request`() = test { + val category = BackupCategory.SETTINGS + val error = VssException.AuthException( + "error performing external request: error sending request for url (http://127.0.0.1:1/lnurl_auth/auth)", + ) + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever(vssBackupClient.putObject(eq(category.name), any())) + .thenReturn(Result.failure(error)) + .thenReturn(Result.success(VssItem(category.name, byteArrayOf(), 1))) + try { + assertEquals(error, sut.triggerBackup(category).exceptionOrNull()) + assertTrue(statuses.value.getValue(category).isRequired) + assertFalse(statuses.value.getValue(category).running) + advanceTimeBy(10_000) + runCurrent() + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + advanceTimeBy(5_000) + runCurrent() + verify(vssBackupClient, times(2)).setup(any()) + verify(vssBackupClient, times(2)).putObject(eq(category.name), any()) + assertFalse(statuses.value.getValue(category).isRequired) + advanceTimeBy(600_000) + runCurrent() + verify(vssBackupClient, times(2)).putObject(eq(category.name), any()) + } finally { + sut.reset() + } + } + + @Test + fun `cold explicit authorization denial stays pending without automatic retry`() = test { + val category = BackupCategory.SETTINGS + val error = VssException.AuthException("authorization was refused: LNURL Auth failed") + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever(vssBackupClient.putObject(eq(category.name), any())).thenReturn(Result.failure(error)) + try { + assertEquals(error, sut.triggerBackup(category).exceptionOrNull()) + advanceTimeBy(600_000) + runCurrent() + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + assertTrue(statuses.value.getValue(category).isRequired) + assertFalse(statuses.value.getValue(category).running) + } finally { + sut.reset() + } + } + + @Test + fun `released VSS connection failure is temporary without broad unknown retry`() { + assertTrue( + VssException.UnknownException("error sending request for url (http://127.0.0.1:1/putObjects)") + .isTemporaryBackupFailure(), + ) + listOf( + "invalid ciphertext", + "permission denied", + "HTTP status client error (401 Unauthorized)", + "error sending request for url (not a URL)", + "prefix error sending request for url (https://example.com)", + "error sending request for url (https://example.com/getObject)", + "error sending request for url (file://example.com/putObjects)", + "error sending request for url (https:///putObjects)", + "error sending request for url (https://example.com/putObjects) suffix", + ).forEach { assertFalse(VssException.UnknownException(it).isTemporaryBackupFailure()) } + } + + @Test + fun `settings backup retries the released VSS connection error and acknowledges recovery`() = test { + whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(3_000)) + val category = BackupCategory.SETTINGS + val backupStatuses = MutableStateFlow(mapOf(category to BackupItemStatus(synced = 1_000, required = 2_000))) + stubBackupStatuses(backupStatuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var attempts = 0 + whenever(vssBackupClient.putObject(eq(category.name), any())).doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + Result.failure( + VssException.UnknownException("error sending request for url (http://127.0.0.1:1/putObjects)") + ) + } else { + Result.success(VssItem(category.name, it.getArgument(1), 1)) + } + } + try { + sut.startObservingBackups() + runCurrent() + advanceTimeBy(5_000) + runCurrent() + assertEquals(1, attempts) + assertTrue(backupStatuses.value.getValue(category).isRequired) + advanceTimeBy(20_000) + runCurrent() + assertEquals(2, attempts) + assertFalse(backupStatuses.value.getValue(category).isRequired) + } finally { + sut.stopObservingBackups() + } + } + + @Test + fun `wallet backup retries after a busy Paykit shared state`() = test { + whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(3_000)) + whenever { privatePaykitRepo.backupSnapshot() } + .thenReturn(Result.failure(PaykitException.SharedStateBusy("shared_state_busy", "locked"))) + .thenReturn(Result.success(PrivatePaykitBackupSnapshot(null, 0))) + val backupStatuses = MutableStateFlow( + mapOf( + BackupCategory.WALLET to BackupItemStatus( + synced = 1_000, + required = 2_000, + ), + ) + ) + val allowWalletClear = CompletableDeferred().apply { complete(Unit) } + stubBackupStatuses(backupStatuses, allowWalletClear) {} + stubBackupObservers() + + try { + sut.startObservingBackups() + runCurrent() + advanceTimeBy(5_000) + runCurrent() + verify(privatePaykitRepo, times(1)).backupSnapshot() + verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + + advanceTimeBy(20_000) + runCurrent() + + verify(privatePaykitRepo, times(2)).backupSnapshot() + verify(vssBackupClient).putObject(eq(BackupCategory.WALLET.name), any()) + assertFalse(backupStatuses.value.getValue(BackupCategory.WALLET).isRequired) + } finally { + sut.stopObservingBackups() + } + } + + @Test + fun `reset prevents an interrupted observer setup from initializing the old LDK client`() = test { + val setupStarted = CompletableDeferred() + val allowSetup = CompletableDeferred() + val backupStatuses = MutableStateFlow(emptyMap()) + val allowWalletClear = CompletableDeferred().apply { complete(Unit) } + stubBackupStatuses(backupStatuses, allowWalletClear) {} + stubBackupObservers() + whenever { vssBackupClient.setupWithRetry(any(), any(), any()) }.doSuspendableAnswer { + setupStarted.complete(Unit) + allowSetup.await() + Result.success(Unit) + } + + sut.startObservingBackups() + setupStarted.await() + sut.setWiping(true) + val reset = async { sut.reset() } + runCurrent() + allowSetup.complete(Unit) + reset.await() + runCurrent() + + verify(vssBackupClientLdk, never()).setup() + } + + @Test + fun `consumed settings change survives observation stop`() = test { + assertConsumedSettingsChangeSurvivesStop(reset = false) + } + + @Test + fun `consumed settings change survives reset when wipe does not clear data`() = test { + assertConsumedSettingsChangeSurvivesStop(reset = true) + } + + private suspend fun TestScope.assertConsumedSettingsChangeSurvivesStop(reset: Boolean) { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000, synced = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + sut.startObservingBackups() + runCurrent() + + val entered = CompletableDeferred() + val release = CompletableDeferred() + var holdNextWrite = true + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + if (holdNextWrite) { + holdNextWrite = false + entered.complete(Unit) + release.await() + } + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } + } + settingsData.update { it.copy(selectedCurrency = "EUR") } + entered.await() + val resetJob = if (reset) async { sut.reset() } else null + if (!reset) sut.stopObservingBackups() + try { + runCurrent() + release.complete(Unit) + resetJob?.await() + runCurrent() + assertTrue(statuses.value.getValue(category).isRequired) + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + val payload = argumentCaptor() + verify(vssBackupClient).putObject(eq(category.name), payload.capture()) + assertTrue(payload.firstValue.decodeToString().contains("EUR")) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + release.complete(Unit) + resetJob?.join() + sut.reset() + } + } + + @Test + fun `reset drains cancelled setup even after stop and restart`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + val statuses = MutableStateFlow(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + whenever { vssBackupClient.setupWithRetry(any(), any(), any()) }.doSuspendableAnswer { + entered.complete(Unit) + withContext(NonCancellable) { release.await() } + Result.success(Unit) + } + + sut.startObservingBackups() + entered.await() + sut.stopObservingBackups() + sut.startObservingBackups() + sut.setWiping(true) + val reset = async { sut.reset() } + try { + runCurrent() + assertFalse(reset.isCompleted) + verify(vssBackupClient, times(1)).setupWithRetry(any(), any(), any()) + reset.cancel() + runCurrent() + assertFalse(reset.isCompleted) + } finally { + release.complete(Unit) + reset.join() + } + verify(vssBackupClientLdk, never()).setup() + } + + @Test + fun `restart waits for prior upload cleanup before scheduling replacement`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + entered.complete(Unit) + try { + awaitCancellation() + } finally { + withContext(NonCancellable) { release.await() } + } + } + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + entered.await() + sut.stopObservingBackups() + sut.startObservingBackups() + try { + runCurrent() + assertTrue(statuses.value.getValue(category).running) + verify(vssBackupClient, times(1)).setupWithRetry(any(), any(), any()) + assertEquals(1, attempts) + release.complete(Unit) + runCurrent() + verify(vssBackupClient, times(2)).setupWithRetry(any(), any(), any()) + advanceTimeBy(5_000) + runCurrent() + assertEquals(2, attempts) + assertFalse(statuses.value.getValue(category).running) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `settings changed during restart cleanup remain pending without another edit`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + val statuses = MutableStateFlow( + mapOf( + BackupCategory.WIDGETS to BackupItemStatus(required = 1_000), + BackupCategory.SETTINGS to BackupItemStatus(required = 1_000, synced = 1_000), + ) + ) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var widgetAttempts = 0 + whenever { vssBackupClient.putObject(eq(BackupCategory.WIDGETS.name), any()) }.doSuspendableAnswer { + widgetAttempts += 1 + if (widgetAttempts == 1) { + entered.complete(Unit) + try { + awaitCancellation() + } finally { + withContext(NonCancellable) { release.await() } + } + } + Result.success(VssItem(key = BackupCategory.WIDGETS.name, value = byteArrayOf(), version = 1)) + } + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + entered.await() + sut.stopObservingBackups() + sut.startObservingBackups() + try { + runCurrent() + settingsData.update { it.copy(selectedCurrency = "EUR") } + runCurrent() + verify(vssBackupClient, never()).putObject(eq(BackupCategory.SETTINGS.name), any()) + release.complete(Unit) + runCurrent() + advanceTimeBy(5_000) + runCurrent() + val payload = argumentCaptor() + verify(vssBackupClient).putObject(eq(BackupCategory.SETTINGS.name), payload.capture()) + assertTrue(payload.firstValue.decodeToString().contains("EUR")) + assertFalse(statuses.value.getValue(BackupCategory.SETTINGS).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `temporary retry waits for other category cleanup during restart`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + val statuses = MutableStateFlow( + mapOf( + BackupCategory.WIDGETS to BackupItemStatus(required = 1_000), + BackupCategory.WALLET to BackupItemStatus(required = 1_000), + ) + ) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + whenever { privatePaykitRepo.backupSnapshot() } + .thenReturn(Result.failure(PaykitException.SharedStateBusy("shared_state_busy", "locked"))) + .thenReturn(Result.success(PrivatePaykitBackupSnapshot(null, 0))) + var widgetAttempts = 0 + whenever { vssBackupClient.putObject(eq(BackupCategory.WIDGETS.name), any()) }.doSuspendableAnswer { + widgetAttempts += 1 + if (widgetAttempts == 1) { + entered.complete(Unit) + try { + awaitCancellation() + } finally { + withContext(NonCancellable) { release.await() } + } + } + Result.success(VssItem(key = BackupCategory.WIDGETS.name, value = byteArrayOf(), version = 1)) + } + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + entered.await() + verify(privatePaykitRepo).backupSnapshot() + sut.stopObservingBackups() + sut.startObservingBackups() + try { + advanceTimeBy(20_000) + runCurrent() + verify(privatePaykitRepo, times(1)).backupSnapshot() + verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + release.complete(Unit) + runCurrent() + advanceTimeBy(5_000) + runCurrent() + verify(vssBackupClient).putObject(eq(BackupCategory.WALLET.name), any()) + assertFalse(statuses.value.getValue(BackupCategory.WALLET).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `full backup requested during observer cleanup resumes without restart`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + entered.complete(Unit) + try { + awaitCancellation() + } finally { + withContext(NonCancellable) { release.await() } + } + } + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + entered.await() + sut.stopObservingBackups() + sut.scheduleFullBackup() + try { + runCurrent() + assertTrue(statuses.value.getValue(category).running) + verify(vssBackupClient, times(1)).setupWithRetry(any(), any(), any()) + assertEquals(1, attempts) + release.complete(Unit) + runCurrent() + verify(vssBackupClient, times(1)).setupWithRetry(any(), any(), any()) + advanceTimeBy(5_000) + runCurrent() + assertEquals(2, attempts) + assertFalse(statuses.value.getValue(category).running) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `manual finalization cannot recreate status entries after reset`() = test { + val data = MutableStateFlow(AppCacheData()) + val store = mock>() + whenever(store.data).thenReturn(data) + val entered = CompletableDeferred() + val release = CompletableDeferred() + var holdNextWrite = false + whenever { store.updateData(any()) }.doSuspendableAnswer { + val transform = it.getArgument AppCacheData>(0) + if (holdNextWrite) { + holdNextWrite = false + entered.complete(Unit) + withContext(NonCancellable) { release.await() } + } + transform(data.value).also { value -> data.value = value } + } + whenever { vssBackupClient.putObject(eq(BackupCategory.SETTINGS.name), any()) }.doSuspendableAnswer { + holdNextWrite = true + Result.success(VssItem(key = BackupCategory.SETTINGS.name, value = byteArrayOf(), version = 1)) + } + val repo = createSut(backupCache = CacheStore(store)) + val manual = async { repo.triggerBackup(BackupCategory.SETTINGS) } + entered.await() + val reset = async { repo.reset() } + try { + runCurrent() + assertFalse(reset.isCompleted) + } finally { + release.complete(Unit) + reset.await() + } + data.value = AppCacheData() + assertTrue(manual.await().isFailure) + assertTrue(data.value.backupStatuses.isEmpty()) + } + + @Test + fun `reset drains manual upload so failed wipe can resume pending backups`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + entered.complete(Unit) + withContext(NonCancellable) { release.await() } + return@doSuspendableAnswer Result.failure(AppError("upload interrupted")) + } + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + val manual = async { sut.triggerBackup(category) } + entered.await() + val reset = async { sut.reset() } + try { + runCurrent() + assertFalse(reset.isCompleted) + } finally { + release.complete(Unit) + reset.await() + manual.await() + } + try { + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + assertEquals(2, attempts) + assertFalse(statuses.value.getValue(category).running) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + sut.reset() + } + } + + @Test + fun `failed explicit USDT callback retries backup after observation stops without acknowledging Core`() = test { + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + sut.startObservingBackups() + sut.stopObservingBackups() + runCurrent() + val service = UsdtService(keychain, Provider { sut }) + whenever(usdtRepo.backupSnapshot()).thenReturn("same-signed-operation") + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .thenReturn( + Result.failure( + VssException.UnknownException("error sending request for url (http://127.0.0.1:1/putObjects)") + ) + ) + .thenReturn(Result.success(VssItem(key = "recovery", value = byteArrayOf(), version = 1))) + try { + assertTrue(runCatching { service.recoveryBackup.persist("same-signed-operation") }.isFailure) + assertEquals(0L, service.recoveryGeneration) + advanceTimeBy(15_001) + runCurrent() + assertFalse(statuses.value.getValue(BackupCategory.USDT_RECOVERY).isRequired) + assertEquals(0L, service.recoveryGeneration) + service.recoveryBackup.persist("same-signed-operation") + assertEquals(1L, service.recoveryGeneration) + val keys = argumentCaptor() + val payloads = argumentCaptor() + verify(vssBackupClient, times(3)).putObject(keys.capture(), payloads.capture()) + assertEquals(1, keys.allValues.distinct().size) + assertTrue(payloads.allValues.all { it.contentEquals(payloads.firstValue) }) + verify(privatePaykitRepo, never()).backupSnapshot() + } finally { + sut.reset() + } + } + + @Test + fun `manual backup failure retries without observation`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever { vssBackupClient.putObject(eq(category.name), any()) } + .thenReturn(Result.failure(VssException.NetworkException("offline"))) + .thenReturn(Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1))) + try { + assertTrue(sut.triggerBackup(category).isFailure) + advanceTimeBy(15_001) + runCurrent() + verify(vssBackupClient, times(2)).putObject(eq(category.name), any()) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + sut.reset() + } + } + + @Test + fun `reset and wipe revoke explicit retry admission before client replacement`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever { vssBackupClient.putObject(eq(category.name), any()) } + .thenReturn(Result.failure(VssException.NetworkException("offline"))) + .thenReturn(Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1))) + assertTrue(sut.triggerBackup(category).isFailure) + sut.setWiping(true) + sut.reset() + advanceTimeBy(20_000) + runCurrent() + verify(vssBackupClient, times(1)).putObject(eq(category.name), any()) + assertTrue(sut.triggerBackup(category).isFailure) + sut.setWiping(false) + sut.triggerBackup(category).getOrThrow() + advanceTimeBy(20_000) + runCurrent() + verify(vssBackupClient, times(2)).putObject(eq(category.name), any()) + assertFalse(statuses.value.getValue(category).isRequired) + sut.reset() + } + + @Test + fun `explicit retry retains newer required data until the successor succeeds`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts == 1) return@doSuspendableAnswer Result.failure(VssException.NetworkException("offline")) + if (attempts == 2) { + statuses.update { data -> + val current = data.getValue(category) + data + (category to current.copy(required = current.required + 1)) + } + } + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + try { + assertTrue(sut.triggerBackup(category).isFailure) + advanceTimeBy(20_001) + runCurrent() + assertEquals(3, attempts) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + sut.reset() + } + } + + @Test + fun `full request survives coalescing with a cancelled observer upload`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + val entered = CompletableDeferred() + val release = CompletableDeferred() + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + entered.complete(Unit) + try { + awaitCancellation() + } finally { + withContext(NonCancellable) { release.await() } + } + } + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + entered.await() + sut.scheduleFullBackup() + sut.stopObservingBackups() + try { + advanceTimeBy(20_000) + runCurrent() + assertEquals(1, attempts) + release.complete(Unit) + runCurrent() + advanceTimeBy(5_000) + runCurrent() + assertEquals(2, attempts) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `permanent failure and caller cancellation do not admit explicit retry`() = test { + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + val entered = CompletableDeferred() + var attempts = 0 + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + Result.failure(VssException.UnknownException("invalid credentials")) + } else { + entered.complete(Unit) + awaitCancellation() + } + } + assertTrue(sut.persistUsdtRecoveryBackup("signed").isFailure) + advanceTimeBy(600_000) + runCurrent() + assertEquals(1, attempts) + val caller = launch { sut.persistUsdtRecoveryBackup("signed") } + entered.await() + caller.cancelAndJoin() + advanceTimeBy(600_000) + runCurrent() + assertEquals(2, attempts) + assertTrue(statuses.value.getValue(BackupCategory.USDT_RECOVERY).isRequired) + sut.reset() + } + + @Test + fun `manual success follows a consumed change after observers stop`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000, synced = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + sut.startObservingBackups() + runCurrent() + val entered = CompletableDeferred() + val release = CompletableDeferred() + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + entered.complete(Unit) + release.await() + } + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + val manual = async { sut.triggerBackup(category) } + entered.await() + settingsData.update { it.copy(selectedCurrency = "EUR") } + runCurrent() + sut.stopObservingBackups() + try { + release.complete(Unit) + manual.await().getOrThrow() + advanceTimeBy(5_001) + runCurrent() + assertEquals(2, attempts) + assertFalse(statuses.value.getValue(category).isRequired) + val payloads = argumentCaptor() + verify(vssBackupClient, times(2)).putObject(eq(category.name), payloads.capture()) + assertTrue(payloads.secondValue.decodeToString().contains("EUR")) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `consumed change cannot orphan a retained explicit retry timer`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000, synced = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + sut.startObservingBackups() + runCurrent() + whenever { vssBackupClient.putObject(eq(category.name), any()) } + .thenReturn(Result.failure(VssException.NetworkException("offline"))) + .thenReturn(Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1))) + assertTrue(sut.triggerBackup(category).isFailure) + val entered = CompletableDeferred() + val release = CompletableDeferred() + var holdNextWrite = true + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + if (holdNextWrite) { + holdNextWrite = false + entered.complete(Unit) + release.await() + } + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } + } + settingsData.update { it.copy(selectedCurrency = "EUR") } + entered.await() + sut.stopObservingBackups() + try { + release.complete(Unit) + runCurrent() + advanceTimeBy(15_001) + runCurrent() + verify(vssBackupClient, times(2)).putObject(eq(category.name), any()) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `older retry timer cannot shorten a newer explicit failure backoff`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts < 3) { + Result.failure(VssException.NetworkException("offline")) + } else { + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + } + try { + assertTrue(sut.triggerBackup(category).isFailure) + advanceTimeBy(5_000) + assertTrue(sut.triggerBackup(category).isFailure) + advanceTimeBy(15_000) + runCurrent() + assertEquals(2, attempts) + advanceTimeBy(10_001) + runCurrent() + assertEquals(3, attempts) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + sut.reset() + } + } + + @Test + fun `stale timer continuation respects a concurrent explicit failure backoff`() = test { + val category = BackupCategory.SETTINGS + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + var attempts = 0 + whenever { vssBackupClient.putObject(eq(category.name), any()) }.doSuspendableAnswer { + attempts += 1 + if (attempts < 3) { + Result.failure(VssException.NetworkException("offline")) + } else { + Result.success(VssItem(key = category.name, value = byteArrayOf(), version = 1)) + } + } + assertTrue(sut.triggerBackup(category).isFailure) + val entered = CompletableDeferred() + val release = CompletableDeferred() + var holdNextReturn = true + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } + if (holdNextReturn) { + holdNextReturn = false + entered.complete(Unit) + release.await() + } + } + advanceTimeBy(10_000) + runCurrent() + entered.await() + try { + assertTrue(sut.triggerBackup(category).isFailure) + release.complete(Unit) + runCurrent() + advanceTimeBy(5_001) + runCurrent() + assertEquals(2, attempts) + advanceTimeBy(20_000) + runCurrent() + assertEquals(3, attempts) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `committed recovery revision survives cancellation after an older permanent failure`() = test { + val category = BackupCategory.USDT_RECOVERY + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + whenever(usdtRepo.backupSnapshot()).thenReturn("signed") + var attempts = 0 + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .doSuspendableAnswer { + attempts++ + if (attempts == 1) { + Result.failure(VssException.UnknownException("invalid data")) + } else { + Result.success(VssItem(key = "recovery", value = byteArrayOf(), version = 1)) + } + } + assertTrue(sut.persistUsdtRecoveryBackup("signed").isFailure) + val failedRevision = statuses.value.getValue(category).required + sut.startObservingBackups() + runCurrent() + advanceTimeBy(10_000) + runCurrent() + assertEquals(1, attempts) + val committed = CompletableDeferred() + val release = CompletableDeferred() + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } + val status = statuses.value.getValue(category) + if (!committed.isCompleted && status.required > failedRevision && !status.running) { + committed.complete(Unit) + release.await() + } + } + val service = UsdtService(keychain, Provider { sut }) + val callback = launch { service.recoveryBackup.persist("signed") } + try { + committed.await() + callback.cancel() + runCurrent() + release.complete(Unit) + callback.join() + assertEquals(0L, service.recoveryGeneration) + assertTrue(statuses.value.getValue(category).required > failedRevision) + advanceTimeBy(10_000) + runCurrent() + assertEquals(2, attempts) + assertFalse(statuses.value.getValue(category).isRequired) + assertEquals(0L, service.recoveryGeneration) + } finally { + release.complete(Unit) + callback.cancelAndJoin() + sut.reset() + } + } + + @Test + fun `explicit recovery callback bypasses scheduled debounce and owns its acknowledgement`() = test { + val category = BackupCategory.USDT_RECOVERY + val statuses = MutableStateFlow(mapOf(category to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + whenever(usdtRepo.backupSnapshot()).thenReturn("background") + val uploads = mutableListOf() + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .doSuspendableAnswer { + val payload = json.decodeFromString(it.getArgument(1).decodeToString()) + uploads += payload.usdtWallet + Result.success(VssItem(key = "recovery", value = byteArrayOf(), version = 1)) + } + sut.startObservingBackups() + runCurrent() + advanceTimeBy(150) + val service = UsdtService(keychain, Provider { sut }) + val callback = async { service.recoveryBackup.persist("signed") } + try { + runCurrent() + assertTrue(callback.isCompleted, "Explicit recovery must not wait for scheduled debounce") + callback.await() + assertEquals(listOf("signed"), uploads) + assertEquals(1L, service.recoveryGeneration) + advanceTimeBy(5_000) + runCurrent() + assertEquals(listOf("signed"), uploads) + } finally { + callback.cancelAndJoin() + sut.reset() + } + } + + @Test + fun `cancelled queued USDT callback retains its required revision`() = test { + val category = BackupCategory.USDT_RECOVERY + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + val release = CompletableDeferred() + val uploads = mutableListOf() + val service = UsdtService(keychain, Provider { sut }) + whenever(usdtRepo.backupSnapshot()).thenReturn("older") + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .doSuspendableAnswer { + val payload = json.decodeFromString(it.getArgument(1).decodeToString()) + uploads += payload.usdtWallet + release.await() + Result.success(VssItem(key = "recovery", value = byteArrayOf(), version = 1)) + } + val older = async { sut.triggerBackup(category) } + runCurrent() + val olderRequired = statuses.value.getValue(category).required + val queued = launch { service.recoveryBackup.persist("newer") } + try { + runCurrent() + queued.cancelAndJoin() + release.complete(Unit) + older.await().getOrThrow() + runCurrent() + val status = statuses.value.getValue(category) + assertTrue(status.required > olderRequired) + assertTrue(status.isRequired) + assertEquals(0L, service.recoveryGeneration) + assertEquals(listOf("older"), uploads) + } finally { + queued.cancelAndJoin() + release.complete(Unit) + sut.reset() + } + } + + @Test + fun `queued USDT callback cannot satisfy a later cancelled revision`() = test { + val category = BackupCategory.USDT_RECOVERY + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + val release = CompletableDeferred() + val uploads = mutableListOf() + whenever(usdtRepo.backupSnapshot()).thenReturn("older") + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .doSuspendableAnswer { + val payload = json.decodeFromString(it.getArgument(1).decodeToString()) + uploads += payload.usdtWallet + if (payload.usdtWallet == "older") release.await() + Result.success(VssItem(key = "recovery", value = byteArrayOf(), version = 1)) + } + val older = async { sut.triggerBackup(category) } + runCurrent() + val first = async { sut.persistUsdtRecoveryBackup("first") } + runCurrent() + val second = launch { sut.persistUsdtRecoveryBackup("second") } + try { + runCurrent() + second.cancelAndJoin() + release.complete(Unit) + older.await().getOrThrow() + first.await().getOrThrow() runCurrent() - assertTrue(uploadStarted.isCompleted) - assertFalse(metadataRead.isCompleted) - assertFalse(restore.isCompleted) - - finishUpload.complete(Unit) - backup.await().getOrThrow() - restore.await().getOrThrow() - assertTrue(metadataRead.isCompleted) + assertTrue(statuses.value.getValue(category).isRequired) + assertEquals(listOf("older", "first"), uploads) } finally { - finishCapture.complete(Unit) - finishUpload.complete(Unit) - backup.cancelAndJoin() - restore.cancelAndJoin() + second.cancelAndJoin() + release.complete(Unit) + sut.reset() } } @Test - fun `automatic wallet backup marks failure when Paykit snapshot fails`() = test { - whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(3_000)) - whenever { privatePaykitRepo.backupSnapshot() } - .thenReturn(Result.failure(BackupRepoTestError("paykit session missing capabilities"))) - val backupStatuses = MutableStateFlow( - mapOf( - BackupCategory.WALLET to BackupItemStatus( - synced = 1_000, - required = 2_000, - ), - ) - ) - val allowWalletClear = CompletableDeferred().apply { complete(Unit) } - stubBackupStatuses(backupStatuses, allowWalletClear) {} - stubBackupObservers() - + fun `older reserved callback preserves a newer failure backoff`() = test { + val category = BackupCategory.USDT_RECOVERY + val statuses = MutableStateFlow>(emptyMap()) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + whenever(usdtRepo.backupSnapshot()).thenReturn("newer") + val uploads = mutableListOf() + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .doSuspendableAnswer { + val payload = json.decodeFromString(it.getArgument(1).decodeToString()) + uploads += payload.usdtWallet + if (uploads.size == 1) { + Result.failure(VssException.NetworkException("offline")) + } else { + Result.success(VssItem(key = "recovery", value = byteArrayOf(), version = 1)) + } + } + val entered = CompletableDeferred() + val release = CompletableDeferred() + var holdNextReturn = true + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } + if (holdNextReturn) { + holdNextReturn = false + entered.complete(Unit) + release.await() + } + } + val older = async { sut.persistUsdtRecoveryBackup("older") } + entered.await() try { - sut.startObservingBackups() - runCurrent() + assertTrue(sut.persistUsdtRecoveryBackup("newer").isFailure) advanceTimeBy(5_000) + release.complete(Unit) + older.await().getOrThrow() runCurrent() - - verify(privatePaykitRepo).backupSnapshot() - verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) - val status = backupStatuses.value.getValue(BackupCategory.WALLET) - assertTrue(status.isRequired) - assertFalse(status.running) - + advanceTimeBy(5_001) + runCurrent() + assertEquals(listOf("newer", "older"), uploads) + assertTrue(statuses.value.getValue(category).isRequired) advanceTimeBy(5_000) runCurrent() + assertEquals(listOf("newer", "older", "newer"), uploads) + assertFalse(statuses.value.getValue(category).isRequired) + } finally { + release.complete(Unit) + sut.reset() + } + } - // still one attempt: the failed timestamp suppresses a retry until data changes again - verify(privatePaykitRepo, times(1)).backupSnapshot() - verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + @Test + fun `USDT callback does not acknowledge reset cancellation and retries the same recovery record`() = test { + val entered = CompletableDeferred() + val release = CompletableDeferred() + val service = UsdtService(keychain, Provider { sut }) + var attempts = 0 + whenever { vssBackupClient.putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) } + .doSuspendableAnswer { + attempts += 1 + if (attempts == 1) { + entered.complete(Unit) + withContext(NonCancellable) { release.await() } + } + Result.success(VssItem(key = "recovery", value = byteArrayOf(), version = 1)) + } + val pending = async { runCatching { service.recoveryBackup.persist("same-signed-operation") } } + entered.await() + val reset = async { sut.reset() } + try { + runCurrent() + assertFalse(reset.isCompleted) } finally { - sut.stopObservingBackups() + release.complete(Unit) + reset.await() } + assertTrue(pending.await().isFailure) + assertEquals(0L, service.recoveryGeneration) + service.recoveryBackup.persist("same-signed-operation") + assertEquals(1L, service.recoveryGeneration) + val keys = argumentCaptor() + val payloads = argumentCaptor() + verify(vssBackupClient, times(2)).putObject(keys.capture(), payloads.capture()) + assertEquals(keys.firstValue, keys.secondValue) + assertTrue(payloads.firstValue.contentEquals(payloads.secondValue)) + sut.reset() } @Test - fun `wallet backup retries after a busy Paykit shared state`() = test { - whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(3_000)) - whenever { privatePaykitRepo.backupSnapshot() } - .thenReturn(Result.failure(PaykitException.SharedStateBusy("shared_state_busy", "locked"))) - .thenReturn(Result.success(PrivatePaykitBackupSnapshot(null, 0))) - val backupStatuses = MutableStateFlow( - mapOf( - BackupCategory.WALLET to BackupItemStatus( - synced = 1_000, - required = 2_000, - ), - ) - ) - val allowWalletClear = CompletableDeferred().apply { complete(Unit) } - stubBackupStatuses(backupStatuses, allowWalletClear) {} + fun `explicit backups are rejected throughout wallet wipe`() = test { + sut.setWiping(true) + sut.reset() + assertTrue(sut.triggerBackup(BackupCategory.SETTINGS).isFailure) + assertTrue(sut.persistWalletBackup().isFailure) + assertTrue(sut.persistUsdtRecoveryBackup("signed").isFailure) + sut.scheduleFullBackup() + verify(vssBackupClient, never()).setup(any()) + verify(vssBackupClient, never()).putObject(any(), any()) + verify(cacheStore, never()).update(any()) + sut.setWiping(false) + sut.triggerBackup(BackupCategory.SETTINGS).getOrThrow() + verify(vssBackupClient).putObject(eq(BackupCategory.SETTINGS.name), any()) + } + + @Test + fun `explicit USDT recovery backup works after observers stop`() = test { stubBackupObservers() + stubBackupStatuses(MutableStateFlow(emptyMap()), CompletableDeferred(Unit)) {} + sut.startObservingBackups() + sut.stopObservingBackups() + whenever(privatePaykitRepo.backupSnapshot()).thenReturn( + Result.failure(PaykitException.SharedStateBusy("shared_state_busy", "lock unavailable")), + ) + sut.persistUsdtRecoveryBackup("same-signed-operation").getOrThrow() + verify(vssBackupClient).putObject(argThat { startsWith(UsdtRecoveryBackupStore.PREFIX) }, any()) + verify(privatePaykitRepo, never()).backupSnapshot() + sut.reset() + } + @Test + fun `wipe admission drains a backup queued at the real SDK operation barrier`() = test { + val sdkLock = PaykitSdkOperationLock() + val held = CompletableDeferred() + val release = CompletableDeferred() + val exportQueued = CompletableDeferred() + val blocker = async { + sdkLock.withLock { + held.complete(Unit) + release.await() + } + } + held.await() + val statuses = MutableStateFlow(mapOf(BackupCategory.WALLET to BackupItemStatus(required = 1_000))) + stubBackupStatuses(statuses, CompletableDeferred(Unit)) {} + stubBackupObservers() + var exported = false + whenever { privatePaykitRepo.backupSnapshot() }.doSuspendableAnswer { + exportQueued.complete(Unit) + sdkLock.withLock(PaykitSdkOperationLock.Priority.Background) { + exported = true + Result.success(PrivatePaykitBackupSnapshot(null, 0)) + } + } + sut.startObservingBackups() + advanceTimeBy(5_000) + runCurrent() + exportQueued.await() + val wipe = async { + sut.setWiping(true) + connectionRecoveryFixture(settingsStore, keychain, vssClients).withIdentityChange { + sdkLock.withWalletWipe { + sut.reset() + } + } + } + runCurrent() try { - sut.startObservingBackups() - runCurrent() - advanceTimeBy(5_000) - runCurrent() - verify(privatePaykitRepo, times(1)).backupSnapshot() + release.complete(Unit) + withTimeout(5.seconds) { wipe.await() } + blocker.await() + assertFalse(exported) verify(vssBackupClient, never()).putObject(eq(BackupCategory.WALLET.name), any()) + } finally { + release.complete(Unit) + sut.reset() + } + } - advanceTimeBy(20_000) - runCurrent() - - verify(privatePaykitRepo, times(2)).backupSnapshot() - verify(vssBackupClient).putObject(eq(BackupCategory.WALLET.name), any()) - assertFalse(backupStatuses.value.getValue(BackupCategory.WALLET).isRequired) + @Test + fun `concurrent lifecycle and scheduling drains with a real cache store`() = test { + stubBackupObservers() + val directory = Files.createTempDirectory("backup-lifecycle-test").toFile() + val dispatcher = Executors.newFixedThreadPool(2).asCoroutineDispatcher() + val dataJob = SupervisorJob() + val store = DataStoreFactory.create( + serializer = AppCacheSerializer, + scope = CoroutineScope(dispatcher + dataJob), + produceFile = { directory.resolve("cache.json") }, + ) + val repo = createSut(backupCache = CacheStore(store), dispatcher = dispatcher) + val setupCalls = AtomicInteger() + whenever { vssBackupClient.setupWithRetry(any(), any(), any()) }.doSuspendableAnswer { + setupCalls.incrementAndGet() + Result.success(Unit) + } + try { + withContext(dispatcher) { + withTimeout(10.seconds) { + val start = CompletableDeferred() + val tasks = listOf( + async { + start.await() + repeat(25) { + repo.startObservingBackups() + repo.scheduleFullBackup() + yield() + } + }, + async { + start.await() + repeat(25) { + repo.stopObservingBackups() + yield() + } + }, + ) + start.complete(Unit) + tasks.awaitAll() + repo.reset() + val beforeRestart = setupCalls.get() + val restarted = CompletableDeferred() + whenever { vssBackupClient.setupWithRetry(any(), any(), any()) }.doSuspendableAnswer { + setupCalls.incrementAndGet() + restarted.complete(Unit) + Result.success(Unit) + } + repeat(25) { repo.startObservingBackups() } + restarted.await() + repo.reset() + assertEquals(beforeRestart + 1, setupCalls.get()) + } + } } finally { - sut.stopObservingBackups() + repo.reset() + dataJob.cancelAndJoin() + dispatcher.close() + directory.deleteRecursively() } } @@ -2749,21 +4495,20 @@ class BackupRepoTest : BaseUnitTest() { val finishSetup = CompletableDeferred() val finishCleanup = CompletableDeferred() var cleanupEntered = false + var startupUpdates = 0 whenever(vssBackupClient.setupWithRetry(any(), any(), any())).doSuspendableAnswer { finishSetup.await() Result.success(Unit) } whenever(cacheStore.backupStatuses).thenReturn(statuses) - whenever(cacheStore.updateBackupStatus(any(), any())).doSuspendableAnswer { - val updatedCategory = it.getArgument(0) - val transform = it.getArgument<(BackupItemStatus) -> BackupItemStatus>(1) - if (updatedCategory == category && !cleanupEntered) { + whenever(cacheStore.update(any())).doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + // Hold this category's startup update before its persisted-state transform runs. + if (++startupUpdates == BackupCategory.entries.indexOf(category) + 1) { cleanupEntered = true finishCleanup.await() } - statuses.update { current -> - current + (updatedCategory to transform(current[updatedCategory] ?: BackupItemStatus())) - } + statuses.update { current -> transform(AppCacheData(backupStatuses = current)).backupStatuses } } stubBackupObservers() try { @@ -2902,6 +4647,57 @@ class BackupRepoTest : BaseUnitTest() { } } + @Test + fun `full backup follows newer required data after observers coalesce`() = test { + whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(3_000)) + val backupStatuses = MutableStateFlow( + mapOf( + BackupCategory.SETTINGS to BackupItemStatus( + synced = 1_000, + required = 2_000, + ), + ) + ) + var uploadAttempts = 0 + whenever { vssBackupClient.putObject(eq(BackupCategory.SETTINGS.name), any()) } + .doSuspendableAnswer { + uploadAttempts += 1 + if (uploadAttempts == 1) { + backupStatuses.update { statuses -> + val status = statuses.getValue(BackupCategory.SETTINGS) + statuses + (BackupCategory.SETTINGS to status.copy(required = 4_000)) + } + whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(5_000)) + Result.success(VssItem(key = BackupCategory.SETTINGS.name, value = byteArrayOf(), version = 1)) + } else { + Result.success( + VssItem( + key = BackupCategory.SETTINGS.name, + value = byteArrayOf(), + version = 1, + ) + ) + } + } + val allowWalletClear = CompletableDeferred().apply { complete(Unit) } + stubBackupStatuses(backupStatuses, allowWalletClear) {} + stubBackupObservers() + + try { + sut.scheduleFullBackup() + sut.startObservingBackups() + runCurrent() + advanceTimeBy(5_000) + runCurrent() + advanceTimeBy(5_000) + runCurrent() + + verify(vssBackupClient, times(2)).putObject(eq(BackupCategory.SETTINGS.name), any()) + } finally { + sut.stopObservingBackups() + } + } + @Test fun `failed automatic backup retries newer required timestamp`() = test { whenever(clock.now()).thenReturn(Instant.fromEpochMilliseconds(3_000)) @@ -3495,6 +5291,83 @@ class BackupRepoTest : BaseUnitTest() { } } + @Test + fun `full activity backup survives observer stop while an unchanged result is being recorded`() = test { + assertFullActivityBackupDuringStatusWrite(afterTransform = false) + } + + @Test + fun `full activity backup survives observer stop before a clean status is published`() = test { + assertFullActivityBackupDuringStatusWrite(afterTransform = true) + } + + private suspend fun TestScope.assertFullActivityBackupDuringStatusWrite(afterTransform: Boolean) { + val category = BackupCategory.ACTIVITY + val changes = MutableStateFlow(0L) + val recording = CompletableDeferred() + val resume = CompletableDeferred() + val storeLock = Mutex() + val statusesBeforePut = mutableListOf() + var fullRequested = false + var uploaded = false + var captured = false + stubBackupObservers() + stubActivityBackupReads() + whenever(activityRepo.activitiesChanged).thenReturn(changes) + + try { + sut.startObservingBackups() + runCurrent() + sut.triggerBackup(category).getOrThrow() + whenever(vssBackupClient.putObject(eq(category.name), any())).doSuspendableAnswer { + uploaded = true + Result.success(VssItem(key = category.name, value = it.getArgument(1), version = 2)) + } + whenever(activityRepo.getClosedChannels()).doSuspendableAnswer { + captured = true + Result.success(emptyList()) + } + whenever(cacheStore.update(any())).doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + storeLock.withLock { + if (captured && !recording.isCompleted) { + val updated = if (afterTransform) transform(cacheData.value) else null + recording.complete(Unit) + resume.await() + cacheData.value = updated ?: transform(cacheData.value) + } else { + cacheData.update(transform) + } + if (fullRequested && !uploaded) { + statusesBeforePut += cacheData.value.backupStatuses.getValue(category) + } + } + } + changes.update { it + 1 } + advanceTimeBy(5_000) + runCurrent() + assertTrue(recording.isCompleted) + verify(vssBackupClient).putObject(eq(category.name), any()) + + fullRequested = true + sut.scheduleFullBackup() + sut.stopObservingBackups() + resume.complete(Unit) + advanceTimeBy(5_000) + runCurrent() + + verify(vssBackupClient, times(2)).putObject(eq(category.name), any()) + assertFalse(cacheData.value.backupStatuses.getValue(category).isRequired) + if (!afterTransform) { + assertTrue(statusesBeforePut.isNotEmpty()) + assertTrue(statusesBeforePut.all { it.isRequired }, "Full request lost: $statusesBeforePut") + } + } finally { + resume.complete(Unit) + sut.stopObservingBackups() + } + } + @Test fun `a new wallet owner uploads activity even if its payload matches the previous owner`() = test { val category = BackupCategory.ACTIVITY @@ -3751,7 +5624,7 @@ class BackupRepoTest : BaseUnitTest() { holdDownload = false val restartedReservations = createReservationRepo(reservationStore) - sut = createSut(restartedReservations) + sut = createSut(reservationRepo = restartedReservations) sut.setRestorePending(false) assertTrue(sut.performFullRestoreFromLatestBackup().isSuccess) @@ -3853,7 +5726,7 @@ class BackupRepoTest : BaseUnitTest() { val transform = it.getArgument<(PrivatePaykitReservationData) -> PrivatePaykitReservationData>(0) reservationData.update(transform) } - sut = createSut(createReservationRepo(reservationStore)) + sut = createSut(reservationRepo = createReservationRepo(reservationStore)) return reservationStore } @@ -4032,17 +5905,18 @@ class BackupRepoTest : BaseUnitTest() { onWalletClearDelayed: () -> Unit, ) { whenever(cacheStore.backupStatuses).thenReturn(backupStatuses) - whenever { cacheStore.updateBackupStatus(any(), any()) }.doSuspendableAnswer { - val category = it.getArgument(0) - val transform = it.getArgument<(BackupItemStatus) -> BackupItemStatus>(1) - if (category == BackupCategory.WALLET && !allowWalletClear.isCompleted) { - onWalletClearDelayed() - allowWalletClear.await() - } - backupStatuses.update { statuses -> - val currentStatus = statuses[category] ?: BackupItemStatus() - statuses + (category to transform(currentStatus)) + whenever { cacheStore.update(any()) }.doSuspendableAnswer { + val transform = it.getArgument<(AppCacheData) -> AppCacheData>(0) + if (!allowWalletClear.isCompleted) { + val current = AppCacheData(backupStatuses = backupStatuses.value) + if (transform(current).backupStatuses[BackupCategory.WALLET] != + current.backupStatuses[BackupCategory.WALLET] + ) { + onWalletClearDelayed() + allowWalletClear.await() + } } + backupStatuses.update { statuses -> transform(AppCacheData(backupStatuses = statuses)).backupStatuses } } } @@ -4079,14 +5953,14 @@ class BackupRepoTest : BaseUnitTest() { } private fun createSut( + backupCache: CacheStore = cacheStore, + dispatcher: CoroutineDispatcher = testDispatcher, reservationRepo: PrivatePaykitAddressReservationRepo = privatePaykitAddressReservationRepo, - ioDispatcher: CoroutineDispatcher = testDispatcher, ) = BackupRepo( context = context, - ioDispatcher = ioDispatcher, - cacheStore = cacheStore, + ioDispatcher = dispatcher, + cacheStore = backupCache, vssBackupClient = vssBackupClient, - vssClients = vssClients, vssBackupClientLdk = vssBackupClientLdk, settingsStore = settingsStore, widgetsStore = widgetsStore, @@ -4117,6 +5991,7 @@ class BackupRepoTest : BaseUnitTest() { clock = clock, db = db, connectionRecovery = connectionRecoveryFixture(settingsStore, keychain, vssClients), + vssClients = vssClients, ) private class BackupRepoTestError(message: String) : AppError(message) diff --git a/app/src/test/java/to/bitkit/repositories/PubkyRepoTest.kt b/app/src/test/java/to/bitkit/repositories/PubkyRepoTest.kt index 4b0f5cf4d2..5b5383a3ed 100644 --- a/app/src/test/java/to/bitkit/repositories/PubkyRepoTest.kt +++ b/app/src/test/java/to/bitkit/repositories/PubkyRepoTest.kt @@ -1552,6 +1552,26 @@ class PubkyRepoTest : BaseUnitTest() { verifyBlocking(pubkyService) { deletePubkyProfile("revision-Test", VALID_SELF_KEY) } } + @Test + fun `pubky app deletion does not renew the session or replay an identity error`() = test { + val secretKey = "local_secret" + authenticateForTesting(publicKey = VALID_SELF_KEY) + whenever(keychain.loadString(Keychain.Key.PUBKY_SECRET_KEY.name)).thenReturn(secretKey) + whenever(pubkyService.publicKeyFromSecret(secretKey)).thenReturn(VALID_SELF_KEY.removePrefix("pubky")) + val identityError = PaykitException.Identity("identity_error", "Missing capabilities") + whenever(pubkyService.deletePubkyProfile("revision-Test", VALID_SELF_KEY)) + .thenAnswer { throw identityError } + .thenReturn(Unit) + + val result = sut.deleteProfile() + + assertSame(identityError, result.exceptionOrNull()) + verify(pubkyService).deletePubkyProfile("revision-Test", VALID_SELF_KEY) + verify(pubkyService, never()).signIn(any()) + verify(pubkyService, never()).deletePaykitProfile() + verify(pubkyService, never()).signOut() + } + @Test fun `deleteProfile is not followed by a profile copy write from a load under way`() = test { authenticateForTesting(publicKey = VALID_SELF_KEY) @@ -2259,36 +2279,46 @@ class PubkyRepoTest : BaseUnitTest() { } @Test - fun `deleteProfileWithSessionRetry should refresh session and retry delete`() = test { - val expiredSession = "expired_session" - val newSession = "new_session" + fun `profile deletion does not replay after an uncertain response`() = test { val secretKey = "local_secret" - authenticateForTesting(publicKey = VALID_SELF_KEY, secret = expiredSession) - whenever(keychain.loadString(Keychain.Key.PAYKIT_SESSION.name)).thenReturn(expiredSession, newSession) + authenticateForTesting(publicKey = VALID_SELF_KEY, secret = "test_session") whenever(keychain.loadString(Keychain.Key.PUBKY_SECRET_KEY.name)).thenReturn(secretKey) - whenever(pubkyService.deletePaykitProfile()).thenAnswer { throw TestAppError("Expired") }.thenReturn(Unit) - whenever(pubkyService.signIn(secretKey)).thenReturn(Unit) whenever(pubkyService.publicKeyFromSecret(secretKey)).thenReturn(VALID_SELF_KEY.removePrefix("pubky")) + val uncertainResponse = TestAppError("Connection closed after profile deletion") + whenever(pubkyService.deletePaykitProfile()).thenAnswer { throw uncertainResponse }.thenReturn(Unit) - val result = sut.deleteProfileWithSessionRetry() + val result = sut.deleteProfile() - assertTrue(result.isSuccess) - verifyBlocking(pubkyService, times(2)) { deletePaykitProfile() } + assertSame(uncertainResponse, result.exceptionOrNull()) + verify(pubkyService).deletePaykitProfile() + verify(pubkyService, never()).signIn(any()) + verify(pubkyService, never()).signOut() } @Test - fun `deleteProfileWithSessionRetry should return failure when session cannot refresh`() = test { - val expiredSession = "expired_session" - authenticateForTesting(publicKey = VALID_SELF_KEY, secret = expiredSession) - whenever(keychain.loadString(Keychain.Key.PAYKIT_SESSION.name)).thenReturn(expiredSession) - whenever(keychain.loadString(Keychain.Key.PUBKY_SECRET_KEY.name)).thenReturn(null) - whenever(pubkyService.deletePaykitProfile()).thenAnswer { throw TestAppError("Expired") } - - val result = sut.deleteProfileWithSessionRetry() + fun `failed profile deletion does not restore an identity after sign out`() = test { + val secretKey = "local_secret" + authenticateForTesting(publicKey = VALID_SELF_KEY, secret = "test_session") + whenever(keychain.loadString(Keychain.Key.PUBKY_SECRET_KEY.name)).thenReturn(secretKey) + whenever(pubkyService.publicKeyFromSecret(secretKey)).thenReturn(VALID_SELF_KEY.removePrefix("pubky")) + val deletionStarted = CompletableDeferred() + val finishDeletion = CompletableDeferred() + val uncertainResponse = TestAppError("Connection closed after profile deletion") + whenever(pubkyService.deletePaykitProfile()).doSuspendableAnswer { + deletionStarted.complete(Unit) + withContext(NonCancellable) { finishDeletion.await() } + throw uncertainResponse + } + val deletion = async { sut.deleteProfile() } + deletionStarted.await() + sut.signOut().getOrThrow() + finishDeletion.complete(Unit) - assertTrue(result.isFailure) - verifyBlocking(pubkyService) { deletePaykitProfile() } - verifyBlocking(pubkyService, never()) { signIn(any()) } + assertSame(uncertainResponse, deletion.await().exceptionOrNull()) + assertNull(sut.publicKey.value) + assertFalse(sut.isAuthenticated.value) + verify(pubkyService, never()).signIn(any()) + verify(pubkyService).deletePaykitProfile() } @Test @@ -4558,31 +4588,6 @@ class PubkyRepoTest : BaseUnitTest() { assertFalse(profileSetupPending.value) } - @Test - fun `refreshSessionIfPossible should refresh session when local secret key exists`() = test { - val secretKey = "local_secret" - whenever(keychain.loadString(Keychain.Key.PUBKY_SECRET_KEY.name)).thenReturn(secretKey) - whenever(pubkyService.signIn(secretKey)).thenReturn(Unit) - whenever(pubkyService.publicKeyFromSecret(secretKey)).thenReturn(VALID_SELF_KEY.removePrefix("pubky")) - - val result = sut.refreshSessionIfPossible() - - assertEquals(true, result.getOrNull()) - assertEquals(VALID_SELF_KEY, sut.publicKey.value) - assertTrue(sut.isAuthenticated.value) - } - - @Test - fun `refreshSessionIfPossible should return false when local secret key is missing`() = test { - whenever(keychain.loadString(Keychain.Key.PUBKY_SECRET_KEY.name)).thenReturn(null) - - val result = sut.refreshSessionIfPossible() - - assertEquals(false, result.getOrNull()) - assertNull(sut.publicKey.value) - assertFalse(sut.isAuthenticated.value) - } - @Test fun `restoreSessionBackupState should derive local secret key for local seed backups`() = test { whenever(keychain.loadString(Keychain.Key.BIP39_MNEMONIC.name)).thenReturn("test mnemonic") @@ -5486,7 +5491,7 @@ class PubkyRepoTest : BaseUnitTest() { } @Test - fun `a contact edit started before the session of its identity is refreshed still saves`() = test { + fun `a contact edit started before the session of its identity is restored still saves`() = test { val store = stubGatedPubkyStore() authenticateForTesting(publicKey = VALID_SELF_KEY) whenever(keychain.loadString(Keychain.Key.PUBKY_SECRET_KEY.name)).thenReturn("local_secret") @@ -5494,7 +5499,7 @@ class PubkyRepoTest : BaseUnitTest() { whenever(pubkyService.publicKeyFromSecret("local_secret")).thenReturn(VALID_SELF_KEY.removePrefix("pubky")) val signIn = checkNotNull(sut.currentSignIn()) - assertEquals(true, sut.refreshSessionIfPossible().getOrNull()) + sut.initialize().getOrThrow() val result = sut.updateContact(signIn, VALID_CONTACT_KEY_A, "Alice", "", null, emptyList(), listOf("Friend")) assertTrue(result.isSuccess) diff --git a/app/src/test/java/to/bitkit/ui/screens/profile/EditProfileViewModelTest.kt b/app/src/test/java/to/bitkit/ui/screens/profile/EditProfileViewModelTest.kt index 83107c23de..61bad7ecd1 100644 --- a/app/src/test/java/to/bitkit/ui/screens/profile/EditProfileViewModelTest.kt +++ b/app/src/test/java/to/bitkit/ui/screens/profile/EditProfileViewModelTest.kt @@ -62,7 +62,7 @@ class EditProfileViewModelTest : BaseUnitTest() { sut.showDeleteConfirmation() advanceUntilIdle() verify(pubkyRepo, times(1)).forgetUnrestoredIdentity() - verify(pubkyRepo, never()).deleteProfileWithSessionRetry() + verify(pubkyRepo, never()).deleteProfile() verify(pubkyRepo, never()).saveProfile(any(), any(), any(), any(), anyOrNull(), any()) assertFalse(sut.uiState.value.showDeleteDialog) @@ -115,7 +115,7 @@ class EditProfileViewModelTest : BaseUnitTest() { val sut = createSut() advanceUntilIdle() val result = CompletableDeferred>() - whenever(pubkyRepo.deleteProfileWithSessionRetry()).doSuspendableAnswer { result.await() } + whenever(pubkyRepo.deleteProfile()).doSuspendableAnswer { result.await() } sut.showDeleteConfirmation() sut.deleteProfile() @@ -131,7 +131,7 @@ class EditProfileViewModelTest : BaseUnitTest() { sut.disconnectProfile() sut.showDeleteConfirmation() advanceUntilIdle() - verify(pubkyRepo, times(1)).deleteProfileWithSessionRetry() + verify(pubkyRepo, times(1)).deleteProfile() verify(pubkyRepo, never()).saveProfile(any(), any(), any(), any(), anyOrNull(), any()) verify(pubkyRepo, never()).signOut() assertFalse(sut.uiState.value.showDeleteDialog) @@ -165,7 +165,7 @@ class EditProfileViewModelTest : BaseUnitTest() { sut.showDeleteConfirmation() advanceUntilIdle() verify(pubkyRepo, times(1)).saveProfile(any(), any(), any(), any(), anyOrNull(), any()) - verify(pubkyRepo, never()).deleteProfileWithSessionRetry() + verify(pubkyRepo, never()).deleteProfile() verify(pubkyRepo, never()).forgetUnrestoredIdentity() assertFalse(sut.uiState.value.showDeleteDialog) @@ -187,7 +187,7 @@ class EditProfileViewModelTest : BaseUnitTest() { @Test fun `deleteProfile should emit success when repository delete succeeds`() = test { - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn(Result.success(Unit)) + whenever(pubkyRepo.deleteProfile()).thenReturn(Result.success(Unit)) val sut = createSut() advanceUntilIdle() @@ -201,14 +201,14 @@ class EditProfileViewModelTest : BaseUnitTest() { assertFalse(sut.uiState.value.showDeleteFailureDialog) inOrder(privatePaykitRepo, pubkyRepo).apply { verify(privatePaykitRepo).removePublishedEndpointsForCleanup(any(), deferAppSync = eq(true)) - verify(pubkyRepo).deleteProfileWithSessionRetry() + verify(pubkyRepo).deleteProfile() verify(privatePaykitRepo).closeAndClear() } } @Test fun `profile created in Bitkit keeps delete copy`() = test { - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn(Result.success(Unit)) + whenever(pubkyRepo.deleteProfile()).thenReturn(Result.success(Unit)) val sut = createSut() advanceUntilIdle() @@ -222,7 +222,7 @@ class EditProfileViewModelTest : BaseUnitTest() { @Test fun `ring identity is unlinked with the same cleanup`() = test { - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn(Result.success(Unit)) + whenever(pubkyRepo.deleteProfile()).thenReturn(Result.success(Unit)) val sut = createSut(isRingIdentity = true) advanceUntilIdle() @@ -238,7 +238,7 @@ class EditProfileViewModelTest : BaseUnitTest() { verify(context, never()).getString(R.string.profile__delete_success) inOrder(privatePaykitRepo, pubkyRepo).apply { verify(privatePaykitRepo).removePublishedEndpointsForCleanup(any(), deferAppSync = eq(true)) - verify(pubkyRepo).deleteProfileWithSessionRetry() + verify(pubkyRepo).deleteProfile() verify(privatePaykitRepo).closeAndClear() } } @@ -260,7 +260,7 @@ class EditProfileViewModelTest : BaseUnitTest() { @Test fun `retryDeleteProfile should emit success when repository delete succeeds`() = test { - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn(Result.success(Unit)) + whenever(pubkyRepo.deleteProfile()).thenReturn(Result.success(Unit)) val sut = createSut() advanceUntilIdle() @@ -274,14 +274,14 @@ class EditProfileViewModelTest : BaseUnitTest() { assertFalse(sut.uiState.value.showDeleteFailureDialog) inOrder(privatePaykitRepo, pubkyRepo).apply { verify(privatePaykitRepo).removePublishedEndpointsForCleanup(any(), deferAppSync = eq(true)) - verify(pubkyRepo).deleteProfileWithSessionRetry() + verify(pubkyRepo).deleteProfile() verify(privatePaykitRepo).closeAndClear() } } @Test fun `deleteProfile should show retry dialog when delete still fails`() = test { - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn( + whenever(pubkyRepo.deleteProfile()).thenReturn( Result.failure(TestAppError("expired session")), ) @@ -299,7 +299,7 @@ class EditProfileViewModelTest : BaseUnitTest() { @Test fun `disconnectProfile should emit disconnect success`() = test { - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn( + whenever(pubkyRepo.deleteProfile()).thenReturn( Result.failure(TestAppError("expired session")), ) whenever(pubkyRepo.signOut()).thenReturn(Result.success(Unit)) @@ -358,7 +358,7 @@ class EditProfileViewModelTest : BaseUnitTest() { val sut = createSut() whenever { privatePaykitRepo.removePublishedEndpointsForCleanup(any(), deferAppSync = eq(true)) } .thenReturn(Result.failure(TestAppError("cleanup failed"))) - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn(Result.success(Unit)) + whenever(pubkyRepo.deleteProfile()).thenReturn(Result.success(Unit)) advanceUntilIdle() sut.effects.test { @@ -371,14 +371,14 @@ class EditProfileViewModelTest : BaseUnitTest() { assertFalse(sut.uiState.value.isDeleting) inOrder(privatePaykitRepo, pubkyRepo).apply { verify(privatePaykitRepo).removePublishedEndpointsForCleanup(any(), deferAppSync = eq(true)) - verify(pubkyRepo).deleteProfileWithSessionRetry() + verify(pubkyRepo).deleteProfile() verify(privatePaykitRepo).closeAndClear() } } @Test fun `dismissDeleteFailureDialog should hide retry dialog`() = test { - whenever(pubkyRepo.deleteProfileWithSessionRetry()).thenReturn( + whenever(pubkyRepo.deleteProfile()).thenReturn( Result.failure(TestAppError("expired session")), ) diff --git a/changelog.d/next/1688.fixed.md b/changelog.d/next/1688.fixed.md new file mode 100644 index 0000000000..fb97b9ecd4 --- /dev/null +++ b/changelog.d/next/1688.fixed.md @@ -0,0 +1 @@ +Prevent uncertain profile deletion failures from automatically repeating contact cleanup and deletion. diff --git a/changelog.d/next/1689.fixed.md b/changelog.d/next/1689.fixed.md new file mode 100644 index 0000000000..ce494b30fa --- /dev/null +++ b/changelog.d/next/1689.fixed.md @@ -0,0 +1 @@ +Keep interrupted backups within their wallet lifecycle, retry newer pending data after failures, and let explicit backups recover without waiting for the Lightning node or background delay, while preventing retired requests from acting on a replacement wallet. diff --git a/journeys/backup-restore/wallet-snapshot-consistency.md b/journeys/backup-restore/wallet-snapshot-consistency.md index 6b58ef5e31..4dccf9d071 100644 --- a/journeys/backup-restore/wallet-snapshot-consistency.md +++ b/journeys/backup-restore/wallet-snapshot-consistency.md @@ -25,3 +25,43 @@ funded personal wallet or shorten payment safety waits. Record the tested commit, gate locations, attempts and upload counts, and the restored state. Unit coverage does not establish a device pass or a maximum backup delay during sustained activity. + +## Backup observation shutdown + +The following controls require the same dedicated lifecycle/capture fixture; +these are not ordinary UI journey capabilities. Use isolated fixture accounts; +do not reset a funded wallet. + +1. Hold a scheduled backup in its cancellable debounce, stop observation, and + request a manual metadata backup while cancellation cleanup remains held. + Verify a same-category manual upload and observation restart wait for cleanup. + Change synchronized settings after requesting restart while cleanup is still + held. Release cleanup; verify those settings and pending wallet work upload + without another edit, and no upload or client setup ran before the drain. + Repeat with a retained temporary-failure retry for another category becoming + due during the wait; that automatic retry must wait too. +2. Request a new USDT recovery backup during cleanup and again after observation + is stopped. Verify both can finish without restarting Lightning, and transient + backup failures retain bounded retry without repeating the payment. +3. Repeatedly start and immediately stop observation. Change a wallet record only + after the stopped listeners finish; verify no new automatic upload starts. + Restart observation and verify a single set of listeners resumes pending work. +4. Hold setup, upload or final status persistence, then request fixture wallet + reset. Verify both VSS clients remain unchanged until old work finishes, even + if the reset caller is canceled. Once reset returns, simulate failed node stop + and resume observation without clearing the cache; pending work must recover. +5. Keep wallet wipe admission closed after reset returns. Request manual retry, + full backup and USDT persistence; all must avoid new uploads/status changes. + After wipe finishes, new explicit backups work with observation stopped. +6. Change required data during an explicit full backup while observation runs. + Verify the new revision uploads after the first job completes without another + mutation. + +Android stop invalidates observation immediately; reset awaits all retained work, +and restart awaits prior observer cleanup. iOS awaits its asynchronous shutdown +boundary. These platform differences do not make observation a prerequisite for +explicit USDT persistence. Component tests cover controlled cancellation, the real +SDK operation barrier, and concurrent calls with a real DataStore. They do not +establish a complete device fault journey or a deployed Paykit/VSS outage pass. + +For the consumption-to-status interval, begin with synchronized settings, change the currency, and hold the required-status cache write. Stop observation or request a fixture reset that leaves the data intact, then release the write and restart without another edit. The changed settings must upload once and become synchronized. Reset must drain the write before a later cache clear; an acknowledged upload need not repeat. diff --git a/journeys/profile/delete-profile-uncertain-response.xml b/journeys/profile/delete-profile-uncertain-response.xml new file mode 100644 index 0000000000..fe825c36ff --- /dev/null +++ b/journeys/profile/delete-profile-uncertain-response.xml @@ -0,0 +1,17 @@ + + + Requires a disposable test identity and a controlled homeserver proxy; never use a personal profile. + Configure the proxy to accept one profile deletion but lose its acknowledgement, retaining a redacted + request counter. This is a fault-injection fixture requirement, not evidence that this journey has run. + Normal SDK recovery may retry an individual request only after a confirmed authentication rejection; + the app must not restart contact cleanup and profile deletion because a request has an uncertain outcome. + + + Open Edit Profile and confirm deletion + Drop the acknowledgement after the server accepts the profile deletion + Verify failure options are shown and no success notification or onboarding navigation occurs + Wait through the network timeout and verify the app starts no second contact cleanup, profile deletion or sign-in + Cancel the failure dialog and verify no mutation is retried + Restart the app without choosing Retry and verify it does not automatically repeat the deletion + +