From cc4ff1dc949eb1465eec7b7402c063c7e390e261 Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Thu, 10 Jul 2025 22:58:04 +0530 Subject: [PATCH 01/10] wip --- executors/src/eoa/store.rs | 277 ++++++++++++++++++++++++++++- executors/src/eoa/worker.rs | 253 +++++++++++++------------- server/src/execution_router/mod.rs | 7 +- 3 files changed, 402 insertions(+), 135 deletions(-) diff --git a/executors/src/eoa/store.rs b/executors/src/eoa/store.rs index d377300..ff343fa 100644 --- a/executors/src/eoa/store.rs +++ b/executors/src/eoa/store.rs @@ -403,6 +403,22 @@ impl EoaExecutorStore { None => format!("eoa_executor:health:{chain_id}:{eoa}"), } } + + /// Name of the sorted set for completed transactions per EOA (for pruning) + fn completed_transactions_per_eoa_key_name(&self, eoa: Address, chain_id: u64) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:completed:{chain_id}:{eoa}"), + None => format!("eoa_executor:completed:{chain_id}:{eoa}"), + } + } + + /// Name of the sorted set for completed transactions globally (for pruning) + fn completed_transactions_global_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:completed_global"), + None => "eoa_executor:completed_global".to_string(), + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1417,6 +1433,8 @@ impl EoaExecutorStore { let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); let tx_data_key = self.transaction_data_key_name(transaction_id); + let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); + let completed_global_key = self.completed_transactions_global_key_name(); let now = chrono::Utc::now().timestamp_millis().max(0) as u64; // Remove this hash:id from submitted @@ -1430,6 +1448,10 @@ impl EoaExecutorStore { pipeline.hset(&tx_data_key, "completed_at", now); pipeline.hset(&tx_data_key, "receipt", receipt); pipeline.hset(&tx_data_key, "status", "confirmed"); + + // Add to completed transactions tracking for pruning + pipeline.zadd(&completed_eoa_key, transaction_id, now); + pipeline.zadd(&completed_global_key, transaction_id, now); }) .await } @@ -1484,7 +1506,7 @@ impl EoaExecutorStore { eoa: Address, chain_id: u64, worker_id: &str, - failures: Vec, + failures: Vec, ) -> Result<(), TransactionStoreError> { if failures.is_empty() { return Ok(()); @@ -1532,6 +1554,8 @@ impl EoaExecutorStore { self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); + let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); + let completed_global_key = self.completed_transactions_global_key_name(); let now = chrono::Utc::now().timestamp_millis().max(0) as u64; for success in &successes { @@ -1548,6 +1572,10 @@ impl EoaExecutorStore { pipeline.hset(&tx_data_key, "completed_at", now); pipeline.hset(&tx_data_key, "receipt", &success.receipt_data); pipeline.hset(&tx_data_key, "status", "confirmed"); + + // Add to completed transactions tracking for pruning + pipeline.zadd(&completed_eoa_key, &success.transaction_id, now); + pipeline.zadd(&completed_global_key, &success.transaction_id, now); } }) .await @@ -1713,6 +1741,197 @@ impl EoaExecutorStore { Ok(()) } + + /// Get count of submitted transactions awaiting confirmation + pub async fn get_submitted_transactions_count( + &self, + eoa: Address, + chain_id: u64, + ) -> Result { + let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); + let mut conn = self.redis.clone(); + + let count: u64 = conn.zcard(&submitted_key).await?; + Ok(count) + } + + /// Fail a transaction that's in the pending queue + pub async fn fail_pending_transaction( + &self, + eoa: Address, + chain_id: u64, + worker_id: &str, + transaction_id: &str, + failure_reason: &str, + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { + let pending_key = self.pending_transactions_list_name(eoa, chain_id); + let tx_data_key = self.transaction_data_key_name(transaction_id); + let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); + let completed_global_key = self.completed_transactions_global_key_name(); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Remove from pending queue + pipeline.lrem(&pending_key, 0, transaction_id); + + // Update transaction data with failure + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", failure_reason); + pipeline.hset(&tx_data_key, "status", "failed"); + + // Add to completed transactions tracking for pruning + pipeline.zadd(&completed_eoa_key, transaction_id, now); + pipeline.zadd(&completed_global_key, transaction_id, now); + }) + .await + } + + /// Fail a transaction that's in the borrowed state (we know the nonce) + pub async fn fail_borrowed_transaction( + &self, + eoa: Address, + chain_id: u64, + worker_id: &str, + transaction_id: &str, + nonce: u64, + failure_reason: &str, + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { + let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); + let tx_data_key = self.transaction_data_key_name(transaction_id); + let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); + let completed_global_key = self.completed_transactions_global_key_name(); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Remove from borrowed state using the known nonce + pipeline.hdel(&borrowed_key, nonce.to_string()); + + // Update transaction data with failure + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", failure_reason); + pipeline.hset(&tx_data_key, "status", "failed"); + + // Add to completed transactions tracking for pruning + pipeline.zadd(&completed_eoa_key, transaction_id, now); + pipeline.zadd(&completed_global_key, transaction_id, now); + }) + .await + } + + /// Prune old completed transactions for a specific EOA if it exceeds the cap + pub async fn prune_completed_transactions_for_eoa( + &self, + eoa: Address, + chain_id: u64, + max_per_eoa: u64, + batch_size: u64, + ) -> Result { + let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); + let completed_global_key = self.completed_transactions_global_key_name(); + let mut conn = self.redis.clone(); + + // Check current count + let current_count: u64 = conn.zcard(&completed_eoa_key).await?; + if current_count <= max_per_eoa { + return Ok(0); // No pruning needed + } + + let to_remove = current_count - max_per_eoa; + let batch_to_remove = to_remove.min(batch_size); + + // Get oldest transactions (lowest scores) + let oldest_transactions: Vec = conn + .zrange(&completed_eoa_key, 0, (batch_to_remove - 1) as isize) + .await?; + + if oldest_transactions.is_empty() { + return Ok(0); + } + + // Remove transaction data and tracking + for transaction_id in &oldest_transactions { + let tx_data_key = self.transaction_data_key_name(transaction_id); + let _: () = conn.del(&tx_data_key).await?; + } + + // Remove from tracking sets + for transaction_id in &oldest_transactions { + let _: () = conn.zrem(&completed_eoa_key, transaction_id).await?; + let _: () = conn.zrem(&completed_global_key, transaction_id).await?; + } + + Ok(oldest_transactions.len() as u64) + } + + /// Prune old completed transactions globally if it exceeds the global cap + pub async fn prune_completed_transactions_globally( + &self, + max_global: u64, + batch_size: u64, + ) -> Result { + let completed_global_key = self.completed_transactions_global_key_name(); + let mut conn = self.redis.clone(); + + // Check current count + let current_count: u64 = conn.zcard(&completed_global_key).await?; + if current_count <= max_global { + return Ok(0); // No pruning needed + } + + let to_remove = current_count - max_global; + let batch_to_remove = to_remove.min(batch_size); + + // Get oldest transactions (lowest scores) + let oldest_transactions: Vec = conn + .zrange(&completed_global_key, 0, (batch_to_remove - 1) as isize) + .await?; + + if oldest_transactions.is_empty() { + return Ok(0); + } + + // Remove transaction data + for transaction_id in &oldest_transactions { + let tx_data_key = self.transaction_data_key_name(transaction_id); + let _: () = conn.del(&tx_data_key).await?; + } + + // Remove from global tracking + for transaction_id in &oldest_transactions { + let _: () = conn.zrem(&completed_global_key, transaction_id).await?; + } + + // Also remove from per-EOA tracking sets (we need to scan for these) + // Note: This is less efficient but necessary to keep consistency + self.remove_transactions_from_all_eoa_tracking(&oldest_transactions) + .await?; + + Ok(oldest_transactions.len() as u64) + } + + /// Helper to remove transactions from all EOA tracking sets + async fn remove_transactions_from_all_eoa_tracking( + &self, + transaction_ids: &[String], + ) -> Result<(), TransactionStoreError> { + let mut conn = self.redis.clone(); + let pattern = match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:completed:*"), + None => "eoa_executor:completed:*".to_string(), + }; + + // Get all EOA-specific completed transaction keys + let keys: Vec = conn.keys(&pattern).await?; + + // Remove transactions from each EOA tracking set + for key in keys { + for transaction_id in transaction_ids { + let _: () = conn.zrem(&key, transaction_id).await?; + } + } + + Ok(()) + } } // Additional error types @@ -1974,7 +2193,7 @@ impl<'a> ScopedEoaExecutorStore<'a> { /// Efficiently batch fail and requeue multiple transactions pub async fn batch_fail_and_requeue_transactions( &self, - failures: Vec, + failures: Vec, ) -> Result<(), TransactionStoreError> { self.store .batch_fail_and_requeue_transactions(self.eoa, self.chain_id, &self.worker_id, failures) @@ -2153,4 +2372,58 @@ impl<'a> ScopedEoaExecutorStore<'a> { ) -> Result, TransactionStoreError> { self.store.get_transaction_data(transaction_id).await } + + /// Get count of submitted transactions awaiting confirmation + pub async fn get_submitted_transactions_count(&self) -> Result { + self.store + .get_submitted_transactions_count(self.eoa, self.chain_id) + .await + } + + /// Fail a transaction that's in the pending queue + pub async fn fail_pending_transaction( + &self, + transaction_id: &str, + failure_reason: &str, + ) -> Result<(), TransactionStoreError> { + self.store + .fail_pending_transaction( + self.eoa, + self.chain_id, + &self.worker_id, + transaction_id, + failure_reason, + ) + .await + } + + /// Fail a transaction that's in the borrowed state (we know the nonce) + pub async fn fail_borrowed_transaction( + &self, + transaction_id: &str, + nonce: u64, + failure_reason: &str, + ) -> Result<(), TransactionStoreError> { + self.store + .fail_borrowed_transaction( + self.eoa, + self.chain_id, + &self.worker_id, + transaction_id, + nonce, + failure_reason, + ) + .await + } + + /// Prune old completed transactions for this EOA if it exceeds the cap + pub async fn prune_completed_transactions( + &self, + max_per_eoa: u64, + batch_size: u64, + ) -> Result { + self.store + .prune_completed_transactions_for_eoa(self.eoa, self.chain_id, max_per_eoa, batch_size) + .await + } } diff --git a/executors/src/eoa/worker.rs b/executors/src/eoa/worker.rs index f4939ba..436d148 100644 --- a/executors/src/eoa/worker.rs +++ b/executors/src/eoa/worker.rs @@ -12,7 +12,7 @@ use engine_core::error::EngineError; use engine_core::signer::AccountSigner; use engine_core::transaction::TransactionTypeData; use engine_core::{ - chain::{Chain, ChainService, RpcCredentials}, + chain::{Chain, ChainService}, credentials::SigningCredential, error::{AlloyRpcErrorToEngineError, RpcErrorKind}, signer::{EoaSigner, EoaSigningOptions}, @@ -47,7 +47,7 @@ const NONCE_STALL_TIMEOUT: u64 = 300_000; // 5 minutes in milliseconds - after t pub struct EoaExecutorWorkerJobData { pub eoa_address: Address, pub chain_id: u64, - pub worker_id: String, + pub noop_signing_credential: SigningCredential, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -83,12 +83,18 @@ pub enum EoaExecutorWorkerError { #[error("Transaction build failed: {message}")] TransactionBuildFailed { message: String }, - #[error("RPC error: {message}")] + #[error("RPC error encountered during generic operation: {message}")] RpcError { message: String, inner_error: EngineError, }, + #[error("Error encountered when broadcasting transaction: {message}")] + TransactionSendError { + message: String, + inner_error: EngineError, + }, + #[error("Signature parsing failed: {message}")] SignatureParsingFailed { message: String }, @@ -237,7 +243,7 @@ struct PreparedTransaction { // ========== CONFIRMATION FLOW DATA STRUCTURES ========== #[derive(Debug, Clone)] -struct PendingTransaction { +struct SubmittedTransaction { nonce: u64, hash: String, transaction_id: String, @@ -252,7 +258,7 @@ struct ConfirmedTransaction { } #[derive(Debug, Clone)] -struct FailedTransaction { +struct ReplacedTransaction { hash: String, transaction_id: String, } @@ -266,7 +272,7 @@ pub struct TransactionSuccess { } #[derive(Debug, Clone)] -pub struct TransactionFailure { +pub struct TransactionReplacement { pub hash: String, pub transaction_id: String, } @@ -329,7 +335,7 @@ where &self.store, data.eoa_address, data.chain_id, - data.worker_id.clone(), + job.lease_token.clone(), ) .await .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; @@ -353,7 +359,7 @@ where self.release_eoa_lock( job.job.data.eoa_address, job.job.data.chain_id, - &job.job.data.worker_id, + &job.lease_token, ) .await; } @@ -368,7 +374,7 @@ where self.release_eoa_lock( job.job.data.eoa_address, job.job.data.chain_id, - &job.job.data.worker_id, + &job.lease_token, ) .await; } @@ -383,7 +389,7 @@ where self.release_eoa_lock( job.job.data.eoa_address, job.job.data.chain_id, - &job.job.data.worker_id, + &job.lease_token, ) .await; } @@ -433,13 +439,17 @@ where .await .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? .len(); + let submitted_count = scoped + .get_submitted_transactions_count() + .await + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; // NACK here is a yield, when you think of the queue as a distributed EOA scheduler - if pending_count > 0 || borrowed_count > 0 || recycled_count > 0 { + if pending_count > 0 || borrowed_count > 0 || recycled_count > 0 || submitted_count > 0 { return Err(EoaExecutorWorkerError::WorkRemaining { message: format!( - "Work remaining: {} pending, {} borrowed, {} recycled", - pending_count, borrowed_count, recycled_count + "Work remaining: {} pending, {} borrowed, {} recycled, {} submitted", + pending_count, borrowed_count, recycled_count, submitted_count ), }) .map_err_nack(Some(Duration::from_secs(2)), RequeuePosition::Last); @@ -513,6 +523,7 @@ where // Process results sequentially for Redis state changes let mut recovered_count = 0; + // TODO: both borrowed -> submitted and borrowed -> recycled need to be batched instead of sequential for (borrowed, send_result) in rebroadcast_results { let nonce = borrowed.signed_transaction.nonce(); @@ -520,11 +531,7 @@ where Ok(_) => { // Transaction was sent successfully scoped - .move_borrowed_to_submitted( - nonce, - &format!("{:?}", borrowed.hash), - &borrowed.transaction_id, - ) + .move_borrowed_to_submitted(nonce, &borrowed.hash, &borrowed.transaction_id) .await?; tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted"); } @@ -535,7 +542,7 @@ where scoped .move_borrowed_to_submitted( nonce, - &format!("{:?}", borrowed.hash), + &borrowed.hash, &borrowed.transaction_id, ) .await?; @@ -603,6 +610,8 @@ where Ok(cached_nonce) => cached_nonce, }; + let submitted_count = scoped.get_submitted_transactions_count().await?; + // no nonce progress if current_chain_nonce == cached_nonce { let current_health = self.get_eoa_health(scoped, chain).await?; @@ -610,7 +619,8 @@ where // No nonce progress - check if we should attempt gas bumping for stalled nonce let time_since_movement = now.saturating_sub(current_health.last_nonce_movement_at); - if time_since_movement > NONCE_STALL_TIMEOUT { + // if there are waiting transactions, we can attempt a gas bump + if time_since_movement > NONCE_STALL_TIMEOUT && submitted_count > 0 { tracing::info!( time_since_movement = time_since_movement, stall_timeout = NONCE_STALL_TIMEOUT, @@ -641,18 +651,18 @@ where ); // Get all pending transactions below the current chain nonce - let pending_txs = self - .get_pending_transactions_below_nonce(scoped, current_chain_nonce) + let waiting_txs = self + .get_submitted_transactions_below_nonce(scoped, current_chain_nonce) .await?; - if pending_txs.is_empty() { - tracing::debug!("No pending transactions to confirm"); + if waiting_txs.is_empty() { + tracing::debug!("No waiting transactions to confirm"); return Ok((0, 0)); } // Fetch receipts and categorize transactions - let (confirmed_txs, failed_txs) = self - .fetch_and_categorize_transactions(chain, pending_txs) + let (confirmed_txs, replaced_txs) = self + .fetch_and_categorize_transactions(chain, waiting_txs) .await; // Process confirmed transactions @@ -695,9 +705,9 @@ where 0 }; - // Process failed transactions - let failed_count = if !failed_txs.is_empty() { - let failures: Vec = failed_txs + // Process replaced transactions + let replaced_count = if !replaced_txs.is_empty() { + let replacements: Vec = replaced_txs .into_iter() .map(|tx| { tracing::warn!( @@ -705,15 +715,17 @@ where hash = %tx.hash, "Transaction failed, requeued" ); - TransactionFailure { + TransactionReplacement { hash: tx.hash, transaction_id: tx.transaction_id, } }) .collect(); - let count = failures.len() as u32; - scoped.batch_fail_and_requeue_transactions(failures).await?; + let count = replacements.len() as u32; + scoped + .batch_fail_and_requeue_transactions(replacements) + .await?; count } else { 0 @@ -725,20 +737,11 @@ where .await?; // Synchronize nonces to ensure consistency - if let Err(e) = self - .store - .synchronize_nonces_with_chain( - scoped.eoa(), - scoped.chain_id(), - scoped.worker_id(), - current_chain_nonce, - ) - .await - { - tracing::warn!(error = %e, "Failed to synchronize nonces with chain"); - } + scoped + .synchronize_nonces_with_chain(current_chain_nonce) + .await?; - Ok((confirmed_count, failed_count)) + Ok((confirmed_count, replaced_count)) } // ========== SEND FLOW ========== @@ -753,30 +756,33 @@ where let now = chrono::Utc::now().timestamp_millis().max(0) as u64; // Update balance if it's stale - if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { - let balance = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get balance: {}", engine_error), - inner_error: engine_error, - } - })?; + // TODO: refactor this, very ugly + if health.balance <= health.balance_threshold { + if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { + let balance = chain + .provider() + .get_balance(scoped.eoa()) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get balance: {}", engine_error), + inner_error: engine_error, + } + })?; - health.balance = balance; - health.balance_fetched_at = now; - scoped.update_health_data(&health).await?; - } + health.balance = balance; + health.balance_fetched_at = now; + scoped.update_health_data(&health).await?; + } - if health.balance <= health.balance_threshold { - tracing::warn!( - "EOA has insufficient balance (<= {} wei), skipping send flow", - health.balance_threshold - ); - return Ok(0); + if health.balance <= health.balance_threshold { + tracing::warn!( + "EOA has insufficient balance (<= {} wei), skipping send flow", + health.balance_threshold + ); + return Ok(0); + } } let mut total_sent = 0; @@ -1178,7 +1184,7 @@ where .map(|(i, prepared)| async move { // Add delay for ordering (except first transaction) if i > 0 { - sleep(Duration::from_millis(50)).await; // 50ms delay between consecutive nonces + sleep(Duration::from_millis(50 * i as u64)).await; // 50ms delay between consecutive nonces } let result = chain @@ -1195,12 +1201,12 @@ where let mut sent_count = 0; for (prepared, send_result) in send_results { match send_result { - Ok(_) => { + Ok(pending) => { // Transaction sent successfully match scoped .move_borrowed_to_submitted( prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), + &pending.tx_hash().to_string(), &prepared.transaction_id, ) .await @@ -1334,13 +1340,14 @@ where scoped: &ScopedEoaExecutorStore<'_>, chain: &impl Chain, nonce: u64, - ) -> Result { + credential: SigningCredential, + ) -> Result { // Create a minimal transaction to consume the recycled nonce // Send 0 ETH to self with minimal gas let eoa = scoped.eoa(); // Build no-op transaction (send 0 to self) - let mut tx_request = AlloyTransactionRequest::default() + let tx_request = AlloyTransactionRequest::default() .with_from(eoa) .with_to(eoa) // Send to self .with_value(U256::ZERO) // Send 0 value @@ -1349,43 +1356,24 @@ where .with_nonce(nonce) .with_gas_limit(21000); // Minimal gas for basic transfer - // Estimate gas to ensure the transaction is valid - match chain.provider().estimate_gas(tx_request.clone()).await { - Ok(gas_limit) => { - tx_request = tx_request.with_gas_limit(gas_limit); - } - Err(e) => { - tracing::warn!( - nonce = nonce, - error = %e, - "Failed to estimate gas for no-op transaction" - ); - return Ok(false); + let tx_request = self.estimate_gas_fees(chain, tx_request).await?; + let built_tx = tx_request.build_typed_tx().map_err(|e| { + EoaExecutorWorkerError::TransactionBuildFailed { + message: format!("Failed to build typed transaction for no-op: {e:?}"), } - } + })?; - // Build typed transaction - let typed_tx = match tx_request.build_typed_tx() { - Ok(tx) => tx, - Err(e) => { - tracing::warn!( - nonce = nonce, - error = ?e, - "Failed to build typed transaction for no-op" - ); - return Ok(false); - } - }; + let tx = self.sign_transaction(eoa, credential, built_tx).await?; - // Get signing credential from health or use default approach - // For no-op transactions, we need to find a valid signing credential - // This is a limitation of the current design - no-op transactions - // need access to signing credentials which are transaction-specific - tracing::warn!( - nonce = nonce, - "No-op transaction requires signing credential access - recycled nonce will remain unconsumed" - ); - Ok(false) + chain + .provider() + .send_tx_envelope(tx.into()) + .await + .map_err(|e| EoaExecutorWorkerError::TransactionSendError { + message: format!("Failed to send no-op transaction: {e:?}"), + inner_error: e.to_engine_error(chain), + }) + .map(|pending| pending.tx_hash().to_string()) } // ========== GAS BUMP METHODS ========== @@ -1473,7 +1461,14 @@ where } }; let bumped_typed_tx = self.apply_gas_bump_to_typed_transaction(typed_tx, 120); // 20% increase - let bumped_tx = match self.sign_transaction(bumped_typed_tx, &tx_data).await { + let bumped_tx = match self + .sign_transaction( + tx_data.user_request.from, + tx_data.user_request.signing_credential, + bumped_typed_tx, + ) + .await + { Ok(tx) => tx, Err(e) => { tracing::warn!( @@ -1593,34 +1588,34 @@ where // ========== CONFIRMATION FLOW HELPERS ========== - /// Get pending transactions below the given nonce - async fn get_pending_transactions_below_nonce( + /// Get submitted transactions below the given nonce + async fn get_submitted_transactions_below_nonce( &self, scoped: &ScopedEoaExecutorStore<'_>, nonce: u64, - ) -> Result, EoaExecutorWorkerError> { - let pending_hashes = scoped.get_hashes_below_nonce(nonce).await?; + ) -> Result, EoaExecutorWorkerError> { + let submitted_hashes = scoped.get_hashes_below_nonce(nonce).await?; - let pending_txs = pending_hashes + let submitted_txs = submitted_hashes .into_iter() - .map(|(nonce, hash, transaction_id)| PendingTransaction { + .map(|(nonce, hash, transaction_id)| SubmittedTransaction { nonce, hash, transaction_id, }) .collect(); - Ok(pending_txs) + Ok(submitted_txs) } - /// Fetch receipts for all pending transactions and categorize them + /// Fetch receipts for all submitted transactions and categorize them async fn fetch_and_categorize_transactions( &self, chain: &impl Chain, - pending_txs: Vec, - ) -> (Vec, Vec) { + submitted_txs: Vec, + ) -> (Vec, Vec) { // Fetch all receipts in parallel - let receipt_futures: Vec<_> = pending_txs + let receipt_futures: Vec<_> = submitted_txs .iter() .filter_map(|tx| match tx.hash.parse::() { Ok(hash_bytes) => Some(async move { @@ -1651,7 +1646,7 @@ where }); } Ok(None) | Err(_) => { - failed_txs.push(FailedTransaction { + failed_txs.push(ReplacedTransaction { hash: tx.hash.clone(), transaction_id: tx.transaction_id.clone(), }); @@ -1855,21 +1850,18 @@ where async fn sign_transaction( &self, + from: Address, + credential: SigningCredential, typed_tx: TypedTransaction, - tx_data: &TransactionData, ) -> Result, EoaExecutorWorkerError> { let signing_options = EoaSigningOptions { - from: tx_data.user_request.from, - chain_id: Some(tx_data.user_request.chain_id), + from, + chain_id: typed_tx.chain_id(), }; let signature = self .eoa_signer - .sign_transaction( - signing_options, - typed_tx.clone(), - tx_data.user_request.signing_credential.clone(), - ) + .sign_transaction(signing_options, typed_tx.clone(), credential) .await .map_err(|engine_error| EoaExecutorWorkerError::SigningError { message: format!("Failed to sign transaction: {}", engine_error), @@ -1892,7 +1884,12 @@ where chain: &impl Chain, ) -> Result, EoaExecutorWorkerError> { let typed_tx = self.build_typed_transaction(tx_data, nonce, chain).await?; - self.sign_transaction(typed_tx, tx_data).await + self.sign_transaction( + tx_data.user_request.from, + tx_data.user_request.signing_credential.clone(), + typed_tx, + ) + .await } fn apply_gas_bump_to_typed_transaction( diff --git a/server/src/execution_router/mod.rs b/server/src/execution_router/mod.rs index 446ea07..b44fb32 100644 --- a/server/src/execution_router/mod.rs +++ b/server/src/execution_router/mod.rs @@ -404,7 +404,7 @@ impl ExecutionRouter { data: transaction.data.clone(), gas_limit: transaction.gas_limit, webhook_options: webhook_options.clone(), - signing_credential, + signing_credential: signing_credential.clone(), rpc_credentials, transaction_type_data: transaction.transaction_type_data.clone(), }; @@ -429,10 +429,7 @@ impl ExecutionRouter { let eoa_job_data = EoaExecutorWorkerJobData { eoa_address: eoa_execution_options.from, chain_id: base_execution_options.chain_id, - worker_id: format!( - "eoa_{}_{}", - eoa_execution_options.from, base_execution_options.chain_id - ), + noop_signing_credential: signing_credential, }; // Create idempotent job for this EOA:chain - only one will exist From 81d56cc9ce081199b7484ad4f2ea17b845d0ec6c Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Sat, 12 Jul 2025 05:49:07 +0530 Subject: [PATCH 02/10] refactor eoa store --- executors/src/eoa/mod.rs | 1 + executors/src/eoa/store.rs | 2429 -------------------------- executors/src/eoa/store/atomic.rs | 812 +++++++++ executors/src/eoa/store/error.rs | 22 + executors/src/eoa/store/mod.rs | 719 ++++++++ executors/src/eoa/store/submitted.rs | 276 +++ executors/src/eoa/worker.rs | 325 ++-- server/src/execution_router/mod.rs | 14 +- server/src/main.rs | 6 +- server/src/queue/manager.rs | 16 +- 10 files changed, 1980 insertions(+), 2640 deletions(-) delete mode 100644 executors/src/eoa/store.rs create mode 100644 executors/src/eoa/store/atomic.rs create mode 100644 executors/src/eoa/store/error.rs create mode 100644 executors/src/eoa/store/mod.rs create mode 100644 executors/src/eoa/store/submitted.rs diff --git a/executors/src/eoa/mod.rs b/executors/src/eoa/mod.rs index c7186f7..9b44404 100644 --- a/executors/src/eoa/mod.rs +++ b/executors/src/eoa/mod.rs @@ -1,6 +1,7 @@ pub mod error_classifier; pub mod store; pub mod worker; + pub use error_classifier::{EoaErrorMapper, EoaExecutionError, RecoveryStrategy}; pub use store::{EoaExecutorStore, EoaTransactionRequest}; pub use worker::{EoaExecutorWorker, EoaExecutorWorkerJobData}; diff --git a/executors/src/eoa/store.rs b/executors/src/eoa/store.rs deleted file mode 100644 index ff343fa..0000000 --- a/executors/src/eoa/store.rs +++ /dev/null @@ -1,2429 +0,0 @@ -use alloy::consensus::{Signed, Transaction, TypedTransaction}; -use alloy::network::AnyTransactionReceipt; -use alloy::primitives::{Address, B256, Bytes, U256}; -use chrono; -use engine_core::chain::RpcCredentials; -use engine_core::credentials::SigningCredential; -use engine_core::execution_options::WebhookOptions; -use engine_core::transaction::TransactionTypeData; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::future::Future; -use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; - -pub trait SafeRedisTransaction: Send + Sync { - fn name(&self) -> &str; - fn operation(&self, pipeline: &mut Pipeline); - fn validation( - &self, - conn: &mut ConnectionManager, - ) -> impl Future> + Send; - fn watch_keys(&self) -> Vec; -} - -struct MovePendingToBorrowedWithRecycledNonce { - recycled_key: String, - pending_key: String, - transaction_id: String, - borrowed_key: String, - nonce: u64, - prepared_tx_json: String, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonce { - fn name(&self) -> &str { - "pending->borrowed with recycled nonce" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove nonce from recycled set (we know it exists) - pipeline.zrem(&self.recycled_key, self.nonce); - // Remove transaction from pending (we know it exists) - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.recycled_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check if nonce exists in recycled set - let nonce_score: Option = conn.zscore(&self.recycled_key, self.nonce).await?; - if nonce_score.is_none() { - return Err(TransactionStoreError::NonceNotInRecycledSet { nonce: self.nonce }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MovePendingToBorrowedWithNewNonce { - optimistic_key: String, - pending_key: String, - nonce: u64, - prepared_tx_json: String, - transaction_id: String, - borrowed_key: String, - eoa: Address, - chain_id: u64, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithNewNonce { - fn name(&self) -> &str { - "pending->borrowed with new nonce" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Increment optimistic nonce - pipeline.incr(&self.optimistic_key, 1); - // Remove transaction from pending - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.optimistic_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check current optimistic nonce - let current_optimistic: Option = conn.get(&self.optimistic_key).await?; - let current_nonce = match current_optimistic { - Some(nonce) => nonce, - None => { - return Err(TransactionStoreError::NonceSyncRequired { - eoa: self.eoa, - chain_id: self.chain_id, - }); - } - }; - - if current_nonce != self.nonce { - return Err(TransactionStoreError::OptimisticNonceChanged { - expected: self.nonce, - actual: current_nonce, - }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MoveBorrowedToSubmitted { - nonce: u64, - hash: String, - transaction_id: String, - borrowed_key: String, - submitted_key: String, - hash_to_id_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToSubmitted { - fn name(&self) -> &str { - "borrowed->submitted" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add to submitted with hash:id format - let hash_id_value = format!("{}:{}", self.hash, self.transaction_id); - pipeline.zadd(&self.submitted_key, &hash_id_value, self.nonce); - - // Still maintain hash-to-ID mapping for backward compatibility and external lookups - pipeline.set(&self.hash_to_id_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - -struct MoveBorrowedToRecycled { - nonce: u64, - transaction_id: String, - borrowed_key: String, - recycled_key: String, - pending_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToRecycled { - fn name(&self) -> &str { - "borrowed->recycled" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add nonce to recycled set (with timestamp as score) - pipeline.zadd(&self.recycled_key, self.nonce, self.nonce); - - // Add transaction back to pending - pipeline.lpush(&self.pending_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - -/// The actual user request data -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct EoaTransactionRequest { - pub transaction_id: String, - pub chain_id: u64, - - pub from: Address, - pub to: Option
, - pub value: U256, - pub data: Bytes, - - #[serde(alias = "gas")] - pub gas_limit: Option, - - pub webhook_options: Option>, - - pub signing_credential: SigningCredential, - pub rpc_credentials: RpcCredentials, - - #[serde(flatten)] - pub transaction_type_data: Option, -} - -/// Active attempt for a transaction (full alloy transaction + metadata) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TransactionAttempt { - pub transaction_id: String, - pub details: Signed, - pub sent_at: u64, // Unix timestamp in milliseconds - pub attempt_number: u32, -} - -/// Transaction data for a transaction_id -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TransactionData { - pub transaction_id: String, - pub user_request: EoaTransactionRequest, - pub receipt: Option, - pub attempts: Vec, -} - -pub struct BorrowedTransaction { - pub transaction_id: String, - pub data: Signed, - pub borrowed_at: chrono::DateTime, -} - -/// Transaction store focused on transaction_id operations and nonce indexing -pub struct EoaExecutorStore { - pub redis: ConnectionManager, - pub namespace: Option, -} - -impl EoaExecutorStore { - pub fn new(redis: ConnectionManager, namespace: Option) -> Self { - Self { redis, namespace } - } - - /// Name of the key for the transaction data - /// - /// Transaction data is stored as a Redis HSET with the following fields: - /// - "user_request": JSON string containing EoaTransactionRequest - /// - "receipt": JSON string containing AnyTransactionReceipt (optional) - /// - "status": String status ("confirmed", "failed", etc.) - /// - "completed_at": String Unix timestamp (optional) - fn transaction_data_key_name(&self, transaction_id: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_data:{transaction_id}"), - None => format!("eoa_executor:_tx_data:{transaction_id}"), - } - } - - /// Name of the list for transaction attempts - /// - /// Attempts are stored as a separate Redis LIST where each element is a JSON blob - /// of a TransactionAttempt. This allows efficient append operations. - fn transaction_attempts_list_name(&self, transaction_id: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_attempts:{transaction_id}"), - None => format!("eoa_executor:tx_attempts:{transaction_id}"), - } - } - - /// Name of the list for pending transactions - fn pending_transactions_list_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:pending_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:pending_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the zset for submitted transactions. nonce -> hash:id - /// Same transaction might appear multiple times in the zset with different nonces/gas prices (and thus different hashes) - fn submitted_transactions_zset_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:submitted_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:submitted_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the key that maps transaction hash to transaction id - fn transaction_hash_to_id_key_name(&self, hash: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_hash_to_id:{hash}"), - None => format!("eoa_executor:tx_hash_to_id:{hash}"), - } - } - - /// Name of the hashmap that maps `transaction_id` -> `BorrowedTransactionData` - /// - /// This is used for crash recovery. Before submitting a transaction, we atomically move from pending to this borrowed hashmap. - /// - /// On worker recovery, if any borrowed transactions are found, we rebroadcast them and move back to pending or submitted - /// - /// If there's no crash, happy path moves borrowed transactions back to pending or submitted - fn borrowed_transactions_hashmap_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:borrowed_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:borrowed_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the set that contains recycled nonces. - /// - /// If a transaction was submitted but failed (ie, we know with certainty it didn't enter the mempool), - /// - /// we add the nonce to this set. - /// - /// These nonces are used with priority, before any other nonces. - fn recycled_nonces_set_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:recycled_nonces:{chain_id}:{eoa}"), - None => format!("eoa_executor:recycled_nonces:{chain_id}:{eoa}"), - } - } - - /// Optimistic nonce key name. - /// - /// This is used for optimistic nonce tracking. - /// - /// We store the nonce of the last successfuly sent transaction for each EOA. - /// - /// We increment this nonce for each new transaction. - /// - /// !IMPORTANT! When sending a transaction, we use this nonce as the assigned nonce, NOT the incremented nonce. - fn optimistic_transaction_count_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:optimistic_nonce:{chain_id}:{eoa}"), - None => format!("eoa_executor:optimistic_nonce:{chain_id}:{eoa}"), - } - } - - /// Name of the key that contains the nonce of the last fetched ONCHAIN transaction count for each EOA. - /// - /// This is a cache for the actual transaction count, which is fetched from the RPC. - /// - /// The nonce for the NEXT transaction is the ONCHAIN transaction count (NOT + 1) - /// - /// Eg: transaction count is 0, so we use nonce 0 for sending the next transaction. Once successful, transaction count will be 1. - fn last_transaction_count_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:last_tx_nonce:{chain_id}:{eoa}"), - None => format!("eoa_executor:last_tx_nonce:{chain_id}:{eoa}"), - } - } - - /// EOA health key name. - /// - /// EOA health stores: - /// - cached balance, the timestamp of the last balance fetch - /// - timestamp of the last successful transaction confirmation - /// - timestamp of the last 5 nonce resets - fn eoa_health_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:health:{chain_id}:{eoa}"), - None => format!("eoa_executor:health:{chain_id}:{eoa}"), - } - } - - /// Name of the sorted set for completed transactions per EOA (for pruning) - fn completed_transactions_per_eoa_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:completed:{chain_id}:{eoa}"), - None => format!("eoa_executor:completed:{chain_id}:{eoa}"), - } - } - - /// Name of the sorted set for completed transactions globally (for pruning) - fn completed_transactions_global_key_name(&self) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:completed_global"), - None => "eoa_executor:completed_global".to_string(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EoaHealth { - pub balance: U256, - /// Update the balance threshold when we see out of funds errors - pub balance_threshold: U256, - pub balance_fetched_at: u64, - pub last_confirmation_at: u64, - pub last_nonce_movement_at: u64, // Track when nonce last moved for gas bump detection - pub nonce_resets: Vec, // Last 5 reset timestamps -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BorrowedTransactionData { - pub transaction_id: String, - pub signed_transaction: Signed, - pub hash: String, - pub borrowed_at: u64, -} - -/// Type of nonce allocation for transaction processing -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum NonceType { - /// Nonce was recycled from a previously failed transaction - Recycled(u64), - /// Nonce was incremented from the current optimistic counter - Incremented(u64), -} - -impl NonceType { - /// Get the nonce value regardless of type - pub fn nonce(&self) -> u64 { - match self { - NonceType::Recycled(nonce) => *nonce, - NonceType::Incremented(nonce) => *nonce, - } - } - - /// Check if this is a recycled nonce - pub fn is_recycled(&self) -> bool { - matches!(self, NonceType::Recycled(_)) - } - - /// Check if this is an incremented nonce - pub fn is_incremented(&self) -> bool { - matches!(self, NonceType::Incremented(_)) - } -} - -impl EoaExecutorStore { - // ========== BOILERPLATE REDUCTION PATTERN ========== - // - // This implementation uses a helper method `execute_with_watch_and_retry` to reduce - // boilerplate in atomic Redis operations. The pattern separates: - // 1. Validation phase: async closure that checks preconditions - // 2. Pipeline phase: sync closure that builds Redis commands - // - // Benefits: - // - Eliminates ~80 lines of boilerplate per method - // - Centralizes retry logic, lock checking, and error handling - // - Makes individual methods focus on business logic - // - Reduces chance of bugs in WATCH/MULTI/EXEC handling - // - // See examples in: - // - atomic_move_pending_to_borrowed_with_recycled_nonce_v2() - // - atomic_move_pending_to_borrowed_with_new_nonce() - // - move_borrowed_to_submitted() - // - move_borrowed_to_recycled() - - /// Aggressively acquire EOA lock, forcefully taking over from stalled workers - pub async fn acquire_eoa_lock_aggressively( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - ) -> Result<(), TransactionStoreError> { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // First try normal acquisition - let acquired: bool = conn.set_nx(&lock_key, worker_id).await?; - if acquired { - return Ok(()); - } - // Lock exists, forcefully take it over - tracing::warn!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Forcefully taking over EOA lock from stalled worker" - ); - // Force set - no expiry, only released by explicit takeover - let _: () = conn.set(&lock_key, worker_id).await?; - Ok(()) - } - - /// Release EOA lock following the spec's finally pattern - pub async fn release_eoa_lock( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - ) -> Result<(), TransactionStoreError> { - // Use existing utility method that handles all the atomic lock checking - match self - .with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - pipeline.del(&lock_key); - }) - .await - { - Ok(()) => { - tracing::debug!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Successfully released EOA lock" - ); - Ok(()) - } - Err(TransactionStoreError::LockLost { .. }) => { - // Lock was already taken over, which is fine for release - tracing::debug!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Lock already released or taken over by another worker" - ); - Ok(()) - } - Err(e) => { - // Other errors shouldn't fail the worker, just log - tracing::warn!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - error = %e, - "Failed to release EOA lock" - ); - Ok(()) - } - } - } - - /// Helper to execute atomic operations with proper retry logic and watch handling - /// - /// This helper centralizes all the boilerplate for WATCH/MULTI/EXEC operations: - /// - Retry logic with exponential backoff - /// - Lock ownership validation - /// - WATCH key management - /// - Error handling and UNWATCH cleanup - /// - /// ## Usage: - /// Implement the `SafeRedisTransaction` trait for your operation, then call this method. - /// The trait separates validation (async) from pipeline operations (sync) for clean patterns. - /// - /// ## Example: - /// ```rust - /// let safe_tx = MovePendingToBorrowedWithNewNonce { - /// nonce: expected_nonce, - /// prepared_tx_json, - /// transaction_id, - /// borrowed_key, - /// optimistic_key, - /// pending_key, - /// eoa, - /// chain_id, - /// }; - /// - /// self.execute_with_watch_and_retry(eoa, chain_id, worker_id, &safe_tx).await?; - /// ``` - /// - /// ## When to use this helper: - /// - Operations that implement `SafeRedisTransaction` trait - /// - Need atomic WATCH/MULTI/EXEC with retry logic - /// - Want centralized lock checking and error handling - /// - /// ## When NOT to use this helper: - /// - Simple operations that can use `with_lock_check` instead - /// - Operations that don't need WATCH on multiple keys - /// - Read-only operations that don't modify state - async fn execute_with_watch_and_retry( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - safe_tx: &impl SafeRedisTransaction, - ) -> Result<(), TransactionStoreError> { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for {} on {}:{}", - MAX_RETRIES, - safe_tx.name(), - eoa, - chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - operation = safe_tx.name(), - "Retrying atomic operation" - ); - } - - // WATCH all specified keys including lock - let mut watch_cmd = twmq::redis::cmd("WATCH"); - watch_cmd.arg(&lock_key); - for key in safe_tx.watch_keys() { - watch_cmd.arg(key); - } - let _: () = watch_cmd.query_async(&mut conn).await?; - - // Check lock ownership - let current_owner: Option = conn.get(&lock_key).await?; - if current_owner.as_deref() != Some(worker_id) { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - - // Execute validation - match safe_tx.validation(&mut conn).await { - Ok(()) => { - // Build and execute pipeline - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - safe_tx.operation(&mut pipeline); - - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(()), // Success - Err(_) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // State changed, retry - retry_count += 1; - continue; - } - } - } - Err(e) => { - // Validation failed, unwatch and return error - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(e); - } - } - } - } - - /// Example of how to refactor a complex method using the helper to reduce boilerplate - /// This shows the pattern for atomic_move_pending_to_borrowed_with_recycled_nonce - pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let safe_tx = MovePendingToBorrowedWithRecycledNonce { - recycled_key: self.recycled_nonces_set_name(eoa, chain_id), - pending_key: self.pending_transactions_list_name(eoa, chain_id), - transaction_id: transaction_id.to_string(), - borrowed_key: self.borrowed_transactions_hashmap_name(eoa, chain_id), - nonce, - prepared_tx_json: serde_json::to_string(prepared_tx)?, - }; - - self.execute_with_watch_and_retry(eoa, chain_id, worker_id, &safe_tx) - .await?; - - Ok(()) - } - - /// Atomically move specific transaction from pending to borrowed with new nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_new_nonce( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - expected_nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let prepared_tx_json = serde_json::to_string(prepared_tx)?; - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MovePendingToBorrowedWithNewNonce { - nonce: expected_nonce, - prepared_tx_json, - transaction_id, - borrowed_key, - optimistic_key, - pending_key, - eoa, - chain_id, - }, - ) - .await - } - - /// Generic helper that handles WATCH + retry logic for atomic operations - /// The operation closure receives a mutable connection and should: - /// 1. Perform any validation (return early errors if needed) - /// 2. Build and execute the pipeline - /// 3. Return the result - pub async fn with_atomic_operation( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - watch_keys: Vec, - operation_name: &str, - operation: F, - ) -> Result - where - F: Fn(&mut ConnectionManager) -> Fut, - Fut: std::future::Future>, - { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for {} on {}:{}", - MAX_RETRIES, operation_name, eoa, chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - operation = operation_name, - "Retrying atomic operation" - ); - } - - // WATCH all specified keys (lock is always included) - let mut watch_cmd = twmq::redis::cmd("WATCH"); - watch_cmd.arg(&lock_key); - for key in &watch_keys { - watch_cmd.arg(key); - } - let _: () = watch_cmd.query_async(&mut conn).await?; - - // Check if we still own the lock - let current_owner: Option = conn.get(&lock_key).await?; - match current_owner { - Some(owner) if owner == worker_id => { - // We still own it, proceed - } - _ => { - // Lost ownership - immediately fail - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - } - - // Execute operation (includes validation and pipeline execution) - match operation(&mut conn).await { - Ok(result) => return Ok(result), - Err(TransactionStoreError::LockLost { .. }) => { - // Lock was lost during operation, propagate immediately - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - Err(TransactionStoreError::WatchFailed) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Our lock is fine, retry - retry_count += 1; - continue; - } - Err(other_error) => { - // Other errors propagate immediately (validation failures, etc.) - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(other_error); - } - } - } - } - - /// Wrapper that executes operations with lock validation using WATCH/MULTI/EXEC - pub async fn with_lock_check( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - operation: F, - ) -> Result - where - F: Fn(&mut Pipeline) -> R, - T: From, - { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for lock check on {}:{}", - MAX_RETRIES, eoa, chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - "Retrying lock check operation" - ); - } - - // WATCH the EOA lock - let _: () = twmq::redis::cmd("WATCH") - .arg(&lock_key) - .query_async(&mut conn) - .await?; - - // Check if we still own the lock - let current_owner: Option = conn.get(&lock_key).await?; - match current_owner { - Some(owner) if owner == worker_id => { - // We still own it, proceed - } - _ => { - // Lost ownership - immediately fail - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - } - - // Build pipeline with operation - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - let result = operation(&mut pipeline); - - // Execute with WATCH protection - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(T::from(result)), - Err(_) => { - // WATCH failed, check if it was our lock or someone else's - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Our lock is fine, someone else's WATCH failed - retry - retry_count += 1; - continue; - } - } - } - } - - // ========== ATOMIC OPERATIONS ========== - - /// Peek all borrowed transactions without removing them - pub async fn peek_borrowed_transactions( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let borrowed_map: HashMap = conn.hgetall(&borrowed_key).await?; - let mut result = Vec::new(); - - for (_nonce_str, transaction_json) in borrowed_map { - let borrowed_data: BorrowedTransactionData = serde_json::from_str(&transaction_json)?; - result.push(borrowed_data); - } - - Ok(result) - } - - /// Atomically move borrowed transaction to submitted state - /// Returns error if transaction not found in borrowed state - pub async fn move_borrowed_to_submitted( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - nonce: u64, - hash: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let hash = hash.to_string(); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MoveBorrowedToSubmitted { - nonce, - hash: hash.to_string(), - transaction_id, - borrowed_key, - submitted_key, - hash_to_id_key, - }, - ) - .await - } - - /// Atomically move borrowed transaction back to recycled nonces and pending queue - /// Returns error if transaction not found in borrowed state - pub async fn move_borrowed_to_recycled( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - nonce: u64, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MoveBorrowedToRecycled { - nonce, - transaction_id, - borrowed_key, - recycled_key, - pending_key, - }, - ) - .await - } - - /// Get all hashes below a certain nonce from submitted transactions - /// Returns (nonce, hash, transaction_id) tuples - pub async fn get_hashes_below_nonce( - &self, - eoa: Address, - chain_id: u64, - below_nonce: u64, - ) -> Result, TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Get all entries with nonce < below_nonce - let results: Vec<(String, u64)> = conn - .zrangebyscore_withscores(&submitted_key, 0, below_nonce - 1) - .await?; - - let mut parsed_results = Vec::new(); - for (hash_id_value, nonce) in results { - // Parse hash:id format - if let Some((hash, transaction_id)) = hash_id_value.split_once(':') { - parsed_results.push((nonce, hash.to_string(), transaction_id.to_string())); - } else { - // Fallback for old format (just hash) - look up transaction ID - if let Some(transaction_id) = - self.get_transaction_id_for_hash(&hash_id_value).await? - { - parsed_results.push((nonce, hash_id_value, transaction_id)); - } - } - } - - Ok(parsed_results) - } - - /// Get all transaction IDs for a specific nonce - pub async fn get_transaction_ids_for_nonce( - &self, - eoa: Address, - chain_id: u64, - nonce: u64, - ) -> Result, TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Get all members with the exact nonce - let members: Vec = conn - .zrangebyscore(&submitted_key, nonce, nonce) - .await - .map_err(|e| TransactionStoreError::RedisError { - message: format!("Failed to get transaction IDs for nonce {}: {}", nonce, e), - })?; - - let mut transaction_ids = Vec::new(); - for value in members { - // Parse the value as hash:id format, with fallback to old format - if let Some((_, transaction_id)) = value.split_once(':') { - // New format: hash:id - transaction_ids.push(transaction_id.to_string()); - } else { - // Old format: just hash - look up transaction ID - if let Some(transaction_id) = self.get_transaction_id_for_hash(&value).await? { - transaction_ids.push(transaction_id); - } - } - } - - Ok(transaction_ids) - } - - /// Remove all hashes for a transaction and requeue it - /// Returns error if no hashes found for this transaction in submitted state - /// NOTE: This method keeps the original boilerplate pattern because it needs to pass - /// complex data (transaction_hashes) from validation to pipeline phase. - /// The helper pattern works best for simple validation that doesn't need to pass data. - pub async fn fail_and_requeue_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for fail and requeue transaction {}:{} tx:{}", - MAX_RETRIES, eoa, chain_id, transaction_id - ), - }); - } - - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - } - - // WATCH lock and submitted state - let _: () = twmq::redis::cmd("WATCH") - .arg(&lock_key) - .arg(&submitted_key) - .query_async(&mut conn) - .await?; - - // Check lock ownership - let current_owner: Option = conn.get(&lock_key).await?; - if current_owner.as_deref() != Some(worker_id) { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - - // Find all hashes for this transaction that actually exist in submitted - let all_hash_id_values: Vec = conn.zrange(&submitted_key, 0, -1).await?; - let mut transaction_hashes = Vec::new(); - - for hash_id_value in all_hash_id_values { - // Parse hash:id format - if let Some((hash, tx_id)) = hash_id_value.split_once(':') { - if tx_id == transaction_id { - transaction_hashes.push(hash.to_string()); - } - } else { - // Fallback for old format (just hash) - look up transaction ID - if let Some(tx_id) = self.get_transaction_id_for_hash(&hash_id_value).await? { - if tx_id == transaction_id { - transaction_hashes.push(hash_id_value); - } - } - } - } - - if transaction_hashes.is_empty() { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::TransactionNotInSubmittedState { - transaction_id: transaction_id.to_string(), - }); - } - - // Transaction has hashes in submitted, proceed with atomic removal and requeue - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - - // Remove all hash:id values for this transaction (we know they exist) - for hash in &transaction_hashes { - // Remove the hash:id value from the zset - let hash_id_value = format!("{}:{}", hash, transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Also remove the separate hash-to-ID mapping for backward compatibility - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - pipeline.del(&hash_to_id_key); - } - - // Add back to pending - pipeline.lpush(&pending_key, transaction_id); - - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(()), // Success - Err(_) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Submitted state changed, retry - retry_count += 1; - continue; - } - } - } - } - - /// Check EOA health (balance, etc.) - pub async fn check_eoa_health( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let health_key = self.eoa_health_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let health_json: Option = conn.get(&health_key).await?; - if let Some(json) = health_json { - let health: EoaHealth = serde_json::from_str(&json)?; - Ok(Some(health)) - } else { - Ok(None) - } - } - - /// Update EOA health data - pub async fn update_health_data( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - health: &EoaHealth, - ) -> Result<(), TransactionStoreError> { - let health_json = serde_json::to_string(health)?; - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, &health_json); - }) - .await - } - - /// Update cached transaction count - pub async fn update_cached_transaction_count( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_count: u64, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - pipeline.set(&tx_count_key, transaction_count); - }) - .await - } - - /// Peek recycled nonces without removing them - pub async fn peek_recycled_nonces( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let nonces: Vec = conn.zrange(&recycled_key, 0, -1).await?; - Ok(nonces) - } - - /// Peek at pending transactions without removing them (safe for planning) - pub async fn peek_pending_transactions( - &self, - eoa: Address, - chain_id: u64, - limit: u64, - ) -> Result, TransactionStoreError> { - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Use LRANGE to peek without removing - let transaction_ids: Vec = - conn.lrange(&pending_key, 0, (limit as isize) - 1).await?; - Ok(transaction_ids) - } - - /// Get inflight budget (how many new transactions can be sent) - pub async fn get_inflight_budget( - &self, - eoa: Address, - chain_id: u64, - max_inflight: u64, - ) -> Result { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let last_tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Read both values atomically to avoid race conditions - let (optimistic_nonce, last_tx_count): (Option, Option) = twmq::redis::pipe() - .get(&optimistic_key) - .get(&last_tx_count_key) - .query_async(&mut conn) - .await?; - - let optimistic = match optimistic_nonce { - Some(nonce) => nonce, - None => return Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - }; - let last_count = match last_tx_count { - Some(count) => count, - None => return Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - }; - - let current_inflight = optimistic.saturating_sub(last_count); - let available_budget = max_inflight.saturating_sub(current_inflight); - - Ok(available_budget) - } - - /// Get current optimistic nonce (without incrementing) - pub async fn get_optimistic_nonce( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let current: Option = conn.get(&optimistic_key).await?; - match current { - Some(nonce) => Ok(nonce), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Lock key name for EOA processing - fn eoa_lock_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:lock:{chain_id}:{eoa}"), - None => format!("eoa_executor:lock:{chain_id}:{eoa}"), - } - } - - /// Get transaction ID for a given hash - pub async fn get_transaction_id_for_hash( - &self, - hash: &str, - ) -> Result, TransactionStoreError> { - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let mut conn = self.redis.clone(); - - let transaction_id: Option = conn.get(&hash_to_id_key).await?; - Ok(transaction_id) - } - - /// Get transaction data by transaction ID - pub async fn get_transaction_data( - &self, - transaction_id: &str, - ) -> Result, TransactionStoreError> { - let tx_data_key = self.transaction_data_key_name(transaction_id); - let mut conn = self.redis.clone(); - - // Get the hash data (the transaction data is stored as a hash) - let hash_data: HashMap = conn.hgetall(&tx_data_key).await?; - - if hash_data.is_empty() { - return Ok(None); - } - - // Extract user_request from the hash data - let user_request_json = hash_data.get("user_request").ok_or_else(|| { - TransactionStoreError::TransactionNotFound { - transaction_id: transaction_id.to_string(), - } - })?; - - let user_request: EoaTransactionRequest = serde_json::from_str(user_request_json)?; - - // Extract receipt if present - let receipt = hash_data - .get("receipt") - .and_then(|receipt_str| serde_json::from_str(receipt_str).ok()); - - // Extract attempts from separate list - let attempts_key = self.transaction_attempts_list_name(transaction_id); - let attempts_json_list: Vec = conn.lrange(&attempts_key, 0, -1).await?; - let mut attempts = Vec::new(); - for attempt_json in attempts_json_list { - if let Ok(attempt) = serde_json::from_str::(&attempt_json) { - attempts.push(attempt); - } - } - - Ok(Some(TransactionData { - transaction_id: transaction_id.to_string(), - user_request, - receipt, - attempts, - })) - } - - /// Mark transaction as successful and remove from submitted - pub async fn succeed_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - hash: &str, - receipt: &str, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let tx_data_key = self.transaction_data_key_name(transaction_id); - let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); - let completed_global_key = self.completed_transactions_global_key_name(); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Remove this hash:id from submitted - let hash_id_value = format!("{}:{}", hash, transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove hash mapping - pipeline.del(&hash_to_id_key); - - // Update transaction data with success - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "receipt", receipt); - pipeline.hset(&tx_data_key, "status", "confirmed"); - - // Add to completed transactions tracking for pruning - pipeline.zadd(&completed_eoa_key, transaction_id, now); - pipeline.zadd(&completed_global_key, transaction_id, now); - }) - .await - } - - /// Add a gas bump attempt (new hash) to submitted transactions - pub async fn add_gas_bump_attempt( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - signed_transaction: Signed, - ) -> Result<(), TransactionStoreError> { - let new_hash = signed_transaction.hash().to_string(); - let nonce = signed_transaction.nonce(); - - // Create new attempt - let new_attempt = TransactionAttempt { - transaction_id: transaction_id.to_string(), - details: signed_transaction, - sent_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - attempt_number: 0, // Will be set correctly when reading all attempts - }; - - // Serialize the new attempt - let attempt_json = serde_json::to_string(&new_attempt)?; - - // Get key names - let attempts_list_key = self.transaction_attempts_list_name(transaction_id); - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(&new_hash); - let hash_id_value = format!("{}:{}", new_hash, transaction_id); - - // Now perform the atomic update - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - // Add new hash:id to submitted (keeping old ones) - pipeline.zadd(&submitted_key, &hash_id_value, nonce); - - // Still maintain separate hash-to-ID mapping for backward compatibility - pipeline.set(&hash_to_id_key, transaction_id); - - // Simply push the new attempt to the attempts list - pipeline.lpush(&attempts_list_key, &attempt_json); - }) - .await - } - - /// Efficiently batch fail and requeue multiple transactions - /// This avoids hash-to-ID lookups since we already have both pieces of information - pub async fn batch_fail_and_requeue_transactions( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - failures: Vec, - ) -> Result<(), TransactionStoreError> { - if failures.is_empty() { - return Ok(()); - } - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - - // Remove all hash:id values from submitted - for failure in &failures { - let hash_id_value = format!("{}:{}", failure.hash, failure.transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove separate hash-to-ID mapping - let hash_to_id_key = self.transaction_hash_to_id_key_name(&failure.hash); - pipeline.del(&hash_to_id_key); - } - - // Add unique transaction IDs back to pending (avoid duplicates) - let mut unique_tx_ids = std::collections::HashSet::new(); - for failure in &failures { - unique_tx_ids.insert(&failure.transaction_id); - } - - for transaction_id in unique_tx_ids { - pipeline.lpush(&pending_key, transaction_id); - } - }) - .await - } - - /// Efficiently batch succeed multiple transactions - /// This avoids hash-to-ID lookups since we already have both pieces of information - pub async fn batch_succeed_transactions( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - successes: Vec, - ) -> Result<(), TransactionStoreError> { - if successes.is_empty() { - return Ok(()); - } - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); - let completed_global_key = self.completed_transactions_global_key_name(); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - for success in &successes { - // Remove hash:id from submitted - let hash_id_value = format!("{}:{}", success.hash, success.transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove separate hash-to-ID mapping - let hash_to_id_key = self.transaction_hash_to_id_key_name(&success.hash); - pipeline.del(&hash_to_id_key); - - // Update transaction data with success (following existing Redis hash pattern) - let tx_data_key = self.transaction_data_key_name(&success.transaction_id); - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "receipt", &success.receipt_data); - pipeline.hset(&tx_data_key, "status", "confirmed"); - - // Add to completed transactions tracking for pruning - pipeline.zadd(&completed_eoa_key, &success.transaction_id, now); - pipeline.zadd(&completed_global_key, &success.transaction_id, now); - } - }) - .await - } - - // ========== SEND FLOW ========== - - /// Get cached transaction count - pub async fn get_cached_transaction_count( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let count: Option = conn.get(&tx_count_key).await?; - match count { - Some(count) => Ok(count), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Peek next available nonce (recycled or new) - pub async fn peek_next_available_nonce( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - // Check recycled nonces first - let recycled = self.peek_recycled_nonces(eoa, chain_id).await?; - if !recycled.is_empty() { - return Ok(NonceType::Recycled(recycled[0])); - } - - // Get next optimistic nonce - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let current_optimistic: Option = conn.get(&optimistic_key).await?; - - match current_optimistic { - Some(nonce) => Ok(NonceType::Incremented(nonce)), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Synchronize nonces with the chain - /// - /// Part of standard nonce management flow, called in the confirm stage when chain nonce advances, and we need to update our cached nonce - pub async fn synchronize_nonces_with_chain( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - current_chain_tx_count: u64, - ) -> Result<(), TransactionStoreError> { - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // First, read current health data - let current_health = self.check_eoa_health(eoa, chain_id).await?; - - // Prepare health update if health data exists - let health_update = if let Some(mut health) = current_health { - health.last_nonce_movement_at = now; - health.last_confirmation_at = now; - Some(serde_json::to_string(&health)?) - } else { - None - }; - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - - // Update cached transaction count - pipeline.set(&tx_count_key, current_chain_tx_count); - - // Update health data only if it exists - if let Some(ref health_json) = health_update { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, health_json); - } - }) - .await - } - - /// Reset nonces to specified value - /// - /// This is called when we have too many recycled nonces and detect something wrong - /// We want to start fresh, with the chain nonce as the new optimistic nonce - pub async fn reset_nonces( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - current_chain_tx_count: u64, - ) -> Result<(), TransactionStoreError> { - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - let current_health = self.check_eoa_health(eoa, chain_id).await?; - - // Prepare health update if health data exists - let health_update = if let Some(mut health) = current_health { - health.nonce_resets.push(now); - Some(serde_json::to_string(&health)?) - } else { - None - }; - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let cached_nonce_key = self.last_transaction_count_key_name(eoa, chain_id); - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - - // Update health data only if it exists - if let Some(ref health_json) = health_update { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, health_json); - } - - // Reset the optimistic nonce - pipeline.set(&optimistic_key, current_chain_tx_count); - - // Reset the cached nonce - pipeline.set(&cached_nonce_key, current_chain_tx_count); - - // Reset the recycled nonces - pipeline.del(recycled_key); - }) - .await - } - - /// Add a transaction to the pending queue and store its data - /// This is called when a new transaction request comes in for an EOA - pub async fn add_transaction( - &self, - transaction_request: EoaTransactionRequest, - ) -> Result<(), TransactionStoreError> { - let transaction_id = &transaction_request.transaction_id; - let eoa = transaction_request.from; - let chain_id = transaction_request.chain_id; - - let tx_data_key = self.transaction_data_key_name(transaction_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - - // Store transaction data as JSON in the user_request field of the hash - let user_request_json = serde_json::to_string(&transaction_request)?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - let mut conn = self.redis.clone(); - - // Use a pipeline to atomically store data and add to pending queue - let mut pipeline = twmq::redis::pipe(); - - // Store transaction data - pipeline.hset(&tx_data_key, "user_request", &user_request_json); - pipeline.hset(&tx_data_key, "status", "pending"); - pipeline.hset(&tx_data_key, "created_at", now); - - // Add to pending queue - pipeline.lpush(&pending_key, transaction_id); - - pipeline.query_async::<()>(&mut conn).await?; - - Ok(()) - } - - /// Get count of submitted transactions awaiting confirmation - pub async fn get_submitted_transactions_count( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let count: u64 = conn.zcard(&submitted_key).await?; - Ok(count) - } - - /// Fail a transaction that's in the pending queue - pub async fn fail_pending_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - failure_reason: &str, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let tx_data_key = self.transaction_data_key_name(transaction_id); - let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); - let completed_global_key = self.completed_transactions_global_key_name(); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Remove from pending queue - pipeline.lrem(&pending_key, 0, transaction_id); - - // Update transaction data with failure - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "failure_reason", failure_reason); - pipeline.hset(&tx_data_key, "status", "failed"); - - // Add to completed transactions tracking for pruning - pipeline.zadd(&completed_eoa_key, transaction_id, now); - pipeline.zadd(&completed_global_key, transaction_id, now); - }) - .await - } - - /// Fail a transaction that's in the borrowed state (we know the nonce) - pub async fn fail_borrowed_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - nonce: u64, - failure_reason: &str, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let tx_data_key = self.transaction_data_key_name(transaction_id); - let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); - let completed_global_key = self.completed_transactions_global_key_name(); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Remove from borrowed state using the known nonce - pipeline.hdel(&borrowed_key, nonce.to_string()); - - // Update transaction data with failure - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "failure_reason", failure_reason); - pipeline.hset(&tx_data_key, "status", "failed"); - - // Add to completed transactions tracking for pruning - pipeline.zadd(&completed_eoa_key, transaction_id, now); - pipeline.zadd(&completed_global_key, transaction_id, now); - }) - .await - } - - /// Prune old completed transactions for a specific EOA if it exceeds the cap - pub async fn prune_completed_transactions_for_eoa( - &self, - eoa: Address, - chain_id: u64, - max_per_eoa: u64, - batch_size: u64, - ) -> Result { - let completed_eoa_key = self.completed_transactions_per_eoa_key_name(eoa, chain_id); - let completed_global_key = self.completed_transactions_global_key_name(); - let mut conn = self.redis.clone(); - - // Check current count - let current_count: u64 = conn.zcard(&completed_eoa_key).await?; - if current_count <= max_per_eoa { - return Ok(0); // No pruning needed - } - - let to_remove = current_count - max_per_eoa; - let batch_to_remove = to_remove.min(batch_size); - - // Get oldest transactions (lowest scores) - let oldest_transactions: Vec = conn - .zrange(&completed_eoa_key, 0, (batch_to_remove - 1) as isize) - .await?; - - if oldest_transactions.is_empty() { - return Ok(0); - } - - // Remove transaction data and tracking - for transaction_id in &oldest_transactions { - let tx_data_key = self.transaction_data_key_name(transaction_id); - let _: () = conn.del(&tx_data_key).await?; - } - - // Remove from tracking sets - for transaction_id in &oldest_transactions { - let _: () = conn.zrem(&completed_eoa_key, transaction_id).await?; - let _: () = conn.zrem(&completed_global_key, transaction_id).await?; - } - - Ok(oldest_transactions.len() as u64) - } - - /// Prune old completed transactions globally if it exceeds the global cap - pub async fn prune_completed_transactions_globally( - &self, - max_global: u64, - batch_size: u64, - ) -> Result { - let completed_global_key = self.completed_transactions_global_key_name(); - let mut conn = self.redis.clone(); - - // Check current count - let current_count: u64 = conn.zcard(&completed_global_key).await?; - if current_count <= max_global { - return Ok(0); // No pruning needed - } - - let to_remove = current_count - max_global; - let batch_to_remove = to_remove.min(batch_size); - - // Get oldest transactions (lowest scores) - let oldest_transactions: Vec = conn - .zrange(&completed_global_key, 0, (batch_to_remove - 1) as isize) - .await?; - - if oldest_transactions.is_empty() { - return Ok(0); - } - - // Remove transaction data - for transaction_id in &oldest_transactions { - let tx_data_key = self.transaction_data_key_name(transaction_id); - let _: () = conn.del(&tx_data_key).await?; - } - - // Remove from global tracking - for transaction_id in &oldest_transactions { - let _: () = conn.zrem(&completed_global_key, transaction_id).await?; - } - - // Also remove from per-EOA tracking sets (we need to scan for these) - // Note: This is less efficient but necessary to keep consistency - self.remove_transactions_from_all_eoa_tracking(&oldest_transactions) - .await?; - - Ok(oldest_transactions.len() as u64) - } - - /// Helper to remove transactions from all EOA tracking sets - async fn remove_transactions_from_all_eoa_tracking( - &self, - transaction_ids: &[String], - ) -> Result<(), TransactionStoreError> { - let mut conn = self.redis.clone(); - let pattern = match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:completed:*"), - None => "eoa_executor:completed:*".to_string(), - }; - - // Get all EOA-specific completed transaction keys - let keys: Vec = conn.keys(&pattern).await?; - - // Remove transactions from each EOA tracking set - for key in keys { - for transaction_id in transaction_ids { - let _: () = conn.zrem(&key, transaction_id).await?; - } - } - - Ok(()) - } -} - -// Additional error types -#[derive(Debug, thiserror::Error, Serialize, Deserialize, Clone)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] -pub enum TransactionStoreError { - #[error("Redis error: {message}")] - RedisError { message: String }, - - #[error("Serialization error: {message}")] - DeserError { message: String, text: String }, - - #[error("Transaction not found: {transaction_id}")] - TransactionNotFound { transaction_id: String }, - - #[error("Lost EOA lock: {eoa}:{chain_id} worker: {worker_id}")] - LockLost { - eoa: Address, - chain_id: u64, - worker_id: String, - }, - - #[error("Internal error - worker should quit: {message}")] - InternalError { message: String }, - - #[error("Transaction {transaction_id} not in borrowed state for nonce {nonce}")] - TransactionNotInBorrowedState { transaction_id: String, nonce: u64 }, - - #[error("Hash {hash} not found in submitted transactions")] - HashNotInSubmittedState { hash: String }, - - #[error("Transaction {transaction_id} has no hashes in submitted state")] - TransactionNotInSubmittedState { transaction_id: String }, - - #[error("Nonce {nonce} not available in recycled set")] - NonceNotInRecycledSet { nonce: u64 }, - - #[error("Transaction {transaction_id} not found in pending queue")] - TransactionNotInPendingQueue { transaction_id: String }, - - #[error("Optimistic nonce changed: expected {expected}, found {actual}")] - OptimisticNonceChanged { expected: u64, actual: u64 }, - - #[error("WATCH failed - state changed during operation")] - WatchFailed, - - #[error( - "Nonce synchronization required for {eoa}:{chain_id} - no cached transaction count available" - )] - NonceSyncRequired { eoa: Address, chain_id: u64 }, -} - -impl From for TransactionStoreError { - fn from(error: twmq::redis::RedisError) -> Self { - TransactionStoreError::RedisError { - message: error.to_string(), - } - } -} - -impl From for TransactionStoreError { - fn from(error: serde_json::Error) -> Self { - TransactionStoreError::DeserError { - message: error.to_string(), - text: error.to_string(), - } - } -} - -const MAX_RETRIES: u32 = 10; -const RETRY_BASE_DELAY_MS: u64 = 10; - -/// Scoped transaction store for a specific EOA, chain, and worker -/// -/// This wrapper eliminates the need to repeatedly pass EOA, chain_id, and worker_id -/// to every method call. It provides the same interface as TransactionStore but with -/// these parameters already bound. -/// -/// ## Usage: -/// ```rust -/// let scoped = ScopedTransactionStore::build(store, eoa, chain_id, worker_id).await?; -/// -/// // Much cleaner method calls: -/// scoped.peek_pending_transactions(limit).await?; -/// scoped.move_borrowed_to_submitted(nonce, hash, tx_id, attempt).await?; -/// ``` -pub struct ScopedEoaExecutorStore<'a> { - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, -} - -impl<'a> ScopedEoaExecutorStore<'a> { - /// Build a scoped transaction store for a specific EOA, chain, and worker - /// - /// This acquires the lock for the given EOA/chain. - /// If the lock is not acquired, returns a LockLost error. - #[tracing::instrument(skip_all, fields(eoa = %eoa, chain_id = chain_id, worker_id = %worker_id))] - pub async fn build( - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, - ) -> Result { - // 1. ACQUIRE LOCK AGGRESSIVELY - tracing::info!("Acquiring EOA lock aggressively"); - store - .acquire_eoa_lock_aggressively(eoa, chain_id, &worker_id) - .await - .map_err(|e| { - tracing::error!("Failed to acquire EOA lock: {}", e); - TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.clone(), - } - })?; - - Ok(Self { - store, - eoa, - chain_id, - worker_id, - }) - } - - /// Create a scoped store without lock validation (for read-only operations) - pub fn new_unchecked( - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, - ) -> Self { - Self { - store, - eoa, - chain_id, - worker_id, - } - } - - // ========== ATOMIC OPERATIONS ========== - - /// Atomically move specific transaction from pending to borrowed with recycled nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( - &self, - transaction_id: &str, - nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - self.store - .atomic_move_pending_to_borrowed_with_recycled_nonce( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - nonce, - prepared_tx, - ) - .await - } - - /// Atomically move specific transaction from pending to borrowed with new nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_new_nonce( - &self, - transaction_id: &str, - expected_nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - self.store - .atomic_move_pending_to_borrowed_with_new_nonce( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - expected_nonce, - prepared_tx, - ) - .await - } - - /// Peek all borrowed transactions without removing them - pub async fn peek_borrowed_transactions( - &self, - ) -> Result, TransactionStoreError> { - self.store - .peek_borrowed_transactions(self.eoa, self.chain_id) - .await - } - - /// Atomically move borrowed transaction to submitted state - pub async fn move_borrowed_to_submitted( - &self, - nonce: u64, - hash: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .move_borrowed_to_submitted( - self.eoa, - self.chain_id, - &self.worker_id, - nonce, - hash, - transaction_id, - ) - .await - } - - /// Atomically move borrowed transaction back to recycled nonces and pending queue - pub async fn move_borrowed_to_recycled( - &self, - nonce: u64, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .move_borrowed_to_recycled( - self.eoa, - self.chain_id, - &self.worker_id, - nonce, - transaction_id, - ) - .await - } - - /// Get all hashes below a certain nonce from submitted transactions - /// Returns (nonce, hash, transaction_id) tuples - pub async fn get_hashes_below_nonce( - &self, - below_nonce: u64, - ) -> Result, TransactionStoreError> { - self.store - .get_hashes_below_nonce(self.eoa, self.chain_id, below_nonce) - .await - } - - /// Get all transaction IDs for a specific nonce - pub async fn get_transaction_ids_for_nonce( - &self, - nonce: u64, - ) -> Result, TransactionStoreError> { - self.store - .get_transaction_ids_for_nonce(self.eoa, self.chain_id, nonce) - .await - } - - /// Remove all hashes for a transaction and requeue it - pub async fn fail_and_requeue_transaction( - &self, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .fail_and_requeue_transaction(self.eoa, self.chain_id, &self.worker_id, transaction_id) - .await - } - - /// Efficiently batch fail and requeue multiple transactions - pub async fn batch_fail_and_requeue_transactions( - &self, - failures: Vec, - ) -> Result<(), TransactionStoreError> { - self.store - .batch_fail_and_requeue_transactions(self.eoa, self.chain_id, &self.worker_id, failures) - .await - } - - /// Efficiently batch succeed multiple transactions - pub async fn batch_succeed_transactions( - &self, - successes: Vec, - ) -> Result<(), TransactionStoreError> { - self.store - .batch_succeed_transactions(self.eoa, self.chain_id, &self.worker_id, successes) - .await - } - - // ========== EOA HEALTH & NONCE MANAGEMENT ========== - - /// Check EOA health (balance, etc.) - pub async fn check_eoa_health(&self) -> Result, TransactionStoreError> { - self.store.check_eoa_health(self.eoa, self.chain_id).await - } - - /// Update EOA health data - pub async fn update_health_data( - &self, - health: &EoaHealth, - ) -> Result<(), TransactionStoreError> { - self.store - .update_health_data(self.eoa, self.chain_id, &self.worker_id, health) - .await - } - - /// Update cached transaction count - pub async fn update_cached_transaction_count( - &self, - transaction_count: u64, - ) -> Result<(), TransactionStoreError> { - self.store - .update_cached_transaction_count( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_count, - ) - .await - } - - /// Peek recycled nonces without removing them - pub async fn peek_recycled_nonces(&self) -> Result, TransactionStoreError> { - self.store - .peek_recycled_nonces(self.eoa, self.chain_id) - .await - } - - /// Peek at pending transactions without removing them - pub async fn peek_pending_transactions( - &self, - limit: u64, - ) -> Result, TransactionStoreError> { - self.store - .peek_pending_transactions(self.eoa, self.chain_id, limit) - .await - } - - /// Get inflight budget (how many new transactions can be sent) - pub async fn get_inflight_budget( - &self, - max_inflight: u64, - ) -> Result { - self.store - .get_inflight_budget(self.eoa, self.chain_id, max_inflight) - .await - } - - /// Get current optimistic nonce (without incrementing) - pub async fn get_optimistic_nonce(&self) -> Result { - self.store - .get_optimistic_nonce(self.eoa, self.chain_id) - .await - } - - /// Mark transaction as successful and remove from submitted - pub async fn succeed_transaction( - &self, - transaction_id: &str, - hash: &str, - receipt: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .succeed_transaction( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - hash, - receipt, - ) - .await - } - - /// Add a gas bump attempt (new hash) to submitted transactions - pub async fn add_gas_bump_attempt( - &self, - transaction_id: &str, - signed_transaction: Signed, - ) -> Result<(), TransactionStoreError> { - self.store - .add_gas_bump_attempt( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - signed_transaction, - ) - .await - } - - pub async fn synchronize_nonces_with_chain( - &self, - nonce: u64, - ) -> Result<(), TransactionStoreError> { - self.store - .synchronize_nonces_with_chain(self.eoa, self.chain_id, &self.worker_id, nonce) - .await - } - - pub async fn reset_nonces(&self, nonce: u64) -> Result<(), TransactionStoreError> { - self.store - .reset_nonces(self.eoa, self.chain_id, &self.worker_id, nonce) - .await - } - - // ========== READ-ONLY OPERATIONS ========== - - /// Get cached transaction count - pub async fn get_cached_transaction_count(&self) -> Result { - self.store - .get_cached_transaction_count(self.eoa, self.chain_id) - .await - } - - /// Peek next available nonce (recycled or new) - pub async fn peek_next_available_nonce(&self) -> Result { - self.store - .peek_next_available_nonce(self.eoa, self.chain_id) - .await - } - - // ========== ACCESSORS ========== - - /// Get the EOA address this store is scoped to - pub fn eoa(&self) -> Address { - self.eoa - } - - /// Get the chain ID this store is scoped to - pub fn chain_id(&self) -> u64 { - self.chain_id - } - - /// Get the worker ID this store is scoped to - pub fn worker_id(&self) -> &str { - &self.worker_id - } - - /// Get a reference to the underlying transaction store - pub fn inner(&self) -> &EoaExecutorStore { - self.store - } - - /// Get transaction data by transaction ID - pub async fn get_transaction_data( - &self, - transaction_id: &str, - ) -> Result, TransactionStoreError> { - self.store.get_transaction_data(transaction_id).await - } - - /// Get count of submitted transactions awaiting confirmation - pub async fn get_submitted_transactions_count(&self) -> Result { - self.store - .get_submitted_transactions_count(self.eoa, self.chain_id) - .await - } - - /// Fail a transaction that's in the pending queue - pub async fn fail_pending_transaction( - &self, - transaction_id: &str, - failure_reason: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .fail_pending_transaction( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - failure_reason, - ) - .await - } - - /// Fail a transaction that's in the borrowed state (we know the nonce) - pub async fn fail_borrowed_transaction( - &self, - transaction_id: &str, - nonce: u64, - failure_reason: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .fail_borrowed_transaction( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - nonce, - failure_reason, - ) - .await - } - - /// Prune old completed transactions for this EOA if it exceeds the cap - pub async fn prune_completed_transactions( - &self, - max_per_eoa: u64, - batch_size: u64, - ) -> Result { - self.store - .prune_completed_transactions_for_eoa(self.eoa, self.chain_id, max_per_eoa, batch_size) - .await - } -} diff --git a/executors/src/eoa/store/atomic.rs b/executors/src/eoa/store/atomic.rs new file mode 100644 index 0000000..eb931c9 --- /dev/null +++ b/executors/src/eoa/store/atomic.rs @@ -0,0 +1,812 @@ +use alloy::{ + consensus::{Signed, TypedTransaction}, + primitives::Address, +}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::eoa::{ + EoaExecutorStore, + store::{ + BorrowedTransactionData, ConfirmedTransaction, EoaHealth, TransactionAttempt, + TransactionStoreError, + submitted::{CleanSubmittedTransactions, CleanupReport, SubmittedTransaction}, + }, +}; + +const MAX_RETRIES: u32 = 10; +const RETRY_BASE_DELAY_MS: u64 = 10; + +pub trait SafeRedisTransaction: Send + Sync { + type ValidationData; + type OperationResult; + + fn name(&self) -> &str; + fn operation( + &self, + pipeline: &mut Pipeline, + validation_data: Self::ValidationData, + ) -> Self::OperationResult; + fn validation( + &self, + conn: &mut ConnectionManager, + ) -> impl Future> + Send; + fn watch_keys(&self) -> Vec; +} + +struct MovePendingToBorrowedWithRecycledNonce { + recycled_key: String, + pending_key: String, + transaction_id: String, + borrowed_key: String, + nonce: u64, + prepared_tx_json: String, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonce { + type ValidationData = (); + type OperationResult = (); + + fn name(&self) -> &str { + "pending->borrowed with recycled nonce" + } + + fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { + // Remove nonce from recycled set (we know it exists) + pipeline.zrem(&self.recycled_key, self.nonce); + // Remove transaction from pending (we know it exists) + pipeline.lrem(&self.pending_key, 0, &self.transaction_id); + // Store borrowed transaction + pipeline.hset( + &self.borrowed_key, + self.nonce.to_string(), + &self.prepared_tx_json, + ); + } + + fn watch_keys(&self) -> Vec { + vec![self.recycled_key.clone(), self.pending_key.clone()] + } + + async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { + // Check if nonce exists in recycled set + let nonce_score: Option = conn.zscore(&self.recycled_key, self.nonce).await?; + if nonce_score.is_none() { + return Err(TransactionStoreError::NonceNotInRecycledSet { nonce: self.nonce }); + } + + // Check if transaction exists in pending + let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; + if !pending_transactions.contains(&self.transaction_id) { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: self.transaction_id.clone(), + }); + } + + Ok(()) + } +} + +struct MovePendingToBorrowedWithNewNonce { + optimistic_key: String, + pending_key: String, + nonce: u64, + prepared_tx_json: String, + transaction_id: String, + borrowed_key: String, + eoa: Address, + chain_id: u64, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithNewNonce { + type ValidationData = (); + type OperationResult = (); + + fn name(&self) -> &str { + "pending->borrowed with new nonce" + } + + fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { + // Increment optimistic nonce + pipeline.incr(&self.optimistic_key, 1); + // Remove transaction from pending + pipeline.lrem(&self.pending_key, 0, &self.transaction_id); + // Store borrowed transaction + pipeline.hset( + &self.borrowed_key, + self.nonce.to_string(), + &self.prepared_tx_json, + ); + } + + fn watch_keys(&self) -> Vec { + vec![self.optimistic_key.clone(), self.pending_key.clone()] + } + + async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { + // Check current optimistic nonce + let current_optimistic: Option = conn.get(&self.optimistic_key).await?; + let current_nonce = match current_optimistic { + Some(nonce) => nonce, + None => { + return Err(TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + }); + } + }; + + if current_nonce != self.nonce { + return Err(TransactionStoreError::OptimisticNonceChanged { + expected: self.nonce, + actual: current_nonce, + }); + } + + // Check if transaction exists in pending + let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; + if !pending_transactions.contains(&self.transaction_id) { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: self.transaction_id.clone(), + }); + } + + Ok(()) + } +} + +struct MoveBorrowedToSubmitted { + nonce: u64, + hash: String, + transaction_id: String, + borrowed_key: String, + submitted_key: String, + hash_to_id_key: String, +} + +impl SafeRedisTransaction for MoveBorrowedToSubmitted { + type ValidationData = (); + type OperationResult = (); + + fn name(&self) -> &str { + "borrowed->submitted" + } + + fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { + // Remove from borrowed (we know it exists) + pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); + + // Add to submitted with hash:id format + let hash_id_value = format!("{}:{}", self.hash, self.transaction_id); + pipeline.zadd(&self.submitted_key, &hash_id_value, self.nonce); + + // Still maintain hash-to-ID mapping for backward compatibility and external lookups + pipeline.set(&self.hash_to_id_key, &self.transaction_id); + } + + fn watch_keys(&self) -> Vec { + vec![self.borrowed_key.clone()] + } + + async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { + // Validate that borrowed transaction actually exists + let borrowed_tx: Option = conn + .hget(&self.borrowed_key, self.nonce.to_string()) + .await?; + if borrowed_tx.is_none() { + return Err(TransactionStoreError::TransactionNotInBorrowedState { + transaction_id: self.transaction_id.clone(), + nonce: self.nonce, + }); + } + Ok(()) + } +} + +struct MoveBorrowedToRecycled { + nonce: u64, + transaction_id: String, + borrowed_key: String, + recycled_key: String, + pending_key: String, +} + +impl SafeRedisTransaction for MoveBorrowedToRecycled { + type ValidationData = (); + type OperationResult = (); + + fn name(&self) -> &str { + "borrowed->recycled" + } + + fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { + // Remove from borrowed (we know it exists) + pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); + + // Add nonce to recycled set (with timestamp as score) + pipeline.zadd(&self.recycled_key, self.nonce, self.nonce); + + // Add transaction back to pending + pipeline.lpush(&self.pending_key, &self.transaction_id); + } + + fn watch_keys(&self) -> Vec { + vec![self.borrowed_key.clone()] + } + + async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { + // Validate that borrowed transaction actually exists + let borrowed_tx: Option = conn + .hget(&self.borrowed_key, self.nonce.to_string()) + .await?; + if borrowed_tx.is_none() { + return Err(TransactionStoreError::TransactionNotInBorrowedState { + transaction_id: self.transaction_id.clone(), + nonce: self.nonce, + }); + } + Ok(()) + } +} + +/// Atomic transaction store that owns the base store and provides atomic operations +/// +/// This store is created by calling `acquire_lock()` on the base store and provides +/// access to both atomic (lock-protected) and non-atomic operations. +/// +/// ## Usage: +/// ```rust +/// let base_store = EoaExecutorStore::new(redis, namespace, ); +/// let atomic_store = base_store.acquire_lock(worker_id).await?; +/// +/// // Atomic operations: +/// atomic_store.move_borrowed_to_submitted(nonce, hash, tx_id).await?; +/// +/// // Non-atomic operations via deref: +/// atomic_store.peek_pending_transactions(limit).await?; +/// ``` +pub struct AtomicEoaExecutorStore { + pub store: EoaExecutorStore, + pub worker_id: String, +} + +impl std::ops::Deref for AtomicEoaExecutorStore { + type Target = EoaExecutorStore; + + fn deref(&self) -> &Self::Target { + &self.store + } +} + +impl AtomicEoaExecutorStore { + /// Get the EOA address this store is scoped to + pub fn eoa(&self) -> Address { + self.store.eoa + } + + /// Get the chain ID this store is scoped to + pub fn chain_id(&self) -> u64 { + self.store.chain_id + } + + /// Get the worker ID this store is scoped to + pub fn worker_id(&self) -> &str { + &self.worker_id + } + + /// Release EOA lock following the spec's finally pattern + pub async fn release_eoa_lock(self) -> Result { + // Use existing utility method that handles all the atomic lock checking + match self + .with_lock_check(|pipeline| { + let lock_key = self.eoa_lock_key_name(); + pipeline.del(&lock_key); + }) + .await + { + Ok(()) => { + tracing::debug!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + "Successfully released EOA lock" + ); + Ok(self.store) + } + Err(TransactionStoreError::LockLost { .. }) => { + // Lock was already taken over, which is fine for release + tracing::debug!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + "Lock already released or taken over by another worker" + ); + Ok(self.store) + } + Err(e) => { + // Other errors shouldn't fail the worker, just log + tracing::warn!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + error = %e, + "Failed to release EOA lock" + ); + Ok(self.store) + } + } + } + + /// Example of how to refactor a complex method using the helper to reduce boilerplate + /// This shows the pattern for atomic_move_pending_to_borrowed_with_recycled_nonce + pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( + &self, + transaction_id: &str, + nonce: u64, + prepared_tx: &BorrowedTransactionData, + ) -> Result<(), TransactionStoreError> { + let safe_tx = MovePendingToBorrowedWithRecycledNonce { + recycled_key: self.recycled_nonces_set_name(), + pending_key: self.pending_transactions_zset_name(), + transaction_id: transaction_id.to_string(), + borrowed_key: self.borrowed_transactions_hashmap_name(), + nonce, + prepared_tx_json: serde_json::to_string(prepared_tx)?, + }; + + self.execute_with_watch_and_retry(&safe_tx).await?; + + Ok(()) + } + + /// Atomically move specific transaction from pending to borrowed with new nonce allocation + pub async fn atomic_move_pending_to_borrowed_with_new_nonce( + &self, + transaction_id: &str, + expected_nonce: u64, + prepared_tx: &BorrowedTransactionData, + ) -> Result<(), TransactionStoreError> { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let pending_key = self.pending_transactions_zset_name(); + let prepared_tx_json = serde_json::to_string(prepared_tx)?; + let transaction_id = transaction_id.to_string(); + + self.execute_with_watch_and_retry(&MovePendingToBorrowedWithNewNonce { + nonce: expected_nonce, + prepared_tx_json, + transaction_id, + borrowed_key, + optimistic_key, + pending_key, + eoa: self.eoa, + chain_id: self.chain_id, + }) + .await + } + + /// Wrapper that executes operations with lock validation using WATCH/MULTI/EXEC + pub async fn with_lock_check(&self, operation: F) -> Result + where + F: Fn(&mut Pipeline) -> R, + T: From, + { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + let mut retry_count = 0; + + loop { + if retry_count >= MAX_RETRIES { + return Err(TransactionStoreError::InternalError { + message: format!( + "Exceeded max retries ({}) for lock check on {}:{}", + MAX_RETRIES, + self.eoa(), + self.chain_id() + ), + }); + } + + // Exponential backoff after first retry + if retry_count > 0 { + let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + tracing::debug!( + retry_count = retry_count, + delay_ms = delay_ms, + eoa = %self.eoa(), + chain_id = self.chain_id(), + "Retrying lock check operation" + ); + } + + // WATCH the EOA lock + let _: () = twmq::redis::cmd("WATCH") + .arg(&lock_key) + .query_async(&mut conn) + .await?; + + // Check if we still own the lock + let current_owner: Option = conn.get(&lock_key).await?; + match current_owner { + Some(owner) if owner == self.worker_id() => { + // We still own it, proceed + } + _ => { + // Lost ownership - immediately fail + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(self.eoa_lock_lost_error()); + } + } + + // Build pipeline with operation + let mut pipeline = twmq::redis::pipe(); + pipeline.atomic(); + let result = operation(&mut pipeline); + + // Execute with WATCH protection + match pipeline + .query_async::>(&mut conn) + .await + { + Ok(_) => return Ok(T::from(result)), + Err(_) => { + // WATCH failed, check if it was our lock or someone else's + let still_own_lock: Option = conn.get(&lock_key).await?; + if still_own_lock.as_deref() != Some(self.worker_id()) { + return Err(self.eoa_lock_lost_error()); + } + // Our lock is fine, someone else's WATCH failed - retry + retry_count += 1; + continue; + } + } + } + } + + /// Helper to execute atomic operations with proper retry logic and watch handling + /// + /// This helper centralizes all the boilerplate for WATCH/MULTI/EXEC operations: + /// - Retry logic with exponential backoff + /// - Lock ownership validation + /// - WATCH key management + /// - Error handling and UNWATCH cleanup + /// + /// ## Usage: + /// Implement the `SafeRedisTransaction` trait for your operation, then call this method. + /// The trait separates validation (async) from pipeline operations (sync) for clean patterns. + /// + /// ## Example: + /// ```rust + /// let safe_tx = MovePendingToBorrowedWithNewNonce { + /// nonce: expected_nonce, + /// prepared_tx_json, + /// transaction_id, + /// borrowed_key, + /// optimistic_key, + /// pending_key, + /// eoa, + /// chain_id, + /// }; + /// + /// self.execute_with_watch_and_retry(, worker_id, &safe_tx).await?; + /// ``` + /// + /// ## When to use this helper: + /// - Operations that implement `SafeRedisTransaction` trait + /// - Need atomic WATCH/MULTI/EXEC with retry logic + /// - Want centralized lock checking and error handling + /// + /// ## When NOT to use this helper: + /// - Simple operations that can use `with_lock_check` instead + /// - Operations that don't need WATCH on multiple keys + /// - Read-only operations that don't modify state + async fn execute_with_watch_and_retry( + &self, + safe_tx: &T, + ) -> Result { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + let mut retry_count = 0; + + loop { + if retry_count >= MAX_RETRIES { + return Err(TransactionStoreError::InternalError { + message: format!( + "Exceeded max retries ({}) for {} on {}:{}", + MAX_RETRIES, + safe_tx.name(), + self.eoa, + self.chain_id + ), + }); + } + + // Exponential backoff after first retry + if retry_count > 0 { + let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + tracing::debug!( + retry_count = retry_count, + delay_ms = delay_ms, + eoa = %self.eoa, + chain_id = self.chain_id, + operation = safe_tx.name(), + "Retrying atomic operation" + ); + } + + // WATCH all specified keys including lock + let mut watch_cmd = twmq::redis::cmd("WATCH"); + watch_cmd.arg(&lock_key); + for key in safe_tx.watch_keys() { + watch_cmd.arg(key); + } + let _: () = watch_cmd.query_async(&mut conn).await?; + + // Check lock ownership + let current_owner: Option = conn.get(&lock_key).await?; + if current_owner.as_deref() != Some(self.worker_id()) { + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(TransactionStoreError::LockLost { + eoa: self.eoa, + chain_id: self.chain_id, + worker_id: self.worker_id().to_string(), + }); + } + + // Execute validation + match safe_tx.validation(&mut conn).await { + Ok(validation_data) => { + // Build and execute pipeline + let mut pipeline = twmq::redis::pipe(); + pipeline.atomic(); + let result = safe_tx.operation(&mut pipeline, validation_data); + + match pipeline + .query_async::>(&mut conn) + .await + { + Ok(_) => return Ok(result), // Success + Err(_) => { + // WATCH failed, check if it was our lock + let still_own_lock: Option = conn.get(&lock_key).await?; + if still_own_lock.as_deref() != Some(self.worker_id()) { + return Err(TransactionStoreError::LockLost { + eoa: self.eoa, + chain_id: self.chain_id, + worker_id: self.worker_id().to_string(), + }); + } + // State changed, retry + retry_count += 1; + continue; + } + } + } + Err(e) => { + // Validation failed, unwatch and return error + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(e); + } + } + } + } + + /// Atomically move borrowed transaction to submitted state + /// Returns error if transaction not found in borrowed state + pub async fn atomic_move_borrowed_to_submitted( + &self, + nonce: u64, + hash: &str, + transaction_id: &str, + ) -> Result<(), TransactionStoreError> { + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let submitted_key = self.submitted_transactions_zset_name(); + let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); + let hash = hash.to_string(); + let transaction_id = transaction_id.to_string(); + + self.execute_with_watch_and_retry(&MoveBorrowedToSubmitted { + nonce, + hash: hash.to_string(), + transaction_id, + borrowed_key, + submitted_key, + hash_to_id_key, + }) + .await + } + + /// Atomically move borrowed transaction back to recycled nonces and pending queue + /// Returns error if transaction not found in borrowed state + pub async fn atomic_move_borrowed_to_recycled( + &self, + nonce: u64, + transaction_id: &str, + ) -> Result<(), TransactionStoreError> { + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let recycled_key = self.recycled_nonces_set_name(); + let pending_key = self.pending_transactions_zset_name(); + let transaction_id = transaction_id.to_string(); + + self.execute_with_watch_and_retry(&MoveBorrowedToRecycled { + nonce, + transaction_id, + borrowed_key, + recycled_key, + pending_key, + }) + .await + } + + /// Update EOA health data + pub async fn update_health_data( + &self, + health: &EoaHealth, + ) -> Result<(), TransactionStoreError> { + let health_json = serde_json::to_string(health)?; + self.with_lock_check(|pipeline| { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, &health_json); + }) + .await + } + + /// Synchronize nonces with the chain + /// + /// Part of standard nonce management flow, called in the confirm stage when chain nonce advances, and we need to update our cached nonce + pub async fn update_cached_transaction_count( + &self, + current_chain_tx_count: u64, + ) -> Result<(), TransactionStoreError> { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // First, read current health data + let current_health = self.check_eoa_health().await?; + + // Prepare health update if health data exists + let health_update = if let Some(mut health) = current_health { + health.last_nonce_movement_at = now; + health.last_confirmation_at = now; + Some(serde_json::to_string(&health)?) + } else { + None + }; + + self.with_lock_check(|pipeline| { + let tx_count_key = self.last_transaction_count_key_name(); + + // Update cached transaction count + pipeline.set(&tx_count_key, current_chain_tx_count); + + // Update health data only if it exists + if let Some(ref health_json) = health_update { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, health_json); + } + }) + .await + } + + /// Add a gas bump attempt (new hash) to submitted transactions + pub async fn add_gas_bump_attempt( + &self, + submitted_transaction: &SubmittedTransaction, + signed_transaction: Signed, + ) -> Result<(), TransactionStoreError> { + let new_hash = signed_transaction.hash().to_string(); + + // Create new attempt + let new_attempt = TransactionAttempt { + transaction_id: submitted_transaction.transaction_id.clone(), + details: signed_transaction, + sent_at: chrono::Utc::now().timestamp_millis().max(0) as u64, + attempt_number: 0, // Will be set correctly when reading all attempts + }; + + // Serialize the new attempt + let attempt_json = serde_json::to_string(&new_attempt)?; + + // Get key names + let attempts_list_key = + self.transaction_attempts_list_name(&submitted_transaction.transaction_id); + let submitted_key = self.submitted_transactions_zset_name(); + + let hash_to_id_key = self.transaction_hash_to_id_key_name(&new_hash); + + let (submitted_transaction_string, nonce) = + submitted_transaction.to_redis_string_with_nonce(); + + // Now perform the atomic update + self.with_lock_check(|pipeline| { + // Add new hash:id to submitted (keeping old ones) + pipeline.zadd(&submitted_key, &submitted_transaction_string, nonce); + + // Still maintain separate hash-to-ID mapping for backward compatibility + pipeline.set(&hash_to_id_key, &submitted_transaction.transaction_id); + + // Simply push the new attempt to the attempts list + pipeline.lpush(&attempts_list_key, &attempt_json); + }) + .await + } + + /// Reset nonces to specified value + /// + /// This is called when we have too many recycled nonces and detect something wrong + /// We want to start fresh, with the chain nonce as the new optimistic nonce + pub async fn reset_nonces( + &self, + current_chain_tx_count: u64, + ) -> Result<(), TransactionStoreError> { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + let current_health = self.check_eoa_health().await?; + + // Prepare health update if health data exists + let health_update = if let Some(mut health) = current_health { + health.nonce_resets.push(now); + Some(serde_json::to_string(&health)?) + } else { + None + }; + + self.with_lock_check(|pipeline| { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let cached_nonce_key = self.last_transaction_count_key_name(); + let recycled_key = self.recycled_nonces_set_name(); + + // Update health data only if it exists + if let Some(ref health_json) = health_update { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, health_json); + } + + // Reset the optimistic nonce + pipeline.set(&optimistic_key, current_chain_tx_count); + + // Reset the cached nonce + pipeline.set(&cached_nonce_key, current_chain_tx_count); + + // Reset the recycled nonces + pipeline.del(recycled_key); + }) + .await + } + + /// Fail a transaction that's in the borrowed state (we know the nonce) + pub async fn fail_borrowed_transaction( + &self, + transaction_id: &str, + nonce: u64, + failure_reason: &str, + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(|pipeline| { + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let tx_data_key = self.transaction_data_key_name(transaction_id); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Remove from borrowed state using the known nonce + pipeline.hdel(&borrowed_key, nonce.to_string()); + + // Update transaction data with failure + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", failure_reason); + pipeline.hset(&tx_data_key, "status", "failed"); + }) + .await + } + + pub async fn clean_submitted_transactions( + &self, + confirmed_transactions: &[ConfirmedTransaction], + last_confirmed_nonce: u64, + ) -> Result { + self.execute_with_watch_and_retry(&CleanSubmittedTransactions { + confirmed_transactions, + last_confirmed_nonce, + keys: &self.keys, + }) + .await + } +} diff --git a/executors/src/eoa/store/error.rs b/executors/src/eoa/store/error.rs new file mode 100644 index 0000000..275541f --- /dev/null +++ b/executors/src/eoa/store/error.rs @@ -0,0 +1,22 @@ +use crate::eoa::EoaExecutorStore; +use crate::eoa::store::TransactionStoreError; +use crate::eoa::store::atomic::AtomicEoaExecutorStore; + +impl AtomicEoaExecutorStore { + pub fn eoa_lock_lost_error(&self) -> TransactionStoreError { + TransactionStoreError::LockLost { + eoa: self.eoa(), + chain_id: self.chain_id(), + worker_id: self.worker_id().to_string(), + } + } +} + +impl EoaExecutorStore { + pub fn nonce_sync_required_error(&self) -> TransactionStoreError { + TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + } + } +} diff --git a/executors/src/eoa/store/mod.rs b/executors/src/eoa/store/mod.rs new file mode 100644 index 0000000..49dfcaa --- /dev/null +++ b/executors/src/eoa/store/mod.rs @@ -0,0 +1,719 @@ +use alloy::consensus::{Signed, TypedTransaction}; +use alloy::network::AnyTransactionReceipt; +use alloy::primitives::{Address, Bytes, U256}; +use chrono; +use engine_core::chain::RpcCredentials; +use engine_core::credentials::SigningCredential; +use engine_core::execution_options::WebhookOptions; +use engine_core::transaction::TransactionTypeData; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use twmq::redis::{AsyncCommands, aio::ConnectionManager}; + +mod atomic; +mod submitted; + +pub mod error; +pub use atomic::AtomicEoaExecutorStore; +pub use submitted::{CleanupReport, SubmittedTransaction}; + +use crate::eoa::store::submitted::SubmittedTransactionStringWithNonce; + +pub const NO_OP_TRANSACTION_ID: &str = "noop"; + +#[derive(Debug, Clone)] +pub struct ReplacedTransaction { + pub hash: String, + pub transaction_id: String, +} + +#[derive(Debug, Clone)] +pub struct ConfirmedTransaction { + pub hash: String, + pub transaction_id: String, + pub receipt_data: String, +} + +/// The actual user request data +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct EoaTransactionRequest { + pub transaction_id: String, + pub chain_id: u64, + + pub from: Address, + pub to: Option
, + pub value: U256, + pub data: Bytes, + + #[serde(alias = "gas")] + pub gas_limit: Option, + + pub webhook_options: Option>, + + pub signing_credential: SigningCredential, + pub rpc_credentials: RpcCredentials, + + #[serde(flatten)] + pub transaction_type_data: Option, +} + +/// Active attempt for a transaction (full alloy transaction + metadata) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TransactionAttempt { + pub transaction_id: String, + pub details: Signed, + pub sent_at: u64, // Unix timestamp in milliseconds + pub attempt_number: u32, +} + +/// Transaction data for a transaction_id +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TransactionData { + pub transaction_id: String, + pub user_request: EoaTransactionRequest, + pub receipt: Option, + pub attempts: Vec, + pub created_at: u64, // Unix timestamp in milliseconds +} + +pub struct BorrowedTransaction { + pub transaction_id: String, + pub data: Signed, + pub borrowed_at: chrono::DateTime, +} + +/// Transaction store focused on transaction_id operations and nonce indexing +pub struct EoaExecutorStore { + pub redis: ConnectionManager, + pub keys: EoaExecutorStoreKeys, +} + +pub struct EoaExecutorStoreKeys { + pub eoa: Address, + pub chain_id: u64, + pub namespace: Option, +} + +impl EoaExecutorStoreKeys { + pub fn new(eoa: Address, chain_id: u64, namespace: Option) -> Self { + Self { + eoa, + chain_id, + namespace, + } + } + + /// Lock key name for EOA processing + pub fn eoa_lock_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:lock:{}:{}", self.chain_id, self.eoa), + None => format!("eoa_executor:lock:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the key for the transaction data + /// + /// Transaction data is stored as a Redis HSET with the following fields: + /// - "user_request": JSON string containing EoaTransactionRequest + /// - "receipt": JSON string containing AnyTransactionReceipt (optional) + /// - "status": String status ("confirmed", "failed", etc.) + /// - "completed_at": String Unix timestamp (optional) + /// - "created_at": String Unix timestamp (optional) + pub fn transaction_data_key_name(&self, transaction_id: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_data:{transaction_id}"), + None => format!("eoa_executor:_tx_data:{transaction_id}"), + } + } + + /// Name of the list for transaction attempts + /// + /// Attempts are stored as a separate Redis LIST where each element is a JSON blob + /// of a TransactionAttempt. This allows efficient append operations. + pub fn transaction_attempts_list_name(&self, transaction_id: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_attempts:{transaction_id}"), + None => format!("eoa_executor:tx_attempts:{transaction_id}"), + } + } + + /// Name of the list for pending transactions + pub fn pending_transactions_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:pending_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:pending_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the zset for submitted transactions. nonce -> hash:id + /// + /// Same transaction might appear multiple times in the zset with different nonces/gas prices (and thus different hashes) + pub fn submitted_transactions_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:submitted_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:submitted_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the key that maps transaction hash to transaction id + pub fn transaction_hash_to_id_key_name(&self, hash: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_hash_to_id:{hash}"), + None => format!("eoa_executor:tx_hash_to_id:{hash}"), + } + } + + /// Name of the hashmap that maps `transaction_id` -> `BorrowedTransactionData` + /// + /// This is used for crash recovery. Before submitting a transaction, we atomically move from pending to this borrowed hashmap. + /// + /// On worker recovery, if any borrowed transactions are found, we rebroadcast them and move back to pending or submitted + /// + /// If there's no crash, happy path moves borrowed transactions back to pending or submitted + pub fn borrowed_transactions_hashmap_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:borrowed_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:borrowed_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the set that contains recycled nonces. + /// + /// If a transaction was submitted but failed (ie, we know with certainty it didn't enter the mempool), + /// + /// we add the nonce to this set. + /// + /// These nonces are used with priority, before any other nonces. + pub fn recycled_nonces_set_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:recycled_nonces:{}:{}", + self.chain_id, self.eoa + ), + None => format!( + "eoa_executor:recycled_nonces:{}:{}", + self.chain_id, self.eoa + ), + } + } + + /// Optimistic nonce key name. + /// + /// This is used for optimistic nonce tracking. + /// + /// We store the nonce of the last successfuly sent transaction for each EOA. + /// + /// We increment this nonce for each new transaction. + /// + /// !IMPORTANT! When sending a transaction, we use this nonce as the assigned nonce, NOT the incremented nonce. + pub fn optimistic_transaction_count_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:optimistic_nonce:{}:{}", + self.chain_id, self.eoa + ), + None => format!( + "eoa_executor:optimistic_nonce:{}:{}", + self.chain_id, self.eoa + ), + } + } + + /// Name of the key that contains the nonce of the last fetched ONCHAIN transaction count for each EOA. + /// + /// This is a cache for the actual transaction count, which is fetched from the RPC. + /// + /// The nonce for the NEXT transaction is the ONCHAIN transaction count (NOT + 1) + /// + /// Eg: transaction count is 0, so we use nonce 0 for sending the next transaction. Once successful, transaction count will be 1. + pub fn last_transaction_count_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:last_tx_nonce:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:last_tx_nonce:{}:{}", self.chain_id, self.eoa), + } + } + + /// EOA health key name. + /// + /// EOA health stores: + /// - cached balance, the timestamp of the last balance fetch + /// - timestamp of the last successful transaction confirmation + /// - timestamp of the last 5 nonce resets + pub fn eoa_health_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:health:{}:{}", self.chain_id, self.eoa), + None => format!("eoa_executor:health:{}:{}", self.chain_id, self.eoa), + } + } +} + +impl EoaExecutorStore { + pub fn new( + redis: ConnectionManager, + namespace: Option, + eoa: Address, + chain_id: u64, + ) -> Self { + Self { + redis, + keys: EoaExecutorStoreKeys { + eoa, + chain_id, + namespace, + }, + } + } +} + +impl std::ops::Deref for EoaExecutorStore { + type Target = EoaExecutorStoreKeys; + fn deref(&self) -> &Self::Target { + &self.keys + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaHealth { + pub balance: U256, + /// Update the balance threshold when we see out of funds errors + pub balance_threshold: U256, + pub balance_fetched_at: u64, + pub last_confirmation_at: u64, + pub last_nonce_movement_at: u64, // Track when nonce last moved for gas bump detection + pub nonce_resets: Vec, // Last 5 reset timestamps +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BorrowedTransactionData { + pub transaction_id: String, + pub signed_transaction: Signed, + pub hash: String, + pub borrowed_at: u64, +} + +/// Type of nonce allocation for transaction processing +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NonceType { + /// Nonce was recycled from a previously failed transaction + Recycled(u64), + /// Nonce was incremented from the current optimistic counter + Incremented(u64), +} + +impl NonceType { + /// Get the nonce value regardless of type + pub fn nonce(&self) -> u64 { + match self { + NonceType::Recycled(nonce) => *nonce, + NonceType::Incremented(nonce) => *nonce, + } + } + + /// Check if this is a recycled nonce + pub fn is_recycled(&self) -> bool { + matches!(self, NonceType::Recycled(_)) + } + + /// Check if this is an incremented nonce + pub fn is_incremented(&self) -> bool { + matches!(self, NonceType::Incremented(_)) + } +} + +impl EoaExecutorStore { + /// Aggressively acquire EOA lock, forcefully taking over from stalled workers + /// + /// Creates an AtomicEoaExecutorStore that owns the lock. + pub async fn acquire_eoa_lock_aggressively( + self, + worker_id: &str, + ) -> Result { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + + // First try normal acquisition + let acquired: bool = conn.set_nx(&lock_key, worker_id).await?; + if acquired { + return Ok(AtomicEoaExecutorStore { + store: self, + worker_id: worker_id.to_string(), + }); + } + // Lock exists, forcefully take it over + tracing::warn!( + eoa = %self.eoa, + chain_id = %self.chain_id, + worker_id = %worker_id, + "Forcefully taking over EOA lock from stalled worker" + ); + // Force set - no expiry, only released by explicit takeover + let _: () = conn.set(&lock_key, worker_id).await?; + Ok(AtomicEoaExecutorStore { + store: self, + worker_id: worker_id.to_string(), + }) + } + + /// Peek all borrowed transactions without removing them + pub async fn peek_borrowed_transactions( + &self, + ) -> Result, TransactionStoreError> { + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let mut conn = self.redis.clone(); + + let borrowed_map: HashMap = conn.hgetall(&borrowed_key).await?; + let mut result = Vec::new(); + + for (_nonce_str, transaction_json) in borrowed_map { + let borrowed_data: BorrowedTransactionData = serde_json::from_str(&transaction_json)?; + result.push(borrowed_data); + } + + Ok(result) + } + + /// Get all hashes below a certain nonce from submitted transactions + /// Returns (nonce, hash, transaction_id) tuples + pub async fn get_submitted_transactions_below_nonce( + &self, + below_nonce: u64, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + // Get all entries with nonce < below_nonce + let results: Vec = conn + .zrangebyscore_withscores(&submitted_key, 0, below_nonce - 1) + .await?; + + let submitted_txs: Vec = + SubmittedTransaction::from_redis_strings(&results); + + Ok(submitted_txs) + } + + /// Get all transaction IDs for a specific nonce + pub async fn get_submitted_transactions_for_nonce( + &self, + nonce: u64, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let results: Vec = conn + .zrangebyscore_withscores(&submitted_key, nonce, nonce) + .await?; + + let submitted_txs: Vec = + SubmittedTransaction::from_redis_strings(&results); + + Ok(submitted_txs) + } + + /// Check EOA health (balance, etc.) + pub async fn check_eoa_health(&self) -> Result, TransactionStoreError> { + let mut conn = self.redis.clone(); + + let health_json: Option = conn.get(self.eoa_health_key_name()).await?; + if let Some(json) = health_json { + let health: EoaHealth = serde_json::from_str(&json)?; + Ok(Some(health)) + } else { + Ok(None) + } + } + + /// Peek recycled nonces without removing them + pub async fn peek_recycled_nonces(&self) -> Result, TransactionStoreError> { + let recycled_key = self.recycled_nonces_set_name(); + let mut conn = self.redis.clone(); + + let nonces: Vec = conn.zrange(&recycled_key, 0, -1).await?; + Ok(nonces) + } + + /// Peek at pending transactions without removing them (safe for planning) + pub async fn peek_pending_transactions( + &self, + limit: u64, + ) -> Result, TransactionStoreError> { + let pending_key = self.pending_transactions_zset_name(); + let mut conn = self.redis.clone(); + + // Use LRANGE to peek without removing + let transaction_ids: Vec = + conn.lrange(&pending_key, 0, (limit as isize) - 1).await?; + Ok(transaction_ids) + } + + /// Get inflight budget (how many new transactions can be sent) + pub async fn get_inflight_budget( + &self, + max_inflight: u64, + ) -> Result { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let last_tx_count_key = self.last_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + // Read both values atomically to avoid race conditions + let (optimistic_nonce, last_tx_count): (Option, Option) = twmq::redis::pipe() + .get(&optimistic_key) + .get(&last_tx_count_key) + .query_async(&mut conn) + .await?; + + let optimistic = match optimistic_nonce { + Some(nonce) => nonce, + None => return Err(self.nonce_sync_required_error()), + }; + let last_count = match last_tx_count { + Some(count) => count, + None => return Err(self.nonce_sync_required_error()), + }; + + let current_inflight = optimistic.saturating_sub(last_count); + let available_budget = max_inflight.saturating_sub(current_inflight); + + Ok(available_budget) + } + + /// Get current optimistic nonce (without incrementing) + pub async fn get_optimistic_nonce(&self) -> Result { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + let current: Option = conn.get(&optimistic_key).await?; + match current { + Some(nonce) => Ok(nonce), + None => Err(self.nonce_sync_required_error()), + } + } + /// Get transaction ID for a given hash + pub async fn get_transaction_id_for_hash( + &self, + hash: &str, + ) -> Result, TransactionStoreError> { + let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); + let mut conn = self.redis.clone(); + + let transaction_id: Option = conn.get(&hash_to_id_key).await?; + Ok(transaction_id) + } + + /// Get transaction data by transaction ID + pub async fn get_transaction_data( + &self, + transaction_id: &str, + ) -> Result, TransactionStoreError> { + let tx_data_key = self.transaction_data_key_name(transaction_id); + let mut conn = self.redis.clone(); + + // Get the hash data (the transaction data is stored as a hash) + let hash_data: HashMap = conn.hgetall(&tx_data_key).await?; + + if hash_data.is_empty() { + return Ok(None); + } + + // Extract user_request from the hash data + let user_request_json = hash_data.get("user_request").ok_or_else(|| { + TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + } + })?; + + let user_request: EoaTransactionRequest = serde_json::from_str(user_request_json)?; + + // Extract receipt if present + let receipt = hash_data + .get("receipt") + .and_then(|receipt_str| serde_json::from_str(receipt_str).ok()); + + let created_at = hash_data.get("created_at").ok_or_else(|| { + TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + } + })?; + + // todo: in case of non-existent created_at, we should return a default value + let created_at = + created_at + .parse::() + .map_err(|_| TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + })?; + + // Extract attempts from separate list + let attempts_key = self.transaction_attempts_list_name(transaction_id); + let attempts_json_list: Vec = conn.lrange(&attempts_key, 0, -1).await?; + let mut attempts = Vec::new(); + for attempt_json in attempts_json_list { + if let Ok(attempt) = serde_json::from_str::(&attempt_json) { + attempts.push(attempt); + } + } + + Ok(Some(TransactionData { + transaction_id: transaction_id.to_string(), + created_at, + user_request, + receipt, + attempts, + })) + } + + /// Get cached transaction count + pub async fn get_cached_transaction_count(&self) -> Result { + let tx_count_key = self.last_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + let count: Option = conn.get(&tx_count_key).await?; + match count { + Some(count) => Ok(count), + None => Err(self.nonce_sync_required_error()), + } + } + + /// Add a transaction to the pending queue and store its data + /// This is called when a new transaction request comes in for an EOA + pub async fn add_transaction( + &self, + transaction_request: EoaTransactionRequest, + ) -> Result<(), TransactionStoreError> { + let transaction_id = &transaction_request.transaction_id; + + let tx_data_key = self.transaction_data_key_name(transaction_id); + let pending_key = self.pending_transactions_zset_name(); + + // Store transaction data as JSON in the user_request field of the hash + let user_request_json = serde_json::to_string(&transaction_request)?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + let mut conn = self.redis.clone(); + + // Use a pipeline to atomically store data and add to pending queue + let mut pipeline = twmq::redis::pipe(); + + // Store transaction data + pipeline.hset(&tx_data_key, "user_request", &user_request_json); + pipeline.hset(&tx_data_key, "status", "pending"); + pipeline.hset(&tx_data_key, "created_at", now); + + // Add to pending queue + pipeline.zadd(&pending_key, transaction_id, now); + + pipeline.query_async::<()>(&mut conn).await?; + + Ok(()) + } + + /// Get count of submitted transactions awaiting confirmation + pub async fn get_submitted_transactions_count(&self) -> Result { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let count: u64 = conn.zcard(&submitted_key).await?; + Ok(count) + } + + /// Get the submitted transactions for the highest nonce value + /// + /// Internally submissions are stored in a zset by nonce -> hash:id + /// + /// This will return all hash:id pairs for the highest nonce + #[tracing::instrument(skip_all)] + pub async fn get_highest_submitted_nonce_tranasactions( + &self, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let highest_nonce_txs: Vec = + conn.zrange_withscores(&submitted_key, -1, -1).await?; + + let submitted_txs: Vec = + SubmittedTransaction::from_redis_strings(&highest_nonce_txs); + + Ok(submitted_txs) + } +} + +// Additional error types +#[derive(Debug, thiserror::Error, Serialize, Deserialize, Clone)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] +pub enum TransactionStoreError { + #[error("Redis error: {message}")] + RedisError { message: String }, + + #[error("Serialization error: {message}")] + DeserError { message: String, text: String }, + + #[error("Transaction not found: {transaction_id}")] + TransactionNotFound { transaction_id: String }, + + #[error("Lost EOA lock: {eoa}:{chain_id} worker: {worker_id}")] + LockLost { + eoa: Address, + chain_id: u64, + worker_id: String, + }, + + #[error("Internal error - worker should quit: {message}")] + InternalError { message: String }, + + #[error("Transaction {transaction_id} not in borrowed state for nonce {nonce}")] + TransactionNotInBorrowedState { transaction_id: String, nonce: u64 }, + + #[error("Hash {hash} not found in submitted transactions")] + HashNotInSubmittedState { hash: String }, + + #[error("Transaction {transaction_id} has no hashes in submitted state")] + TransactionNotInSubmittedState { transaction_id: String }, + + #[error("Nonce {nonce} not available in recycled set")] + NonceNotInRecycledSet { nonce: u64 }, + + #[error("Transaction {transaction_id} not found in pending queue")] + TransactionNotInPendingQueue { transaction_id: String }, + + #[error("Optimistic nonce changed: expected {expected}, found {actual}")] + OptimisticNonceChanged { expected: u64, actual: u64 }, + + #[error("WATCH failed - state changed during operation")] + WatchFailed, + + #[error( + "Nonce synchronization required for {eoa}:{chain_id} - no cached transaction count available" + )] + NonceSyncRequired { eoa: Address, chain_id: u64 }, +} + +impl From for TransactionStoreError { + fn from(error: twmq::redis::RedisError) -> Self { + TransactionStoreError::RedisError { + message: error.to_string(), + } + } +} + +impl From for TransactionStoreError { + fn from(error: serde_json::Error) -> Self { + TransactionStoreError::DeserError { + message: error.to_string(), + text: error.to_string(), + } + } +} diff --git a/executors/src/eoa/store/submitted.rs b/executors/src/eoa/store/submitted.rs new file mode 100644 index 0000000..d1196c7 --- /dev/null +++ b/executors/src/eoa/store/submitted.rs @@ -0,0 +1,276 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; + +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::eoa::store::{ + ConfirmedTransaction, EoaExecutorStoreKeys, TransactionStoreError, atomic::SafeRedisTransaction, +}; + +#[derive(Debug, Clone)] +pub struct SubmittedTransaction { + pub nonce: u64, + pub hash: String, + pub transaction_id: String, + pub queued_at: u64, +} + +pub type SubmittedTransactionStringWithNonce = (String, u64); + +impl SubmittedTransaction { + pub fn from_redis_strings(redis_strings: &[SubmittedTransactionStringWithNonce]) -> Vec { + redis_strings + .iter() + .filter_map(|tx| { + let parts: Vec<&str> = tx.0.split(':').collect(); + if parts.len() == 3 { + if let Ok(queued_at) = parts[2].parse::() { + Some(SubmittedTransaction { + hash: parts[0].to_string(), + transaction_id: parts[1].to_string(), + nonce: tx.1, + queued_at, + }) + } else { + tracing::error!("Invalid queued_at timestamp: {}", tx.0); + None + } + } else { + tracing::error!( + "Invalid transaction format, expected 3 parts separated by ':': {}", + tx.0 + ); + None + } + }) + .collect() + } + + /// Returns the string representation of the submitted transaction with the nonce + /// + /// This is used to add the transaction to the submitted state in Redis + /// + /// The format is: + /// + /// ```text + /// hash:transaction_id:queued_at + /// ``` + /// + /// The nonce is the value of the transaction in the submitted state, and is used as the score of the submitted zset + pub fn to_redis_string_with_nonce(&self) -> SubmittedTransactionStringWithNonce { + ( + format!("{}:{}:{}", self.hash, self.transaction_id, self.queued_at), + self.nonce, + ) + } +} + +pub struct CleanSubmittedTransactions<'a> { + pub last_confirmed_nonce: u64, + pub confirmed_transactions: &'a [ConfirmedTransaction], + pub keys: &'a EoaExecutorStoreKeys, +} + +#[derive(Debug, Default)] +pub struct CleanupReport { + pub total_hashes_processed: usize, + pub unique_transaction_ids: usize, + pub noop_count: usize, + pub moved_to_success: usize, + pub moved_to_pending: usize, + + /// Any transaction ID values that have multiple nonces in the submitted state + pub cross_nonce_violations: Vec<(String, Vec)>, // (transaction_id, nonces) + + /// Any nonces that have multiple confirmations (very rare, indicates re-org) + pub per_nonce_violations: Vec<(u64, Vec)>, // (nonce, confirmed_hashes) + + /// Any nonces that have no confirmations (transactions we sent got replaced by a different one uknown to us) + pub nonces_without_receipts: Vec<(u64, Vec)>, // (nonce, hashes) +} + +/// This operation takes a list of confirmed transactions and the last confirmed nonce +/// +/// It will fetch all submitted transactions with a nonce less than or equal to the last confirmed nonce. +/// For each nonce: +/// - it will go through all the hashes for that nonce +/// - if the hash is in the confirmed transactions, it will be removed from submitted to success +/// - if the hash is not in the confirmed transactions, it will be removed from submitted to pending +/// +/// It will also deduplicate transactions by ID, so if any of the hashes for that ID are in the confirmed transactions, +/// this hash will not be moved back to pending. +/// +/// ***IMPORTANT***: This should not happen with different nonces. A transaction ID should only appear once in the submitted state. +/// Multiple submissions for the same transaction ID with different nonces can cause duplicate transactions +/// Multiple submissions for the same transaction ID with the same nonce is fine, because this indicated gas bumps. +impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { + type ValidationData = Vec; + type OperationResult = CleanupReport; + + fn name(&self) -> &str { + "clean submitted transactions" + } + + fn watch_keys(&self) -> Vec { + vec![self.keys.submitted_transactions_zset_name()] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + let submitted_txs: Vec = conn + .zrange_withscores( + self.keys.submitted_transactions_zset_name(), + 0, + self.last_confirmed_nonce as isize, + ) + .await?; + + let submitted_txs = SubmittedTransaction::from_redis_strings(&submitted_txs); + Ok(submitted_txs) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + submitted_txs: Self::ValidationData, + ) -> Self::OperationResult { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Build confirmed lookups + let confirmed_hashes: HashSet<&str> = self + .confirmed_transactions + .iter() + .map(|tx| tx.hash.as_str()) + .collect(); + + let confirmed_ids: BTreeMap<&str, &ConfirmedTransaction> = self + .confirmed_transactions + .iter() + .map(|tx| (tx.transaction_id.as_str(), tx)) + .collect(); + + // Detect violations and get grouped data + let (_, _, mut report) = detect_violations(&submitted_txs, &confirmed_hashes); + + // Process every hash and track unique IDs + let mut processed_ids = HashSet::new(); + + let mut replaced_transactions = Vec::with_capacity(submitted_txs.len()); + + for tx in &submitted_txs { + // Clean up this hash from Redis (happens for ALL hashes) + let (submitted_tx_redis_string, _nonce) = tx.clone().to_redis_string_with_nonce(); + + pipeline.zrem( + self.keys.submitted_transactions_zset_name(), + &submitted_tx_redis_string, + ); + pipeline.del(self.keys.transaction_hash_to_id_key_name(&tx.hash)); + + // Process each unique transaction_id once + if processed_ids.insert(&tx.transaction_id) { + match ( + tx.transaction_id.as_str(), + confirmed_ids.get(tx.transaction_id.as_str()), + ) { + // if the transaction id is noop, we don't do anything + ("noop", _) => report.noop_count += 1, + + // in case of a valid ID, we check if it's in the confirmed transactions + // if it is confirmed, we succeed it and queue success jobs + (id, Some(confirmed_tx)) => { + let data_key_name = self.keys.transaction_data_key_name(id); + pipeline.hset(&data_key_name, "status", "confirmed"); + pipeline.hset(&data_key_name, "completed_at", now); + pipeline.hset(&data_key_name, "receipt", confirmed_tx.receipt_data.clone()); + + // TODO: + // queue success jobs here + + report.moved_to_success += 1; + } + + // if the ID is not in the confirmed transactions, we queue it for pending + _ => { + replaced_transactions.push((&tx.transaction_id, tx.queued_at)); + report.moved_to_pending += 1; + } + } + } + } + + pipeline.zadd_multiple( + self.keys.pending_transactions_zset_name(), + &replaced_transactions, + ); + + // Finalize report stats + report.total_hashes_processed = submitted_txs.len(); + report.unique_transaction_ids = processed_ids.len(); + + report + } +} + +fn detect_violations<'a>( + submitted_txs: &'a [SubmittedTransaction], + confirmed_hashes: &'a HashSet<&str>, +) -> ( + HashMap<&'a str, Vec>, + BTreeMap>, + CleanupReport, +) { + let mut report = CleanupReport::default(); + let mut txs_by_nonce: BTreeMap> = BTreeMap::new(); + let mut transaction_id_to_nonces: HashMap<&str, Vec> = HashMap::new(); + + // Group data + for tx in submitted_txs { + txs_by_nonce.entry(tx.nonce).or_default().push(tx); + transaction_id_to_nonces + .entry(&tx.transaction_id) + .or_default() + .push(tx.nonce); + } + + // Check cross-nonce violations + for (transaction_id, nonces) in &transaction_id_to_nonces { + let mut unique_nonces = nonces.clone(); + unique_nonces.sort(); + unique_nonces.dedup(); + if unique_nonces.len() > 1 { + report + .cross_nonce_violations + .push((transaction_id.to_string(), unique_nonces)); + } + } + + // Check per-nonce violations + for (nonce, txs) in &txs_by_nonce { + let confirmed_hashes_for_nonce: Vec = txs + .iter() + .filter(|tx| confirmed_hashes.contains(tx.hash.as_str())) + .map(|tx| tx.hash.clone()) + .collect(); + + if confirmed_hashes_for_nonce.len() > 1 { + report + .per_nonce_violations + .push((*nonce, confirmed_hashes_for_nonce)); + } + } + + // Check nonces without receipts + for (nonce, txs) in &txs_by_nonce { + let has_confirmed = txs + .iter() + .any(|tx| confirmed_hashes.contains(tx.hash.as_str())); + if !has_confirmed { + let hashes: Vec = txs.iter().map(|tx| tx.hash.clone()).collect(); + report.nonces_without_receipts.push((*nonce, hashes)); + } + } + + (transaction_id_to_nonces, txs_by_nonce, report) +} diff --git a/executors/src/eoa/worker.rs b/executors/src/eoa/worker.rs index 436d148..2322340 100644 --- a/executors/src/eoa/worker.rs +++ b/executors/src/eoa/worker.rs @@ -21,6 +21,8 @@ use hex; use serde::{Deserialize, Serialize}; use std::{sync::Arc, time::Duration}; use tokio::time::sleep; +use twmq::redis::AsyncCommands; +use twmq::redis::aio::ConnectionManager; use twmq::{ DurableExecution, FailHookData, NackHookData, SuccessHookData, UserCancellable, error::TwmqError, @@ -29,8 +31,9 @@ use twmq::{ }; use crate::eoa::store::{ - BorrowedTransactionData, EoaExecutorStore, EoaHealth, EoaTransactionRequest, - ScopedEoaExecutorStore, TransactionData, TransactionStoreError, + AtomicEoaExecutorStore, BorrowedTransactionData, CleanupReport, ConfirmedTransaction, + EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, EoaTransactionRequest, ReplacedTransaction, + SubmittedTransaction, TransactionData, TransactionStoreError, }; // ========== SPEC-COMPLIANT CONSTANTS ========== @@ -233,48 +236,19 @@ fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { } } -// ========== PREPARED TRANSACTION ========== #[derive(Debug, Clone)] -struct PreparedTransaction { - transaction_id: String, - signed_tx: Signed, - nonce: u64, -} - -// ========== CONFIRMATION FLOW DATA STRUCTURES ========== -#[derive(Debug, Clone)] -struct SubmittedTransaction { - nonce: u64, - hash: String, - transaction_id: String, -} - -#[derive(Debug, Clone)] -struct ConfirmedTransaction { - nonce: u64, - hash: String, - transaction_id: String, - receipt: alloy::rpc::types::TransactionReceipt, -} - -#[derive(Debug, Clone)] -struct ReplacedTransaction { - hash: String, - transaction_id: String, -} - -// ========== STORE BATCH OPERATION TYPES ========== -#[derive(Debug, Clone)] -pub struct TransactionSuccess { +struct ConfirmedTransactionWithRichReceipt { + pub nonce: u64, pub hash: String, pub transaction_id: String, - pub receipt_data: String, + pub receipt: alloy::rpc::types::TransactionReceipt, } #[derive(Debug, Clone)] -pub struct TransactionReplacement { - pub hash: String, +pub struct PreparedTransaction { pub transaction_id: String, + pub signed_tx: Signed, + pub nonce: u64, } // ========== MAIN WORKER ========== @@ -299,7 +273,9 @@ where CS: ChainService + Send + Sync + 'static, { pub chain_service: Arc, - pub store: Arc, + pub redis: ConnectionManager, + pub namespace: Option, + pub eoa_signer: Arc, pub max_inflight: u64, // Note: Spec uses MAX_INFLIGHT_PER_EOA constant pub max_recycled_nonces: u64, // Note: Spec uses MAX_RECYCLED_THRESHOLD constant @@ -331,12 +307,13 @@ where .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; // 2. CREATE SCOPED STORE (acquires lock) - let scoped = ScopedEoaExecutorStore::build( - &self.store, + let scoped = EoaExecutorStore::new( + self.redis.clone(), + self.namespace.clone(), data.eoa_address, data.chain_id, - job.lease_token.clone(), ) + .acquire_eoa_lock_aggressively(&job.lease_token) .await .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; @@ -355,13 +332,7 @@ where _success_data: SuccessHookData<'_, Self::Output>, _tx: &mut TransactionContext<'_>, ) { - // Release EOA lock on success - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.lease_token, - ) - .await; + self.release_eoa_lock(&job.job.data).await; } async fn on_nack( @@ -370,13 +341,7 @@ where _nack_data: NackHookData<'_, Self::ErrorData>, _tx: &mut TransactionContext<'_>, ) { - // Release EOA lock on nack - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.lease_token, - ) - .await; + self.release_eoa_lock(&job.job.data).await; } async fn on_fail( @@ -385,13 +350,7 @@ where _fail_data: FailHookData<'_, Self::ErrorData>, _tx: &mut TransactionContext<'_>, ) { - // Release EOA lock on fail - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.lease_token, - ) - .await; + self.release_eoa_lock(&job.job.data).await; } } @@ -402,7 +361,7 @@ where /// Execute the main EOA worker workflow async fn execute_main_workflow( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> JobResult { // 1. CRASH RECOVERY @@ -412,7 +371,7 @@ where .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; // 2. CONFIRM FLOW - let (confirmed, failed) = self + let confirmations_report = self .confirm_flow(scoped, chain) .await .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; @@ -458,19 +417,26 @@ where // Only succeed if no work remains Ok(EoaExecutorWorkerResult { recovered_transactions: recovered, - confirmed_transactions: confirmed, - failed_transactions: failed, + confirmed_transactions: confirmations_report.moved_to_success as u32, + failed_transactions: confirmations_report.moved_to_pending as u32, sent_transactions: sent, }) } /// Release EOA lock following the spec's finally pattern - async fn release_eoa_lock(&self, eoa: Address, chain_id: u64, worker_id: &str) { - if let Err(e) = self.store.release_eoa_lock(eoa, chain_id, worker_id).await { + async fn release_eoa_lock(&self, job_data: &EoaExecutorWorkerJobData) { + let keys = EoaExecutorStoreKeys::new( + job_data.eoa_address, + job_data.chain_id, + self.namespace.clone(), + ); + + let lock_key = keys.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + if let Err(e) = conn.del::<&str, ()>(&lock_key).await { tracing::error!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, + eoa = %job_data.eoa_address, + chain_id = %job_data.chain_id, error = %e, "Failed to release EOA lock" ); @@ -481,7 +447,7 @@ where #[tracing::instrument(skip_all)] async fn recover_borrowed_state( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { let mut borrowed_transactions = scoped.peek_borrowed_transactions().await?; @@ -531,7 +497,11 @@ where Ok(_) => { // Transaction was sent successfully scoped - .move_borrowed_to_submitted(nonce, &borrowed.hash, &borrowed.transaction_id) + .atomic_move_borrowed_to_submitted( + nonce, + &borrowed.hash, + &borrowed.transaction_id, + ) .await?; tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted"); } @@ -540,7 +510,7 @@ where SendErrorClassification::PossiblySent => { // Transaction possibly sent, move to submitted scoped - .move_borrowed_to_submitted( + .atomic_move_borrowed_to_submitted( nonce, &borrowed.hash, &borrowed.transaction_id, @@ -551,7 +521,7 @@ where SendErrorClassification::DeterministicFailure => { // Transaction is broken, recycle nonce and requeue scoped - .move_borrowed_to_recycled(nonce, &borrowed.transaction_id) + .atomic_move_borrowed_to_recycled(nonce, &borrowed.transaction_id) .await?; tracing::warn!(transaction_id = %borrowed.transaction_id, nonce = nonce, error = %e, "Recycled failed transaction"); @@ -583,9 +553,9 @@ where #[tracing::instrument(skip_all)] async fn confirm_flow( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, - ) -> Result<(u32, u32), EoaExecutorWorkerError> { + ) -> Result { // Get fresh on-chain transaction count let current_chain_nonce = chain .provider() @@ -641,7 +611,7 @@ where } tracing::debug!("No nonce progress, skipping confirm flow"); - return Ok((0, 0)); + return Ok(CleanupReport::default()); } tracing::info!( @@ -651,104 +621,69 @@ where ); // Get all pending transactions below the current chain nonce - let waiting_txs = self - .get_submitted_transactions_below_nonce(scoped, current_chain_nonce) + let waiting_txs = scoped + .get_submitted_transactions_below_nonce(current_chain_nonce) .await?; if waiting_txs.is_empty() { tracing::debug!("No waiting transactions to confirm"); - return Ok((0, 0)); + return Ok(CleanupReport::default()); } // Fetch receipts and categorize transactions let (confirmed_txs, replaced_txs) = self - .fetch_and_categorize_transactions(chain, waiting_txs) + .fetch_confirmed_transaction_receipts(chain, waiting_txs) .await; // Process confirmed transactions - let confirmed_count = if !confirmed_txs.is_empty() { - let successes: Vec = confirmed_txs - .into_iter() - .map(|tx| { - let receipt_data = match serde_json::to_string(&tx.receipt) { - Ok(receipt_json) => receipt_json, - Err(e) => { - tracing::warn!( - transaction_id = %tx.transaction_id, - hash = %tx.hash, - error = %e, - "Failed to serialize receipt as JSON, using debug format" - ); - format!("{:?}", tx.receipt) - } - }; - - tracing::info!( - transaction_id = %tx.transaction_id, - nonce = tx.nonce, - hash = %tx.hash, - "Transaction confirmed" - ); - - TransactionSuccess { - hash: tx.hash, - transaction_id: tx.transaction_id, - receipt_data, + let successes: Vec = confirmed_txs + .into_iter() + .map(|tx| { + let receipt_data = match serde_json::to_string(&tx.receipt) { + Ok(receipt_json) => receipt_json, + Err(e) => { + tracing::warn!( + transaction_id = %tx.transaction_id, + hash = %tx.hash, + error = %e, + "Failed to serialize receipt as JSON, using debug format" + ); + format!("{:?}", tx.receipt) } - }) - .collect(); + }; - let count = successes.len() as u32; - scoped.batch_succeed_transactions(successes).await?; - count - } else { - 0 - }; + tracing::info!( + transaction_id = %tx.transaction_id, + nonce = tx.nonce, + hash = %tx.hash, + "Transaction confirmed" + ); - // Process replaced transactions - let replaced_count = if !replaced_txs.is_empty() { - let replacements: Vec = replaced_txs - .into_iter() - .map(|tx| { - tracing::warn!( - transaction_id = %tx.transaction_id, - hash = %tx.hash, - "Transaction failed, requeued" - ); - TransactionReplacement { - hash: tx.hash, - transaction_id: tx.transaction_id, - } - }) - .collect(); + ConfirmedTransaction { + hash: tx.hash, + transaction_id: tx.transaction_id, + receipt_data, + } + }) + .collect(); - let count = replacements.len() as u32; - scoped - .batch_fail_and_requeue_transactions(replacements) - .await?; - count - } else { - 0 - }; + let report = scoped + .clean_submitted_transactions(&successes, current_chain_nonce - 1) + .await?; // Update cached transaction count scoped .update_cached_transaction_count(current_chain_nonce) .await?; - // Synchronize nonces to ensure consistency - scoped - .synchronize_nonces_with_chain(current_chain_nonce) - .await?; - - Ok((confirmed_count, replaced_count)) + Ok(report) } // ========== SEND FLOW ========== #[tracing::instrument(skip_all)] async fn send_flow( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { // 1. Get EOA health (initializes if needed) and check if we should update balance @@ -811,7 +746,7 @@ where async fn process_recycled_nonces( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { let recycled_nonces = scoped.peek_recycled_nonces().await?; @@ -820,11 +755,18 @@ where return Ok(0); } - // Get pending transactions (one per recycled nonce) + // Get pending transactions let pending_txs = scoped .peek_pending_transactions(recycled_nonces.len() as u64) .await?; + // let highest_submitted_nonce_txs = + // scoped.get_highest_submitted_nonce_tranasactions().await?; + + // let highest_submitted_nonce = highest_submitted_nonce_txs + // .first() + // .and_then(|tx| Some(tx.nonce)); + // 1. SEQUENTIAL REDIS: Collect nonce-transaction pairs let mut nonce_tx_pairs = Vec::new(); for (i, nonce) in recycled_nonces.into_iter().enumerate() { @@ -961,9 +903,9 @@ where Ok(_) => { // Transaction sent successfully match scoped - .move_borrowed_to_submitted( + .atomic_move_borrowed_to_submitted( prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), + &prepared.signed_tx.hash().to_string(), &prepared.transaction_id, ) .await @@ -991,7 +933,7 @@ where SendErrorClassification::PossiblySent => { // Move to submitted state match scoped - .move_borrowed_to_submitted( + .atomic_move_borrowed_to_submitted( prepared.nonce, &format!("{:?}", prepared.signed_tx.hash()), &prepared.transaction_id, @@ -1018,7 +960,10 @@ where SendErrorClassification::DeterministicFailure => { // Recycle nonce and requeue transaction match scoped - .move_borrowed_to_recycled(prepared.nonce, &prepared.transaction_id) + .atomic_move_borrowed_to_recycled( + prepared.nonce, + &prepared.transaction_id, + ) .await { Ok(()) => { @@ -1066,7 +1011,7 @@ where async fn process_new_transactions( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, budget: u64, ) -> Result { @@ -1204,7 +1149,7 @@ where Ok(pending) => { // Transaction sent successfully match scoped - .move_borrowed_to_submitted( + .atomic_move_borrowed_to_submitted( prepared.nonce, &pending.tx_hash().to_string(), &prepared.transaction_id, @@ -1234,7 +1179,7 @@ where SendErrorClassification::PossiblySent => { // Move to submitted state match scoped - .move_borrowed_to_submitted( + .atomic_move_borrowed_to_submitted( prepared.nonce, &format!("{:?}", prepared.signed_tx.hash()), &prepared.transaction_id, @@ -1261,7 +1206,10 @@ where SendErrorClassification::DeterministicFailure => { // Recycle nonce and requeue transaction match scoped - .move_borrowed_to_recycled(prepared.nonce, &prepared.transaction_id) + .atomic_move_borrowed_to_recycled( + prepared.nonce, + &prepared.transaction_id, + ) .await { Ok(()) => { @@ -1310,7 +1258,7 @@ where // ========== TRANSACTION BUILDING & SENDING ========== async fn build_and_sign_single_transaction( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, transaction_id: &str, nonce: u64, chain: &impl Chain, @@ -1337,7 +1285,7 @@ where async fn send_noop_transaction( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, nonce: u64, credential: SigningCredential, @@ -1381,7 +1329,7 @@ where /// Attempt to gas bump a stalled transaction for the next expected nonce async fn attempt_gas_bump_for_stalled_nonce( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, expected_nonce: u64, ) -> Result { @@ -1391,9 +1339,11 @@ where ); // Get all transaction IDs for this nonce - let transaction_ids = scoped.get_transaction_ids_for_nonce(expected_nonce).await?; + let submitted_transactions = scoped + .get_submitted_transactions_for_nonce(expected_nonce) + .await?; - if transaction_ids.is_empty() { + if submitted_transactions.is_empty() { tracing::debug!( nonce = expected_nonce, "No transactions found for stalled nonce" @@ -1405,7 +1355,7 @@ where let mut newest_transaction: Option<(String, TransactionData)> = None; let mut newest_submitted_at = 0u64; - for transaction_id in transaction_ids { + for SubmittedTransaction { transaction_id, .. } in submitted_transactions { if let Some(tx_data) = scoped.get_transaction_data(&transaction_id).await? { // Find the most recent attempt for this transaction if let Some(latest_attempt) = tx_data.attempts.last() { @@ -1483,7 +1433,15 @@ where // Record the gas bump attempt scoped - .add_gas_bump_attempt(&transaction_id, bumped_tx.clone()) + .add_gas_bump_attempt( + &SubmittedTransaction { + nonce: expected_nonce, + hash: bumped_tx.hash().to_string(), + transaction_id: transaction_id.to_string(), + queued_at: tx_data.created_at, + }, + bumped_tx.clone(), + ) .await?; // Send the bumped transaction @@ -1519,7 +1477,7 @@ where /// This method ensures the health data is always available for the worker async fn get_eoa_health( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { let store_health = scoped.check_eoa_health().await?; @@ -1563,7 +1521,7 @@ where #[tracing::instrument(skip_all, fields(eoa = %scoped.eoa(), chain_id = %chain.chain_id()))] async fn update_balance_threshold( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result<(), EoaExecutorWorkerError> { let mut health = self.get_eoa_health(scoped, chain).await?; @@ -1586,34 +1544,15 @@ where Ok(()) } - // ========== CONFIRMATION FLOW HELPERS ========== - - /// Get submitted transactions below the given nonce - async fn get_submitted_transactions_below_nonce( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - nonce: u64, - ) -> Result, EoaExecutorWorkerError> { - let submitted_hashes = scoped.get_hashes_below_nonce(nonce).await?; - - let submitted_txs = submitted_hashes - .into_iter() - .map(|(nonce, hash, transaction_id)| SubmittedTransaction { - nonce, - hash, - transaction_id, - }) - .collect(); - - Ok(submitted_txs) - } - /// Fetch receipts for all submitted transactions and categorize them - async fn fetch_and_categorize_transactions( + async fn fetch_confirmed_transaction_receipts( &self, chain: &impl Chain, submitted_txs: Vec, - ) -> (Vec, Vec) { + ) -> ( + Vec, + Vec, + ) { // Fetch all receipts in parallel let receipt_futures: Vec<_> = submitted_txs .iter() @@ -1638,7 +1577,7 @@ where for (tx, receipt_result) in receipt_results { match receipt_result { Ok(Some(receipt)) => { - confirmed_txs.push(ConfirmedTransaction { + confirmed_txs.push(ConfirmedTransactionWithRichReceipt { nonce: tx.nonce, hash: tx.hash.clone(), transaction_id: tx.transaction_id.clone(), diff --git a/server/src/execution_router/mod.rs b/server/src/execution_router/mod.rs index b44fb32..9280dd1 100644 --- a/server/src/execution_router/mod.rs +++ b/server/src/execution_router/mod.rs @@ -26,7 +26,7 @@ use engine_executors::{ transaction_registry::TransactionRegistry, webhook::WebhookJobHandler, }; -use twmq::{Queue, error::TwmqError}; +use twmq::{Queue, error::TwmqError, redis::aio::ConnectionManager}; use vault_sdk::VaultClient; use vault_types::{ RegexRule, Rule, @@ -37,11 +37,12 @@ use vault_types::{ use crate::chains::ThirdwebChainService; pub struct ExecutionRouter { + pub redis: ConnectionManager, + pub namespace: Option, pub webhook_queue: Arc>, pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, pub eoa_executor_queue: Arc>>, - pub eoa_executor_store: Arc, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -409,8 +410,15 @@ impl ExecutionRouter { transaction_type_data: transaction.transaction_type_data.clone(), }; + let eoa_executor_store = EoaExecutorStore::new( + self.redis.clone(), + self.namespace.clone(), + eoa_execution_options.from, + base_execution_options.chain_id, + ); + // Add transaction to the store - self.eoa_executor_store + eoa_executor_store .add_transaction(eoa_transaction_request) .await .map_err(|e| TwmqError::Runtime { diff --git a/server/src/main.rs b/server/src/main.rs index 995317b..cd4dc4c 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -51,9 +51,10 @@ async fn main() -> anyhow::Result<()> { iaw_client: iaw_client.clone(), }); let eoa_signer = Arc::new(EoaSigner::new(vault_client.clone(), iaw_client)); + let redis_client = twmq::redis::Client::open(config.redis.url.as_str())?; let queue_manager = QueueManager::new( - &config.redis, + redis_client.clone(), &config.queue, chains.clone(), signer.clone(), @@ -74,11 +75,12 @@ async fn main() -> anyhow::Result<()> { .build()?; let execution_router = ExecutionRouter { + namespace: config.queue.execution_namespace.clone(), + redis: redis_client.get_connection_manager().await?, webhook_queue: queue_manager.webhook_queue.clone(), external_bundler_send_queue: queue_manager.external_bundler_send_queue.clone(), userop_confirm_queue: queue_manager.userop_confirm_queue.clone(), eoa_executor_queue: queue_manager.eoa_executor_queue.clone(), - eoa_executor_store: queue_manager.eoa_executor_store.clone(), eip7702_send_queue: queue_manager.eip7702_send_queue.clone(), eip7702_confirm_queue: queue_manager.eip7702_confirm_queue.clone(), transaction_registry: queue_manager.transaction_registry.clone(), diff --git a/server/src/queue/manager.rs b/server/src/queue/manager.rs index 04e216e..53ca78b 100644 --- a/server/src/queue/manager.rs +++ b/server/src/queue/manager.rs @@ -26,7 +26,6 @@ pub struct QueueManager { pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, pub eoa_executor_queue: Arc>>, - pub eoa_executor_store: Arc, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -48,27 +47,18 @@ const EOA_EXECUTOR_QUEUE_NAME: &str = "eoa_executor"; impl QueueManager { pub async fn new( - redis_config: &RedisConfig, + redis_client: twmq::redis::Client, queue_config: &QueueConfig, chain_service: Arc, userop_signer: Arc, eoa_signer: Arc, ) -> Result { - // Create Redis clients - let redis_client = twmq::redis::Client::open(redis_config.url.as_str())?; - // Create transaction registry let transaction_registry = Arc::new(TransactionRegistry::new( redis_client.get_connection_manager().await?, queue_config.execution_namespace.clone(), )); - // Create EOA executor store - let eoa_executor_store = Arc::new(EoaExecutorStore::new( - redis_client.get_connection_manager().await?, - queue_config.execution_namespace.clone(), - )); - // Create deployment cache and lock let deployment_cache = RedisDeploymentCache::new(redis_client.clone()).await?; let deployment_lock = RedisDeploymentLock::new(redis_client.clone()).await?; @@ -221,8 +211,9 @@ impl QueueManager { // Create EOA executor queue let eoa_executor_handler = EoaExecutorWorker { chain_service: chain_service.clone(), - store: eoa_executor_store.clone(), eoa_signer: eoa_signer.clone(), + namespace: queue_config.execution_namespace.clone(), + redis: redis_client.get_connection_manager().await?, max_inflight: 100, max_recycled_nonces: 50, }; @@ -241,7 +232,6 @@ impl QueueManager { external_bundler_send_queue, userop_confirm_queue, eoa_executor_queue, - eoa_executor_store, eip7702_send_queue, eip7702_confirm_queue, transaction_registry, From 67a7ab7eb5fec67a6c4ec1ee81dd56e2ab1eaf87 Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Sat, 12 Jul 2025 07:00:17 +0530 Subject: [PATCH 03/10] add event notification integration points --- executors/src/eoa/events.rs | 119 +++++++++++++++++++++++++++ executors/src/eoa/mod.rs | 1 + executors/src/eoa/store/submitted.rs | 4 +- executors/src/eoa/worker.rs | 9 +- executors/src/webhook/envelope.rs | 29 +++++++ executors/src/webhook/mod.rs | 86 ++++++++++++++++++- server/src/queue/manager.rs | 8 +- 7 files changed, 246 insertions(+), 10 deletions(-) create mode 100644 executors/src/eoa/events.rs diff --git a/executors/src/eoa/events.rs b/executors/src/eoa/events.rs new file mode 100644 index 0000000..354679a --- /dev/null +++ b/executors/src/eoa/events.rs @@ -0,0 +1,119 @@ +use std::fmt::Display; + +use engine_core::error::EngineError; +use serde::{Deserialize, Serialize}; +use twmq::job::RequeuePosition; + +use crate::{ + eoa::{ + store::{SubmittedTransaction, TransactionData}, + worker::ConfirmedTransactionWithRichReceipt, + }, + webhook::envelope::{ + BareWebhookNotificationEnvelope, SerializableNackData, SerializableSuccessData, StageEvent, + }, +}; + +pub struct EoaExecutorEvent { + pub transaction_data: TransactionData, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaSendAttemptNackData { + pub nonce: u64, + pub error: EngineError, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum EoaExecutorStage { + SendAttempt, + SendAttemptNack, + TransactionReplaced, + TransactionConfirmed, +} + +impl Display for EoaExecutorStage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EoaExecutorStage::SendAttempt => write!(f, "send_attempt"), + EoaExecutorStage::SendAttemptNack => write!(f, "send_attempt_nack"), + EoaExecutorStage::TransactionReplaced => write!(f, "transaction_replaced"), + EoaExecutorStage::TransactionConfirmed => write!(f, "transaction_confirmed"), + } + } +} + +const EXECUTOR_NAME: &str = "eoa"; + +impl EoaExecutorEvent { + pub fn send_attempt_success_envelopes( + &self, + submitted_transaction: SubmittedTransaction, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::SendAttempt.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: submitted_transaction.clone(), + }, + } + } + + pub fn send_attempt_nack_envelopes( + &self, + nonce: u64, + error: EngineError, + attempt_number: u32, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::SendAttemptNack.to_string(), + event_type: StageEvent::Nack, + payload: SerializableNackData { + error: EoaSendAttemptNackData { + nonce, + error: error.clone(), + }, + delay_ms: None, + position: RequeuePosition::Last, + attempt_number, + max_attempts: None, + next_retry_at: None, + }, + } + } + + pub fn transaction_replaced_envelopes( + &self, + replaced_transaction: SubmittedTransaction, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::TransactionReplaced.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: replaced_transaction.clone(), + }, + } + } + + pub fn transaction_confirmed_envelopes( + &self, + confirmed_transaction: ConfirmedTransactionWithRichReceipt, + ) -> BareWebhookNotificationEnvelope> + { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::TransactionConfirmed.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: confirmed_transaction.clone(), + }, + } + } +} diff --git a/executors/src/eoa/mod.rs b/executors/src/eoa/mod.rs index 9b44404..89dd7e2 100644 --- a/executors/src/eoa/mod.rs +++ b/executors/src/eoa/mod.rs @@ -1,4 +1,5 @@ pub mod error_classifier; +pub mod events; pub mod store; pub mod worker; diff --git a/executors/src/eoa/store/submitted.rs b/executors/src/eoa/store/submitted.rs index d1196c7..104a107 100644 --- a/executors/src/eoa/store/submitted.rs +++ b/executors/src/eoa/store/submitted.rs @@ -1,12 +1,14 @@ use std::collections::{BTreeMap, HashMap, HashSet}; +use serde::{Deserialize, Serialize}; use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; use crate::eoa::store::{ ConfirmedTransaction, EoaExecutorStoreKeys, TransactionStoreError, atomic::SafeRedisTransaction, }; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct SubmittedTransaction { pub nonce: u64, pub hash: String, diff --git a/executors/src/eoa/worker.rs b/executors/src/eoa/worker.rs index 2322340..a5e2465 100644 --- a/executors/src/eoa/worker.rs +++ b/executors/src/eoa/worker.rs @@ -21,6 +21,7 @@ use hex; use serde::{Deserialize, Serialize}; use std::{sync::Arc, time::Duration}; use tokio::time::sleep; +use twmq::Queue; use twmq::redis::AsyncCommands; use twmq::redis::aio::ConnectionManager; use twmq::{ @@ -35,6 +36,7 @@ use crate::eoa::store::{ EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, EoaTransactionRequest, ReplacedTransaction, SubmittedTransaction, TransactionData, TransactionStoreError, }; +use crate::webhook::WebhookJobHandler; // ========== SPEC-COMPLIANT CONSTANTS ========== const MAX_INFLIGHT_PER_EOA: u64 = 100; // Default from spec @@ -236,8 +238,9 @@ fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { } } -#[derive(Debug, Clone)] -struct ConfirmedTransactionWithRichReceipt { +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConfirmedTransactionWithRichReceipt { pub nonce: u64, pub hash: String, pub transaction_id: String, @@ -273,6 +276,8 @@ where CS: ChainService + Send + Sync + 'static, { pub chain_service: Arc, + pub webhook_queue: Arc>, + pub redis: ConnectionManager, pub namespace: Option, diff --git a/executors/src/webhook/envelope.rs b/executors/src/webhook/envelope.rs index 1887615..b4b3e06 100644 --- a/executors/src/webhook/envelope.rs +++ b/executors/src/webhook/envelope.rs @@ -41,6 +41,35 @@ pub struct WebhookNotificationEnvelope { pub delivery_target_url: Option, } +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct BareWebhookNotificationEnvelope { + pub transaction_id: String, + pub event_type: StageEvent, + pub executor_name: String, + pub stage_name: String, + pub payload: T, +} + +impl BareWebhookNotificationEnvelope { + pub fn into_webhook_notification_envelope( + self, + timestamp: u64, + delivery_target_url: String, + ) -> WebhookNotificationEnvelope { + WebhookNotificationEnvelope { + notification_id: Uuid::new_v4().to_string(), + transaction_id: self.transaction_id, + timestamp, + executor_name: self.executor_name, + stage_name: self.stage_name, + event_type: self.event_type, + payload: self.payload, + delivery_target_url: Some(delivery_target_url), + } + } +} + // --- Serializable Hook Data Wrappers --- // These wrap the hook data to make them serializable (removing lifetimes) #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/executors/src/webhook/mod.rs b/executors/src/webhook/mod.rs index 3408d26..8d2512f 100644 --- a/executors/src/webhook/mod.rs +++ b/executors/src/webhook/mod.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; +use std::env; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use engine_core::execution_options::WebhookOptions; use hex; use hmac::{Hmac, Mac}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; @@ -9,7 +11,10 @@ use serde::{Deserialize, Serialize}; use twmq::error::TwmqError; use twmq::hooks::TransactionContext; use twmq::job::{BorrowedJob, JobError, JobResult, RequeuePosition, ToJobResult}; -use twmq::{DurableExecution, FailHookData, NackHookData, SuccessHookData, UserCancellable}; +use twmq::{DurableExecution, FailHookData, NackHookData, Queue, SuccessHookData, UserCancellable}; +use uuid::Uuid; + +use crate::webhook::envelope::{BareWebhookNotificationEnvelope, WebhookNotificationEnvelope}; pub mod envelope; @@ -113,7 +118,10 @@ impl DurableExecution for WebhookJobHandler { type JobData = WebhookJobPayload; #[tracing::instrument(skip_all, fields(queue = "webhook", job_id = job.job.id))] - async fn process(&self, job: &BorrowedJob) -> JobResult { + async fn process( + &self, + job: &BorrowedJob, + ) -> JobResult { let payload = &job.job.data; let mut request_headers = HeaderMap::new(); @@ -423,3 +431,77 @@ impl DurableExecution for WebhookJobHandler { ); } } + +pub fn queue_webhook_envelopes( + envelope: BareWebhookNotificationEnvelope, + webhook_options: Vec, + tx: &mut TransactionContext<'_>, + webhook_queue: Arc>, +) -> Result<(), TwmqError> { + let now = chrono::Utc::now().timestamp().min(0) as u64; + let serialised_webhook_envelopes = + webhook_options + .iter() + .map(|webhook_option| { + let webhook_notification_envelope = envelope + .clone() + .into_webhook_notification_envelope(now, webhook_option.url.clone()); + let serialised_envelope = serde_json::to_string(&webhook_notification_envelope)?; + Ok(( + serialised_envelope, + webhook_notification_envelope, + webhook_option.clone(), + )) + }) + .collect::, WebhookOptions)>, + serde_json::Error, + >>()?; + + let webhook_payloads = serialised_webhook_envelopes + .into_iter() + .map( + |(serialised_envelope, webhook_notification_envelope, webhook_option)| { + let payload = WebhookJobPayload { + url: webhook_option.url, + body: serialised_envelope, + headers: Some( + [ + ("Content-Type".to_string(), "application/json".to_string()), + ( + "User-Agent".to_string(), + format!("{}/{}", envelope.executor_name, envelope.stage_name), + ), + ] + .into_iter() + .collect(), + ), + hmac_secret: webhook_option.secret, // TODO: Add HMAC support if needed + http_method: Some("POST".to_string()), + }; + return (payload, webhook_notification_envelope); + }, + ) + .collect::>(); + + for (payload, webhook_notification_envelope) in webhook_payloads { + let mut webhook_job = webhook_queue.clone().job(payload); + webhook_job.options.id = format!( + "{}_{}_webhook", + webhook_notification_envelope.transaction_id, + webhook_notification_envelope.notification_id + ); + + tx.queue_job(webhook_job)?; + tracing::info!( + transaction_id = %webhook_notification_envelope.transaction_id, + executor = %webhook_notification_envelope.executor_name, + stage = %webhook_notification_envelope.stage_name, + event = ?webhook_notification_envelope.event_type, + notification_id = %webhook_notification_envelope.notification_id, + "Queued webhook notification" + ); + } + + Ok(()) +} diff --git a/server/src/queue/manager.rs b/server/src/queue/manager.rs index 53ca78b..20fe806 100644 --- a/server/src/queue/manager.rs +++ b/server/src/queue/manager.rs @@ -5,7 +5,7 @@ use alloy::transports::http::reqwest; use engine_core::error::EngineError; use engine_executors::{ eip7702_executor::{confirm::Eip7702ConfirmationHandler, send::Eip7702SendHandler}, - eoa::{EoaExecutorStore, EoaExecutorWorker}, + eoa::EoaExecutorWorker, external_bundler::{ confirm::UserOpConfirmationHandler, deployment::{RedisDeploymentCache, RedisDeploymentLock}, @@ -16,10 +16,7 @@ use engine_executors::{ }; use twmq::{Queue, queue::QueueOptions, shutdown::ShutdownHandle}; -use crate::{ - chains::ThirdwebChainService, - config::{QueueConfig, RedisConfig}, -}; +use crate::{chains::ThirdwebChainService, config::QueueConfig}; pub struct QueueManager { pub webhook_queue: Arc>, @@ -212,6 +209,7 @@ impl QueueManager { let eoa_executor_handler = EoaExecutorWorker { chain_service: chain_service.clone(), eoa_signer: eoa_signer.clone(), + webhook_queue: webhook_queue.clone(), namespace: queue_config.execution_namespace.clone(), redis: redis_client.get_connection_manager().await?, max_inflight: 100, From 1c7297410fec961b050fe4ea05159597ed2159ea Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Sat, 12 Jul 2025 07:05:16 +0530 Subject: [PATCH 04/10] wip --- executors/src/eoa/events.rs | 39 +++++++++++++++++++++++++----------- executors/src/webhook/mod.rs | 4 +--- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/executors/src/eoa/events.rs b/executors/src/eoa/events.rs index 354679a..b9d62be 100644 --- a/executors/src/eoa/events.rs +++ b/executors/src/eoa/events.rs @@ -1,16 +1,16 @@ use std::fmt::Display; -use engine_core::error::EngineError; use serde::{Deserialize, Serialize}; use twmq::job::RequeuePosition; use crate::{ eoa::{ store::{SubmittedTransaction, TransactionData}, - worker::ConfirmedTransactionWithRichReceipt, + worker::{ConfirmedTransactionWithRichReceipt, EoaExecutorWorkerError}, }, webhook::envelope::{ - BareWebhookNotificationEnvelope, SerializableNackData, SerializableSuccessData, StageEvent, + BareWebhookNotificationEnvelope, SerializableFailData, SerializableNackData, + SerializableSuccessData, StageEvent, }, }; @@ -21,13 +21,12 @@ pub struct EoaExecutorEvent { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EoaSendAttemptNackData { pub nonce: u64, - pub error: EngineError, + pub error: EoaExecutorWorkerError, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EoaExecutorStage { SendAttempt, - SendAttemptNack, TransactionReplaced, TransactionConfirmed, } @@ -36,7 +35,6 @@ impl Display for EoaExecutorStage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { EoaExecutorStage::SendAttempt => write!(f, "send_attempt"), - EoaExecutorStage::SendAttemptNack => write!(f, "send_attempt_nack"), EoaExecutorStage::TransactionReplaced => write!(f, "transaction_replaced"), EoaExecutorStage::TransactionConfirmed => write!(f, "transaction_confirmed"), } @@ -46,7 +44,7 @@ impl Display for EoaExecutorStage { const EXECUTOR_NAME: &str = "eoa"; impl EoaExecutorEvent { - pub fn send_attempt_success_envelopes( + pub fn send_attempt_success_envelope( &self, submitted_transaction: SubmittedTransaction, ) -> BareWebhookNotificationEnvelope> { @@ -61,16 +59,16 @@ impl EoaExecutorEvent { } } - pub fn send_attempt_nack_envelopes( + pub fn send_attempt_nack_envelope( &self, nonce: u64, - error: EngineError, + error: EoaExecutorWorkerError, attempt_number: u32, ) -> BareWebhookNotificationEnvelope> { BareWebhookNotificationEnvelope { transaction_id: self.transaction_data.transaction_id.clone(), executor_name: EXECUTOR_NAME.to_string(), - stage_name: EoaExecutorStage::SendAttemptNack.to_string(), + stage_name: EoaExecutorStage::SendAttempt.to_string(), event_type: StageEvent::Nack, payload: SerializableNackData { error: EoaSendAttemptNackData { @@ -86,7 +84,7 @@ impl EoaExecutorEvent { } } - pub fn transaction_replaced_envelopes( + pub fn transaction_replaced_envelope( &self, replaced_transaction: SubmittedTransaction, ) -> BareWebhookNotificationEnvelope> { @@ -101,7 +99,7 @@ impl EoaExecutorEvent { } } - pub fn transaction_confirmed_envelopes( + pub fn transaction_confirmed_envelope( &self, confirmed_transaction: ConfirmedTransactionWithRichReceipt, ) -> BareWebhookNotificationEnvelope> @@ -116,4 +114,21 @@ impl EoaExecutorEvent { }, } } + + pub fn transaction_failed_envelope( + &self, + error: EoaExecutorWorkerError, + final_attempt_number: u32, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::SendAttempt.to_string(), + event_type: StageEvent::Failure, + payload: SerializableFailData { + error: error.clone(), + final_attempt_number, + }, + } + } } diff --git a/executors/src/webhook/mod.rs b/executors/src/webhook/mod.rs index 8d2512f..73c7f22 100644 --- a/executors/src/webhook/mod.rs +++ b/executors/src/webhook/mod.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::env; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -12,7 +11,6 @@ use twmq::error::TwmqError; use twmq::hooks::TransactionContext; use twmq::job::{BorrowedJob, JobError, JobResult, RequeuePosition, ToJobResult}; use twmq::{DurableExecution, FailHookData, NackHookData, Queue, SuccessHookData, UserCancellable}; -use uuid::Uuid; use crate::webhook::envelope::{BareWebhookNotificationEnvelope, WebhookNotificationEnvelope}; @@ -479,7 +477,7 @@ pub fn queue_webhook_envelopes( hmac_secret: webhook_option.secret, // TODO: Add HMAC support if needed http_method: Some("POST".to_string()), }; - return (payload, webhook_notification_envelope); + (payload, webhook_notification_envelope) }, ) .collect::>(); From 09259467145c0fa3bf69179f04414a3e12ea31f2 Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Sat, 12 Jul 2025 07:25:47 +0530 Subject: [PATCH 05/10] typo --- executors/src/webhook/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executors/src/webhook/mod.rs b/executors/src/webhook/mod.rs index 73c7f22..271c4b6 100644 --- a/executors/src/webhook/mod.rs +++ b/executors/src/webhook/mod.rs @@ -436,7 +436,7 @@ pub fn queue_webhook_envelopes( tx: &mut TransactionContext<'_>, webhook_queue: Arc>, ) -> Result<(), TwmqError> { - let now = chrono::Utc::now().timestamp().min(0) as u64; + let now = chrono::Utc::now().timestamp().max(0) as u64; let serialised_webhook_envelopes = webhook_options .iter() From 18625172a64884a4651a88264a4ab83439510440 Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Sat, 12 Jul 2025 07:52:15 +0530 Subject: [PATCH 06/10] Add transaction context creation from Redis pipeline - Introduced `transaction_context_from_pipeline` method to allow atomic job queuing within an existing Redis transaction. - This enhancement improves the flexibility of transaction handling in the queue implementation. --- twmq/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/twmq/src/lib.rs b/twmq/src/lib.rs index be6733c..177fc0e 100644 --- a/twmq/src/lib.rs +++ b/twmq/src/lib.rs @@ -161,6 +161,15 @@ impl Queue { } } + /// Create a TransactionContext from an existing Redis pipeline + /// This allows queueing jobs atomically within an existing transaction + pub fn transaction_context_from_pipeline<'a>( + &self, + pipeline: &'a mut redis::Pipeline, + ) -> hooks::TransactionContext<'a> { + hooks::TransactionContext::new(pipeline, self.name.clone()) + } + // Get queue name pub fn name(&self) -> &str { &self.name From d2334396cf25dd036c47b79333e7e9eb96a62b2a Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Mon, 14 Jul 2025 02:15:49 +0530 Subject: [PATCH 07/10] Add thirdweb-core dependency and enhance transaction processing - Added `thirdweb-core` as a dependency in `Cargo.toml` and updated `Cargo.lock`. - Refactored transaction handling in the EOA executor to improve error handling and processing of borrowed transactions. - Introduced new methods for moving pending transactions to borrowed state using incremented and recycled nonces. - Enhanced submission result processing to include detailed reporting and webhook event queuing for transaction outcomes. These changes aim to improve the robustness and flexibility of transaction management within the EOA executor. --- Cargo.lock | 1 + executors/Cargo.toml | 1 + executors/src/eoa/store/atomic.rs | 393 +++------- executors/src/eoa/store/borrowed.rs | 383 ++++++++++ executors/src/eoa/store/mod.rs | 60 +- executors/src/eoa/store/pending.rs | 257 +++++++ executors/src/eoa/store/submitted.rs | 2 +- executors/src/eoa/worker.rs | 1000 +++++++++++++------------- 8 files changed, 1263 insertions(+), 834 deletions(-) create mode 100644 executors/src/eoa/store/borrowed.rs create mode 100644 executors/src/eoa/store/pending.rs diff --git a/Cargo.lock b/Cargo.lock index 3b82182..4a739a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,6 +2192,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "thirdweb-core", "thiserror 2.0.12", "tokio", "tracing", diff --git a/executors/Cargo.toml b/executors/Cargo.toml index 2148d9e..283ef56 100644 --- a/executors/Cargo.toml +++ b/executors/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] hex = "0.4.3" alloy = { version = "1.0.8", features = ["serde"] } +thirdweb-core = { version = "0.1.0", path = "../thirdweb-core" } hmac = "0.12.1" reqwest = "0.12.15" serde = "1.0.219" diff --git a/executors/src/eoa/store/atomic.rs b/executors/src/eoa/store/atomic.rs index eb931c9..cea982d 100644 --- a/executors/src/eoa/store/atomic.rs +++ b/executors/src/eoa/store/atomic.rs @@ -1,16 +1,25 @@ +use std::sync::Arc; + use alloy::{ consensus::{Signed, TypedTransaction}, primitives::Address, }; use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; -use crate::eoa::{ - EoaExecutorStore, - store::{ - BorrowedTransactionData, ConfirmedTransaction, EoaHealth, TransactionAttempt, - TransactionStoreError, - submitted::{CleanSubmittedTransactions, CleanupReport, SubmittedTransaction}, +use crate::{ + eoa::{ + EoaExecutorStore, + store::{ + BorrowedTransactionData, ConfirmedTransaction, EoaHealth, TransactionAttempt, + TransactionStoreError, + borrowed::{BorrowedProcessingReport, ProcessBorrowedTransactions, SubmissionResult}, + pending::{ + MovePendingToBorrowedWithIncrementedNonces, MovePendingToBorrowedWithRecycledNonces, + }, + submitted::{CleanSubmittedTransactions, CleanupReport, SubmittedTransaction}, + }, }, + webhook::WebhookJobHandler, }; const MAX_RETRIES: u32 = 10; @@ -33,221 +42,6 @@ pub trait SafeRedisTransaction: Send + Sync { fn watch_keys(&self) -> Vec; } -struct MovePendingToBorrowedWithRecycledNonce { - recycled_key: String, - pending_key: String, - transaction_id: String, - borrowed_key: String, - nonce: u64, - prepared_tx_json: String, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonce { - type ValidationData = (); - type OperationResult = (); - - fn name(&self) -> &str { - "pending->borrowed with recycled nonce" - } - - fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { - // Remove nonce from recycled set (we know it exists) - pipeline.zrem(&self.recycled_key, self.nonce); - // Remove transaction from pending (we know it exists) - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.recycled_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check if nonce exists in recycled set - let nonce_score: Option = conn.zscore(&self.recycled_key, self.nonce).await?; - if nonce_score.is_none() { - return Err(TransactionStoreError::NonceNotInRecycledSet { nonce: self.nonce }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MovePendingToBorrowedWithNewNonce { - optimistic_key: String, - pending_key: String, - nonce: u64, - prepared_tx_json: String, - transaction_id: String, - borrowed_key: String, - eoa: Address, - chain_id: u64, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithNewNonce { - type ValidationData = (); - type OperationResult = (); - - fn name(&self) -> &str { - "pending->borrowed with new nonce" - } - - fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { - // Increment optimistic nonce - pipeline.incr(&self.optimistic_key, 1); - // Remove transaction from pending - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.optimistic_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check current optimistic nonce - let current_optimistic: Option = conn.get(&self.optimistic_key).await?; - let current_nonce = match current_optimistic { - Some(nonce) => nonce, - None => { - return Err(TransactionStoreError::NonceSyncRequired { - eoa: self.eoa, - chain_id: self.chain_id, - }); - } - }; - - if current_nonce != self.nonce { - return Err(TransactionStoreError::OptimisticNonceChanged { - expected: self.nonce, - actual: current_nonce, - }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MoveBorrowedToSubmitted { - nonce: u64, - hash: String, - transaction_id: String, - borrowed_key: String, - submitted_key: String, - hash_to_id_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToSubmitted { - type ValidationData = (); - type OperationResult = (); - - fn name(&self) -> &str { - "borrowed->submitted" - } - - fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add to submitted with hash:id format - let hash_id_value = format!("{}:{}", self.hash, self.transaction_id); - pipeline.zadd(&self.submitted_key, &hash_id_value, self.nonce); - - // Still maintain hash-to-ID mapping for backward compatibility and external lookups - pipeline.set(&self.hash_to_id_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - -struct MoveBorrowedToRecycled { - nonce: u64, - transaction_id: String, - borrowed_key: String, - recycled_key: String, - pending_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToRecycled { - type ValidationData = (); - type OperationResult = (); - - fn name(&self) -> &str { - "borrowed->recycled" - } - - fn operation(&self, pipeline: &mut Pipeline, _: Self::ValidationData) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add nonce to recycled set (with timestamp as score) - pipeline.zadd(&self.recycled_key, self.nonce, self.nonce); - - // Add transaction back to pending - pipeline.lpush(&self.pending_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - /// Atomic transaction store that owns the base store and provides atomic operations /// /// This store is created by calling `acquire_lock()` on the base store and provides @@ -336,50 +130,34 @@ impl AtomicEoaExecutorStore { } } - /// Example of how to refactor a complex method using the helper to reduce boilerplate - /// This shows the pattern for atomic_move_pending_to_borrowed_with_recycled_nonce - pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( + /// Atomically move multiple pending transactions to borrowed state using incremented nonces + /// + /// The transactions must have sequential nonces starting from the current optimistic count. + /// This operation validates nonce ordering and atomically moves all transactions. + pub async fn atomic_move_pending_to_borrowed_with_incremented_nonces( &self, - transaction_id: &str, - nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let safe_tx = MovePendingToBorrowedWithRecycledNonce { - recycled_key: self.recycled_nonces_set_name(), - pending_key: self.pending_transactions_zset_name(), - transaction_id: transaction_id.to_string(), - borrowed_key: self.borrowed_transactions_hashmap_name(), - nonce, - prepared_tx_json: serde_json::to_string(prepared_tx)?, - }; - - self.execute_with_watch_and_retry(&safe_tx).await?; - - Ok(()) + transactions: &[BorrowedTransactionData], + ) -> Result { + self.execute_with_watch_and_retry(&MovePendingToBorrowedWithIncrementedNonces { + transactions, + keys: &self.keys, + eoa: self.eoa, + chain_id: self.chain_id, + }) + .await } - /// Atomically move specific transaction from pending to borrowed with new nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_new_nonce( + /// Atomically move multiple pending transactions to borrowed state using recycled nonces + /// + /// All nonces must exist in the recycled nonces set. This operation validates nonce + /// availability and atomically moves all transactions. + pub async fn atomic_move_pending_to_borrowed_with_recycled_nonces( &self, - transaction_id: &str, - expected_nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let optimistic_key = self.optimistic_transaction_count_key_name(); - let borrowed_key = self.borrowed_transactions_hashmap_name(); - let pending_key = self.pending_transactions_zset_name(); - let prepared_tx_json = serde_json::to_string(prepared_tx)?; - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry(&MovePendingToBorrowedWithNewNonce { - nonce: expected_nonce, - prepared_tx_json, - transaction_id, - borrowed_key, - optimistic_key, - pending_key, - eoa: self.eoa, - chain_id: self.chain_id, + transactions: &[BorrowedTransactionData], + ) -> Result { + self.execute_with_watch_and_retry(&MovePendingToBorrowedWithRecycledNonces { + transactions, + keys: &self.keys, }) .await } @@ -592,53 +370,6 @@ impl AtomicEoaExecutorStore { } } - /// Atomically move borrowed transaction to submitted state - /// Returns error if transaction not found in borrowed state - pub async fn atomic_move_borrowed_to_submitted( - &self, - nonce: u64, - hash: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(); - let submitted_key = self.submitted_transactions_zset_name(); - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let hash = hash.to_string(); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry(&MoveBorrowedToSubmitted { - nonce, - hash: hash.to_string(), - transaction_id, - borrowed_key, - submitted_key, - hash_to_id_key, - }) - .await - } - - /// Atomically move borrowed transaction back to recycled nonces and pending queue - /// Returns error if transaction not found in borrowed state - pub async fn atomic_move_borrowed_to_recycled( - &self, - nonce: u64, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(); - let recycled_key = self.recycled_nonces_set_name(); - let pending_key = self.pending_transactions_zset_name(); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry(&MoveBorrowedToRecycled { - nonce, - transaction_id, - borrowed_key, - recycled_key, - pending_key, - }) - .await - } - /// Update EOA health data pub async fn update_health_data( &self, @@ -754,7 +485,7 @@ impl AtomicEoaExecutorStore { self.with_lock_check(|pipeline| { let optimistic_key = self.optimistic_transaction_count_key_name(); let cached_nonce_key = self.last_transaction_count_key_name(); - let recycled_key = self.recycled_nonces_set_name(); + let recycled_key = self.recycled_nonces_zset_name(); // Update health data only if it exists if let Some(ref health_json) = health_update { @@ -797,6 +528,34 @@ impl AtomicEoaExecutorStore { .await } + /// Fail a transaction that's in the pending state (remove from pending and fail) + /// This is used for deterministic failures during preparation that should not retry + pub async fn fail_pending_transaction( + &self, + transaction_id: &str, + failure_reason: &str, + webhook_queue: Arc>, + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(|pipeline| { + let pending_key = self.pending_transactions_zset_name(); + let tx_data_key = self.transaction_data_key_name(transaction_id); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Remove from pending state + pipeline.zrem(&pending_key, transaction_id); + + // Update transaction data with failure + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", failure_reason); + pipeline.hset(&tx_data_key, "status", "failed"); + + // TODO: Queue webhook event for failed transaction + // let webhook_job = WebhookJobHandler::new(...); + // webhook_queue.push(webhook_job); + }) + .await + } + pub async fn clean_submitted_transactions( &self, confirmed_transactions: &[ConfirmedTransaction], @@ -809,4 +568,20 @@ impl AtomicEoaExecutorStore { }) .await } + + /// Process borrowed transactions with given submission results + /// This method moves transactions from borrowed state to submitted/pending/failed states + /// based on the submission results, and queues appropriate webhook events + pub async fn process_borrowed_transactions( + &self, + results: Vec, + webhook_queue: Arc>, + ) -> Result { + self.execute_with_watch_and_retry(&ProcessBorrowedTransactions { + results, + keys: &self.keys, + webhook_queue, + }) + .await + } } diff --git a/executors/src/eoa/store/borrowed.rs b/executors/src/eoa/store/borrowed.rs new file mode 100644 index 0000000..3eceb35 --- /dev/null +++ b/executors/src/eoa/store/borrowed.rs @@ -0,0 +1,383 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use alloy::consensus::Transaction; +use alloy::primitives::Address; +use serde::{Deserialize, Serialize}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; +use twmq::{Queue, hooks::TransactionContext}; + +use crate::eoa::{ + events::EoaExecutorEvent, + store::{ + BorrowedTransactionData, EoaExecutorStoreKeys, TransactionData, TransactionStoreError, + atomic::SafeRedisTransaction, submitted::SubmittedTransaction, + }, + worker::EoaExecutorWorkerError, +}; +use crate::webhook::{WebhookJobHandler, queue_webhook_envelopes}; + +/// Error information for NACK operations (retryable errors) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SubmissionErrorNack { + pub transaction_id: String, + pub error: EoaExecutorWorkerError, + pub user_data: Option, +} + +/// Error information for FAIL operations (permanent failures) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SubmissionErrorFail { + pub transaction_id: String, + pub error: EoaExecutorWorkerError, + pub user_data: Option, +} + +/// Result of a submission attempt +#[derive(Debug, Clone)] +pub enum SubmissionResult { + Success(SubmittedTransaction), + Nack(SubmissionErrorNack), + Fail(SubmissionErrorFail), +} + +/// Internal representation where all user data is guaranteed to be present +#[derive(Debug, Clone)] +pub enum SubmissionResultWithUserData { + Success(SubmittedTransaction, TransactionData), + Nack(SubmissionErrorNack, TransactionData), + Fail(SubmissionErrorFail, TransactionData), +} + +impl SubmissionResultWithUserData { + fn transaction_id(&self) -> &str { + match self { + SubmissionResultWithUserData::Success(tx, _) => &tx.transaction_id, + SubmissionResultWithUserData::Nack(err, _) => &err.transaction_id, + SubmissionResultWithUserData::Fail(err, _) => &err.transaction_id, + } + } + + fn user_data(&self) -> &TransactionData { + match self { + SubmissionResultWithUserData::Success(_, data) => data, + SubmissionResultWithUserData::Nack(_, data) => data, + SubmissionResultWithUserData::Fail(_, data) => data, + } + } +} + +/// Batch operation to process borrowed transactions +pub struct ProcessBorrowedTransactions<'a> { + pub results: Vec, + pub keys: &'a EoaExecutorStoreKeys, + pub webhook_queue: Arc>, +} + +#[derive(Debug, Default)] +pub struct BorrowedProcessingReport { + pub total_processed: usize, + pub moved_to_submitted: usize, + pub moved_to_pending: usize, + pub failed_transactions: usize, + pub webhook_events_queued: usize, +} + +impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { + type ValidationData = ( + Vec, + Vec, + ); + type OperationResult = BorrowedProcessingReport; + + fn name(&self) -> &str { + "process borrowed transactions" + } + + fn watch_keys(&self) -> Vec { + vec![self.keys.borrowed_transactions_hashmap_name()] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + // Collect all transaction IDs that need user data + let mut transactions_needing_data = Vec::new(); + let mut results_with_partial_data = Vec::new(); + + for result in &self.results { + match result { + SubmissionResult::Success(tx) => { + transactions_needing_data.push(tx.transaction_id.clone()); + results_with_partial_data.push(result.clone()); + } + SubmissionResult::Nack(err) => { + if err.user_data.is_none() { + transactions_needing_data.push(err.transaction_id.clone()); + } + results_with_partial_data.push(result.clone()); + } + SubmissionResult::Fail(err) => { + if err.user_data.is_none() { + transactions_needing_data.push(err.transaction_id.clone()); + } + results_with_partial_data.push(result.clone()); + } + } + } + + // Batch fetch missing user data + let mut user_data_map = HashMap::new(); + for transaction_id in transactions_needing_data { + let data_key = self.keys.transaction_data_key_name(&transaction_id); + if let Some(data_json) = conn.get::<&str, Option>(&data_key).await? { + let transaction_data: TransactionData = serde_json::from_str(&data_json)?; + user_data_map.insert(transaction_id, transaction_data); + } + } + + // Get all borrowed transactions to validate they exist + let borrowed_transactions_map: HashMap = conn + .hgetall(self.keys.borrowed_transactions_hashmap_name()) + .await?; + + let borrowed_transactions: Vec = borrowed_transactions_map + .into_iter() + .filter_map(|(nonce_str, data_json)| { + let borrowed_data: BorrowedTransactionData = + serde_json::from_str(&data_json).ok()?; + Some(borrowed_data) + }) + .collect(); + + // Convert to results with guaranteed user data + let mut results_with_user_data = Vec::new(); + for result in results_with_partial_data { + match result { + SubmissionResult::Success(tx) => { + if let Some(user_data) = user_data_map.get(&tx.transaction_id) { + results_with_user_data + .push(SubmissionResultWithUserData::Success(tx, user_data.clone())); + } else { + return Err(TransactionStoreError::TransactionNotFound { + transaction_id: tx.transaction_id.clone(), + }); + } + } + SubmissionResult::Nack(mut err) => { + let user_data = if let Some(data) = err.user_data.take() { + data + } else if let Some(data) = user_data_map.get(&err.transaction_id) { + data.clone() + } else { + return Err(TransactionStoreError::TransactionNotFound { + transaction_id: err.transaction_id.clone(), + }); + }; + results_with_user_data.push(SubmissionResultWithUserData::Nack(err, user_data)); + } + SubmissionResult::Fail(mut err) => { + let user_data = if let Some(data) = err.user_data.take() { + data + } else if let Some(data) = user_data_map.get(&err.transaction_id) { + data.clone() + } else { + return Err(TransactionStoreError::TransactionNotFound { + transaction_id: err.transaction_id.clone(), + }); + }; + results_with_user_data.push(SubmissionResultWithUserData::Fail(err, user_data)); + } + } + } + + Ok((results_with_user_data, borrowed_transactions)) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + validation_data: Self::ValidationData, + ) -> Self::OperationResult { + let (results_with_user_data, borrowed_transactions) = validation_data; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Create borrowed transactions lookup by transaction_id + let borrowed_by_id: HashMap = borrowed_transactions + .iter() + .map(|tx| (tx.transaction_id.clone(), tx)) + .collect(); + + let mut report = BorrowedProcessingReport::default(); + + for result in &results_with_user_data { + let transaction_id = result.transaction_id(); + let user_data = result.user_data(); + + // Find the corresponding borrowed transaction to get the nonce + let borrowed_tx = match borrowed_by_id.get(transaction_id) { + Some(tx) => tx, + None => { + // Transaction not in borrowed state, skip + continue; + } + }; + + let nonce = borrowed_tx.signed_transaction.nonce(); + + // We'll set attempt_number to 1 for simplicity in the operation phase + // The actual attempt tracking is handled by the attempts list + let attempt_number = 1; + + // Define attempts_key for all match arms + let attempts_key = self.keys.transaction_attempts_list_name(transaction_id); + + match result { + SubmissionResultWithUserData::Success(tx, user_data) => { + // Remove from borrowed + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + nonce.to_string(), + ); + + // Add to submitted + let (submitted_tx_redis_string, nonce) = tx.to_redis_string_with_nonce(); + pipeline.zadd( + self.keys.submitted_transactions_zset_name(), + &submitted_tx_redis_string, + nonce, + ); + + // Update hash-to-ID mapping + let hash_to_id_key = self.keys.transaction_hash_to_id_key_name(&tx.hash); + pipeline.set(&hash_to_id_key, &tx.transaction_id); + + // Update transaction data status + let tx_data_key = self.keys.transaction_data_key_name(&tx.transaction_id); + pipeline.hset(&tx_data_key, "status", "submitted"); + + // Add attempt to attempts list + let attempt_json = + serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); + pipeline.lpush(&attempts_key, &attempt_json); + + // Queue webhook event + let event = EoaExecutorEvent { + transaction_data: user_data.clone(), + }; + let envelope = event.send_attempt_success_envelope(tx.clone()); + if let Some(webhook_options) = &user_data.user_request.webhook_options { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for success: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.moved_to_submitted += 1; + } + SubmissionResultWithUserData::Nack(err, user_data) => { + // Remove from borrowed + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + nonce.to_string(), + ); + + // Add back to pending + pipeline.zadd( + self.keys.pending_transactions_zset_name(), + &err.transaction_id, + now, + ); + + // Update transaction data status + let tx_data_key = self.keys.transaction_data_key_name(&err.transaction_id); + pipeline.hset(&tx_data_key, "status", "pending"); + + // Add attempt to attempts list + let attempt_json = + serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); + pipeline.lpush(&attempts_key, &attempt_json); + + // Queue webhook event + let event = EoaExecutorEvent { + transaction_data: user_data.clone(), + }; + let envelope = + event.send_attempt_nack_envelope(nonce, err.error.clone(), attempt_number); + if let Some(webhook_options) = &user_data.user_request.webhook_options { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for nack: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.moved_to_pending += 1; + } + SubmissionResultWithUserData::Fail(err, user_data) => { + // Remove from borrowed + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + nonce.to_string(), + ); + + // Update transaction data with failure + let tx_data_key = self.keys.transaction_data_key_name(&err.transaction_id); + pipeline.hset(&tx_data_key, "status", "failed"); + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", err.error.to_string()); + + // Add attempt to attempts list + let attempt_json = + serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); + pipeline.lpush(&attempts_key, &attempt_json); + + // Queue webhook event + let event = EoaExecutorEvent { + transaction_data: user_data.clone(), + }; + let envelope = + event.transaction_failed_envelope(err.error.clone(), attempt_number); + if let Some(webhook_options) = &user_data.user_request.webhook_options { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.failed_transactions += 1; + } + } + } + + report.total_processed = results_with_user_data.len(); + report + } +} diff --git a/executors/src/eoa/store/mod.rs b/executors/src/eoa/store/mod.rs index 49dfcaa..a0d3874 100644 --- a/executors/src/eoa/store/mod.rs +++ b/executors/src/eoa/store/mod.rs @@ -1,4 +1,4 @@ -use alloy::consensus::{Signed, TypedTransaction}; +use alloy::consensus::{Signed, Transaction, TypedTransaction}; use alloy::network::AnyTransactionReceipt; use alloy::primitives::{Address, Bytes, U256}; use chrono; @@ -11,10 +11,15 @@ use std::collections::HashMap; use twmq::redis::{AsyncCommands, aio::ConnectionManager}; mod atomic; +mod borrowed; +mod pending; mod submitted; pub mod error; pub use atomic::AtomicEoaExecutorStore; +pub use borrowed::{ + BorrowedProcessingReport, SubmissionErrorFail, SubmissionErrorNack, SubmissionResult, +}; pub use submitted::{CleanupReport, SubmittedTransaction}; use crate::eoa::store::submitted::SubmittedTransactionStringWithNonce; @@ -34,6 +39,21 @@ pub struct ConfirmedTransaction { pub receipt_data: String, } +#[derive(Debug, Clone)] +pub struct PendingTransaction { + pub transaction_id: String, + pub queued_at: u64, +} + +impl From<(String, u64)> for PendingTransaction { + fn from((transaction_id, queued_at): (String, u64)) -> Self { + Self { + transaction_id, + queued_at, + } + } +} + /// The actual user request data #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] @@ -120,6 +140,7 @@ impl EoaExecutorStoreKeys { /// - "status": String status ("confirmed", "failed", etc.) /// - "completed_at": String Unix timestamp (optional) /// - "created_at": String Unix timestamp (optional) + /// - "failure_reason": String failure reason (optional) pub fn transaction_data_key_name(&self, transaction_id: &str) -> String { match &self.namespace { Some(ns) => format!("{ns}:eoa_executor:tx_data:{transaction_id}"), @@ -138,7 +159,9 @@ impl EoaExecutorStoreKeys { } } - /// Name of the list for pending transactions + /// Name of the zset for pending transactions + /// + /// zset contains the `transaction_id` scored by the queued_at timestamp (unix timestamp in milliseconds) pub fn pending_transactions_zset_name(&self) -> String { match &self.namespace { Some(ns) => format!( @@ -194,7 +217,7 @@ impl EoaExecutorStoreKeys { /// we add the nonce to this set. /// /// These nonces are used with priority, before any other nonces. - pub fn recycled_nonces_set_name(&self) -> String { + pub fn recycled_nonces_zset_name(&self) -> String { match &self.namespace { Some(ns) => format!( "{ns}:eoa_executor:recycled_nonces:{}:{}", @@ -300,10 +323,22 @@ pub struct EoaHealth { pub struct BorrowedTransactionData { pub transaction_id: String, pub signed_transaction: Signed, + pub queued_at: u64, pub hash: String, pub borrowed_at: u64, } +impl Into for &BorrowedTransactionData { + fn into(self) -> SubmittedTransaction { + SubmittedTransaction { + nonce: self.signed_transaction.nonce(), + hash: self.signed_transaction.hash().to_string(), + transaction_id: self.transaction_id.clone(), + queued_at: self.queued_at, + } + } +} + /// Type of nonce allocation for transaction processing #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum NonceType { @@ -438,7 +473,7 @@ impl EoaExecutorStore { /// Peek recycled nonces without removing them pub async fn peek_recycled_nonces(&self) -> Result, TransactionStoreError> { - let recycled_key = self.recycled_nonces_set_name(); + let recycled_key = self.recycled_nonces_zset_name(); let mut conn = self.redis.clone(); let nonces: Vec = conn.zrange(&recycled_key, 0, -1).await?; @@ -449,14 +484,19 @@ impl EoaExecutorStore { pub async fn peek_pending_transactions( &self, limit: u64, - ) -> Result, TransactionStoreError> { + ) -> Result, TransactionStoreError> { let pending_key = self.pending_transactions_zset_name(); let mut conn = self.redis.clone(); - // Use LRANGE to peek without removing - let transaction_ids: Vec = - conn.lrange(&pending_key, 0, (limit as isize) - 1).await?; - Ok(transaction_ids) + // Use ZRANGE to peek without removing + let transaction_ids: Vec<(String, u64)> = conn + .zrange_withscores(&pending_key, 0, (limit - 1) as isize) + .await?; + + Ok(transaction_ids + .into_iter() + .map(PendingTransaction::from) + .collect()) } /// Get inflight budget (how many new transactions can be sent) @@ -491,7 +531,7 @@ impl EoaExecutorStore { } /// Get current optimistic nonce (without incrementing) - pub async fn get_optimistic_nonce(&self) -> Result { + pub async fn get_optimistic_transaction_count(&self) -> Result { let optimistic_key = self.optimistic_transaction_count_key_name(); let mut conn = self.redis.clone(); diff --git a/executors/src/eoa/store/pending.rs b/executors/src/eoa/store/pending.rs new file mode 100644 index 0000000..08da0f5 --- /dev/null +++ b/executors/src/eoa/store/pending.rs @@ -0,0 +1,257 @@ +use std::collections::HashSet; + +use alloy::{consensus::Transaction, primitives::Address}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::eoa::store::{ + BorrowedTransactionData, EoaExecutorStoreKeys, TransactionStoreError, + atomic::SafeRedisTransaction, +}; + +/// Atomic operation to move pending transactions to borrowed state using incremented nonces +/// +/// This operation validates that: +/// 1. The nonces in the vector are sequential with no gaps +/// 2. The lowest nonce matches the current optimistic transaction count +/// 3. All transactions exist in the pending queue +/// +/// Then atomically: +/// 1. Removes transactions from pending queue +/// 2. Adds transactions to borrowed state +/// 3. Updates optimistic transaction count to highest nonce + 1 +pub struct MovePendingToBorrowedWithIncrementedNonces<'a> { + pub transactions: &'a [BorrowedTransactionData], + pub keys: &'a EoaExecutorStoreKeys, + pub eoa: Address, + pub chain_id: u64, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithIncrementedNonces<'_> { + type ValidationData = Vec; // serialized borrowed transactions + type OperationResult = usize; // number of transactions processed + + fn name(&self) -> &str { + "pending->borrowed with incremented nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.optimistic_transaction_count_key_name(), + self.keys.borrowed_transactions_hashmap_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + if self.transactions.is_empty() { + return Err(TransactionStoreError::InternalError { + message: "Cannot process empty transaction list".to_string(), + }); + } + + // Get current optimistic nonce + let current_optimistic: Option = conn + .get(self.keys.optimistic_transaction_count_key_name()) + .await?; + let current_nonce = + current_optimistic.ok_or_else(|| TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + })?; + + // Extract and validate nonces + let mut nonces: Vec = self + .transactions + .iter() + .map(|tx| tx.signed_transaction.nonce()) + .collect(); + nonces.sort(); + + // Check that nonces are sequential with no gaps + for (i, &nonce) in nonces.iter().enumerate() { + let expected_nonce = current_nonce + i as u64; + if nonce != expected_nonce { + return Err(TransactionStoreError::InternalError { + message: format!( + "Non-sequential nonces detected: expected {}, found {} at position {}", + expected_nonce, nonce, i + ), + }); + } + } + + // Verify all transactions exist in pending queue using batched ZSCORE calls + if !self.transactions.is_empty() { + let mut pipe = twmq::redis::pipe(); + for tx in self.transactions { + pipe.zscore( + self.keys.pending_transactions_zset_name(), + &tx.transaction_id, + ); + } + let scores: Vec> = pipe.query_async(conn).await?; + + for (tx, score) in self.transactions.iter().zip(scores.iter()) { + if score.is_none() { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + // Pre-serialize all borrowed transaction data + let mut serialized_transactions = Vec::with_capacity(self.transactions.len()); + for tx in self.transactions { + let borrowed_json = + serde_json::to_string(tx).map_err(|e| TransactionStoreError::InternalError { + message: format!("Failed to serialize borrowed transaction: {}", e), + })?; + serialized_transactions.push(borrowed_json); + } + + Ok(serialized_transactions) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + serialized_transactions: Self::ValidationData, + ) -> Self::OperationResult { + let borrowed_key = self.keys.borrowed_transactions_hashmap_name(); + let pending_key = self.keys.pending_transactions_zset_name(); + let optimistic_key = self.keys.optimistic_transaction_count_key_name(); + + for (tx, borrowed_json) in self.transactions.iter().zip(serialized_transactions.iter()) { + let nonce = tx.signed_transaction.nonce(); + + // Remove from pending queue + pipeline.zrem(&pending_key, &tx.transaction_id); + + // Add to borrowed state + pipeline.hset(&borrowed_key, nonce.to_string(), borrowed_json); + } + + // Update optimistic tx count to highest nonce + 1 + if let Some(last_tx) = self.transactions.last() { + let new_optimistic_tx_count = last_tx.signed_transaction.nonce() + 1; + pipeline.set(&optimistic_key, new_optimistic_tx_count); + } + + self.transactions.len() + } +} + +/// Atomic operation to move pending transactions to borrowed state using recycled nonces +/// +/// This operation validates that: +/// 1. All nonces exist in the recycled nonces set +/// 2. All transactions exist in the pending queue +/// +/// Then atomically: +/// 1. Removes nonces from recycled set +/// 2. Removes transactions from pending queue +/// 3. Adds transactions to borrowed state +pub struct MovePendingToBorrowedWithRecycledNonces<'a> { + pub transactions: &'a [BorrowedTransactionData], + pub keys: &'a EoaExecutorStoreKeys, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonces<'_> { + type ValidationData = Vec; // serialized borrowed transactions + type OperationResult = usize; // number of transactions processed + + fn name(&self) -> &str { + "pending->borrowed with recycled nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.recycled_nonces_zset_name(), + self.keys.borrowed_transactions_hashmap_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + if self.transactions.is_empty() { + return Err(TransactionStoreError::InternalError { + message: "Cannot process empty transaction list".to_string(), + }); + } + + // Get all recycled nonces + let recycled_nonces: HashSet = conn + .zrange(self.keys.recycled_nonces_zset_name(), 0, -1) + .await?; + + // Verify all nonces are in recycled set + for tx in self.transactions { + let nonce = tx.signed_transaction.nonce(); + if !recycled_nonces.contains(&nonce) { + return Err(TransactionStoreError::NonceNotInRecycledSet { nonce }); + } + } + + // Verify all transactions exist in pending queue using batched ZSCORE calls + if !self.transactions.is_empty() { + let mut pipe = twmq::redis::pipe(); + for tx in self.transactions { + pipe.zscore( + self.keys.pending_transactions_zset_name(), + &tx.transaction_id, + ); + } + let scores: Vec> = pipe.query_async(conn).await?; + + for (tx, score) in self.transactions.iter().zip(scores.iter()) { + if score.is_none() { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + // Pre-serialize all borrowed transaction data + let mut serialized_transactions = Vec::with_capacity(self.transactions.len()); + for tx in self.transactions { + let borrowed_json = + serde_json::to_string(tx).map_err(|e| TransactionStoreError::InternalError { + message: format!("Failed to serialize borrowed transaction: {}", e), + })?; + serialized_transactions.push(borrowed_json); + } + + Ok(serialized_transactions) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + serialized_transactions: Self::ValidationData, + ) -> Self::OperationResult { + let recycled_key = self.keys.recycled_nonces_zset_name(); + let pending_key = self.keys.pending_transactions_zset_name(); + let borrowed_key = self.keys.borrowed_transactions_hashmap_name(); + + for (tx, borrowed_json) in self.transactions.iter().zip(serialized_transactions.iter()) { + let nonce = tx.signed_transaction.nonce(); + + // Remove nonce from recycled set + pipeline.zrem(&recycled_key, nonce); + + // Remove from pending queue + pipeline.zrem(&pending_key, &tx.transaction_id); + + // Add to borrowed state + pipeline.hset(&borrowed_key, nonce.to_string(), borrowed_json); + } + + self.transactions.len() + } +} diff --git a/executors/src/eoa/store/submitted.rs b/executors/src/eoa/store/submitted.rs index 104a107..943001c 100644 --- a/executors/src/eoa/store/submitted.rs +++ b/executors/src/eoa/store/submitted.rs @@ -121,7 +121,7 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { conn: &mut ConnectionManager, ) -> Result { let submitted_txs: Vec = conn - .zrange_withscores( + .zrangebyscore_withscores( self.keys.submitted_transactions_zset_name(), 0, self.last_confirmed_nonce as isize, diff --git a/executors/src/eoa/worker.rs b/executors/src/eoa/worker.rs index a5e2465..7c488af 100644 --- a/executors/src/eoa/worker.rs +++ b/executors/src/eoa/worker.rs @@ -4,7 +4,7 @@ use alloy::consensus::{ }; use alloy::network::{TransactionBuilder, TransactionBuilder7702}; use alloy::primitives::{Address, B256, Bytes, U256}; -use alloy::providers::Provider; +use alloy::providers::{PendingTransactionBuilder, Provider}; use alloy::rpc::types::TransactionRequest as AlloyTransactionRequest; use alloy::signers::Signature; use alloy::transports::{RpcError, TransportErrorKind}; @@ -20,6 +20,7 @@ use engine_core::{ use hex; use serde::{Deserialize, Serialize}; use std::{sync::Arc, time::Duration}; +use thirdweb_core::iaw::IAWError; use tokio::time::sleep; use twmq::Queue; use twmq::redis::AsyncCommands; @@ -33,7 +34,8 @@ use twmq::{ use crate::eoa::store::{ AtomicEoaExecutorStore, BorrowedTransactionData, CleanupReport, ConfirmedTransaction, - EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, EoaTransactionRequest, ReplacedTransaction, + EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, EoaTransactionRequest, PendingTransaction, + ReplacedTransaction, SubmissionErrorFail, SubmissionErrorNack, SubmissionResult, SubmittedTransaction, TransactionData, TransactionStoreError, }; use crate::webhook::WebhookJobHandler; @@ -46,6 +48,10 @@ const MIN_TRANSACTIONS_PER_EOA: u64 = 1; // Fleet management from spec const HEALTH_CHECK_INTERVAL: u64 = 300; // 5 minutes in seconds const NONCE_STALL_TIMEOUT: u64 = 300_000; // 5 minutes in milliseconds - after this time, attempt gas bump +// Retry constants for preparation phase +const MAX_PREPARATION_RETRIES: u32 = 3; +const PREPARATION_RETRY_DELAY_MS: u64 = 100; + // ========== JOB DATA ========== #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] @@ -144,13 +150,13 @@ impl UserCancellable for EoaExecutorWorkerError { // ========== SIMPLE ERROR CLASSIFICATION ========== #[derive(Debug)] -enum SendErrorClassification { +pub enum SendErrorClassification { PossiblySent, // "nonce too low", "already known" etc DeterministicFailure, // Invalid signature, malformed tx, insufficient funds etc } #[derive(PartialEq, Eq, Debug)] -enum SendContext { +pub enum SendContext { Rebroadcast, InitialBroadcast, } @@ -238,6 +244,38 @@ fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { } } +fn is_retryable_preparation_error(error: &EoaExecutorWorkerError) -> bool { + match error { + EoaExecutorWorkerError::RpcError { inner_error, .. } => { + // extract the RpcErrorKind from the inner error + if let EngineError::RpcError { kind, .. } = inner_error { + is_retryable_rpc_error(kind) + } else { + false + } + } + EoaExecutorWorkerError::ChainServiceError { .. } => true, // Network related + EoaExecutorWorkerError::StoreError { inner_error, .. } => { + matches!(inner_error, TransactionStoreError::RedisError { .. }) + } + EoaExecutorWorkerError::TransactionSimulationFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::TransactionBuildFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::SigningError { inner_error, .. } => match inner_error { + // if vault error, it's not retryable + EngineError::VaultError { .. } => false, + // if iaw error, it's retryable only if it's a network error + EngineError::IawError { error, .. } => matches!(error, IAWError::NetworkError { .. }), + _ => false, + }, + EoaExecutorWorkerError::TransactionNotFound { .. } => false, // Deterministic + EoaExecutorWorkerError::InternalError { .. } => false, // Deterministic + EoaExecutorWorkerError::UserCancelled => false, // Deterministic + EoaExecutorWorkerError::TransactionSendError { .. } => false, // Different context + EoaExecutorWorkerError::SignatureParsingFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::WorkRemaining { .. } => false, // Different context + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ConfirmedTransactionWithRichReceipt { @@ -247,13 +285,6 @@ pub struct ConfirmedTransactionWithRichReceipt { pub receipt: alloy::rpc::types::TransactionReceipt, } -#[derive(Debug, Clone)] -pub struct PreparedTransaction { - pub transaction_id: String, - pub signed_tx: Signed, - pub nonce: u64, -} - // ========== MAIN WORKER ========== /// EOA Executor Worker /// @@ -359,6 +390,55 @@ where } } +impl SubmissionResult { + /// Convert a send result to a SubmissionResult for batch processing + /// This handles the specific RpcError type from alloy + pub fn from_send_result( + borrowed_transaction: &BorrowedTransactionData, + send_result: Result>, + send_context: SendContext, + user_data: Option, + chain: &impl Chain, + ) -> Self { + match send_result { + Ok(_) => SubmissionResult::Success(borrowed_transaction.into()), + Err(ref rpc_error) => { + match classify_send_error(rpc_error, send_context) { + SendErrorClassification::PossiblySent => { + SubmissionResult::Success(borrowed_transaction.into()) + } + SendErrorClassification::DeterministicFailure => { + // Transaction failed, should be retried + let engine_error = rpc_error.to_engine_error(chain); + let error = EoaExecutorWorkerError::TransactionSendError { + message: format!("Transaction send failed: {}", rpc_error), + inner_error: engine_error, + }; + SubmissionResult::Nack(SubmissionErrorNack { + transaction_id: borrowed_transaction.transaction_id.clone(), + error, + user_data, + }) + } + } + } + } + } + + /// Helper method for when we need to create a failure result + pub fn from_failure( + transaction_id: String, + error: EoaExecutorWorkerError, + user_data: Option, + ) -> Self { + SubmissionResult::Fail(SubmissionErrorFail { + transaction_id, + error, + user_data, + }) + } +} + impl EoaExecutorWorker where CS: ChainService + Send + Sync + 'static, @@ -492,66 +572,45 @@ where let rebroadcast_results = futures::future::join_all(rebroadcast_futures).await; - // Process results sequentially for Redis state changes - let mut recovered_count = 0; - // TODO: both borrowed -> submitted and borrowed -> recycled need to be batched instead of sequential - for (borrowed, send_result) in rebroadcast_results { - let nonce = borrowed.signed_transaction.nonce(); + // Convert results to SubmissionResult for batch processing + let submission_results: Vec = rebroadcast_results + .into_iter() + .map(|(borrowed, send_result)| { + SubmissionResult::from_send_result( + borrowed, + send_result, + SendContext::Rebroadcast, + None, // We'll let the batch operation fetch user data + chain, + ) + }) + .collect(); - match send_result { - Ok(_) => { - // Transaction was sent successfully - scoped - .atomic_move_borrowed_to_submitted( - nonce, - &borrowed.hash, - &borrowed.transaction_id, - ) - .await?; - tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted"); - } - Err(e) => { - match classify_send_error(&e, SendContext::Rebroadcast) { - SendErrorClassification::PossiblySent => { - // Transaction possibly sent, move to submitted - scoped - .atomic_move_borrowed_to_submitted( - nonce, - &borrowed.hash, - &borrowed.transaction_id, - ) - .await?; - tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted (possibly sent)"); - } - SendErrorClassification::DeterministicFailure => { - // Transaction is broken, recycle nonce and requeue - scoped - .atomic_move_borrowed_to_recycled(nonce, &borrowed.transaction_id) - .await?; - tracing::warn!(transaction_id = %borrowed.transaction_id, nonce = nonce, error = %e, "Recycled failed transaction"); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - self.update_balance_threshold(scoped, chain).await?; - } + // TODO: Implement post-processing analysis for balance threshold updates and nonce resets + // Currently we lose the granular error handling that was in the individual atomic operations. + // Consider: + // 1. Analyzing submission_results for specific error patterns + // 2. Calling update_balance_threshold if needed + // 3. Detecting nonce reset conditions + // 4. Or move this logic into the batch processor itself - // Check if this should trigger nonce reset - if should_trigger_nonce_reset(&e) { - tracing::warn!( - eoa = %scoped.eoa(), - chain_id = %scoped.chain_id(), - "Nonce too high error detected, may need nonce synchronization" - ); - // The next confirm_flow will fetch fresh nonce and auto-sync - } - } - } - } - } + // Process all results in one batch operation + let report = scoped + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; - recovered_count += 1; - } + // TODO: Handle post-processing updates here if needed + // For now, we skip the individual error analysis that was done in the old atomic approach - Ok(recovered_count) + tracing::info!( + "Recovered {} transactions: {} submitted, {} recycled, {} failed", + report.total_processed, + report.moved_to_submitted, + report.moved_to_pending, + report.failed_transactions + ); + + Ok(report.total_processed as u32) } // ========== CONFIRM FLOW ========== @@ -760,258 +819,185 @@ where return Ok(0); } - // Get pending transactions - let pending_txs = scoped - .peek_pending_transactions(recycled_nonces.len() as u64) - .await?; + let mut total_sent = 0; + let mut remaining_nonces = recycled_nonces; - // let highest_submitted_nonce_txs = - // scoped.get_highest_submitted_nonce_tranasactions().await?; + // Loop to handle preparation failures and refill with new transactions + while !remaining_nonces.is_empty() { + // Get pending transactions to match with recycled nonces + let pending_txs = scoped + .peek_pending_transactions(remaining_nonces.len() as u64) + .await?; + + if pending_txs.is_empty() { + tracing::debug!("No pending transactions available for recycled nonces"); + break; + } - // let highest_submitted_nonce = highest_submitted_nonce_txs - // .first() - // .and_then(|tx| Some(tx.nonce)); + // Pair recycled nonces with pending transactions + let mut build_tasks = Vec::new(); + let mut nonce_tx_pairs = Vec::new(); - // 1. SEQUENTIAL REDIS: Collect nonce-transaction pairs - let mut nonce_tx_pairs = Vec::new(); - for (i, nonce) in recycled_nonces.into_iter().enumerate() { - if let Some(tx_id) = pending_txs.get(i) { - // Get transaction data - if let Some(tx_data) = scoped.get_transaction_data(tx_id).await? { - nonce_tx_pairs.push((nonce, tx_id.clone(), tx_data)); + for (i, nonce) in remaining_nonces.iter().enumerate() { + if let Some(p_tx) = pending_txs.get(i) { + build_tasks.push(self.build_and_sign_single_transaction_with_retries( + scoped, p_tx, *nonce, chain, + )); + nonce_tx_pairs.push((*nonce, p_tx.clone())); } else { - tracing::warn!("Transaction data not found for {}", tx_id); - continue; + // No more pending transactions for this recycled nonce + tracing::debug!("No pending transaction for recycled nonce {}", nonce); + break; } - } else { - // No pending transactions - skip recycled nonces without pending transactions - tracing::debug!("No pending transaction for recycled nonce {}", nonce); - continue; } - } - if nonce_tx_pairs.is_empty() { - return Ok(0); - } + if build_tasks.is_empty() { + break; + } - // 2. PARALLEL BUILD/SIGN: Build and sign all transactions in parallel - let build_futures: Vec<_> = nonce_tx_pairs - .iter() - .map(|(nonce, transaction_id, tx_data)| async move { - let prepared = self - .build_and_sign_transaction(tx_data, *nonce, chain) - .await; - (*nonce, transaction_id, prepared) - }) - .collect(); + // Build and sign all transactions in parallel + let prepared_results = futures::future::join_all(build_tasks).await; - let build_results = futures::future::join_all(build_futures).await; - - // 3. SEQUENTIAL REDIS: Move successfully built transactions to borrowed state - let mut prepared_txs = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (nonce, transaction_id, build_result) in build_results { - match build_result { - Ok(signed_tx) => { - let borrowed_data = BorrowedTransactionData { - transaction_id: transaction_id.clone(), - signed_transaction: signed_tx.clone(), - hash: signed_tx.hash().to_string(), - borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - }; - - // Try to atomically move from pending to borrowed with recycled nonce - match scoped - .atomic_move_pending_to_borrowed_with_recycled_nonce( - transaction_id, - nonce, - &borrowed_data, - ) - .await - { - Ok(()) => { - let prepared = PreparedTransaction { - transaction_id: transaction_id.clone(), - signed_tx, - nonce, - }; - prepared_txs.push(prepared); - } - Err(TransactionStoreError::NonceNotInRecycledSet { .. }) => { - tracing::debug!("Nonce {} was consumed by another worker", nonce); - continue; - } - Err(TransactionStoreError::TransactionNotInPendingQueue { .. }) => { - tracing::debug!("Transaction {} already processed", transaction_id); - continue; - } - Err(e) => { - tracing::error!("Failed to move {} to borrowed: {}", transaction_id, e); - continue; - } + // Separate successful preparations from failures + let mut prepared_txs = Vec::new(); + let mut failed_tx_ids = Vec::new(); + let mut balance_threshold_update_needed = false; + + for (i, result) in prepared_results.into_iter().enumerate() { + match result { + Ok(borrowed_data) => { + prepared_txs.push(borrowed_data); } - } - Err(e) => { - // Accumulate balance threshold issues instead of updating immediately - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; + Err(e) => { + // Track balance threshold issues + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, + .. + } = &e + { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; + + let (_nonce, pending_tx) = &nonce_tx_pairs[i]; + tracing::warn!( + "Failed to build recycled transaction {}: {}", + pending_tx.transaction_id, + e + ); + + // For deterministic build failures, fail the transaction immediately + if !is_retryable_preparation_error(&e) { + failed_tx_ids.push(pending_tx.transaction_id.clone()); } } + } + } + + // Fail deterministic failures from pending state + for tx_id in failed_tx_ids { + if let Err(e) = scoped + .fail_pending_transaction( + &tx_id, + "Deterministic preparation failure", + self.webhook_queue.clone(), + ) + .await + { + tracing::error!("Failed to fail pending transaction {}: {}", tx_id, e); + } + } - tracing::warn!("Failed to build transaction {}: {}", transaction_id, e); - continue; + // Update balance threshold if needed + if balance_threshold_update_needed { + if let Err(e) = self.update_balance_threshold(scoped, chain).await { + tracing::error!( + "Failed to update balance threshold after build failures: {}", + e + ); } } - } - // Update balance threshold once if any build failures were due to balance issues - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after parallel build failures: {}", - e + if prepared_txs.is_empty() { + // No successful preparations, try again with more pending transactions + // Remove the nonces we couldn't use from our list + remaining_nonces = remaining_nonces + .into_iter() + .skip(nonce_tx_pairs.len()) + .collect(); + continue; + } + + // Move prepared transactions to borrowed state with recycled nonces + let moved_count = scoped + .atomic_move_pending_to_borrowed_with_recycled_nonces(&prepared_txs) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = prepared_txs.len(), + "Moved transactions to borrowed state using recycled nonces" + ); + + // Actually send the transactions to the blockchain + let send_tasks: Vec<_> = prepared_txs + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { chain.provider().send_tx_envelope(signed_tx.into()).await } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // Process send results and update states + let mut submission_results = Vec::new(); + for (i, send_result) in send_results.into_iter().enumerate() { + let borrowed_tx = &prepared_txs[i]; + let user_data = scoped + .get_transaction_data(&borrowed_tx.transaction_id) + .await?; + + let submission_result = SubmissionResult::from_send_result( + borrowed_tx, + send_result, + SendContext::InitialBroadcast, + user_data, + chain, ); + submission_results.push(submission_result); } - } - if prepared_txs.is_empty() { - return Ok(0); - } + // Use batch processing to handle all submission results + let processing_report = scoped + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; - // 4. PARALLEL SEND: Send all transactions in parallel - let send_futures: Vec<_> = prepared_txs - .iter() - .map(|prepared| async move { - let result = chain - .provider() - .send_tx_envelope(prepared.signed_tx.clone().into()) - .await; - (prepared, result) - }) - .collect(); + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); - let send_results = futures::future::join_all(send_futures).await; + total_sent += processing_report.moved_to_submitted; - // 5. SEQUENTIAL REDIS: Process results and update states - let mut sent_count = 0; - for (prepared, send_result) in send_results { - match send_result { - Ok(_) => { - // Transaction sent successfully - match scoped - .atomic_move_borrowed_to_submitted( - prepared.nonce, - &prepared.signed_tx.hash().to_string(), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - hash = ?prepared.signed_tx.hash(), - "Successfully sent recycled transaction" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - Err(e) => { - match classify_send_error(&e, SendContext::InitialBroadcast) { - SendErrorClassification::PossiblySent => { - // Move to submitted state - match scoped - .atomic_move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - "Recycled transaction possibly sent" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - SendErrorClassification::DeterministicFailure => { - // Recycle nonce and requeue transaction - match scoped - .atomic_move_borrowed_to_recycled( - prepared.nonce, - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - tracing::warn!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - error = %e, - "Recycled transaction failed, re-recycled nonce" - ); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - if let Err(e) = - self.update_balance_threshold(scoped, chain).await - { - tracing::error!( - "Failed to update balance threshold: {}", - e - ); - } - } - - if should_trigger_nonce_reset(&e) { - tracing::warn!( - nonce = prepared.nonce, - "Nonce too high error detected, may need nonce synchronization" - ); - } - } - Err(e) => { - tracing::error!( - "Failed to move {} back to recycled: {}", - prepared.transaction_id, - e - ); - } - } - } - } - } + // Remove the nonces we successfully processed from our list + remaining_nonces = remaining_nonces.into_iter().skip(moved_count).collect(); + + // If we didn't use all available nonces, we ran out of pending transactions + if moved_count < nonce_tx_pairs.len() { + break; } } - Ok(sent_count) + Ok(total_sent as u32) } async fn process_new_transactions( @@ -1024,256 +1010,240 @@ where return Ok(0); } - // 1. SEQUENTIAL REDIS: Get pending transactions - let pending_txs = scoped.peek_pending_transactions(budget).await?; - if pending_txs.is_empty() { - return Ok(0); - } + let mut total_sent = 0; + let mut remaining_budget = budget; + + // Loop to handle preparation failures and refill with new transactions + while remaining_budget > 0 { + // 1. Get pending transactions + let pending_txs = scoped.peek_pending_transactions(remaining_budget).await?; + if pending_txs.is_empty() { + break; + } - let optimistic_nonce = scoped.get_optimistic_nonce().await?; + let optimistic_nonce = scoped.get_optimistic_transaction_count().await?; + + // 2. Build and sign all transactions in parallel + let build_tasks: Vec<_> = pending_txs + .iter() + .enumerate() + .map(|(i, tx)| { + let expected_nonce = optimistic_nonce + i as u64; + self.build_and_sign_single_transaction_with_retries( + scoped, + tx, + expected_nonce, + chain, + ) + }) + .collect(); + + let prepared_results = futures::future::join_all(build_tasks).await; + + // 3. Separate successful preparations from failures + let mut prepared_txs = Vec::new(); + let mut failed_tx_ids = Vec::new(); + let mut balance_threshold_update_needed = false; + + for (i, result) in prepared_results.into_iter().enumerate() { + match result { + Ok(borrowed_data) => { + prepared_txs.push(borrowed_data); + } + Err(e) => { + // Track balance threshold issues + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, + .. + } = &e + { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } - // 2. PARALLEL BUILD/SIGN: Build and sign all transactions in parallel - let build_tasks: Vec<_> = pending_txs - .iter() - .enumerate() - .map(|(i, tx_id)| { - let expected_nonce = optimistic_nonce + i as u64; - self.build_and_sign_single_transaction(scoped, tx_id, expected_nonce, chain) - }) - .collect(); + let pending_tx = &pending_txs[i]; + tracing::warn!( + "Failed to build transaction {}: {}", + pending_tx.transaction_id, + e + ); - let prepared_results = futures::future::join_all(build_tasks).await; - - // 3. SEQUENTIAL REDIS: Move successful transactions to borrowed state (maintain nonce order) - let mut prepared_txs = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (i, result) in prepared_results.into_iter().enumerate() { - match result { - Ok(prepared) => { - let borrowed_data = BorrowedTransactionData { - transaction_id: prepared.transaction_id.clone(), - signed_transaction: prepared.signed_tx.clone(), - hash: prepared.signed_tx.hash().to_string(), - borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - }; - - match scoped - .atomic_move_pending_to_borrowed_with_new_nonce( - &prepared.transaction_id, - prepared.nonce, - &borrowed_data, - ) - .await - { - Ok(()) => prepared_txs.push(prepared), - Err(TransactionStoreError::OptimisticNonceChanged { .. }) => { - tracing::debug!( - "Nonce changed for transaction {}, skipping", - prepared.transaction_id - ); - break; // Stop processing if nonce changed - } - Err(TransactionStoreError::TransactionNotInPendingQueue { .. }) => { - tracing::debug!( - "Transaction {} already processed, skipping", - prepared.transaction_id - ); - continue; - } - Err(e) => { - tracing::error!( - "Failed to move transaction {} to borrowed: {}", - prepared.transaction_id, - e - ); - continue; + // For deterministic build failures, fail the transaction immediately + if !is_retryable_preparation_error(&e) { + failed_tx_ids.push(pending_tx.transaction_id.clone()); } } } - Err(e) => { - // Accumulate balance threshold issues instead of updating immediately - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } + } - tracing::warn!("Failed to build transaction {}: {}", pending_txs[i], e); - // Individual transaction failure doesn't stop the worker - continue; + // 4. Fail deterministic failures from pending state + for tx_id in failed_tx_ids { + if let Err(e) = scoped + .fail_pending_transaction( + &tx_id, + "Deterministic preparation failure", + self.webhook_queue.clone(), + ) + .await + { + tracing::error!("Failed to fail pending transaction {}: {}", tx_id, e); } } - } - // Update balance threshold once if any build failures were due to balance issues - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after parallel build failures: {}", - e + // Update balance threshold if needed + if balance_threshold_update_needed { + if let Err(e) = self.update_balance_threshold(scoped, chain).await { + tracing::error!( + "Failed to update balance threshold after build failures: {}", + e + ); + } + } + + if prepared_txs.is_empty() { + // No successful preparations, try again with remaining budget + remaining_budget = remaining_budget.saturating_sub(pending_txs.len() as u64); + continue; + } + + // 5. Move prepared transactions to borrowed state + let moved_count = scoped + .atomic_move_pending_to_borrowed_with_incremented_nonces(&prepared_txs) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = prepared_txs.len(), + "Moved transactions to borrowed state using incremented nonces" + ); + + // 6. Actually send the transactions to the blockchain + let send_tasks: Vec<_> = prepared_txs + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { chain.provider().send_tx_envelope(signed_tx.into()).await } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // 7. Process send results and update states + let mut submission_results = Vec::new(); + for (i, send_result) in send_results.into_iter().enumerate() { + let borrowed_tx = &prepared_txs[i]; + let user_data = scoped + .get_transaction_data(&borrowed_tx.transaction_id) + .await?; + + let submission_result = SubmissionResult::from_send_result( + borrowed_tx, + send_result, + SendContext::InitialBroadcast, + user_data, + chain, ); + submission_results.push(submission_result); } - } - if prepared_txs.is_empty() { - return Ok(0); + // 8. Use batch processing to handle all submission results + let processing_report = scoped + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); + + total_sent += processing_report.moved_to_submitted; + remaining_budget = remaining_budget.saturating_sub(moved_count as u64); + + // If we didn't use all our budget, we ran out of pending transactions + if moved_count < pending_txs.len() { + break; + } } - // 4. PARALLEL SEND (but ordered): Send all transactions in parallel but in nonce order - let send_futures: Vec<_> = prepared_txs - .iter() - .enumerate() - .map(|(i, prepared)| async move { - // Add delay for ordering (except first transaction) - if i > 0 { - sleep(Duration::from_millis(50 * i as u64)).await; // 50ms delay between consecutive nonces - } + Ok(total_sent as u32) + } - let result = chain - .provider() - .send_tx_envelope(prepared.signed_tx.clone().into()) - .await; - (prepared, result) - }) - .collect(); + // ========== TRANSACTION BUILDING & SENDING ========== + async fn build_and_sign_single_transaction_with_retries( + &self, + scoped: &AtomicEoaExecutorStore, + pending_transaction: &PendingTransaction, + nonce: u64, + chain: &impl Chain, + ) -> Result { + let mut last_error = None; - let send_results = futures::future::join_all(send_futures).await; - - // 5. SEQUENTIAL REDIS: Process results and update states - let mut sent_count = 0; - for (prepared, send_result) in send_results { - match send_result { - Ok(pending) => { - // Transaction sent successfully - match scoped - .atomic_move_borrowed_to_submitted( - prepared.nonce, - &pending.tx_hash().to_string(), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - hash = ?prepared.signed_tx.hash(), - "Successfully sent new transaction" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - Err(e) => { - match classify_send_error(&e, SendContext::InitialBroadcast) { - SendErrorClassification::PossiblySent => { - // Move to submitted state - match scoped - .atomic_move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - "New transaction possibly sent" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - SendErrorClassification::DeterministicFailure => { - // Recycle nonce and requeue transaction - match scoped - .atomic_move_borrowed_to_recycled( - prepared.nonce, - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - tracing::warn!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - error = %e, - "New transaction failed, recycled nonce" - ); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - if let Err(e) = - self.update_balance_threshold(scoped, chain).await - { - tracing::error!( - "Failed to update balance threshold: {}", - e - ); - } - } - - if should_trigger_nonce_reset(&e) { - tracing::warn!( - nonce = prepared.nonce, - "Nonce too high error detected, may need nonce synchronization" - ); - } - } - Err(e) => { - tracing::error!( - "Failed to move {} to recycled: {}", - prepared.transaction_id, - e - ); - } - } - } + // Internal retry loop for retryable errors + for attempt in 0..=MAX_PREPARATION_RETRIES { + if attempt > 0 { + // Simple exponential backoff + let delay = PREPARATION_RETRY_DELAY_MS * (2_u64.pow(attempt - 1)); + sleep(Duration::from_millis(delay)).await; + + tracing::debug!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + "Retrying transaction preparation" + ); + } + + match self + .build_and_sign_single_transaction(scoped, pending_transaction, nonce, chain) + .await + { + Ok(result) => return Ok(result), + Err(error) => { + if is_retryable_preparation_error(&error) && attempt < MAX_PREPARATION_RETRIES { + tracing::warn!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + error = %error, + "Retryable error during transaction preparation, will retry" + ); + last_error = Some(error); + continue; + } else { + // Either deterministic error or exceeded max retries + return Err(error); } } } } - Ok(sent_count) + // This should never be reached, but just in case + Err( + last_error.unwrap_or_else(|| EoaExecutorWorkerError::InternalError { + message: "Unexpected error in retry loop".to_string(), + }), + ) } - // ========== TRANSACTION BUILDING & SENDING ========== async fn build_and_sign_single_transaction( &self, scoped: &AtomicEoaExecutorStore, - transaction_id: &str, + pending_transaction: &PendingTransaction, nonce: u64, chain: &impl Chain, - ) -> Result { + ) -> Result { // Get transaction data let tx_data = scoped - .get_transaction_data(transaction_id) + .get_transaction_data(&pending_transaction.transaction_id) .await? .ok_or_else(|| EoaExecutorWorkerError::TransactionNotFound { - transaction_id: transaction_id.to_string(), + transaction_id: pending_transaction.transaction_id.clone(), })?; // Build and sign transaction @@ -1281,10 +1251,12 @@ where .build_and_sign_transaction(&tx_data, nonce, chain) .await?; - Ok(PreparedTransaction { - transaction_id: transaction_id.to_string(), - signed_tx, - nonce, + Ok(BorrowedTransactionData { + transaction_id: pending_transaction.transaction_id.clone(), + hash: signed_tx.hash().to_string(), + signed_transaction: signed_tx, + borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, + queued_at: pending_transaction.queued_at, }) } From 1518cd8f0779a02f4e7aa774ae3424623f8604e4 Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Tue, 15 Jul 2025 04:20:06 +0530 Subject: [PATCH 08/10] Refactor signer and transaction handling for improved efficiency - Updated the `SmartAccountSigner` to pass credentials by reference instead of cloning, enhancing performance. - Modified transaction structures to include default serialization options for gas limits and transaction type data. - Enhanced webhook options handling in various job data structures, ensuring consistent default behavior. - Refactored the EOA executor to streamline transaction confirmation and submission processes, improving overall transaction management. These changes aim to optimize resource usage and improve the clarity of transaction handling across the codebase. --- aa-core/src/signer.rs | 8 +- core/src/execution_options/mod.rs | 3 +- core/src/signer.rs | 37 +- core/src/transaction.rs | 10 +- executors/src/eip7702_executor/confirm.rs | 5 +- executors/src/eip7702_executor/send.rs | 9 +- executors/src/eoa/events.rs | 82 +- executors/src/eoa/mod.rs | 2 +- executors/src/eoa/store/atomic.rs | 116 +- executors/src/eoa/store/borrowed.rs | 323 +--- executors/src/eoa/store/hydrate.rs | 157 ++ executors/src/eoa/store/mod.rs | 136 +- executors/src/eoa/store/pending.rs | 26 +- executors/src/eoa/store/submitted.rs | 278 +++- executors/src/eoa/worker.rs | 1848 --------------------- executors/src/eoa/worker/confirm.rs | 360 ++++ executors/src/eoa/worker/error.rs | 262 +++ executors/src/eoa/worker/mod.rs | 501 ++++++ executors/src/eoa/worker/send.rs | 402 +++++ executors/src/eoa/worker/transaction.rs | 427 +++++ executors/src/external_bundler/confirm.rs | 4 +- executors/src/external_bundler/send.rs | 11 +- executors/src/webhook/envelope.rs | 19 +- server/src/execution_router/mod.rs | 20 +- server/src/http/routes/admin/mod.rs | 1 + server/src/http/routes/admin/queue.rs | 135 ++ server/src/http/routes/contract_write.rs | 3 +- server/src/http/routes/mod.rs | 1 + server/src/http/routes/sign_message.rs | 2 +- server/src/http/routes/sign_typed_data.rs | 2 +- server/src/http/server.rs | 3 + server/src/queue/manager.rs | 6 +- thirdweb-core/src/iaw/mod.rs | 24 +- twmq/src/lib.rs | 6 +- 34 files changed, 2904 insertions(+), 2325 deletions(-) create mode 100644 executors/src/eoa/store/hydrate.rs delete mode 100644 executors/src/eoa/worker.rs create mode 100644 executors/src/eoa/worker/confirm.rs create mode 100644 executors/src/eoa/worker/error.rs create mode 100644 executors/src/eoa/worker/mod.rs create mode 100644 executors/src/eoa/worker/send.rs create mode 100644 executors/src/eoa/worker/transaction.rs create mode 100644 server/src/http/routes/admin/mod.rs create mode 100644 server/src/http/routes/admin/queue.rs diff --git a/aa-core/src/signer.rs b/aa-core/src/signer.rs index 968ccd4..cc15b30 100644 --- a/aa-core/src/signer.rs +++ b/aa-core/src/signer.rs @@ -161,7 +161,7 @@ impl SmartAccountSigner { }, message, format, - self.credentials.clone(), + &self.credentials, ) .await } @@ -189,7 +189,7 @@ impl SmartAccountSigner { from: self.options.signer_address, }, typed_data, - self.credentials.clone(), + &self.credentials, ) .await; } @@ -212,7 +212,7 @@ impl SmartAccountSigner { from: self.options.signer_address, }, typed_data, - self.credentials.clone(), + &self.credentials, ) .await } @@ -242,7 +242,7 @@ impl SmartAccountSigner { from: self.options.signer_address, }, &typed_data, - self.credentials.clone(), + &self.credentials, ) .await } diff --git a/core/src/execution_options/mod.rs b/core/src/execution_options/mod.rs index a738b5e..48fd2d9 100644 --- a/core/src/execution_options/mod.rs +++ b/core/src/execution_options/mod.rs @@ -89,7 +89,8 @@ pub struct WebhookOptions { pub struct SendTransactionRequest { pub execution_options: ExecutionOptions, pub params: Vec, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, } /// # QueuedTransaction diff --git a/core/src/signer.rs b/core/src/signer.rs index 5004a92..3b75db3 100644 --- a/core/src/signer.rs +++ b/core/src/signer.rs @@ -169,7 +169,7 @@ pub trait AccountSigner { options: Self::SigningOptions, message: &str, format: MessageFormat, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; /// Sign typed data @@ -177,15 +177,15 @@ pub trait AccountSigner { &self, options: Self::SigningOptions, typed_data: &TypedData, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; /// Sign a transaction fn sign_transaction( &self, options: Self::SigningOptions, - transaction: TypedTransaction, - credentials: SigningCredential, + transaction: &TypedTransaction, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; /// Sign EIP-7702 authorization @@ -195,7 +195,7 @@ pub trait AccountSigner { chain_id: u64, address: Address, nonce: alloy::primitives::U256, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> impl std::future::Future> + Send; } @@ -224,7 +224,7 @@ impl AccountSigner for EoaSigner { options: EoaSigningOptions, message: &str, format: MessageFormat, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> Result { match credentials { SigningCredential::Vault(auth_method) => { @@ -260,7 +260,7 @@ impl AccountSigner for EoaSigner { .sign_message( auth_token, thirdweb_auth, - message.to_string(), + message, options.from, options.chain_id, Some(iaw_format), @@ -280,7 +280,7 @@ impl AccountSigner for EoaSigner { &self, options: EoaSigningOptions, typed_data: &TypedData, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> Result { match &credentials { SigningCredential::Vault(auth_method) => { @@ -301,12 +301,7 @@ impl AccountSigner for EoaSigner { } => { let iaw_result = self .iaw_client - .sign_typed_data( - auth_token.clone(), - thirdweb_auth.clone(), - typed_data.clone(), - options.from, - ) + .sign_typed_data(auth_token, thirdweb_auth, typed_data, options.from) .await .map_err(|e| { tracing::error!("Error signing typed data with EOA (IAW): {:?}", e); @@ -321,14 +316,14 @@ impl AccountSigner for EoaSigner { async fn sign_transaction( &self, options: EoaSigningOptions, - transaction: TypedTransaction, - credentials: SigningCredential, + transaction: &TypedTransaction, + credentials: &SigningCredential, ) -> Result { match credentials { SigningCredential::Vault(auth_method) => { let vault_result = self .vault_client - .sign_transaction(auth_method.clone(), transaction, options.from) + .sign_transaction(auth_method.clone(), transaction.clone(), options.from) .await .map_err(|e| { tracing::error!("Error signing transaction with EOA (Vault): {:?}", e); @@ -343,7 +338,7 @@ impl AccountSigner for EoaSigner { } => { let iaw_result = self .iaw_client - .sign_transaction(auth_token.clone(), thirdweb_auth.clone(), transaction) + .sign_transaction(auth_token, thirdweb_auth, &transaction) .await .map_err(|e| { tracing::error!("Error signing transaction with EOA (IAW): {:?}", e); @@ -361,7 +356,7 @@ impl AccountSigner for EoaSigner { chain_id: u64, address: Address, nonce: U256, - credentials: SigningCredential, + credentials: &SigningCredential, ) -> Result { // Create the Authorization struct that both clients expect let authorization = Authorization { @@ -373,7 +368,7 @@ impl AccountSigner for EoaSigner { SigningCredential::Vault(auth_method) => { let vault_result = self .vault_client - .sign_authorization(auth_method, options.from, authorization) + .sign_authorization(auth_method.clone(), options.from, authorization) .await .map_err(|e| { tracing::error!("Error signing authorization with EOA (Vault): {:?}", e); @@ -389,7 +384,7 @@ impl AccountSigner for EoaSigner { } => { let iaw_result = self .iaw_client - .sign_authorization(auth_token, thirdweb_auth, options.from, authorization) + .sign_authorization(auth_token, thirdweb_auth, options.from, &authorization) .await .map_err(|e| { tracing::error!("Error signing authorization with EOA (IAW): {:?}", e); diff --git a/core/src/transaction.rs b/core/src/transaction.rs index 422f76c..21b7bc3 100644 --- a/core/src/transaction.rs +++ b/core/src/transaction.rs @@ -23,7 +23,7 @@ pub struct InnerTransaction { /// Gas limit for the transaction /// If not provided, engine will estimate the gas limit #[schema(value_type = Option)] - #[serde(default, rename = "gasLimit")] + #[serde(default, rename = "gasLimit", skip_serializing_if = "Option::is_none")] pub gas_limit: Option, /// Transaction type-specific data for different EIP standards @@ -33,7 +33,7 @@ pub struct InnerTransaction { /// Depending on the execution mode chosen, these might be ignored: /// /// - For ERC4337 execution, all gas fee related fields are ignored. Sending signed authorizations is also not supported. - #[serde(flatten)] + #[serde(flatten, default, skip_serializing_if = "Option::is_none")] pub transaction_type_data: Option, } @@ -58,14 +58,17 @@ pub struct Transaction7702Data { /// List of signed authorizations for contract delegation /// Each authorization allows the EOA to temporarily delegate to a smart contract #[schema(value_type = Option>)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub authorization_list: Option>, /// Maximum fee per gas willing to pay (in wei) /// This is the total fee cap including base fee and priority fee + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_fee_per_gas: Option, /// Maximum priority fee per gas willing to pay (in wei) /// This is the tip paid to validators for transaction inclusion + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_priority_fee_per_gas: Option, } @@ -77,10 +80,12 @@ pub struct Transaction7702Data { pub struct Transaction1559Data { /// Maximum fee per gas willing to pay (in wei) /// This is the total fee cap including base fee and priority fee + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_fee_per_gas: Option, /// Maximum priority fee per gas willing to pay (in wei) /// This is the tip paid to validators for transaction inclusion + #[serde(default, skip_serializing_if = "Option::is_none")] pub max_priority_fee_per_gas: Option, } @@ -92,5 +97,6 @@ pub struct Transaction1559Data { pub struct TransactionLegacyData { /// Gas price willing to pay (in wei) /// This is the total price per unit of gas for legacy transactions + #[serde(default, skip_serializing_if = "Option::is_none")] pub gas_price: Option, } diff --git a/executors/src/eip7702_executor/confirm.rs b/executors/src/eip7702_executor/confirm.rs index f1e995c..27f2efc 100644 --- a/executors/src/eip7702_executor/confirm.rs +++ b/executors/src/eip7702_executor/confirm.rs @@ -32,11 +32,12 @@ pub struct Eip7702ConfirmationJobData { pub bundler_transaction_id: String, pub eoa_address: Address, pub rpc_credentials: RpcCredentials, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, } impl HasWebhookOptions for Eip7702ConfirmationJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } diff --git a/executors/src/eip7702_executor/send.rs b/executors/src/eip7702_executor/send.rs index bebf6d2..a9d4f0d 100644 --- a/executors/src/eip7702_executor/send.rs +++ b/executors/src/eip7702_executor/send.rs @@ -46,13 +46,14 @@ pub struct Eip7702SendJobData { pub transactions: Vec, pub eoa_address: Address, pub signing_credential: SigningCredential, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, pub rpc_credentials: RpcCredentials, pub nonce: Option, } impl HasWebhookOptions for Eip7702SendJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } @@ -243,7 +244,7 @@ where .sign_typed_data( signing_options.clone(), &typed_data, - job_data.signing_credential.clone(), + &job_data.signing_credential, ) .await .map_err(|e| Eip7702SendError::SigningError { @@ -272,7 +273,7 @@ where job_data.chain_id, MINIMAL_ACCOUNT_IMPLEMENTATION_ADDRESS, nonce, - job_data.signing_credential.clone(), + &job_data.signing_credential, ) .await .map_err(|e| Eip7702SendError::SigningError { diff --git a/executors/src/eoa/events.rs b/executors/src/eoa/events.rs index b9d62be..5d89542 100644 --- a/executors/src/eoa/events.rs +++ b/executors/src/eoa/events.rs @@ -5,8 +5,8 @@ use twmq::job::RequeuePosition; use crate::{ eoa::{ - store::{SubmittedTransaction, TransactionData}, - worker::{ConfirmedTransactionWithRichReceipt, EoaExecutorWorkerError}, + store::{ConfirmedTransaction, SubmittedTransactionDehydrated}, + worker::error::EoaExecutorWorkerError, }, webhook::envelope::{ BareWebhookNotificationEnvelope, SerializableFailData, SerializableNackData, @@ -15,7 +15,17 @@ use crate::{ }; pub struct EoaExecutorEvent { - pub transaction_data: TransactionData, + pub transaction_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] +pub enum EoaConfirmationError { + #[error( + "Previously submitted attempt for transaction replaced at nonce with different transaction" + )] + #[serde(rename_all = "camelCase")] + TransactionReplaced { nonce: u64, hash: String }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -24,19 +34,22 @@ pub struct EoaSendAttemptNackData { pub error: EoaExecutorWorkerError, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaExecutorConfirmedTransaction { + pub receipt: alloy::rpc::types::TransactionReceipt, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EoaExecutorStage { - SendAttempt, - TransactionReplaced, - TransactionConfirmed, + Send, + Confirmation, } impl Display for EoaExecutorStage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - EoaExecutorStage::SendAttempt => write!(f, "send_attempt"), - EoaExecutorStage::TransactionReplaced => write!(f, "transaction_replaced"), - EoaExecutorStage::TransactionConfirmed => write!(f, "transaction_confirmed"), + EoaExecutorStage::Send => write!(f, "send"), + EoaExecutorStage::Confirmation => write!(f, "confirmation"), } } } @@ -46,12 +59,13 @@ const EXECUTOR_NAME: &str = "eoa"; impl EoaExecutorEvent { pub fn send_attempt_success_envelope( &self, - submitted_transaction: SubmittedTransaction, - ) -> BareWebhookNotificationEnvelope> { + submitted_transaction: SubmittedTransactionDehydrated, + ) -> BareWebhookNotificationEnvelope> + { BareWebhookNotificationEnvelope { - transaction_id: self.transaction_data.transaction_id.clone(), + transaction_id: self.transaction_id.clone(), executor_name: EXECUTOR_NAME.to_string(), - stage_name: EoaExecutorStage::SendAttempt.to_string(), + stage_name: EoaExecutorStage::Send.to_string(), event_type: StageEvent::Success, payload: SerializableSuccessData { result: submitted_transaction.clone(), @@ -66,9 +80,9 @@ impl EoaExecutorEvent { attempt_number: u32, ) -> BareWebhookNotificationEnvelope> { BareWebhookNotificationEnvelope { - transaction_id: self.transaction_data.transaction_id.clone(), + transaction_id: self.transaction_id.clone(), executor_name: EXECUTOR_NAME.to_string(), - stage_name: EoaExecutorStage::SendAttempt.to_string(), + stage_name: EoaExecutorStage::Send.to_string(), event_type: StageEvent::Nack, payload: SerializableNackData { error: EoaSendAttemptNackData { @@ -86,31 +100,41 @@ impl EoaExecutorEvent { pub fn transaction_replaced_envelope( &self, - replaced_transaction: SubmittedTransaction, - ) -> BareWebhookNotificationEnvelope> { + replaced_transaction: SubmittedTransactionDehydrated, + ) -> BareWebhookNotificationEnvelope> { BareWebhookNotificationEnvelope { - transaction_id: self.transaction_data.transaction_id.clone(), + transaction_id: self.transaction_id.clone(), executor_name: EXECUTOR_NAME.to_string(), - stage_name: EoaExecutorStage::TransactionReplaced.to_string(), - event_type: StageEvent::Success, - payload: SerializableSuccessData { - result: replaced_transaction.clone(), + stage_name: EoaExecutorStage::Send.to_string(), + event_type: StageEvent::Nack, + payload: SerializableNackData { + error: EoaConfirmationError::TransactionReplaced { + nonce: replaced_transaction.nonce, + hash: replaced_transaction.hash, + }, + delay_ms: None, + position: RequeuePosition::Last, + attempt_number: 0, + max_attempts: None, + next_retry_at: None, }, } } pub fn transaction_confirmed_envelope( &self, - confirmed_transaction: ConfirmedTransactionWithRichReceipt, - ) -> BareWebhookNotificationEnvelope> + confirmed_transaction: ConfirmedTransaction, + ) -> BareWebhookNotificationEnvelope> { BareWebhookNotificationEnvelope { - transaction_id: self.transaction_data.transaction_id.clone(), + transaction_id: self.transaction_id.clone(), executor_name: EXECUTOR_NAME.to_string(), - stage_name: EoaExecutorStage::TransactionConfirmed.to_string(), + stage_name: EoaExecutorStage::Confirmation.to_string(), event_type: StageEvent::Success, payload: SerializableSuccessData { - result: confirmed_transaction.clone(), + result: EoaExecutorConfirmedTransaction { + receipt: confirmed_transaction.receipt, + }, }, } } @@ -121,9 +145,9 @@ impl EoaExecutorEvent { final_attempt_number: u32, ) -> BareWebhookNotificationEnvelope> { BareWebhookNotificationEnvelope { - transaction_id: self.transaction_data.transaction_id.clone(), + transaction_id: self.transaction_id.clone(), executor_name: EXECUTOR_NAME.to_string(), - stage_name: EoaExecutorStage::SendAttempt.to_string(), + stage_name: EoaExecutorStage::Send.to_string(), event_type: StageEvent::Failure, payload: SerializableFailData { error: error.clone(), diff --git a/executors/src/eoa/mod.rs b/executors/src/eoa/mod.rs index 89dd7e2..d0ab4e6 100644 --- a/executors/src/eoa/mod.rs +++ b/executors/src/eoa/mod.rs @@ -5,4 +5,4 @@ pub mod worker; pub use error_classifier::{EoaErrorMapper, EoaExecutionError, RecoveryStrategy}; pub use store::{EoaExecutorStore, EoaTransactionRequest}; -pub use worker::{EoaExecutorWorker, EoaExecutorWorkerJobData}; +pub use worker::{EoaExecutorJobHandler, EoaExecutorWorkerJobData}; diff --git a/executors/src/eoa/store/atomic.rs b/executors/src/eoa/store/atomic.rs index cea982d..53707a5 100644 --- a/executors/src/eoa/store/atomic.rs +++ b/executors/src/eoa/store/atomic.rs @@ -9,17 +9,22 @@ use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; use crate::{ eoa::{ EoaExecutorStore, + events::EoaExecutorEvent, store::{ - BorrowedTransactionData, ConfirmedTransaction, EoaHealth, TransactionAttempt, - TransactionStoreError, + BorrowedTransactionData, ConfirmedTransaction, EoaHealth, PendingTransaction, + SubmittedTransactionDehydrated, TransactionAttempt, TransactionStoreError, borrowed::{BorrowedProcessingReport, ProcessBorrowedTransactions, SubmissionResult}, pending::{ MovePendingToBorrowedWithIncrementedNonces, MovePendingToBorrowedWithRecycledNonces, }, - submitted::{CleanSubmittedTransactions, CleanupReport, SubmittedTransaction}, + submitted::{ + CleanAndGetRecycledNonces, CleanSubmittedTransactions, CleanupReport, + SubmittedNoopTransaction, SubmittedTransaction, + }, }, + worker::error::EoaExecutorWorkerError, }, - webhook::WebhookJobHandler, + webhook::{WebhookJobHandler, queue_webhook_envelopes}, }; const MAX_RETRIES: u32 = 10; @@ -38,6 +43,7 @@ pub trait SafeRedisTransaction: Send + Sync { fn validation( &self, conn: &mut ConnectionManager, + store: &EoaExecutorStore, ) -> impl Future> + Send; fn watch_keys(&self) -> Vec; } @@ -333,7 +339,7 @@ impl AtomicEoaExecutorStore { } // Execute validation - match safe_tx.validation(&mut conn).await { + match safe_tx.validation(&mut conn, &self.store).await { Ok(validation_data) => { // Build and execute pipeline let mut pipeline = twmq::redis::pipe(); @@ -345,7 +351,8 @@ impl AtomicEoaExecutorStore { .await { Ok(_) => return Ok(result), // Success - Err(_) => { + Err(e) => { + tracing::error!("WATCH failed: {}", e); // WATCH failed, check if it was our lock let still_own_lock: Option = conn.get(&lock_key).await?; if still_own_lock.as_deref() != Some(self.worker_id()) { @@ -393,7 +400,7 @@ impl AtomicEoaExecutorStore { let now = chrono::Utc::now().timestamp_millis().max(0) as u64; // First, read current health data - let current_health = self.check_eoa_health().await?; + let current_health = self.get_eoa_health().await?; // Prepare health update if health data exists let health_update = if let Some(mut health) = current_health { @@ -422,7 +429,7 @@ impl AtomicEoaExecutorStore { /// Add a gas bump attempt (new hash) to submitted transactions pub async fn add_gas_bump_attempt( &self, - submitted_transaction: &SubmittedTransaction, + submitted_transaction: &SubmittedTransactionDehydrated, signed_transaction: Signed, ) -> Result<(), TransactionStoreError> { let new_hash = signed_transaction.hash().to_string(); @@ -472,7 +479,7 @@ impl AtomicEoaExecutorStore { ) -> Result<(), TransactionStoreError> { let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - let current_health = self.check_eoa_health().await?; + let current_health = self.get_eoa_health().await?; // Prepare health update if health data exists let health_update = if let Some(mut health) = current_health { @@ -505,53 +512,44 @@ impl AtomicEoaExecutorStore { .await } - /// Fail a transaction that's in the borrowed state (we know the nonce) - pub async fn fail_borrowed_transaction( - &self, - transaction_id: &str, - nonce: u64, - failure_reason: &str, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(|pipeline| { - let borrowed_key = self.borrowed_transactions_hashmap_name(); - let tx_data_key = self.transaction_data_key_name(transaction_id); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Remove from borrowed state using the known nonce - pipeline.hdel(&borrowed_key, nonce.to_string()); - - // Update transaction data with failure - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "failure_reason", failure_reason); - pipeline.hset(&tx_data_key, "status", "failed"); - }) - .await - } - /// Fail a transaction that's in the pending state (remove from pending and fail) /// This is used for deterministic failures during preparation that should not retry pub async fn fail_pending_transaction( &self, - transaction_id: &str, - failure_reason: &str, + pending_transaction: &PendingTransaction, + error: EoaExecutorWorkerError, webhook_queue: Arc>, ) -> Result<(), TransactionStoreError> { self.with_lock_check(|pipeline| { let pending_key = self.pending_transactions_zset_name(); - let tx_data_key = self.transaction_data_key_name(transaction_id); + let tx_data_key = self.transaction_data_key_name(&pending_transaction.transaction_id); let now = chrono::Utc::now().timestamp_millis().max(0) as u64; // Remove from pending state - pipeline.zrem(&pending_key, transaction_id); + pipeline.zrem(&pending_key, &pending_transaction.transaction_id); // Update transaction data with failure pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "failure_reason", failure_reason); + pipeline.hset(&tx_data_key, "failure_reason", error.to_string()); pipeline.hset(&tx_data_key, "status", "failed"); - // TODO: Queue webhook event for failed transaction - // let webhook_job = WebhookJobHandler::new(...); - // webhook_queue.push(webhook_job); + let event = EoaExecutorEvent { + transaction_id: pending_transaction.transaction_id.clone(), + }; + + let fail_envelope = event.transaction_failed_envelope(error.clone(), 1); + + if !pending_transaction.user_request.webhook_options.is_empty() { + let mut tx_context = webhook_queue.transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + fail_envelope, + pending_transaction.user_request.webhook_options.clone(), + &mut tx_context, + webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } + } }) .await } @@ -560,11 +558,13 @@ impl AtomicEoaExecutorStore { &self, confirmed_transactions: &[ConfirmedTransaction], last_confirmed_nonce: u64, + webhook_queue: Arc>, ) -> Result { self.execute_with_watch_and_retry(&CleanSubmittedTransactions { confirmed_transactions, last_confirmed_nonce, keys: &self.keys, + webhook_queue, }) .await } @@ -577,6 +577,7 @@ impl AtomicEoaExecutorStore { results: Vec, webhook_queue: Arc>, ) -> Result { + dbg!("getting here", &results); self.execute_with_watch_and_retry(&ProcessBorrowedTransactions { results, keys: &self.keys, @@ -584,4 +585,39 @@ impl AtomicEoaExecutorStore { }) .await } + + pub async fn clean_and_get_recycled_nonces(&self) -> Result, TransactionStoreError> { + self.execute_with_watch_and_retry(&CleanAndGetRecycledNonces { keys: &self.keys }) + .await + } + + pub async fn process_noop_transactions( + &self, + noop_transactions: &[SubmittedNoopTransaction], + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(|pipeline| { + let recycled_key = self.recycled_nonces_zset_name(); + let submitted_key = self.submitted_transactions_zset_name(); + + pipeline.zrem( + &recycled_key, + noop_transactions + .iter() + .map(|tx| tx.nonce) + .collect::>(), + ); + + pipeline.zadd_multiple( + submitted_key, + &noop_transactions + .iter() + .map(|tx| { + let (tx_string, nonce) = tx.to_redis_string_with_nonce(); + (nonce, tx_string) + }) + .collect::>(), + ); + }) + .await + } } diff --git a/executors/src/eoa/store/borrowed.rs b/executors/src/eoa/store/borrowed.rs index 3eceb35..f4c5e36 100644 --- a/executors/src/eoa/store/borrowed.rs +++ b/executors/src/eoa/store/borrowed.rs @@ -1,69 +1,36 @@ -use std::collections::HashMap; use std::sync::Arc; -use alloy::consensus::Transaction; -use alloy::primitives::Address; -use serde::{Deserialize, Serialize}; +use twmq::Queue; use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; -use twmq::{Queue, hooks::TransactionContext}; +use crate::eoa::EoaExecutorStore; use crate::eoa::{ events::EoaExecutorEvent, store::{ - BorrowedTransactionData, EoaExecutorStoreKeys, TransactionData, TransactionStoreError, - atomic::SafeRedisTransaction, submitted::SubmittedTransaction, + EoaExecutorStoreKeys, TransactionStoreError, atomic::SafeRedisTransaction, + submitted::SubmittedTransaction, }, - worker::EoaExecutorWorkerError, + worker::error::EoaExecutorWorkerError, }; use crate::webhook::{WebhookJobHandler, queue_webhook_envelopes}; -/// Error information for NACK operations (retryable errors) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SubmissionErrorNack { - pub transaction_id: String, - pub error: EoaExecutorWorkerError, - pub user_data: Option, -} - -/// Error information for FAIL operations (permanent failures) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SubmissionErrorFail { - pub transaction_id: String, - pub error: EoaExecutorWorkerError, - pub user_data: Option, -} - -/// Result of a submission attempt #[derive(Debug, Clone)] -pub enum SubmissionResult { - Success(SubmittedTransaction), - Nack(SubmissionErrorNack), - Fail(SubmissionErrorFail), +pub enum SubmissionResultType { + Success, + Nack(EoaExecutorWorkerError), + Fail(EoaExecutorWorkerError), } -/// Internal representation where all user data is guaranteed to be present +/// Result of a submission attempt #[derive(Debug, Clone)] -pub enum SubmissionResultWithUserData { - Success(SubmittedTransaction, TransactionData), - Nack(SubmissionErrorNack, TransactionData), - Fail(SubmissionErrorFail, TransactionData), +pub struct SubmissionResult { + pub transaction: SubmittedTransaction, + pub result: SubmissionResultType, } -impl SubmissionResultWithUserData { - fn transaction_id(&self) -> &str { - match self { - SubmissionResultWithUserData::Success(tx, _) => &tx.transaction_id, - SubmissionResultWithUserData::Nack(err, _) => &err.transaction_id, - SubmissionResultWithUserData::Fail(err, _) => &err.transaction_id, - } - } - - fn user_data(&self) -> &TransactionData { - match self { - SubmissionResultWithUserData::Success(_, data) => data, - SubmissionResultWithUserData::Nack(_, data) => data, - SubmissionResultWithUserData::Fail(_, data) => data, - } +impl SubmissionResult { + pub fn transaction_id(&self) -> &str { + &self.transaction.transaction_id } } @@ -81,13 +48,11 @@ pub struct BorrowedProcessingReport { pub moved_to_pending: usize, pub failed_transactions: usize, pub webhook_events_queued: usize, + pub ignored_not_in_borrowed: usize, } impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { - type ValidationData = ( - Vec, - Vec, - ); + type ValidationData = Vec; type OperationResult = BorrowedProcessingReport; fn name(&self) -> &str { @@ -101,98 +66,29 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { async fn validation( &self, conn: &mut ConnectionManager, + _store: &EoaExecutorStore, ) -> Result { - // Collect all transaction IDs that need user data - let mut transactions_needing_data = Vec::new(); - let mut results_with_partial_data = Vec::new(); - - for result in &self.results { - match result { - SubmissionResult::Success(tx) => { - transactions_needing_data.push(tx.transaction_id.clone()); - results_with_partial_data.push(result.clone()); - } - SubmissionResult::Nack(err) => { - if err.user_data.is_none() { - transactions_needing_data.push(err.transaction_id.clone()); - } - results_with_partial_data.push(result.clone()); - } - SubmissionResult::Fail(err) => { - if err.user_data.is_none() { - transactions_needing_data.push(err.transaction_id.clone()); - } - results_with_partial_data.push(result.clone()); - } - } - } - - // Batch fetch missing user data - let mut user_data_map = HashMap::new(); - for transaction_id in transactions_needing_data { - let data_key = self.keys.transaction_data_key_name(&transaction_id); - if let Some(data_json) = conn.get::<&str, Option>(&data_key).await? { - let transaction_data: TransactionData = serde_json::from_str(&data_json)?; - user_data_map.insert(transaction_id, transaction_data); - } - } - - // Get all borrowed transactions to validate they exist - let borrowed_transactions_map: HashMap = conn - .hgetall(self.keys.borrowed_transactions_hashmap_name()) + // Get all borrowed transaction IDs + let borrowed_transaction_ids: Vec = conn + .hkeys(self.keys.borrowed_transactions_hashmap_name()) .await?; - let borrowed_transactions: Vec = borrowed_transactions_map - .into_iter() - .filter_map(|(nonce_str, data_json)| { - let borrowed_data: BorrowedTransactionData = - serde_json::from_str(&data_json).ok()?; - Some(borrowed_data) - }) - .collect(); - - // Convert to results with guaranteed user data - let mut results_with_user_data = Vec::new(); - for result in results_with_partial_data { - match result { - SubmissionResult::Success(tx) => { - if let Some(user_data) = user_data_map.get(&tx.transaction_id) { - results_with_user_data - .push(SubmissionResultWithUserData::Success(tx, user_data.clone())); - } else { - return Err(TransactionStoreError::TransactionNotFound { - transaction_id: tx.transaction_id.clone(), - }); - } - } - SubmissionResult::Nack(mut err) => { - let user_data = if let Some(data) = err.user_data.take() { - data - } else if let Some(data) = user_data_map.get(&err.transaction_id) { - data.clone() - } else { - return Err(TransactionStoreError::TransactionNotFound { - transaction_id: err.transaction_id.clone(), - }); - }; - results_with_user_data.push(SubmissionResultWithUserData::Nack(err, user_data)); - } - SubmissionResult::Fail(mut err) => { - let user_data = if let Some(data) = err.user_data.take() { - data - } else if let Some(data) = user_data_map.get(&err.transaction_id) { - data.clone() - } else { - return Err(TransactionStoreError::TransactionNotFound { - transaction_id: err.transaction_id.clone(), - }); - }; - results_with_user_data.push(SubmissionResultWithUserData::Fail(err, user_data)); - } + // Filter submission results to only include those that exist in borrowed state + let mut valid_results = Vec::new(); + for result in &self.results { + let transaction_id = result.transaction_id(); + if borrowed_transaction_ids.contains(&transaction_id.to_string()) { + valid_results.push(result.clone()); + } else { + tracing::warn!( + transaction_id = %transaction_id, + nonce = %result.transaction.nonce, + "Submission result not found in borrowed state, ignoring" + ); } } - Ok((results_with_user_data, borrowed_transactions)) + Ok(valid_results) } fn operation( @@ -200,49 +96,30 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { pipeline: &mut Pipeline, validation_data: Self::ValidationData, ) -> Self::OperationResult { - let (results_with_user_data, borrowed_transactions) = validation_data; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Create borrowed transactions lookup by transaction_id - let borrowed_by_id: HashMap = borrowed_transactions - .iter() - .map(|tx| (tx.transaction_id.clone(), tx)) - .collect(); - + let valid_results = validation_data; let mut report = BorrowedProcessingReport::default(); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - for result in &results_with_user_data { + for result in &valid_results { let transaction_id = result.transaction_id(); - let user_data = result.user_data(); - - // Find the corresponding borrowed transaction to get the nonce - let borrowed_tx = match borrowed_by_id.get(transaction_id) { - Some(tx) => tx, - None => { - // Transaction not in borrowed state, skip - continue; - } - }; - - let nonce = borrowed_tx.signed_transaction.nonce(); - - // We'll set attempt_number to 1 for simplicity in the operation phase - // The actual attempt tracking is handled by the attempts list - let attempt_number = 1; - - // Define attempts_key for all match arms - let attempts_key = self.keys.transaction_attempts_list_name(transaction_id); - - match result { - SubmissionResultWithUserData::Success(tx, user_data) => { - // Remove from borrowed - pipeline.hdel( - self.keys.borrowed_transactions_hashmap_name(), - nonce.to_string(), - ); - - // Add to submitted - let (submitted_tx_redis_string, nonce) = tx.to_redis_string_with_nonce(); + let nonce = result.transaction.nonce; + + // Remove from borrowed hashmap + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + transaction_id, + ); + + // Add attempt to attempts list (using transaction hash as attempt data) + // let attempt_json = serde_json::to_string(&result.transaction.hash).unwrap(); + // todo figure out what to do about attempts + // pipeline.lpush(&attempts_key, &attempt_json); + + match &result.result { + SubmissionResultType::Success => { + // Add to submitted zset + let (submitted_tx_redis_string, nonce) = + result.transaction.to_redis_string_with_nonce(); pipeline.zadd( self.keys.submitted_transactions_zset_name(), &submitted_tx_redis_string, @@ -250,30 +127,30 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { ); // Update hash-to-ID mapping - let hash_to_id_key = self.keys.transaction_hash_to_id_key_name(&tx.hash); - pipeline.set(&hash_to_id_key, &tx.transaction_id); + let hash_to_id_key = self + .keys + .transaction_hash_to_id_key_name(&result.transaction.hash); + + pipeline.set(&hash_to_id_key, transaction_id); // Update transaction data status - let tx_data_key = self.keys.transaction_data_key_name(&tx.transaction_id); + let tx_data_key = self.keys.transaction_data_key_name(transaction_id); pipeline.hset(&tx_data_key, "status", "submitted"); - // Add attempt to attempts list - let attempt_json = - serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); - pipeline.lpush(&attempts_key, &attempt_json); - - // Queue webhook event + // Queue webhook event using user_request from SubmissionResult let event = EoaExecutorEvent { - transaction_data: user_data.clone(), + transaction_id: transaction_id.to_string(), }; - let envelope = event.send_attempt_success_envelope(tx.clone()); - if let Some(webhook_options) = &user_data.user_request.webhook_options { + + let envelope = + event.send_attempt_success_envelope(result.transaction.data.clone()); + if !result.transaction.user_request.webhook_options.is_empty() { let mut tx_context = self .webhook_queue .transaction_context_from_pipeline(pipeline); if let Err(e) = queue_webhook_envelopes( envelope, - webhook_options.clone(), + result.transaction.user_request.webhook_options.clone(), &mut tx_context, self.webhook_queue.clone(), ) { @@ -285,42 +162,31 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { report.moved_to_submitted += 1; } - SubmissionResultWithUserData::Nack(err, user_data) => { - // Remove from borrowed - pipeline.hdel( - self.keys.borrowed_transactions_hashmap_name(), - nonce.to_string(), - ); - + SubmissionResultType::Nack(err) => { // Add back to pending pipeline.zadd( self.keys.pending_transactions_zset_name(), - &err.transaction_id, + transaction_id, now, ); // Update transaction data status - let tx_data_key = self.keys.transaction_data_key_name(&err.transaction_id); + let tx_data_key = self.keys.transaction_data_key_name(transaction_id); pipeline.hset(&tx_data_key, "status", "pending"); - // Add attempt to attempts list - let attempt_json = - serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); - pipeline.lpush(&attempts_key, &attempt_json); - - // Queue webhook event + // Queue webhook event using user_request from SubmissionResult let event = EoaExecutorEvent { - transaction_data: user_data.clone(), + transaction_id: transaction_id.to_string(), }; - let envelope = - event.send_attempt_nack_envelope(nonce, err.error.clone(), attempt_number); - if let Some(webhook_options) = &user_data.user_request.webhook_options { + let envelope = event.send_attempt_nack_envelope(nonce, err.clone(), 1); + + if !result.transaction.user_request.webhook_options.is_empty() { let mut tx_context = self .webhook_queue .transaction_context_from_pipeline(pipeline); if let Err(e) = queue_webhook_envelopes( envelope, - webhook_options.clone(), + result.transaction.user_request.webhook_options.clone(), &mut tx_context, self.webhook_queue.clone(), ) { @@ -332,37 +198,25 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { report.moved_to_pending += 1; } - SubmissionResultWithUserData::Fail(err, user_data) => { - // Remove from borrowed - pipeline.hdel( - self.keys.borrowed_transactions_hashmap_name(), - nonce.to_string(), - ); - - // Update transaction data with failure - let tx_data_key = self.keys.transaction_data_key_name(&err.transaction_id); + SubmissionResultType::Fail(err) => { + // Mark as failed + let tx_data_key = self.keys.transaction_data_key_name(transaction_id); pipeline.hset(&tx_data_key, "status", "failed"); pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "failure_reason", err.error.to_string()); - - // Add attempt to attempts list - let attempt_json = - serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); - pipeline.lpush(&attempts_key, &attempt_json); + pipeline.hset(&tx_data_key, "failure_reason", err.to_string()); - // Queue webhook event + // Queue webhook event using user_request from SubmissionResult let event = EoaExecutorEvent { - transaction_data: user_data.clone(), + transaction_id: transaction_id.to_string(), }; - let envelope = - event.transaction_failed_envelope(err.error.clone(), attempt_number); - if let Some(webhook_options) = &user_data.user_request.webhook_options { + let envelope = event.transaction_failed_envelope(err.clone(), 1); + if !result.transaction.user_request.webhook_options.is_empty() { let mut tx_context = self .webhook_queue .transaction_context_from_pipeline(pipeline); if let Err(e) = queue_webhook_envelopes( envelope, - webhook_options.clone(), + result.transaction.user_request.webhook_options.clone(), &mut tx_context, self.webhook_queue.clone(), ) { @@ -377,7 +231,8 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { } } - report.total_processed = results_with_user_data.len(); + report.total_processed = valid_results.len(); + report.ignored_not_in_borrowed = self.results.len() - valid_results.len(); report } } diff --git a/executors/src/eoa/store/hydrate.rs b/executors/src/eoa/store/hydrate.rs new file mode 100644 index 0000000..74f113a --- /dev/null +++ b/executors/src/eoa/store/hydrate.rs @@ -0,0 +1,157 @@ +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::eoa::{ + EoaExecutorStore, EoaTransactionRequest, + store::{ + BorrowedTransaction, BorrowedTransactionData, NO_OP_TRANSACTION_ID, + SubmittedNoopTransaction, SubmittedTransaction, SubmittedTransactionHydrated, + TransactionStoreError, submitted::SubmittedTransactionDehydrated, + }, +}; + +pub trait Dehydrated { + type Hydrated; + + fn transaction_id(&self) -> &str; + + fn hydrate(self, required_data: R) -> Self::Hydrated; +} + +#[derive(Debug, Clone)] +pub enum SubmittedTransactionHydrator { + Noop, + Real(EoaTransactionRequest), +} + +impl Dehydrated for SubmittedTransactionDehydrated { + type Hydrated = SubmittedTransactionHydrated; + + fn transaction_id(&self) -> &str { + &self.transaction_id + } + + fn hydrate(self, required_data: SubmittedTransactionHydrator) -> SubmittedTransactionHydrated { + match required_data { + SubmittedTransactionHydrator::Noop => { + SubmittedTransactionHydrated::Noop(SubmittedNoopTransaction { + nonce: self.nonce, + hash: self.hash, + }) + } + SubmittedTransactionHydrator::Real(request) => { + SubmittedTransactionHydrated::Real(SubmittedTransaction { + data: self, + user_request: request, + }) + } + } + } +} + +impl Dehydrated for BorrowedTransactionData { + type Hydrated = BorrowedTransaction; + + fn transaction_id(&self) -> &str { + &self.transaction_id + } + + fn hydrate(self, required_data: EoaTransactionRequest) -> BorrowedTransaction { + BorrowedTransaction { + data: self, + user_request: required_data, + } + } +} + +impl EoaExecutorStore { + pub async fn hydrate_all( + &self, + dehydrated: Vec, + ) -> Result, TransactionStoreError> + where + D: Dehydrated, + { + let mut pipe = twmq::redis::pipe(); + + for d in &dehydrated { + pipe.hget( + self.keys.transaction_data_key_name(d.transaction_id()), + "user_request", + ); + } + + let results: Vec = pipe.query_async(&mut self.redis.clone()).await?; + + let mut hydrated = Vec::with_capacity(dehydrated.len()); + for (d, r) in dehydrated.into_iter().zip(results.iter()) { + hydrated.push(d.hydrate(serde_json::from_str::(r)?)); + } + + Ok(hydrated) + } + + pub async fn hydrate(&self, dehydrated: D) -> Result + where + D: Dehydrated, + { + let mut pipe = twmq::redis::pipe(); + pipe.hget( + self.keys + .transaction_data_key_name(dehydrated.transaction_id()), + "user_request", + ); + let result: String = pipe.query_async(&mut self.redis.clone()).await?; + Ok(dehydrated.hydrate(serde_json::from_str::(&result)?)) + } + + pub async fn hydrate_all_submitted( + &self, + dehydrated: Vec, + ) -> Result, TransactionStoreError> + where + D: Dehydrated, + { + let mut pipe = twmq::redis::pipe(); + for d in &dehydrated { + if d.transaction_id() == NO_OP_TRANSACTION_ID { + continue; + } + + pipe.hget( + self.keys.transaction_data_key_name(d.transaction_id()), + "user_request", + ); + } + + let results: Vec = pipe.query_async(&mut self.redis.clone()).await?; + + let id_to_eoa_request = results + .into_iter() + .map(|r| { + let request = serde_json::from_str::(&r)?; + Ok((request.transaction_id.clone(), request)) + }) + .collect::, TransactionStoreError>>()?; + + let mut hydrated = Vec::with_capacity(dehydrated.len()); + + for d in dehydrated { + let id = d.transaction_id(); + if id == NO_OP_TRANSACTION_ID { + hydrated.push(d.hydrate(SubmittedTransactionHydrator::Noop)); + continue; + } + + let request = + id_to_eoa_request + .get(id) + .ok_or(TransactionStoreError::TransactionNotFound { + transaction_id: id.to_string(), + })?; + + hydrated.push(d.hydrate(SubmittedTransactionHydrator::Real(request.clone()))); + } + + Ok(hydrated) + } +} diff --git a/executors/src/eoa/store/mod.rs b/executors/src/eoa/store/mod.rs index a0d3874..1bfb39b 100644 --- a/executors/src/eoa/store/mod.rs +++ b/executors/src/eoa/store/mod.rs @@ -8,6 +8,7 @@ use engine_core::execution_options::WebhookOptions; use engine_core::transaction::TransactionTypeData; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::ops::Deref; use twmq::redis::{AsyncCommands, aio::ConnectionManager}; mod atomic; @@ -15,14 +16,15 @@ mod borrowed; mod pending; mod submitted; +pub mod hydrate; + pub mod error; pub use atomic::AtomicEoaExecutorStore; -pub use borrowed::{ - BorrowedProcessingReport, SubmissionErrorFail, SubmissionErrorNack, SubmissionResult, +pub use borrowed::{BorrowedProcessingReport, SubmissionResult, SubmissionResultType}; +pub use submitted::{ + CleanupReport, SubmittedNoopTransaction, SubmittedTransaction, SubmittedTransactionDehydrated, + SubmittedTransactionHydrated, SubmittedTransactionStringWithNonce, }; -pub use submitted::{CleanupReport, SubmittedTransaction}; - -use crate::eoa::store::submitted::SubmittedTransactionStringWithNonce; pub const NO_OP_TRANSACTION_ID: &str = "noop"; @@ -36,22 +38,18 @@ pub struct ReplacedTransaction { pub struct ConfirmedTransaction { pub hash: String, pub transaction_id: String, - pub receipt_data: String, + pub receipt: alloy::rpc::types::TransactionReceipt, + pub receipt_serialized: String, } +/// (transaction_id, queued_at) +type PendingTransactionStringWithQueuedAt = (String, u64); + #[derive(Debug, Clone)] pub struct PendingTransaction { pub transaction_id: String, pub queued_at: u64, -} - -impl From<(String, u64)> for PendingTransaction { - fn from((transaction_id, queued_at): (String, u64)) -> Self { - Self { - transaction_id, - queued_at, - } - } + pub user_request: EoaTransactionRequest, } /// The actual user request data @@ -69,7 +67,8 @@ pub struct EoaTransactionRequest { #[serde(alias = "gas")] pub gas_limit: Option, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, pub signing_credential: SigningCredential, pub rpc_credentials: RpcCredentials, @@ -97,12 +96,6 @@ pub struct TransactionData { pub created_at: u64, // Unix timestamp in milliseconds } -pub struct BorrowedTransaction { - pub transaction_id: String, - pub data: Signed, - pub borrowed_at: chrono::DateTime, -} - /// Transaction store focused on transaction_id operations and nonce indexing pub struct EoaExecutorStore { pub redis: ConnectionManager, @@ -328,13 +321,35 @@ pub struct BorrowedTransactionData { pub borrowed_at: u64, } -impl Into for &BorrowedTransactionData { - fn into(self) -> SubmittedTransaction { +#[derive(Debug, Clone)] +pub struct BorrowedTransaction { + pub data: BorrowedTransactionData, + pub user_request: EoaTransactionRequest, +} + +impl Deref for BorrowedTransaction { + type Target = BorrowedTransactionData; + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl From for SubmittedTransactionDehydrated { + fn from(data: BorrowedTransactionData) -> Self { + SubmittedTransactionDehydrated { + nonce: data.signed_transaction.nonce(), + hash: data.signed_transaction.hash().to_string(), + transaction_id: data.transaction_id.clone(), + queued_at: data.queued_at, + } + } +} + +impl From for SubmittedTransaction { + fn from(data: BorrowedTransaction) -> Self { SubmittedTransaction { - nonce: self.signed_transaction.nonce(), - hash: self.signed_transaction.hash().to_string(), - transaction_id: self.transaction_id.clone(), - queued_at: self.queued_at, + data: data.data.into(), + user_request: data.user_request.clone(), } } } @@ -412,7 +427,7 @@ impl EoaExecutorStore { let borrowed_map: HashMap = conn.hgetall(&borrowed_key).await?; let mut result = Vec::new(); - for (_nonce_str, transaction_json) in borrowed_map { + for (_transaction_id, transaction_json) in borrowed_map { let borrowed_data: BorrowedTransactionData = serde_json::from_str(&transaction_json)?; result.push(borrowed_data); } @@ -422,20 +437,24 @@ impl EoaExecutorStore { /// Get all hashes below a certain nonce from submitted transactions /// Returns (nonce, hash, transaction_id) tuples - pub async fn get_submitted_transactions_below_nonce( + pub async fn get_submitted_transactions_below_chain_transaction_count( &self, - below_nonce: u64, - ) -> Result, TransactionStoreError> { + count: u64, + ) -> Result, TransactionStoreError> { + if count == 0 { + return Ok(Vec::new()); + } + let submitted_key = self.submitted_transactions_zset_name(); let mut conn = self.redis.clone(); - // Get all entries with nonce < below_nonce + // Get all entries with nonce < transaction count let results: Vec = conn - .zrangebyscore_withscores(&submitted_key, 0, below_nonce - 1) + .zrangebyscore_withscores(&submitted_key, 0, count - 1) .await?; - let submitted_txs: Vec = - SubmittedTransaction::from_redis_strings(&results); + let submitted_txs: Vec = + SubmittedTransactionDehydrated::from_redis_strings(&results); Ok(submitted_txs) } @@ -444,7 +463,7 @@ impl EoaExecutorStore { pub async fn get_submitted_transactions_for_nonce( &self, nonce: u64, - ) -> Result, TransactionStoreError> { + ) -> Result, TransactionStoreError> { let submitted_key = self.submitted_transactions_zset_name(); let mut conn = self.redis.clone(); @@ -452,14 +471,14 @@ impl EoaExecutorStore { .zrangebyscore_withscores(&submitted_key, nonce, nonce) .await?; - let submitted_txs: Vec = - SubmittedTransaction::from_redis_strings(&results); + let submitted_txs: Vec = + SubmittedTransactionDehydrated::from_redis_strings(&results); Ok(submitted_txs) } /// Check EOA health (balance, etc.) - pub async fn check_eoa_health(&self) -> Result, TransactionStoreError> { + pub async fn get_eoa_health(&self) -> Result, TransactionStoreError> { let mut conn = self.redis.clone(); let health_json: Option = conn.get(self.eoa_health_key_name()).await?; @@ -489,14 +508,37 @@ impl EoaExecutorStore { let mut conn = self.redis.clone(); // Use ZRANGE to peek without removing - let transaction_ids: Vec<(String, u64)> = conn + let transaction_ids: Vec = conn .zrange_withscores(&pending_key, 0, (limit - 1) as isize) .await?; - Ok(transaction_ids + let mut pipe = twmq::redis::pipe(); + + for (transaction_id, _) in &transaction_ids { + let tx_data_key = self.transaction_data_key_name(transaction_id); + pipe.hget(&tx_data_key, "user_request"); + } + + let user_requests: Vec = pipe.query_async(&mut conn).await?; + + let user_requests: Vec = user_requests + .into_iter() + .map(|user_request_json| serde_json::from_str(&user_request_json)) + .collect::, serde_json::Error>>()?; + + let pending_transactions: Vec = transaction_ids .into_iter() - .map(PendingTransaction::from) - .collect()) + .zip(user_requests) + .map( + |((transaction_id, queued_at), user_request)| PendingTransaction { + transaction_id, + queued_at, + user_request, + }, + ) + .collect(); + + Ok(pending_transactions) } /// Get inflight budget (how many new transactions can be sent) @@ -677,15 +719,15 @@ impl EoaExecutorStore { #[tracing::instrument(skip_all)] pub async fn get_highest_submitted_nonce_tranasactions( &self, - ) -> Result, TransactionStoreError> { + ) -> Result, TransactionStoreError> { let submitted_key = self.submitted_transactions_zset_name(); let mut conn = self.redis.clone(); let highest_nonce_txs: Vec = conn.zrange_withscores(&submitted_key, -1, -1).await?; - let submitted_txs: Vec = - SubmittedTransaction::from_redis_strings(&highest_nonce_txs); + let submitted_txs: Vec = + SubmittedTransactionDehydrated::from_redis_strings(&highest_nonce_txs); Ok(submitted_txs) } diff --git a/executors/src/eoa/store/pending.rs b/executors/src/eoa/store/pending.rs index 08da0f5..c1c5f67 100644 --- a/executors/src/eoa/store/pending.rs +++ b/executors/src/eoa/store/pending.rs @@ -3,9 +3,12 @@ use std::collections::HashSet; use alloy::{consensus::Transaction, primitives::Address}; use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; -use crate::eoa::store::{ - BorrowedTransactionData, EoaExecutorStoreKeys, TransactionStoreError, - atomic::SafeRedisTransaction, +use crate::eoa::{ + EoaExecutorStore, + store::{ + BorrowedTransactionData, EoaExecutorStoreKeys, TransactionStoreError, + atomic::SafeRedisTransaction, + }, }; /// Atomic operation to move pending transactions to borrowed state using incremented nonces @@ -44,6 +47,7 @@ impl SafeRedisTransaction for MovePendingToBorrowedWithIncrementedNonces<'_> { async fn validation( &self, conn: &mut ConnectionManager, + _store: &EoaExecutorStore, ) -> Result { if self.transactions.is_empty() { return Err(TransactionStoreError::InternalError { @@ -55,11 +59,10 @@ impl SafeRedisTransaction for MovePendingToBorrowedWithIncrementedNonces<'_> { let current_optimistic: Option = conn .get(self.keys.optimistic_transaction_count_key_name()) .await?; - let current_nonce = - current_optimistic.ok_or_else(|| TransactionStoreError::NonceSyncRequired { - eoa: self.eoa, - chain_id: self.chain_id, - })?; + let current_nonce = current_optimistic.ok_or(TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + })?; // Extract and validate nonces let mut nonces: Vec = self @@ -125,13 +128,11 @@ impl SafeRedisTransaction for MovePendingToBorrowedWithIncrementedNonces<'_> { let optimistic_key = self.keys.optimistic_transaction_count_key_name(); for (tx, borrowed_json) in self.transactions.iter().zip(serialized_transactions.iter()) { - let nonce = tx.signed_transaction.nonce(); - // Remove from pending queue pipeline.zrem(&pending_key, &tx.transaction_id); // Add to borrowed state - pipeline.hset(&borrowed_key, nonce.to_string(), borrowed_json); + pipeline.hset(&borrowed_key, &tx.transaction_id, borrowed_json); } // Update optimistic tx count to highest nonce + 1 @@ -177,6 +178,7 @@ impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonces<'_> { async fn validation( &self, conn: &mut ConnectionManager, + _store: &EoaExecutorStore, ) -> Result { if self.transactions.is_empty() { return Err(TransactionStoreError::InternalError { @@ -249,7 +251,7 @@ impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonces<'_> { pipeline.zrem(&pending_key, &tx.transaction_id); // Add to borrowed state - pipeline.hset(&borrowed_key, nonce.to_string(), borrowed_json); + pipeline.hset(&borrowed_key, &tx.transaction_id, borrowed_json); } self.transactions.len() diff --git a/executors/src/eoa/store/submitted.rs b/executors/src/eoa/store/submitted.rs index 943001c..12ef378 100644 --- a/executors/src/eoa/store/submitted.rs +++ b/executors/src/eoa/store/submitted.rs @@ -1,24 +1,100 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + ops::Deref, + sync::Arc, +}; use serde::{Deserialize, Serialize}; use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; -use crate::eoa::store::{ - ConfirmedTransaction, EoaExecutorStoreKeys, TransactionStoreError, atomic::SafeRedisTransaction, +use crate::{ + eoa::{ + EoaExecutorStore, EoaTransactionRequest, + events::EoaExecutorEvent, + store::{ + ConfirmedTransaction, EoaExecutorStoreKeys, NO_OP_TRANSACTION_ID, + TransactionStoreError, atomic::SafeRedisTransaction, + }, + }, + webhook::{WebhookJobHandler, queue_webhook_envelopes}, }; +#[derive(Debug, Clone)] +pub struct SubmittedTransaction { + pub data: SubmittedTransactionDehydrated, + pub user_request: EoaTransactionRequest, +} + +impl Deref for SubmittedTransaction { + type Target = SubmittedTransactionDehydrated; + fn deref(&self) -> &Self::Target { + &self.data + } +} + +#[derive(Debug, Clone)] +pub struct SubmittedNoopTransaction { + pub nonce: u64, + pub hash: String, +} + +pub type SubmittedTransactionStringWithNonce = (String, u64); + +impl SubmittedNoopTransaction { + pub fn to_redis_string_with_nonce(&self) -> SubmittedTransactionStringWithNonce { + ( + format!("{}:{}:0", self.hash, NO_OP_TRANSACTION_ID), + self.nonce, + ) + } +} + +#[derive(Debug, Clone)] +pub enum SubmittedTransactionHydrated { + Noop(SubmittedNoopTransaction), + Real(SubmittedTransaction), +} + +impl SubmittedTransactionHydrated { + pub fn hash(&self) -> &str { + match self { + SubmittedTransactionHydrated::Noop(tx) => &tx.hash, + SubmittedTransactionHydrated::Real(tx) => &tx.hash, + } + } + + pub fn nonce(&self) -> u64 { + match self { + SubmittedTransactionHydrated::Noop(tx) => tx.nonce, + SubmittedTransactionHydrated::Real(tx) => tx.nonce, + } + } + + pub fn transaction_id(&self) -> &str { + match self { + SubmittedTransactionHydrated::Noop(_) => NO_OP_TRANSACTION_ID, + SubmittedTransactionHydrated::Real(tx) => &tx.transaction_id, + } + } + + pub fn to_redis_string_with_nonce(&self) -> SubmittedTransactionStringWithNonce { + match self { + SubmittedTransactionHydrated::Noop(tx) => tx.to_redis_string_with_nonce(), + SubmittedTransactionHydrated::Real(tx) => tx.to_redis_string_with_nonce(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct SubmittedTransaction { +pub struct SubmittedTransactionDehydrated { pub nonce: u64, pub hash: String, pub transaction_id: String, pub queued_at: u64, } -pub type SubmittedTransactionStringWithNonce = (String, u64); - -impl SubmittedTransaction { +impl SubmittedTransactionDehydrated { pub fn from_redis_strings(redis_strings: &[SubmittedTransactionStringWithNonce]) -> Vec { redis_strings .iter() @@ -26,7 +102,7 @@ impl SubmittedTransaction { let parts: Vec<&str> = tx.0.split(':').collect(); if parts.len() == 3 { if let Ok(queued_at) = parts[2].parse::() { - Some(SubmittedTransaction { + Some(SubmittedTransactionDehydrated { hash: parts[0].to_string(), transaction_id: parts[1].to_string(), nonce: tx.1, @@ -70,6 +146,11 @@ pub struct CleanSubmittedTransactions<'a> { pub last_confirmed_nonce: u64, pub confirmed_transactions: &'a [ConfirmedTransaction], pub keys: &'a EoaExecutorStoreKeys, + pub webhook_queue: Arc>, +} + +pub struct CleanAndGetRecycledNonces<'a> { + pub keys: &'a EoaExecutorStoreKeys, } #[derive(Debug, Default)] @@ -105,7 +186,7 @@ pub struct CleanupReport { /// Multiple submissions for the same transaction ID with different nonces can cause duplicate transactions /// Multiple submissions for the same transaction ID with the same nonce is fine, because this indicated gas bumps. impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { - type ValidationData = Vec; + type ValidationData = Vec; type OperationResult = CleanupReport; fn name(&self) -> &str { @@ -119,6 +200,7 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { async fn validation( &self, conn: &mut ConnectionManager, + store: &EoaExecutorStore, ) -> Result { let submitted_txs: Vec = conn .zrangebyscore_withscores( @@ -128,8 +210,9 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { ) .await?; - let submitted_txs = SubmittedTransaction::from_redis_strings(&submitted_txs); - Ok(submitted_txs) + let submitted_txs = SubmittedTransactionDehydrated::from_redis_strings(&submitted_txs); + let hydrated = store.hydrate_all_submitted(submitted_txs).await?; + Ok(hydrated) } fn operation( @@ -146,10 +229,10 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { .map(|tx| tx.hash.as_str()) .collect(); - let confirmed_ids: BTreeMap<&str, &ConfirmedTransaction> = self + let confirmed_ids: BTreeMap<&str, ConfirmedTransaction> = self .confirmed_transactions .iter() - .map(|tx| (tx.transaction_id.as_str(), tx)) + .map(|tx| (tx.transaction_id.as_str(), tx.clone())) .collect(); // Detect violations and get grouped data @@ -168,16 +251,13 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { self.keys.submitted_transactions_zset_name(), &submitted_tx_redis_string, ); - pipeline.del(self.keys.transaction_hash_to_id_key_name(&tx.hash)); + pipeline.del(self.keys.transaction_hash_to_id_key_name(tx.hash())); // Process each unique transaction_id once - if processed_ids.insert(&tx.transaction_id) { - match ( - tx.transaction_id.as_str(), - confirmed_ids.get(tx.transaction_id.as_str()), - ) { + if processed_ids.insert(tx.transaction_id()) { + match (tx.transaction_id(), confirmed_ids.get(tx.transaction_id())) { // if the transaction id is noop, we don't do anything - ("noop", _) => report.noop_count += 1, + (NO_OP_TRANSACTION_ID, _) => report.noop_count += 1, // in case of a valid ID, we check if it's in the confirmed transactions // if it is confirmed, we succeed it and queue success jobs @@ -185,26 +265,82 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { let data_key_name = self.keys.transaction_data_key_name(id); pipeline.hset(&data_key_name, "status", "confirmed"); pipeline.hset(&data_key_name, "completed_at", now); - pipeline.hset(&data_key_name, "receipt", confirmed_tx.receipt_data.clone()); - - // TODO: - // queue success jobs here + pipeline.hset( + &data_key_name, + "receipt", + confirmed_tx.receipt_serialized.clone(), + ); + + if let SubmittedTransactionHydrated::Real(tx) = tx { + if !tx.user_request.webhook_options.is_empty() { + let event = EoaExecutorEvent { + transaction_id: tx.transaction_id.clone(), + }; + + let success_envelope = + event.transaction_confirmed_envelope(confirmed_tx.clone()); + + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + success_envelope, + tx.user_request.webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } + } + } report.moved_to_success += 1; } // if the ID is not in the confirmed transactions, we queue it for pending _ => { - replaced_transactions.push((&tx.transaction_id, tx.queued_at)); - report.moved_to_pending += 1; + if let SubmittedTransactionHydrated::Real(tx) = tx { + // zadd_multiple expects (score, member) + replaced_transactions.push((tx.queued_at, tx.transaction_id.clone())); + + if !tx.user_request.webhook_options.is_empty() { + let event = EoaExecutorEvent { + transaction_id: tx.transaction_id.clone(), + }; + + let success_envelope = + event.transaction_replaced_envelope(tx.data.clone()); + + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + success_envelope, + tx.user_request.webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } + } + + report.moved_to_pending += 1; + } } } } } - pipeline.zadd_multiple( - self.keys.pending_transactions_zset_name(), - &replaced_transactions, + if !replaced_transactions.is_empty() { + pipeline.zadd_multiple( + self.keys.pending_transactions_zset_name(), + &replaced_transactions, + ); + } + + pipeline.set( + self.keys.last_transaction_count_key_name(), + self.last_confirmed_nonce + 1, ); // Finalize report stats @@ -216,7 +352,7 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { } fn detect_violations<'a>( - submitted_txs: &'a [SubmittedTransaction], + submitted_txs: &'a [SubmittedTransactionHydrated], confirmed_hashes: &'a HashSet<&str>, ) -> ( HashMap<&'a str, Vec>, @@ -227,8 +363,16 @@ fn detect_violations<'a>( let mut txs_by_nonce: BTreeMap> = BTreeMap::new(); let mut transaction_id_to_nonces: HashMap<&str, Vec> = HashMap::new(); + let real_submitted_txs: Vec<&SubmittedTransaction> = submitted_txs + .iter() + .filter_map(|tx| match tx { + SubmittedTransactionHydrated::Real(tx) => Some(tx), + SubmittedTransactionHydrated::Noop(_) => None, + }) + .collect(); + // Group data - for tx in submitted_txs { + for tx in real_submitted_txs { txs_by_nonce.entry(tx.nonce).or_default().push(tx); transaction_id_to_nonces .entry(&tx.transaction_id) @@ -276,3 +420,75 @@ fn detect_violations<'a>( (transaction_id_to_nonces, txs_by_nonce, report) } + +impl SafeRedisTransaction for CleanAndGetRecycledNonces<'_> { + type ValidationData = (u64, Vec); + type OperationResult = Vec; + + fn name(&self) -> &str { + "clean and get recycled nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.recycled_nonces_zset_name(), + self.keys.last_transaction_count_key_name(), + self.keys.submitted_transactions_zset_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + _store: &EoaExecutorStore, + ) -> Result { + // get the highest submitted nonce + let highest_submitted: Vec = conn + .zrange_withscores(self.keys.submitted_transactions_zset_name(), -1, -1) + .await?; + + let highest_submitted_nonce = highest_submitted.first().map(|tx| tx.1); + + let highest_submitted_nonce = match highest_submitted_nonce { + Some(nonce) => nonce, + None => { + let cached_tx_count: Option = conn + .get(self.keys.last_transaction_count_key_name()) + .await?; + + let Some(count) = cached_tx_count else { + return Err(TransactionStoreError::NonceSyncRequired { + eoa: self.keys.eoa, + chain_id: self.keys.chain_id, + }); + }; + count - 1 + } + }; + + let recycled_nonces: Vec = conn + .zrange(self.keys.recycled_nonces_zset_name(), 0, -1) + .await?; + + let recycled_nonces = recycled_nonces + .into_iter() + .filter(|nonce| *nonce < highest_submitted_nonce) + .collect(); + + return Ok((highest_submitted_nonce, recycled_nonces)); + } + + fn operation( + &self, + pipeline: &mut Pipeline, + (highest_submitted_nonce, recycled_nonces): Self::ValidationData, + ) -> Self::OperationResult { + pipeline.zrembyscore( + self.keys.recycled_nonces_zset_name(), + highest_submitted_nonce, + "+inf", + ); + + recycled_nonces + } +} diff --git a/executors/src/eoa/worker.rs b/executors/src/eoa/worker.rs deleted file mode 100644 index 7c488af..0000000 --- a/executors/src/eoa/worker.rs +++ /dev/null @@ -1,1848 +0,0 @@ -use alloy::consensus::{ - SignableTransaction, Signed, Transaction, TxEip4844Variant, TxEip4844WithSidecar, - TypedTransaction, -}; -use alloy::network::{TransactionBuilder, TransactionBuilder7702}; -use alloy::primitives::{Address, B256, Bytes, U256}; -use alloy::providers::{PendingTransactionBuilder, Provider}; -use alloy::rpc::types::TransactionRequest as AlloyTransactionRequest; -use alloy::signers::Signature; -use alloy::transports::{RpcError, TransportErrorKind}; -use engine_core::error::EngineError; -use engine_core::signer::AccountSigner; -use engine_core::transaction::TransactionTypeData; -use engine_core::{ - chain::{Chain, ChainService}, - credentials::SigningCredential, - error::{AlloyRpcErrorToEngineError, RpcErrorKind}, - signer::{EoaSigner, EoaSigningOptions}, -}; -use hex; -use serde::{Deserialize, Serialize}; -use std::{sync::Arc, time::Duration}; -use thirdweb_core::iaw::IAWError; -use tokio::time::sleep; -use twmq::Queue; -use twmq::redis::AsyncCommands; -use twmq::redis::aio::ConnectionManager; -use twmq::{ - DurableExecution, FailHookData, NackHookData, SuccessHookData, UserCancellable, - error::TwmqError, - hooks::TransactionContext, - job::{BorrowedJob, JobResult, RequeuePosition, ToJobResult}, -}; - -use crate::eoa::store::{ - AtomicEoaExecutorStore, BorrowedTransactionData, CleanupReport, ConfirmedTransaction, - EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, EoaTransactionRequest, PendingTransaction, - ReplacedTransaction, SubmissionErrorFail, SubmissionErrorNack, SubmissionResult, - SubmittedTransaction, TransactionData, TransactionStoreError, -}; -use crate::webhook::WebhookJobHandler; - -// ========== SPEC-COMPLIANT CONSTANTS ========== -const MAX_INFLIGHT_PER_EOA: u64 = 100; // Default from spec -const MAX_RECYCLED_THRESHOLD: u64 = 50; // Circuit breaker from spec -const TARGET_TRANSACTIONS_PER_EOA: u64 = 10; // Fleet management from spec -const MIN_TRANSACTIONS_PER_EOA: u64 = 1; // Fleet management from spec -const HEALTH_CHECK_INTERVAL: u64 = 300; // 5 minutes in seconds -const NONCE_STALL_TIMEOUT: u64 = 300_000; // 5 minutes in milliseconds - after this time, attempt gas bump - -// Retry constants for preparation phase -const MAX_PREPARATION_RETRIES: u32 = 3; -const PREPARATION_RETRY_DELAY_MS: u64 = 100; - -// ========== JOB DATA ========== -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct EoaExecutorWorkerJobData { - pub eoa_address: Address, - pub chain_id: u64, - pub noop_signing_credential: SigningCredential, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct EoaExecutorWorkerResult { - pub recovered_transactions: u32, - pub confirmed_transactions: u32, - pub failed_transactions: u32, - pub sent_transactions: u32, -} - -#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] -pub enum EoaExecutorWorkerError { - #[error("Chain service error for chainId {chain_id}: {message}")] - ChainServiceError { chain_id: u64, message: String }, - - #[error("Store error: {message}")] - StoreError { - message: String, - inner_error: TransactionStoreError, - }, - - #[error("Transaction not found: {transaction_id}")] - TransactionNotFound { transaction_id: String }, - - #[error("Transaction simulation failed: {message}")] - TransactionSimulationFailed { - message: String, - inner_error: EngineError, - }, - - #[error("Transaction build failed: {message}")] - TransactionBuildFailed { message: String }, - - #[error("RPC error encountered during generic operation: {message}")] - RpcError { - message: String, - inner_error: EngineError, - }, - - #[error("Error encountered when broadcasting transaction: {message}")] - TransactionSendError { - message: String, - inner_error: EngineError, - }, - - #[error("Signature parsing failed: {message}")] - SignatureParsingFailed { message: String }, - - #[error("Transaction signing failed: {message}")] - SigningError { - message: String, - inner_error: EngineError, - }, - - #[error("Work still remaining: {message}")] - WorkRemaining { message: String }, - - #[error("Internal error: {message}")] - InternalError { message: String }, - - #[error("User cancelled")] - UserCancelled, -} - -impl From for EoaExecutorWorkerError { - fn from(error: TwmqError) -> Self { - EoaExecutorWorkerError::InternalError { - message: format!("Queue error: {}", error), - } - } -} - -impl From for EoaExecutorWorkerError { - fn from(error: TransactionStoreError) -> Self { - EoaExecutorWorkerError::StoreError { - message: error.to_string(), - inner_error: error, - } - } -} - -impl UserCancellable for EoaExecutorWorkerError { - fn user_cancelled() -> Self { - EoaExecutorWorkerError::UserCancelled - } -} - -// ========== SIMPLE ERROR CLASSIFICATION ========== -#[derive(Debug)] -pub enum SendErrorClassification { - PossiblySent, // "nonce too low", "already known" etc - DeterministicFailure, // Invalid signature, malformed tx, insufficient funds etc -} - -#[derive(PartialEq, Eq, Debug)] -pub enum SendContext { - Rebroadcast, - InitialBroadcast, -} - -#[tracing::instrument(skip_all, fields(error = %error, context = ?context))] -fn classify_send_error( - error: &RpcError, - context: SendContext, -) -> SendErrorClassification { - if !error.is_error_resp() { - return SendErrorClassification::DeterministicFailure; - } - - let error_str = error.to_string().to_lowercase(); - - // Deterministic failures that didn't consume nonce (spec-compliant) - if error_str.contains("invalid signature") - || error_str.contains("malformed transaction") - || (context == SendContext::InitialBroadcast && error_str.contains("insufficient funds")) - || error_str.contains("invalid transaction format") - || error_str.contains("nonce too high") - // Should trigger nonce reset - { - return SendErrorClassification::DeterministicFailure; - } - - // Transaction possibly made it to mempool (spec-compliant) - if error_str.contains("nonce too low") - || error_str.contains("already known") - || error_str.contains("replacement transaction underpriced") - { - return SendErrorClassification::PossiblySent; - } - - // Additional common failures that didn't consume nonce - if error_str.contains("malformed") - || error_str.contains("gas limit") - || error_str.contains("intrinsic gas too low") - { - return SendErrorClassification::DeterministicFailure; - } - - tracing::warn!( - "Unknown send error: {}. PLEASE REPORT FOR ADDING CORRECT CLASSIFICATION [NOTIFY]", - error_str - ); - - // Default: assume possibly sent for safety - SendErrorClassification::PossiblySent -} - -fn should_trigger_nonce_reset(error: &RpcError) -> bool { - let error_str = error.to_string().to_lowercase(); - - // "nonce too high" should trigger nonce reset as per spec - error_str.contains("nonce too high") -} - -fn should_update_balance_threshold(error: &EngineError) -> bool { - match error { - EngineError::RpcError { kind, .. } - | EngineError::PaymasterError { kind, .. } - | EngineError::BundlerError { kind, .. } => match kind { - RpcErrorKind::ErrorResp(resp) => { - let message = resp.message.to_lowercase(); - message.contains("insufficient funds") - || message.contains("insufficient balance") - || message.contains("out of gas") - || message.contains("insufficient eth") - || message.contains("balance too low") - || message.contains("not enough funds") - || message.contains("insufficient native token") - } - _ => false, - }, - _ => false, - } -} - -fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { - match kind { - RpcErrorKind::TransportHttpError { status, .. } if *status >= 400 && *status < 500 => false, - RpcErrorKind::UnsupportedFeature { .. } => false, - _ => true, - } -} - -fn is_retryable_preparation_error(error: &EoaExecutorWorkerError) -> bool { - match error { - EoaExecutorWorkerError::RpcError { inner_error, .. } => { - // extract the RpcErrorKind from the inner error - if let EngineError::RpcError { kind, .. } = inner_error { - is_retryable_rpc_error(kind) - } else { - false - } - } - EoaExecutorWorkerError::ChainServiceError { .. } => true, // Network related - EoaExecutorWorkerError::StoreError { inner_error, .. } => { - matches!(inner_error, TransactionStoreError::RedisError { .. }) - } - EoaExecutorWorkerError::TransactionSimulationFailed { .. } => false, // Deterministic - EoaExecutorWorkerError::TransactionBuildFailed { .. } => false, // Deterministic - EoaExecutorWorkerError::SigningError { inner_error, .. } => match inner_error { - // if vault error, it's not retryable - EngineError::VaultError { .. } => false, - // if iaw error, it's retryable only if it's a network error - EngineError::IawError { error, .. } => matches!(error, IAWError::NetworkError { .. }), - _ => false, - }, - EoaExecutorWorkerError::TransactionNotFound { .. } => false, // Deterministic - EoaExecutorWorkerError::InternalError { .. } => false, // Deterministic - EoaExecutorWorkerError::UserCancelled => false, // Deterministic - EoaExecutorWorkerError::TransactionSendError { .. } => false, // Different context - EoaExecutorWorkerError::SignatureParsingFailed { .. } => false, // Deterministic - EoaExecutorWorkerError::WorkRemaining { .. } => false, // Different context - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ConfirmedTransactionWithRichReceipt { - pub nonce: u64, - pub hash: String, - pub transaction_id: String, - pub receipt: alloy::rpc::types::TransactionReceipt, -} - -// ========== MAIN WORKER ========== -/// EOA Executor Worker -/// -/// ## Core Workflow: -/// 1. **Acquire Lock Aggressively** - Takes over stalled workers using force acquisition. This is a lock over EOA:CHAIN -/// 2. **Crash Recovery** - Rebroadcasts borrowed transactions, handles deterministic failures -/// 3. **Confirmation Flow** - Fetches receipts, confirms transactions, handles nonce sync, requeues replaced transactions -/// 4. **Send Flow** - Processes recycled nonces first, then new transactions with in-flight budget control -/// 5. **Lock Release** - Explicit release in finally pattern as per spec -/// -/// ## Key Features: -/// - **Atomic Operations**: All state transitions use Redis WATCH/MULTI/EXEC for durability -/// - **Borrowed State**: Mid-send crash recovery with atomic pending->borrowed->submitted transitions -/// - **Nonce Management**: Optimistic nonce tracking with recycled nonce priority -/// - **Error Classification**: Spec-compliant deterministic vs. possibly-sent error handling -/// - **Circuit Breakers**: Automatic recycled nonce nuking when threshold exceeded -/// - **Health Monitoring**: Balance checking with configurable thresholds -pub struct EoaExecutorWorker -where - CS: ChainService + Send + Sync + 'static, -{ - pub chain_service: Arc, - pub webhook_queue: Arc>, - - pub redis: ConnectionManager, - pub namespace: Option, - - pub eoa_signer: Arc, - pub max_inflight: u64, // Note: Spec uses MAX_INFLIGHT_PER_EOA constant - pub max_recycled_nonces: u64, // Note: Spec uses MAX_RECYCLED_THRESHOLD constant -} - -impl DurableExecution for EoaExecutorWorker -where - CS: ChainService + Send + Sync + 'static, -{ - type Output = EoaExecutorWorkerResult; - type ErrorData = EoaExecutorWorkerError; - type JobData = EoaExecutorWorkerJobData; - - #[tracing::instrument(skip_all, fields(eoa = %job.job.data.eoa_address, chain_id = job.job.data.chain_id))] - async fn process( - &self, - job: &BorrowedJob, - ) -> JobResult { - let data = &job.job.data; - - // 1. GET CHAIN - let chain = self - .chain_service - .get_chain(data.chain_id) - .map_err(|e| EoaExecutorWorkerError::ChainServiceError { - chain_id: data.chain_id, - message: format!("Failed to get chain: {}", e), - }) - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 2. CREATE SCOPED STORE (acquires lock) - let scoped = EoaExecutorStore::new( - self.redis.clone(), - self.namespace.clone(), - data.eoa_address, - data.chain_id, - ) - .acquire_eoa_lock_aggressively(&job.lease_token) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // initiate health data if doesn't exist - self.get_eoa_health(&scoped, &chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // Execute main workflow with proper error handling - self.execute_main_workflow(&scoped, &chain).await - } - - async fn on_success( - &self, - job: &BorrowedJob, - _success_data: SuccessHookData<'_, Self::Output>, - _tx: &mut TransactionContext<'_>, - ) { - self.release_eoa_lock(&job.job.data).await; - } - - async fn on_nack( - &self, - job: &BorrowedJob, - _nack_data: NackHookData<'_, Self::ErrorData>, - _tx: &mut TransactionContext<'_>, - ) { - self.release_eoa_lock(&job.job.data).await; - } - - async fn on_fail( - &self, - job: &BorrowedJob, - _fail_data: FailHookData<'_, Self::ErrorData>, - _tx: &mut TransactionContext<'_>, - ) { - self.release_eoa_lock(&job.job.data).await; - } -} - -impl SubmissionResult { - /// Convert a send result to a SubmissionResult for batch processing - /// This handles the specific RpcError type from alloy - pub fn from_send_result( - borrowed_transaction: &BorrowedTransactionData, - send_result: Result>, - send_context: SendContext, - user_data: Option, - chain: &impl Chain, - ) -> Self { - match send_result { - Ok(_) => SubmissionResult::Success(borrowed_transaction.into()), - Err(ref rpc_error) => { - match classify_send_error(rpc_error, send_context) { - SendErrorClassification::PossiblySent => { - SubmissionResult::Success(borrowed_transaction.into()) - } - SendErrorClassification::DeterministicFailure => { - // Transaction failed, should be retried - let engine_error = rpc_error.to_engine_error(chain); - let error = EoaExecutorWorkerError::TransactionSendError { - message: format!("Transaction send failed: {}", rpc_error), - inner_error: engine_error, - }; - SubmissionResult::Nack(SubmissionErrorNack { - transaction_id: borrowed_transaction.transaction_id.clone(), - error, - user_data, - }) - } - } - } - } - } - - /// Helper method for when we need to create a failure result - pub fn from_failure( - transaction_id: String, - error: EoaExecutorWorkerError, - user_data: Option, - ) -> Self { - SubmissionResult::Fail(SubmissionErrorFail { - transaction_id, - error, - user_data, - }) - } -} - -impl EoaExecutorWorker -where - CS: ChainService + Send + Sync + 'static, -{ - /// Execute the main EOA worker workflow - async fn execute_main_workflow( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - ) -> JobResult { - // 1. CRASH RECOVERY - let recovered = self - .recover_borrowed_state(scoped, chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 2. CONFIRM FLOW - let confirmations_report = self - .confirm_flow(scoped, chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 3. SEND FLOW - let sent = self - .send_flow(scoped, chain) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // 4. CHECK FOR REMAINING WORK - let pending_count = scoped - .peek_pending_transactions(1000) - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? - .len(); - let borrowed_count = scoped - .peek_borrowed_transactions() - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? - .len(); - let recycled_count = scoped - .peek_recycled_nonces() - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? - .len(); - let submitted_count = scoped - .get_submitted_transactions_count() - .await - .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; - - // NACK here is a yield, when you think of the queue as a distributed EOA scheduler - if pending_count > 0 || borrowed_count > 0 || recycled_count > 0 || submitted_count > 0 { - return Err(EoaExecutorWorkerError::WorkRemaining { - message: format!( - "Work remaining: {} pending, {} borrowed, {} recycled, {} submitted", - pending_count, borrowed_count, recycled_count, submitted_count - ), - }) - .map_err_nack(Some(Duration::from_secs(2)), RequeuePosition::Last); - } - - // Only succeed if no work remains - Ok(EoaExecutorWorkerResult { - recovered_transactions: recovered, - confirmed_transactions: confirmations_report.moved_to_success as u32, - failed_transactions: confirmations_report.moved_to_pending as u32, - sent_transactions: sent, - }) - } - - /// Release EOA lock following the spec's finally pattern - async fn release_eoa_lock(&self, job_data: &EoaExecutorWorkerJobData) { - let keys = EoaExecutorStoreKeys::new( - job_data.eoa_address, - job_data.chain_id, - self.namespace.clone(), - ); - - let lock_key = keys.eoa_lock_key_name(); - let mut conn = self.redis.clone(); - if let Err(e) = conn.del::<&str, ()>(&lock_key).await { - tracing::error!( - eoa = %job_data.eoa_address, - chain_id = %job_data.chain_id, - error = %e, - "Failed to release EOA lock" - ); - } - } - - // ========== CRASH RECOVERY ========== - #[tracing::instrument(skip_all)] - async fn recover_borrowed_state( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - ) -> Result { - let mut borrowed_transactions = scoped.peek_borrowed_transactions().await?; - - if borrowed_transactions.is_empty() { - return Ok(0); - } - - tracing::warn!( - "Recovering {} borrowed transactions. This indicates a worker crash or system issue", - borrowed_transactions.len() - ); - - // Sort borrowed transactions by nonce to ensure proper ordering - borrowed_transactions.sort_by_key(|tx| tx.signed_transaction.nonce()); - - // Rebroadcast all transactions in parallel - let rebroadcast_futures: Vec<_> = borrowed_transactions - .iter() - .map(|borrowed| { - let tx_envelope = borrowed.signed_transaction.clone().into(); - let nonce = borrowed.signed_transaction.nonce(); - let transaction_id = borrowed.transaction_id.clone(); - - tracing::info!( - transaction_id = %transaction_id, - nonce = nonce, - "Recovering borrowed transaction" - ); - - async move { - let send_result = chain.provider().send_tx_envelope(tx_envelope).await; - (borrowed, send_result) - } - }) - .collect(); - - let rebroadcast_results = futures::future::join_all(rebroadcast_futures).await; - - // Convert results to SubmissionResult for batch processing - let submission_results: Vec = rebroadcast_results - .into_iter() - .map(|(borrowed, send_result)| { - SubmissionResult::from_send_result( - borrowed, - send_result, - SendContext::Rebroadcast, - None, // We'll let the batch operation fetch user data - chain, - ) - }) - .collect(); - - // TODO: Implement post-processing analysis for balance threshold updates and nonce resets - // Currently we lose the granular error handling that was in the individual atomic operations. - // Consider: - // 1. Analyzing submission_results for specific error patterns - // 2. Calling update_balance_threshold if needed - // 3. Detecting nonce reset conditions - // 4. Or move this logic into the batch processor itself - - // Process all results in one batch operation - let report = scoped - .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) - .await?; - - // TODO: Handle post-processing updates here if needed - // For now, we skip the individual error analysis that was done in the old atomic approach - - tracing::info!( - "Recovered {} transactions: {} submitted, {} recycled, {} failed", - report.total_processed, - report.moved_to_submitted, - report.moved_to_pending, - report.failed_transactions - ); - - Ok(report.total_processed as u32) - } - - // ========== CONFIRM FLOW ========== - #[tracing::instrument(skip_all)] - async fn confirm_flow( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - ) -> Result { - // Get fresh on-chain transaction count - let current_chain_nonce = chain - .provider() - .get_transaction_count(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get transaction count: {}", engine_error), - inner_error: engine_error, - } - })?; - - let cached_nonce = match scoped.get_cached_transaction_count().await { - Err(e) => match e { - TransactionStoreError::NonceSyncRequired { .. } => { - scoped.reset_nonces(current_chain_nonce).await?; - current_chain_nonce - } - _ => return Err(e.into()), - }, - Ok(cached_nonce) => cached_nonce, - }; - - let submitted_count = scoped.get_submitted_transactions_count().await?; - - // no nonce progress - if current_chain_nonce == cached_nonce { - let current_health = self.get_eoa_health(scoped, chain).await?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - // No nonce progress - check if we should attempt gas bumping for stalled nonce - let time_since_movement = now.saturating_sub(current_health.last_nonce_movement_at); - - // if there are waiting transactions, we can attempt a gas bump - if time_since_movement > NONCE_STALL_TIMEOUT && submitted_count > 0 { - tracing::info!( - time_since_movement = time_since_movement, - stall_timeout = NONCE_STALL_TIMEOUT, - current_chain_nonce = current_chain_nonce, - "Nonce has been stalled, attempting gas bump" - ); - - // Attempt gas bump for the next expected nonce - if let Err(e) = self - .attempt_gas_bump_for_stalled_nonce(scoped, chain, current_chain_nonce) - .await - { - tracing::warn!( - error = %e, - "Failed to attempt gas bump for stalled nonce" - ); - } - } - - tracing::debug!("No nonce progress, skipping confirm flow"); - return Ok(CleanupReport::default()); - } - - tracing::info!( - current_chain_nonce = current_chain_nonce, - cached_nonce = cached_nonce, - "Processing confirmations" - ); - - // Get all pending transactions below the current chain nonce - let waiting_txs = scoped - .get_submitted_transactions_below_nonce(current_chain_nonce) - .await?; - - if waiting_txs.is_empty() { - tracing::debug!("No waiting transactions to confirm"); - return Ok(CleanupReport::default()); - } - - // Fetch receipts and categorize transactions - let (confirmed_txs, replaced_txs) = self - .fetch_confirmed_transaction_receipts(chain, waiting_txs) - .await; - - // Process confirmed transactions - let successes: Vec = confirmed_txs - .into_iter() - .map(|tx| { - let receipt_data = match serde_json::to_string(&tx.receipt) { - Ok(receipt_json) => receipt_json, - Err(e) => { - tracing::warn!( - transaction_id = %tx.transaction_id, - hash = %tx.hash, - error = %e, - "Failed to serialize receipt as JSON, using debug format" - ); - format!("{:?}", tx.receipt) - } - }; - - tracing::info!( - transaction_id = %tx.transaction_id, - nonce = tx.nonce, - hash = %tx.hash, - "Transaction confirmed" - ); - - ConfirmedTransaction { - hash: tx.hash, - transaction_id: tx.transaction_id, - receipt_data, - } - }) - .collect(); - - let report = scoped - .clean_submitted_transactions(&successes, current_chain_nonce - 1) - .await?; - - // Update cached transaction count - scoped - .update_cached_transaction_count(current_chain_nonce) - .await?; - - Ok(report) - } - - // ========== SEND FLOW ========== - #[tracing::instrument(skip_all)] - async fn send_flow( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - ) -> Result { - // 1. Get EOA health (initializes if needed) and check if we should update balance - let mut health = self.get_eoa_health(scoped, chain).await?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Update balance if it's stale - // TODO: refactor this, very ugly - if health.balance <= health.balance_threshold { - if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { - let balance = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get balance: {}", engine_error), - inner_error: engine_error, - } - })?; - - health.balance = balance; - health.balance_fetched_at = now; - scoped.update_health_data(&health).await?; - } - - if health.balance <= health.balance_threshold { - tracing::warn!( - "EOA has insufficient balance (<= {} wei), skipping send flow", - health.balance_threshold - ); - return Ok(0); - } - } - - let mut total_sent = 0; - - // 2. Process recycled nonces first - total_sent += self.process_recycled_nonces(scoped, chain).await?; - - // 3. Only proceed to new nonces if we successfully used all recycled nonces - let remaining_recycled = scoped.peek_recycled_nonces().await?.len(); - if remaining_recycled == 0 { - let inflight_budget = scoped.get_inflight_budget(self.max_inflight).await?; - if inflight_budget > 0 { - total_sent += self - .process_new_transactions(scoped, chain, inflight_budget) - .await?; - } - } else { - tracing::warn!( - "Still have {} recycled nonces, not sending new transactions", - remaining_recycled - ); - } - - Ok(total_sent) - } - - async fn process_recycled_nonces( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - ) -> Result { - let recycled_nonces = scoped.peek_recycled_nonces().await?; - - if recycled_nonces.is_empty() { - return Ok(0); - } - - let mut total_sent = 0; - let mut remaining_nonces = recycled_nonces; - - // Loop to handle preparation failures and refill with new transactions - while !remaining_nonces.is_empty() { - // Get pending transactions to match with recycled nonces - let pending_txs = scoped - .peek_pending_transactions(remaining_nonces.len() as u64) - .await?; - - if pending_txs.is_empty() { - tracing::debug!("No pending transactions available for recycled nonces"); - break; - } - - // Pair recycled nonces with pending transactions - let mut build_tasks = Vec::new(); - let mut nonce_tx_pairs = Vec::new(); - - for (i, nonce) in remaining_nonces.iter().enumerate() { - if let Some(p_tx) = pending_txs.get(i) { - build_tasks.push(self.build_and_sign_single_transaction_with_retries( - scoped, p_tx, *nonce, chain, - )); - nonce_tx_pairs.push((*nonce, p_tx.clone())); - } else { - // No more pending transactions for this recycled nonce - tracing::debug!("No pending transaction for recycled nonce {}", nonce); - break; - } - } - - if build_tasks.is_empty() { - break; - } - - // Build and sign all transactions in parallel - let prepared_results = futures::future::join_all(build_tasks).await; - - // Separate successful preparations from failures - let mut prepared_txs = Vec::new(); - let mut failed_tx_ids = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (i, result) in prepared_results.into_iter().enumerate() { - match result { - Ok(borrowed_data) => { - prepared_txs.push(borrowed_data); - } - Err(e) => { - // Track balance threshold issues - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, - .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } - - let (_nonce, pending_tx) = &nonce_tx_pairs[i]; - tracing::warn!( - "Failed to build recycled transaction {}: {}", - pending_tx.transaction_id, - e - ); - - // For deterministic build failures, fail the transaction immediately - if !is_retryable_preparation_error(&e) { - failed_tx_ids.push(pending_tx.transaction_id.clone()); - } - } - } - } - - // Fail deterministic failures from pending state - for tx_id in failed_tx_ids { - if let Err(e) = scoped - .fail_pending_transaction( - &tx_id, - "Deterministic preparation failure", - self.webhook_queue.clone(), - ) - .await - { - tracing::error!("Failed to fail pending transaction {}: {}", tx_id, e); - } - } - - // Update balance threshold if needed - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after build failures: {}", - e - ); - } - } - - if prepared_txs.is_empty() { - // No successful preparations, try again with more pending transactions - // Remove the nonces we couldn't use from our list - remaining_nonces = remaining_nonces - .into_iter() - .skip(nonce_tx_pairs.len()) - .collect(); - continue; - } - - // Move prepared transactions to borrowed state with recycled nonces - let moved_count = scoped - .atomic_move_pending_to_borrowed_with_recycled_nonces(&prepared_txs) - .await?; - - tracing::debug!( - moved_count = moved_count, - total_prepared = prepared_txs.len(), - "Moved transactions to borrowed state using recycled nonces" - ); - - // Actually send the transactions to the blockchain - let send_tasks: Vec<_> = prepared_txs - .iter() - .map(|borrowed_tx| { - let signed_tx = borrowed_tx.signed_transaction.clone(); - async move { chain.provider().send_tx_envelope(signed_tx.into()).await } - }) - .collect(); - - let send_results = futures::future::join_all(send_tasks).await; - - // Process send results and update states - let mut submission_results = Vec::new(); - for (i, send_result) in send_results.into_iter().enumerate() { - let borrowed_tx = &prepared_txs[i]; - let user_data = scoped - .get_transaction_data(&borrowed_tx.transaction_id) - .await?; - - let submission_result = SubmissionResult::from_send_result( - borrowed_tx, - send_result, - SendContext::InitialBroadcast, - user_data, - chain, - ); - submission_results.push(submission_result); - } - - // Use batch processing to handle all submission results - let processing_report = scoped - .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) - .await?; - - tracing::debug!( - "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", - processing_report.total_processed, - processing_report.moved_to_submitted, - processing_report.moved_to_pending, - processing_report.failed_transactions - ); - - total_sent += processing_report.moved_to_submitted; - - // Remove the nonces we successfully processed from our list - remaining_nonces = remaining_nonces.into_iter().skip(moved_count).collect(); - - // If we didn't use all available nonces, we ran out of pending transactions - if moved_count < nonce_tx_pairs.len() { - break; - } - } - - Ok(total_sent as u32) - } - - async fn process_new_transactions( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - budget: u64, - ) -> Result { - if budget == 0 { - return Ok(0); - } - - let mut total_sent = 0; - let mut remaining_budget = budget; - - // Loop to handle preparation failures and refill with new transactions - while remaining_budget > 0 { - // 1. Get pending transactions - let pending_txs = scoped.peek_pending_transactions(remaining_budget).await?; - if pending_txs.is_empty() { - break; - } - - let optimistic_nonce = scoped.get_optimistic_transaction_count().await?; - - // 2. Build and sign all transactions in parallel - let build_tasks: Vec<_> = pending_txs - .iter() - .enumerate() - .map(|(i, tx)| { - let expected_nonce = optimistic_nonce + i as u64; - self.build_and_sign_single_transaction_with_retries( - scoped, - tx, - expected_nonce, - chain, - ) - }) - .collect(); - - let prepared_results = futures::future::join_all(build_tasks).await; - - // 3. Separate successful preparations from failures - let mut prepared_txs = Vec::new(); - let mut failed_tx_ids = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (i, result) in prepared_results.into_iter().enumerate() { - match result { - Ok(borrowed_data) => { - prepared_txs.push(borrowed_data); - } - Err(e) => { - // Track balance threshold issues - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, - .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; - } - } - - let pending_tx = &pending_txs[i]; - tracing::warn!( - "Failed to build transaction {}: {}", - pending_tx.transaction_id, - e - ); - - // For deterministic build failures, fail the transaction immediately - if !is_retryable_preparation_error(&e) { - failed_tx_ids.push(pending_tx.transaction_id.clone()); - } - } - } - } - - // 4. Fail deterministic failures from pending state - for tx_id in failed_tx_ids { - if let Err(e) = scoped - .fail_pending_transaction( - &tx_id, - "Deterministic preparation failure", - self.webhook_queue.clone(), - ) - .await - { - tracing::error!("Failed to fail pending transaction {}: {}", tx_id, e); - } - } - - // Update balance threshold if needed - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after build failures: {}", - e - ); - } - } - - if prepared_txs.is_empty() { - // No successful preparations, try again with remaining budget - remaining_budget = remaining_budget.saturating_sub(pending_txs.len() as u64); - continue; - } - - // 5. Move prepared transactions to borrowed state - let moved_count = scoped - .atomic_move_pending_to_borrowed_with_incremented_nonces(&prepared_txs) - .await?; - - tracing::debug!( - moved_count = moved_count, - total_prepared = prepared_txs.len(), - "Moved transactions to borrowed state using incremented nonces" - ); - - // 6. Actually send the transactions to the blockchain - let send_tasks: Vec<_> = prepared_txs - .iter() - .map(|borrowed_tx| { - let signed_tx = borrowed_tx.signed_transaction.clone(); - async move { chain.provider().send_tx_envelope(signed_tx.into()).await } - }) - .collect(); - - let send_results = futures::future::join_all(send_tasks).await; - - // 7. Process send results and update states - let mut submission_results = Vec::new(); - for (i, send_result) in send_results.into_iter().enumerate() { - let borrowed_tx = &prepared_txs[i]; - let user_data = scoped - .get_transaction_data(&borrowed_tx.transaction_id) - .await?; - - let submission_result = SubmissionResult::from_send_result( - borrowed_tx, - send_result, - SendContext::InitialBroadcast, - user_data, - chain, - ); - submission_results.push(submission_result); - } - - // 8. Use batch processing to handle all submission results - let processing_report = scoped - .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) - .await?; - - tracing::debug!( - "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", - processing_report.total_processed, - processing_report.moved_to_submitted, - processing_report.moved_to_pending, - processing_report.failed_transactions - ); - - total_sent += processing_report.moved_to_submitted; - remaining_budget = remaining_budget.saturating_sub(moved_count as u64); - - // If we didn't use all our budget, we ran out of pending transactions - if moved_count < pending_txs.len() { - break; - } - } - - Ok(total_sent as u32) - } - - // ========== TRANSACTION BUILDING & SENDING ========== - async fn build_and_sign_single_transaction_with_retries( - &self, - scoped: &AtomicEoaExecutorStore, - pending_transaction: &PendingTransaction, - nonce: u64, - chain: &impl Chain, - ) -> Result { - let mut last_error = None; - - // Internal retry loop for retryable errors - for attempt in 0..=MAX_PREPARATION_RETRIES { - if attempt > 0 { - // Simple exponential backoff - let delay = PREPARATION_RETRY_DELAY_MS * (2_u64.pow(attempt - 1)); - sleep(Duration::from_millis(delay)).await; - - tracing::debug!( - transaction_id = %pending_transaction.transaction_id, - attempt = attempt, - "Retrying transaction preparation" - ); - } - - match self - .build_and_sign_single_transaction(scoped, pending_transaction, nonce, chain) - .await - { - Ok(result) => return Ok(result), - Err(error) => { - if is_retryable_preparation_error(&error) && attempt < MAX_PREPARATION_RETRIES { - tracing::warn!( - transaction_id = %pending_transaction.transaction_id, - attempt = attempt, - error = %error, - "Retryable error during transaction preparation, will retry" - ); - last_error = Some(error); - continue; - } else { - // Either deterministic error or exceeded max retries - return Err(error); - } - } - } - } - - // This should never be reached, but just in case - Err( - last_error.unwrap_or_else(|| EoaExecutorWorkerError::InternalError { - message: "Unexpected error in retry loop".to_string(), - }), - ) - } - - async fn build_and_sign_single_transaction( - &self, - scoped: &AtomicEoaExecutorStore, - pending_transaction: &PendingTransaction, - nonce: u64, - chain: &impl Chain, - ) -> Result { - // Get transaction data - let tx_data = scoped - .get_transaction_data(&pending_transaction.transaction_id) - .await? - .ok_or_else(|| EoaExecutorWorkerError::TransactionNotFound { - transaction_id: pending_transaction.transaction_id.clone(), - })?; - - // Build and sign transaction - let signed_tx = self - .build_and_sign_transaction(&tx_data, nonce, chain) - .await?; - - Ok(BorrowedTransactionData { - transaction_id: pending_transaction.transaction_id.clone(), - hash: signed_tx.hash().to_string(), - signed_transaction: signed_tx, - borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - queued_at: pending_transaction.queued_at, - }) - } - - async fn send_noop_transaction( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - nonce: u64, - credential: SigningCredential, - ) -> Result { - // Create a minimal transaction to consume the recycled nonce - // Send 0 ETH to self with minimal gas - let eoa = scoped.eoa(); - - // Build no-op transaction (send 0 to self) - let tx_request = AlloyTransactionRequest::default() - .with_from(eoa) - .with_to(eoa) // Send to self - .with_value(U256::ZERO) // Send 0 value - .with_input(Bytes::new()) // No data - .with_chain_id(scoped.chain_id()) - .with_nonce(nonce) - .with_gas_limit(21000); // Minimal gas for basic transfer - - let tx_request = self.estimate_gas_fees(chain, tx_request).await?; - let built_tx = tx_request.build_typed_tx().map_err(|e| { - EoaExecutorWorkerError::TransactionBuildFailed { - message: format!("Failed to build typed transaction for no-op: {e:?}"), - } - })?; - - let tx = self.sign_transaction(eoa, credential, built_tx).await?; - - chain - .provider() - .send_tx_envelope(tx.into()) - .await - .map_err(|e| EoaExecutorWorkerError::TransactionSendError { - message: format!("Failed to send no-op transaction: {e:?}"), - inner_error: e.to_engine_error(chain), - }) - .map(|pending| pending.tx_hash().to_string()) - } - - // ========== GAS BUMP METHODS ========== - - /// Attempt to gas bump a stalled transaction for the next expected nonce - async fn attempt_gas_bump_for_stalled_nonce( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - expected_nonce: u64, - ) -> Result { - tracing::info!( - nonce = expected_nonce, - "Attempting gas bump for stalled nonce" - ); - - // Get all transaction IDs for this nonce - let submitted_transactions = scoped - .get_submitted_transactions_for_nonce(expected_nonce) - .await?; - - if submitted_transactions.is_empty() { - tracing::debug!( - nonce = expected_nonce, - "No transactions found for stalled nonce" - ); - return Ok(false); - } - - // Load transaction data for all IDs and find the newest one - let mut newest_transaction: Option<(String, TransactionData)> = None; - let mut newest_submitted_at = 0u64; - - for SubmittedTransaction { transaction_id, .. } in submitted_transactions { - if let Some(tx_data) = scoped.get_transaction_data(&transaction_id).await? { - // Find the most recent attempt for this transaction - if let Some(latest_attempt) = tx_data.attempts.last() { - let submitted_at = latest_attempt.sent_at; - if submitted_at > newest_submitted_at { - newest_submitted_at = submitted_at; - newest_transaction = Some((transaction_id, tx_data)); - } - } - } - } - - if let Some((transaction_id, tx_data)) = newest_transaction { - tracing::info!( - transaction_id = %transaction_id, - nonce = expected_nonce, - "Found newest transaction for gas bump" - ); - - // Get the latest attempt to extract gas values from - // Build typed transaction -> manually bump -> sign - let typed_tx = match self - .build_typed_transaction(&tx_data, expected_nonce, chain) - .await - { - Ok(tx) => tx, - Err(e) => { - // Check if this is a balance threshold issue during simulation - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!("Failed to update balance threshold: {}", e); - } - } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!("Failed to update balance threshold: {}", e); - } - } - } - - tracing::warn!( - transaction_id = %transaction_id, - nonce = expected_nonce, - error = %e, - "Failed to build typed transaction for gas bump" - ); - return Ok(false); - } - }; - let bumped_typed_tx = self.apply_gas_bump_to_typed_transaction(typed_tx, 120); // 20% increase - let bumped_tx = match self - .sign_transaction( - tx_data.user_request.from, - tx_data.user_request.signing_credential, - bumped_typed_tx, - ) - .await - { - Ok(tx) => tx, - Err(e) => { - tracing::warn!( - transaction_id = %transaction_id, - nonce = expected_nonce, - error = %e, - "Failed to sign transaction for gas bump" - ); - return Ok(false); - } - }; - - // Record the gas bump attempt - scoped - .add_gas_bump_attempt( - &SubmittedTransaction { - nonce: expected_nonce, - hash: bumped_tx.hash().to_string(), - transaction_id: transaction_id.to_string(), - queued_at: tx_data.created_at, - }, - bumped_tx.clone(), - ) - .await?; - - // Send the bumped transaction - let tx_envelope = bumped_tx.into(); - match chain.provider().send_tx_envelope(tx_envelope).await { - Ok(_) => { - tracing::info!( - transaction_id = %transaction_id, - nonce = expected_nonce, - "Successfully sent gas bumped transaction" - ); - return Ok(true); - } - Err(e) => { - tracing::warn!( - transaction_id = %transaction_id, - nonce = expected_nonce, - error = %e, - "Failed to send gas bumped transaction" - ); - // Don't fail the worker, just log the error - return Ok(false); - } - } - } - - Ok(false) - } - - // ========== HEALTH ACCESSOR ========== - - /// Get EOA health, initializing it if it doesn't exist - /// This method ensures the health data is always available for the worker - async fn get_eoa_health( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - ) -> Result { - let store_health = scoped.check_eoa_health().await?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - match store_health { - Some(health) => Ok(health), - None => { - // Initialize with fresh data from chain - let balance = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!( - "Failed to get balance during initialization: {}", - engine_error - ), - inner_error: engine_error, - } - })?; - - let health = EoaHealth { - balance, - balance_threshold: U256::ZERO, - balance_fetched_at: now, - last_confirmation_at: now, - last_nonce_movement_at: now, - nonce_resets: Vec::new(), - }; - - // Save to store - scoped.update_health_data(&health).await?; - Ok(health) - } - } - } - - #[tracing::instrument(skip_all, fields(eoa = %scoped.eoa(), chain_id = %chain.chain_id()))] - async fn update_balance_threshold( - &self, - scoped: &AtomicEoaExecutorStore, - chain: &impl Chain, - ) -> Result<(), EoaExecutorWorkerError> { - let mut health = self.get_eoa_health(scoped, chain).await?; - - tracing::info!("Updating balance threshold"); - let balance_threshold = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get balance: {}", engine_error), - inner_error: engine_error, - } - })?; - - health.balance_threshold = balance_threshold; - scoped.update_health_data(&health).await?; - Ok(()) - } - - /// Fetch receipts for all submitted transactions and categorize them - async fn fetch_confirmed_transaction_receipts( - &self, - chain: &impl Chain, - submitted_txs: Vec, - ) -> ( - Vec, - Vec, - ) { - // Fetch all receipts in parallel - let receipt_futures: Vec<_> = submitted_txs - .iter() - .filter_map(|tx| match tx.hash.parse::() { - Ok(hash_bytes) => Some(async move { - let receipt = chain.provider().get_transaction_receipt(hash_bytes).await; - (tx, receipt) - }), - Err(_) => { - tracing::warn!("Invalid hash format: {}, skipping", tx.hash); - None - } - }) - .collect(); - - let receipt_results = futures::future::join_all(receipt_futures).await; - - // Categorize transactions - let mut confirmed_txs = Vec::new(); - let mut failed_txs = Vec::new(); - - for (tx, receipt_result) in receipt_results { - match receipt_result { - Ok(Some(receipt)) => { - confirmed_txs.push(ConfirmedTransactionWithRichReceipt { - nonce: tx.nonce, - hash: tx.hash.clone(), - transaction_id: tx.transaction_id.clone(), - receipt, - }); - } - Ok(None) | Err(_) => { - failed_txs.push(ReplacedTransaction { - hash: tx.hash.clone(), - transaction_id: tx.transaction_id.clone(), - }); - } - } - } - - (confirmed_txs, failed_txs) - } - - // ========== HELPER METHODS ========== - async fn estimate_gas_fees( - &self, - chain: &impl Chain, - tx: AlloyTransactionRequest, - ) -> Result { - // Check what fees are missing and need to be estimated - - // If we have gas_price set, we're doing legacy - don't estimate EIP-1559 - if tx.gas_price.is_some() { - return Ok(tx); - } - - // If we have both EIP-1559 fees set, don't estimate - if tx.max_fee_per_gas.is_some() && tx.max_priority_fee_per_gas.is_some() { - return Ok(tx); - } - - // Try EIP-1559 fees first, fall back to legacy if unsupported - match chain.provider().estimate_eip1559_fees().await { - Ok(eip1559_fees) => { - tracing::debug!( - "Using EIP-1559 fees: max_fee={}, max_priority_fee={}", - eip1559_fees.max_fee_per_gas, - eip1559_fees.max_priority_fee_per_gas - ); - - let mut result = tx; - // Only set fees that are missing - if result.max_fee_per_gas.is_none() { - result = result.with_max_fee_per_gas(eip1559_fees.max_fee_per_gas); - } - if result.max_priority_fee_per_gas.is_none() { - result = - result.with_max_priority_fee_per_gas(eip1559_fees.max_priority_fee_per_gas); - } - - Ok(result) - } - Err(eip1559_error) => { - // Check if this is an "unsupported feature" error - if let RpcError::UnsupportedFeature(_) = &eip1559_error { - tracing::debug!("EIP-1559 not supported, falling back to legacy gas price"); - - // Fall back to legacy gas price only if no gas price is set - if tx.authorization_list().is_none() { - match chain.provider().get_gas_price().await { - Ok(gas_price) => { - tracing::debug!("Using legacy gas price: {}", gas_price); - Ok(tx.with_gas_price(gas_price)) - } - Err(legacy_error) => Err(EoaExecutorWorkerError::RpcError { - message: format!( - "Failed to get legacy gas price: {}", - legacy_error - ), - inner_error: legacy_error.to_engine_error(chain), - }), - } - } else { - Err(EoaExecutorWorkerError::TransactionBuildFailed { - message: "EIP7702 transactions not supported on chain".to_string(), - }) - } - } else { - // Other EIP-1559 error - Err(EoaExecutorWorkerError::RpcError { - message: format!("Failed to estimate EIP-1559 fees: {}", eip1559_error), - inner_error: eip1559_error.to_engine_error(chain), - }) - } - } - } - } - - async fn build_typed_transaction( - &self, - tx_data: &TransactionData, - nonce: u64, - chain: &impl Chain, - ) -> Result { - // Build transaction request from stored data - let mut tx_request = AlloyTransactionRequest::default() - .with_from(tx_data.user_request.from) - .with_value(tx_data.user_request.value) - .with_input(tx_data.user_request.data.clone()) - .with_chain_id(tx_data.user_request.chain_id) - .with_nonce(nonce); - - if let Some(to) = tx_data.user_request.to { - tx_request = tx_request.with_to(to); - } - - if let Some(gas_limit) = tx_data.user_request.gas_limit { - tx_request = tx_request.with_gas_limit(gas_limit); - } - - // Handle gas fees - either from user settings or estimation - tx_request = if let Some(type_data) = &tx_data.user_request.transaction_type_data { - // User provided gas settings - respect them first - match type_data { - TransactionTypeData::Eip1559(data) => { - let mut req = tx_request; - if let Some(max_fee) = data.max_fee_per_gas { - req = req.with_max_fee_per_gas(max_fee); - } - if let Some(max_priority) = data.max_priority_fee_per_gas { - req = req.with_max_priority_fee_per_gas(max_priority); - } - - // if either not set, estimate the other one - if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { - req = self.estimate_gas_fees(chain, req).await?; - } - - req - } - TransactionTypeData::Legacy(data) => { - if let Some(gas_price) = data.gas_price { - tx_request.with_gas_price(gas_price) - } else { - // User didn't provide gas price, estimate it - self.estimate_gas_fees(chain, tx_request).await? - } - } - TransactionTypeData::Eip7702(data) => { - let mut req = tx_request; - if let Some(authorization_list) = &data.authorization_list { - req = req.with_authorization_list(authorization_list.clone()); - } - if let Some(max_fee) = data.max_fee_per_gas { - req = req.with_max_fee_per_gas(max_fee); - } - if let Some(max_priority) = data.max_priority_fee_per_gas { - req = req.with_max_priority_fee_per_gas(max_priority); - } - - // if either not set, estimate the other one - if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { - req = self.estimate_gas_fees(chain, req).await?; - } - - req - } - } - } else { - // No user settings - estimate appropriate fees - self.estimate_gas_fees(chain, tx_request).await? - }; - - // Estimate gas if needed - if tx_request.gas.is_none() { - match chain.provider().estimate_gas(tx_request.clone()).await { - Ok(gas_limit) => { - tx_request = tx_request.with_gas_limit(gas_limit * 110 / 100); // 10% buffer - } - Err(e) => { - // Check if this is a revert - if let RpcError::ErrorResp(error_payload) = &e { - if let Some(revert_data) = error_payload.as_revert_data() { - // This is a revert - the transaction is fundamentally broken - // This should fail the individual transaction, not the worker - return Err(EoaExecutorWorkerError::TransactionSimulationFailed { - message: format!( - "Transaction reverted during gas estimation: {} (revert: {})", - error_payload.message, - hex::encode(&revert_data) - ), - inner_error: e.to_engine_error(chain), - }); - } - } - - // Not a revert - could be RPC issue, this should nack the worker - let engine_error = e.to_engine_error(chain); - return Err(EoaExecutorWorkerError::RpcError { - message: format!("Gas estimation failed: {}", engine_error), - inner_error: engine_error, - }); - } - } - } - - // Build typed transaction - tx_request - .build_typed_tx() - .map_err(|e| EoaExecutorWorkerError::TransactionBuildFailed { - message: format!("Failed to build typed transaction: {:?}", e), - }) - } - - async fn sign_transaction( - &self, - from: Address, - credential: SigningCredential, - typed_tx: TypedTransaction, - ) -> Result, EoaExecutorWorkerError> { - let signing_options = EoaSigningOptions { - from, - chain_id: typed_tx.chain_id(), - }; - - let signature = self - .eoa_signer - .sign_transaction(signing_options, typed_tx.clone(), credential) - .await - .map_err(|engine_error| EoaExecutorWorkerError::SigningError { - message: format!("Failed to sign transaction: {}", engine_error), - inner_error: engine_error, - })?; - - let signature = signature.parse::().map_err(|e| { - EoaExecutorWorkerError::SignatureParsingFailed { - message: format!("Failed to parse signature: {}", e), - } - })?; - - Ok(typed_tx.into_signed(signature)) - } - - async fn build_and_sign_transaction( - &self, - tx_data: &TransactionData, - nonce: u64, - chain: &impl Chain, - ) -> Result, EoaExecutorWorkerError> { - let typed_tx = self.build_typed_transaction(tx_data, nonce, chain).await?; - self.sign_transaction( - tx_data.user_request.from, - tx_data.user_request.signing_credential.clone(), - typed_tx, - ) - .await - } - - fn apply_gas_bump_to_typed_transaction( - &self, - mut typed_tx: TypedTransaction, - bump_multiplier: u32, // e.g., 120 for 20% increase - ) -> TypedTransaction { - match &mut typed_tx { - TypedTransaction::Eip1559(tx) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - TypedTransaction::Legacy(tx) => { - tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; - } - TypedTransaction::Eip2930(tx) => { - tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; - } - TypedTransaction::Eip7702(tx) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - TypedTransaction::Eip4844(tx) => match tx { - TxEip4844Variant::TxEip4844(tx) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - TxEip4844Variant::TxEip4844WithSidecar(TxEip4844WithSidecar { tx, .. }) => { - tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; - tx.max_priority_fee_per_gas = - tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; - } - }, - } - typed_tx - } -} diff --git a/executors/src/eoa/worker/confirm.rs b/executors/src/eoa/worker/confirm.rs new file mode 100644 index 0000000..def76d5 --- /dev/null +++ b/executors/src/eoa/worker/confirm.rs @@ -0,0 +1,360 @@ +use alloy::{primitives::B256, providers::Provider}; +use engine_core::{chain::Chain, error::AlloyRpcErrorToEngineError}; +use serde::{Deserialize, Serialize}; + +use crate::eoa::{ + store::{ + CleanupReport, ConfirmedTransaction, ReplacedTransaction, SubmittedTransaction, + SubmittedTransactionDehydrated, TransactionData, TransactionStoreError, + }, + worker::{ + EoaExecutorWorker, + error::{EoaExecutorWorkerError, should_update_balance_threshold}, + }, +}; + +const NONCE_STALL_TIMEOUT: u64 = 300_000; // 5 minutes in milliseconds - after this time, attempt gas bump + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConfirmedTransactionWithRichReceipt { + pub nonce: u64, + pub hash: String, + pub transaction_id: String, + pub receipt: alloy::rpc::types::TransactionReceipt, +} + +impl EoaExecutorWorker { + // ========== CONFIRM FLOW ========== + #[tracing::instrument(skip_all)] + pub async fn confirm_flow(&self) -> Result { + // Get fresh on-chain transaction count + let current_chain_transaction_count = self + .chain + .provider() + .get_transaction_count(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get transaction count: {}", engine_error), + inner_error: engine_error, + } + })?; + + let cached_transaction_count = match self.store.get_cached_transaction_count().await { + Err(e) => match e { + TransactionStoreError::NonceSyncRequired { .. } => { + self.store + .reset_nonces(current_chain_transaction_count) + .await?; + current_chain_transaction_count + } + _ => return Err(e.into()), + }, + Ok(cached_nonce) => cached_nonce, + }; + + let submitted_count = self.store.get_submitted_transactions_count().await?; + + // no nonce progress + if current_chain_transaction_count <= cached_transaction_count { + let current_health = self.get_eoa_health().await?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + // No nonce progress - check if we should attempt gas bumping for stalled nonce + let time_since_movement = now.saturating_sub(current_health.last_nonce_movement_at); + + // if there are waiting transactions, we can attempt a gas bump + if time_since_movement > NONCE_STALL_TIMEOUT && submitted_count > 0 { + tracing::info!( + time_since_movement = time_since_movement, + stall_timeout = NONCE_STALL_TIMEOUT, + current_chain_nonce = current_chain_transaction_count, + "Nonce has been stalled, attempting gas bump" + ); + + // Attempt gas bump for the next expected nonce + if let Err(e) = self + .attempt_gas_bump_for_stalled_nonce(current_chain_transaction_count) + .await + { + tracing::warn!( + error = %e, + "Failed to attempt gas bump for stalled nonce" + ); + } + } + + tracing::debug!("No nonce progress, skipping confirm flow"); + return Ok(CleanupReport::default()); + } + + tracing::info!( + current_chain_nonce = current_chain_transaction_count, + cached_nonce = cached_transaction_count, + "Processing confirmations" + ); + + // Get all pending transactions below the current chain transaction count + // ie, if transaction count is 1, nonce 0 should have mined + let waiting_txs = self + .store + .get_submitted_transactions_below_chain_transaction_count( + current_chain_transaction_count, + ) + .await?; + + if waiting_txs.is_empty() { + tracing::debug!("No waiting transactions to confirm"); + return Ok(CleanupReport::default()); + } + + // Fetch receipts and categorize transactions + let (confirmed_txs, replaced_txs) = + self.fetch_confirmed_transaction_receipts(waiting_txs).await; + + // Process confirmed transactions + let successes: Vec = confirmed_txs + .into_iter() + .map(|tx| { + let receipt_data = match serde_json::to_string(&tx.receipt) { + Ok(receipt_json) => receipt_json, + Err(e) => { + tracing::warn!( + transaction_id = %tx.transaction_id, + hash = %tx.hash, + error = %e, + "Failed to serialize receipt as JSON, using debug format" + ); + format!("{:?}", tx.receipt) + } + }; + + tracing::info!( + transaction_id = %tx.transaction_id, + nonce = tx.nonce, + hash = %tx.hash, + "Transaction confirmed" + ); + + ConfirmedTransaction { + hash: tx.hash, + transaction_id: tx.transaction_id, + receipt: tx.receipt.into(), + receipt_serialized: receipt_data, + } + }) + .collect(); + + let report = self + .store + .clean_submitted_transactions( + &successes, + current_chain_transaction_count - 1, + self.webhook_queue.clone(), + ) + .await?; + + Ok(report) + } + + /// Fetch receipts for all submitted transactions and categorize them + async fn fetch_confirmed_transaction_receipts( + &self, + submitted_txs: Vec, + ) -> ( + Vec, + Vec, + ) { + // Fetch all receipts in parallel + let receipt_futures: Vec<_> = submitted_txs + .iter() + .filter_map(|tx| match tx.hash.parse::() { + Ok(hash_bytes) => Some(async move { + let receipt = self + .chain + .provider() + .get_transaction_receipt(hash_bytes) + .await; + (tx, receipt) + }), + Err(_) => { + tracing::warn!("Invalid hash format: {}, skipping", tx.hash); + None + } + }) + .collect(); + + let receipt_results = futures::future::join_all(receipt_futures).await; + + // Categorize transactions + let mut confirmed_txs = Vec::new(); + let mut failed_txs = Vec::new(); + + for (tx, receipt_result) in receipt_results { + match receipt_result { + Ok(Some(receipt)) => { + confirmed_txs.push(ConfirmedTransactionWithRichReceipt { + nonce: tx.nonce, + hash: tx.hash.clone(), + transaction_id: tx.transaction_id.clone(), + receipt, + }); + } + Ok(None) | Err(_) => { + failed_txs.push(ReplacedTransaction { + hash: tx.hash.clone(), + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + (confirmed_txs, failed_txs) + } + + // ========== GAS BUMP METHODS ========== + + /// Attempt to gas bump a stalled transaction for the next expected nonce + async fn attempt_gas_bump_for_stalled_nonce( + &self, + expected_nonce: u64, + ) -> Result { + tracing::info!( + nonce = expected_nonce, + "Attempting gas bump for stalled nonce" + ); + + // Get all transaction IDs for this nonce + let submitted_transactions = self + .store + .get_submitted_transactions_for_nonce(expected_nonce) + .await?; + + if submitted_transactions.is_empty() { + tracing::debug!( + nonce = expected_nonce, + "No transactions found for stalled nonce, sending noop" + ); + + let noop_tx = self.send_noop_transaction(expected_nonce).await?; + self.store.process_noop_transactions(&[noop_tx]).await?; + return Ok(true); + } + + // Load transaction data for all IDs and find the newest one + let mut newest_transaction: Option<(String, TransactionData)> = None; + let mut newest_submitted_at = 0u64; + + for SubmittedTransactionDehydrated { transaction_id, .. } in submitted_transactions { + if let Some(tx_data) = self.store.get_transaction_data(&transaction_id).await? { + // Find the most recent attempt for this transaction + if let Some(latest_attempt) = tx_data.attempts.last() { + let submitted_at = latest_attempt.sent_at; + if submitted_at > newest_submitted_at { + newest_submitted_at = submitted_at; + newest_transaction = Some((transaction_id, tx_data)); + } + } + } + } + + if let Some((transaction_id, tx_data)) = newest_transaction { + tracing::info!( + transaction_id = %transaction_id, + nonce = expected_nonce, + "Found newest transaction for gas bump" + ); + + // Get the latest attempt to extract gas values from + // Build typed transaction -> manually bump -> sign + let typed_tx = match self + .build_typed_transaction(&tx_data.user_request, expected_nonce) + .await + { + Ok(tx) => tx, + Err(e) => { + // Check if this is a balance threshold issue during simulation + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, .. + } = &e + { + if should_update_balance_threshold(inner_error) { + if let Err(e) = self.update_balance_threshold().await { + tracing::error!("Failed to update balance threshold: {}", e); + } + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + if let Err(e) = self.update_balance_threshold().await { + tracing::error!("Failed to update balance threshold: {}", e); + } + } + } + + tracing::warn!( + transaction_id = %transaction_id, + nonce = expected_nonce, + error = %e, + "Failed to build typed transaction for gas bump" + ); + return Ok(false); + } + }; + let bumped_typed_tx = self.apply_gas_bump_to_typed_transaction(typed_tx, 120); // 20% increase + let bumped_tx = match self + .sign_transaction(bumped_typed_tx, &tx_data.user_request.signing_credential) + .await + { + Ok(tx) => tx, + Err(e) => { + tracing::warn!( + transaction_id = %transaction_id, + nonce = expected_nonce, + error = %e, + "Failed to sign transaction for gas bump" + ); + return Ok(false); + } + }; + + // Record the gas bump attempt + self.store + .add_gas_bump_attempt( + &SubmittedTransactionDehydrated { + nonce: expected_nonce, + hash: bumped_tx.hash().to_string(), + transaction_id: transaction_id.to_string(), + queued_at: tx_data.created_at, + }, + bumped_tx.clone(), + ) + .await?; + + // Send the bumped transaction + let tx_envelope = bumped_tx.into(); + match self.chain.provider().send_tx_envelope(tx_envelope).await { + Ok(_) => { + tracing::info!( + transaction_id = %transaction_id, + nonce = expected_nonce, + "Successfully sent gas bumped transaction" + ); + return Ok(true); + } + Err(e) => { + tracing::warn!( + transaction_id = %transaction_id, + nonce = expected_nonce, + error = %e, + "Failed to send gas bumped transaction" + ); + // Don't fail the worker, just log the error + return Ok(false); + } + } + } + + Ok(false) + } +} diff --git a/executors/src/eoa/worker/error.rs b/executors/src/eoa/worker/error.rs new file mode 100644 index 0000000..45d39b4 --- /dev/null +++ b/executors/src/eoa/worker/error.rs @@ -0,0 +1,262 @@ +use alloy::transports::{RpcError, TransportErrorKind}; +use engine_core::{ + chain::Chain, + error::{AlloyRpcErrorToEngineError, EngineError, RpcErrorKind}, +}; +use serde::{Deserialize, Serialize}; +use thirdweb_core::iaw::IAWError; +use twmq::{UserCancellable, error::TwmqError}; + +use crate::eoa::{ + EoaTransactionRequest, + store::{ + BorrowedTransaction, BorrowedTransactionData, SubmissionResult, SubmissionResultType, + SubmittedTransaction, TransactionStoreError, + }, + worker::EoaExecutorWorkerResult, +}; + +#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] +pub enum EoaExecutorWorkerError { + #[error("Chain service error for chainId {chain_id}: {message}")] + ChainServiceError { chain_id: u64, message: String }, + + #[error("Store error: {message}")] + StoreError { + message: String, + inner_error: TransactionStoreError, + }, + + #[error("Transaction not found: {transaction_id}")] + TransactionNotFound { transaction_id: String }, + + #[error("Transaction simulation failed: {message}")] + TransactionSimulationFailed { + message: String, + inner_error: EngineError, + }, + + #[error("Transaction build failed: {message}")] + TransactionBuildFailed { message: String }, + + #[error("RPC error encountered during generic operation: {message}")] + RpcError { + message: String, + inner_error: EngineError, + }, + + #[error("Error encountered when broadcasting transaction: {message}")] + TransactionSendError { + message: String, + inner_error: EngineError, + }, + + #[error("Signature parsing failed: {message}")] + SignatureParsingFailed { message: String }, + + #[error("Transaction signing failed: {message}")] + SigningError { + message: String, + inner_error: EngineError, + }, + + #[error("Work still remaining: {result:?}")] + WorkRemaining { result: EoaExecutorWorkerResult }, + + #[error("Internal error: {message}")] + InternalError { message: String }, + + #[error("User cancelled")] + UserCancelled, +} + +impl From for EoaExecutorWorkerError { + fn from(error: TwmqError) -> Self { + EoaExecutorWorkerError::InternalError { + message: format!("Queue error: {}", error), + } + } +} + +impl From for EoaExecutorWorkerError { + fn from(error: TransactionStoreError) -> Self { + EoaExecutorWorkerError::StoreError { + message: error.to_string(), + inner_error: error, + } + } +} + +impl UserCancellable for EoaExecutorWorkerError { + fn user_cancelled() -> Self { + EoaExecutorWorkerError::UserCancelled + } +} + +// ========== SIMPLE ERROR CLASSIFICATION ========== +#[derive(Debug)] +pub enum SendErrorClassification { + PossiblySent, // "nonce too low", "already known" etc + DeterministicFailure, // Invalid signature, malformed tx, insufficient funds etc +} + +#[derive(PartialEq, Eq, Debug)] +pub enum SendContext { + Rebroadcast, + InitialBroadcast, +} + +#[tracing::instrument(skip_all, fields(error = %error, context = ?context))] +pub fn classify_send_error( + error: &RpcError, + context: SendContext, +) -> SendErrorClassification { + if !error.is_error_resp() { + return SendErrorClassification::DeterministicFailure; + } + + let error_str = error.to_string().to_lowercase(); + + // Deterministic failures that didn't consume nonce (spec-compliant) + if error_str.contains("invalid signature") + || error_str.contains("malformed transaction") + || (context == SendContext::InitialBroadcast && error_str.contains("insufficient funds")) + || error_str.contains("invalid transaction format") + || error_str.contains("nonce too high") + // Should trigger nonce reset + { + return SendErrorClassification::DeterministicFailure; + } + + // Transaction possibly made it to mempool (spec-compliant) + if error_str.contains("nonce too low") + || error_str.contains("already known") + || error_str.contains("replacement transaction underpriced") + || error_str.contains("transaction already imported") + { + return SendErrorClassification::PossiblySent; + } + + // Additional common failures that didn't consume nonce + if error_str.contains("malformed") + || error_str.contains("gas limit") + || error_str.contains("intrinsic gas too low") + { + return SendErrorClassification::DeterministicFailure; + } + + tracing::warn!( + "Unknown send error: {}. PLEASE REPORT FOR ADDING CORRECT CLASSIFICATION [NOTIFY]", + error_str + ); + + // Default: assume possibly sent for safety + SendErrorClassification::PossiblySent +} + +pub fn should_trigger_nonce_reset(error: &RpcError) -> bool { + let error_str = error.to_string().to_lowercase(); + + // "nonce too high" should trigger nonce reset as per spec + error_str.contains("nonce too high") +} + +pub fn should_update_balance_threshold(error: &EngineError) -> bool { + match error { + EngineError::RpcError { kind, .. } + | EngineError::PaymasterError { kind, .. } + | EngineError::BundlerError { kind, .. } => match kind { + RpcErrorKind::ErrorResp(resp) => { + let message = resp.message.to_lowercase(); + message.contains("insufficient funds") + || message.contains("insufficient balance") + || message.contains("out of gas") + || message.contains("insufficient eth") + || message.contains("balance too low") + || message.contains("not enough funds") + || message.contains("insufficient native token") + } + _ => false, + }, + _ => false, + } +} + +pub fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { + match kind { + RpcErrorKind::TransportHttpError { status, .. } if *status >= 400 && *status < 500 => false, + RpcErrorKind::UnsupportedFeature { .. } => false, + _ => true, + } +} + +pub fn is_retryable_preparation_error(error: &EoaExecutorWorkerError) -> bool { + match error { + EoaExecutorWorkerError::RpcError { inner_error, .. } => { + // extract the RpcErrorKind from the inner error + if let EngineError::RpcError { kind, .. } = inner_error { + is_retryable_rpc_error(kind) + } else { + false + } + } + EoaExecutorWorkerError::ChainServiceError { .. } => true, // Network related + EoaExecutorWorkerError::StoreError { inner_error, .. } => { + matches!(inner_error, TransactionStoreError::RedisError { .. }) + } + EoaExecutorWorkerError::TransactionSimulationFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::TransactionBuildFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::SigningError { inner_error, .. } => match inner_error { + // if vault error, it's not retryable + EngineError::VaultError { .. } => false, + // if iaw error, it's retryable only if it's a network error + EngineError::IawError { error, .. } => matches!(error, IAWError::NetworkError { .. }), + _ => false, + }, + EoaExecutorWorkerError::TransactionNotFound { .. } => false, // Deterministic + EoaExecutorWorkerError::InternalError { .. } => false, // Deterministic + EoaExecutorWorkerError::UserCancelled => false, // Deterministic + EoaExecutorWorkerError::TransactionSendError { .. } => false, // Different context + EoaExecutorWorkerError::SignatureParsingFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::WorkRemaining { .. } => false, // Different context + } +} + +impl SubmissionResult { + /// Convert a send result to a SubmissionResult for batch processing + /// This handles the specific RpcError type from alloy + pub fn from_send_result( + borrowed_transaction: &BorrowedTransaction, + send_result: Result>, + send_context: SendContext, + chain: &impl Chain, + ) -> Self { + match send_result { + Ok(_) => SubmissionResult { + result: SubmissionResultType::Success, + transaction: borrowed_transaction.clone().into(), + }, + Err(ref rpc_error) => { + match classify_send_error(rpc_error, send_context) { + SendErrorClassification::PossiblySent => SubmissionResult { + result: SubmissionResultType::Success, + transaction: borrowed_transaction.clone().into(), + }, + SendErrorClassification::DeterministicFailure => { + // Transaction failed, should be retried + let engine_error = rpc_error.to_engine_error(chain); + let error = EoaExecutorWorkerError::TransactionSendError { + message: format!("Transaction send failed: {}", rpc_error), + inner_error: engine_error, + }; + SubmissionResult { + result: SubmissionResultType::Nack(error), + transaction: borrowed_transaction.clone().into(), + } + } + } + } + } + } +} diff --git a/executors/src/eoa/worker/mod.rs b/executors/src/eoa/worker/mod.rs new file mode 100644 index 0000000..0d36c16 --- /dev/null +++ b/executors/src/eoa/worker/mod.rs @@ -0,0 +1,501 @@ +use alloy::consensus::Transaction; +use alloy::primitives::{Address, U256}; +use alloy::providers::Provider; +use engine_core::{ + chain::{Chain, ChainService}, + credentials::SigningCredential, + error::AlloyRpcErrorToEngineError, + signer::EoaSigner, +}; +use serde::{Deserialize, Serialize}; +use std::{sync::Arc, time::Duration}; +use twmq::Queue; +use twmq::redis::AsyncCommands; +use twmq::redis::aio::ConnectionManager; +use twmq::{ + DurableExecution, FailHookData, NackHookData, SuccessHookData, + hooks::TransactionContext, + job::{BorrowedJob, JobResult, RequeuePosition, ToJobResult}, +}; + +use crate::eoa::store::{ + AtomicEoaExecutorStore, EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, SubmissionResult, +}; +use crate::webhook::WebhookJobHandler; + +pub mod confirm; +pub mod error; +mod send; +mod transaction; + +use error::{EoaExecutorWorkerError, SendContext}; + +// ========== SPEC-COMPLIANT CONSTANTS ========== +const MAX_INFLIGHT_PER_EOA: u64 = 100; // Default from spec +const MAX_RECYCLED_THRESHOLD: u64 = 50; // Circuit breaker from spec +const TARGET_TRANSACTIONS_PER_EOA: u64 = 10; // Fleet management from spec +const MIN_TRANSACTIONS_PER_EOA: u64 = 1; // Fleet management from spec + +// ========== JOB DATA ========== +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct EoaExecutorWorkerJobData { + pub eoa_address: Address, + pub chain_id: u64, + pub noop_signing_credential: SigningCredential, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct EoaExecutorWorkerResult { + // what we did + /// Number of transactions we recovered from borrowed state + pub recovered_transactions: u32, + + /// Number of transactions we confirmed + pub confirmed_transactions: u32, + + /// Number of transactions we failed due to deterministic errors + pub failed_transactions: u32, + + /// Number of transactions we sent + pub sent_transactions: u32, + + /// Number of transactions that got replaced in the mempool and are now pending + pub replaced_transactions: u32, + + // what we have left + /// Number of transactions currently in the submitted state + pub submitted_transactions: u32, + + /// Number of transactions currently in the pending state + pub pending_transactions: u32, + + /// Number of transactions currently in the borrowed state + pub borrowed_transactions: u32, + + /// Number of recycled nonces + pub recycled_nonces: u32, +} + +impl EoaExecutorWorkerResult { + pub fn is_work_remaining(&self) -> bool { + self.pending_transactions > 0 + || self.borrowed_transactions > 0 + || self.recycled_nonces > 0 + || self.submitted_transactions > 0 + } +} + +// ========== MAIN WORKER ========== +/// EOA Executor Worker +/// +/// ## Core Workflow: +/// 1. **Acquire Lock Aggressively** - Takes over stalled workers using force acquisition. This is a lock over EOA:CHAIN +/// 2. **Crash Recovery** - Rebroadcasts borrowed transactions, handles deterministic failures +/// 3. **Confirmation Flow** - Fetches receipts, confirms transactions, handles nonce sync, requeues replaced transactions +/// 4. **Send Flow** - Processes recycled nonces first, then new transactions with in-flight budget control +/// 5. **Lock Release** - Explicit release in finally pattern as per spec +/// +/// ## Key Features: +/// - **Atomic Operations**: All state transitions use Redis WATCH/MULTI/EXEC for durability +/// - **Borrowed State**: Mid-send crash recovery with atomic pending->borrowed->submitted transitions +/// - **Nonce Management**: Optimistic nonce tracking with recycled nonce priority +/// - **Error Classification**: Spec-compliant deterministic vs. possibly-sent error handling +/// - **Circuit Breakers**: Automatic recycled nonce nuking when threshold exceeded +/// - **Health Monitoring**: Balance checking with configurable thresholds +pub struct EoaExecutorJobHandler +where + CS: ChainService + Send + Sync + 'static, +{ + pub chain_service: Arc, + pub webhook_queue: Arc>, + + pub redis: ConnectionManager, + pub namespace: Option, + + pub eoa_signer: Arc, + pub max_inflight: u64, // Note: Spec uses MAX_INFLIGHT_PER_EOA constant + pub max_recycled_nonces: u64, // Note: Spec uses MAX_RECYCLED_THRESHOLD constant +} + +impl DurableExecution for EoaExecutorJobHandler +where + CS: ChainService + Send + Sync + 'static, +{ + type Output = EoaExecutorWorkerResult; + type ErrorData = EoaExecutorWorkerError; + type JobData = EoaExecutorWorkerJobData; + + #[tracing::instrument(skip_all, fields(eoa = %job.job.data.eoa_address, chain_id = job.job.data.chain_id))] + async fn process( + &self, + job: &BorrowedJob, + ) -> JobResult { + let data = &job.job.data; + + // 1. GET CHAIN + let chain = self + .chain_service + .get_chain(data.chain_id) + .map_err(|e| EoaExecutorWorkerError::ChainServiceError { + chain_id: data.chain_id, + message: format!("Failed to get chain: {}", e), + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 2. CREATE SCOPED STORE (acquires lock) + let scoped = EoaExecutorStore::new( + self.redis.clone(), + self.namespace.clone(), + data.eoa_address, + data.chain_id, + ) + .acquire_eoa_lock_aggressively(&job.lease_token) + .await + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + let worker = EoaExecutorWorker { + store: scoped, + chain, + eoa: data.eoa_address, + chain_id: data.chain_id, + noop_signing_credential: data.noop_signing_credential.clone(), + + max_inflight: self.max_inflight, + max_recycled_nonces: self.max_recycled_nonces, + webhook_queue: self.webhook_queue.clone(), + signer: self.eoa_signer.clone(), + }; + + let result = worker.execute_main_workflow().await?; + worker.release_eoa_lock().await; + + if result.is_work_remaining() { + Err(EoaExecutorWorkerError::WorkRemaining { result }) + .map_err_nack(Some(Duration::from_secs(2)), RequeuePosition::Last) + } else { + Ok(result) + } + + // // initiate health data if doesn't exist + // self.get_eoa_health(&scoped, &chain) + // .await + // .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // // Execute main workflow with proper error handling + // self.execute_main_workflow(&scoped, &chain).await + } + + async fn on_success( + &self, + job: &BorrowedJob, + _success_data: SuccessHookData<'_, Self::Output>, + _tx: &mut TransactionContext<'_>, + ) { + self.soft_release_eoa_lock(&job.job.data).await; + } + + async fn on_nack( + &self, + job: &BorrowedJob, + _nack_data: NackHookData<'_, Self::ErrorData>, + _tx: &mut TransactionContext<'_>, + ) { + self.soft_release_eoa_lock(&job.job.data).await; + } + + async fn on_fail( + &self, + job: &BorrowedJob, + _fail_data: FailHookData<'_, Self::ErrorData>, + _tx: &mut TransactionContext<'_>, + ) { + self.soft_release_eoa_lock(&job.job.data).await; + } +} + +impl EoaExecutorJobHandler +where + CS: ChainService + Send + Sync + 'static, +{ + async fn soft_release_eoa_lock(&self, job_data: &EoaExecutorWorkerJobData) { + let keys = EoaExecutorStoreKeys::new( + job_data.eoa_address, + job_data.chain_id, + self.namespace.clone(), + ); + + let lock_key = keys.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + if let Err(e) = conn.del::<&str, ()>(&lock_key).await { + tracing::error!( + eoa = %job_data.eoa_address, + chain_id = %job_data.chain_id, + error = %e, + "Failed to release EOA lock" + ); + } + } +} + +pub struct EoaExecutorWorker { + pub store: AtomicEoaExecutorStore, + pub chain: C, + + pub eoa: Address, + pub chain_id: u64, + pub noop_signing_credential: SigningCredential, + + pub max_inflight: u64, + pub max_recycled_nonces: u64, + + pub webhook_queue: Arc>, + pub signer: Arc, +} + +impl EoaExecutorWorker { + /// Execute the main EOA worker workflow + async fn execute_main_workflow( + &self, + ) -> JobResult { + // 1. CRASH RECOVERY + let recovered = self + .recover_borrowed_state() + .await + .map_err(|e| { + tracing::error!("Error in recover_borrowed_state: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 2. CONFIRM FLOW + let confirmations_report = self + .confirm_flow() + .await + .map_err(|e| { + tracing::error!("Error in confirm flow: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 3. SEND FLOW + let sent = self + .send_flow() + .await + .map_err(|e| { + tracing::error!("Error in send_flow: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + // 4. CHECK FOR REMAINING WORK + let pending_count = self + .store + .peek_pending_transactions(1000) + .await + .map_err(|e| { + tracing::error!("Error in peek_pending_transactions: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? + .len(); + let borrowed_count = self + .store + .peek_borrowed_transactions() + .await + .map_err(|e| { + tracing::error!("Error in peek_borrowed_transactions: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? + .len(); + let recycled_count = self + .store + .peek_recycled_nonces() + .await + .map_err(|e| { + tracing::error!("Error in peek_recycled_nonces: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? + .len(); + let submitted_count = self + .store + .get_submitted_transactions_count() + .await + .map_err(|e| { + tracing::error!("Error in get_submitted_transactions_count: {}", e); + e + }) + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; + + Ok(EoaExecutorWorkerResult { + recovered_transactions: recovered, + confirmed_transactions: confirmations_report.moved_to_success as u32, + failed_transactions: confirmations_report.moved_to_pending as u32, + sent_transactions: sent, + + replaced_transactions: confirmations_report.moved_to_pending as u32, + submitted_transactions: submitted_count as u32, + pending_transactions: pending_count as u32, + borrowed_transactions: borrowed_count as u32, + recycled_nonces: recycled_count as u32, + }) + } + + // ========== CRASH RECOVERY ========== + #[tracing::instrument(skip_all)] + async fn recover_borrowed_state(&self) -> Result { + let borrowed_transactions = self.store.peek_borrowed_transactions().await?; + let mut borrowed_transactions = self.store.hydrate_all(borrowed_transactions).await?; + + if borrowed_transactions.is_empty() { + return Ok(0); + } + + tracing::warn!( + "Recovering {} borrowed transactions. This indicates a worker crash or system issue", + borrowed_transactions.len() + ); + + // Sort borrowed transactions by nonce to ensure proper ordering + borrowed_transactions.sort_by_key(|tx| tx.signed_transaction.nonce()); + + // Rebroadcast all transactions in parallel + let rebroadcast_futures: Vec<_> = borrowed_transactions + .iter() + .map(|borrowed| { + let tx_envelope = borrowed.signed_transaction.clone().into(); + let nonce = borrowed.signed_transaction.nonce(); + let transaction_id = borrowed.transaction_id.clone(); + + tracing::info!( + transaction_id = %transaction_id, + nonce = nonce, + "Recovering borrowed transaction" + ); + + async move { + let send_result = self.chain.provider().send_tx_envelope(tx_envelope).await; + (borrowed, send_result) + } + }) + .collect(); + + let rebroadcast_results = futures::future::join_all(rebroadcast_futures).await; + + // Convert results to SubmissionResult for batch processing + let submission_results: Vec = rebroadcast_results + .into_iter() + .map(|(borrowed, send_result)| { + SubmissionResult::from_send_result( + borrowed, + send_result, + SendContext::Rebroadcast, + &self.chain, + ) + }) + .collect(); + + dbg!(&submission_results); + + // TODO: Implement post-processing analysis for balance threshold updates and nonce resets + // Currently we lose the granular error handling that was in the individual atomic operations. + // Consider: + // 1. Analyzing submission_results for specific error patterns + // 2. Calling update_balance_threshold if needed + // 3. Detecting nonce reset conditions + // 4. Or move this logic into the batch processor itself + + // Process all results in one batch operation + let report = self + .store + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + // TODO: Handle post-processing updates here if needed + // For now, we skip the individual error analysis that was done in the old atomic approach + + tracing::info!( + "Recovered {} transactions: {} submitted, {} recycled, {} failed", + report.total_processed, + report.moved_to_submitted, + report.moved_to_pending, + report.failed_transactions + ); + + Ok(report.total_processed as u32) + } + + // ========== HEALTH ACCESSOR ========== + + /// Get EOA health, initializing it if it doesn't exist + /// This method ensures the health data is always available for the worker + async fn get_eoa_health(&self) -> Result { + let store_health = self.store.get_eoa_health().await?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + match store_health { + Some(health) => Ok(health), + None => { + // Initialize with fresh data from chain + let balance = self + .chain + .provider() + .get_balance(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!( + "Failed to get balance during initialization: {}", + engine_error + ), + inner_error: engine_error, + } + })?; + + let health = EoaHealth { + balance, + balance_threshold: U256::ZERO, + balance_fetched_at: now, + last_confirmation_at: now, + last_nonce_movement_at: now, + nonce_resets: Vec::new(), + }; + + // Save to store + self.store.update_health_data(&health).await?; + Ok(health) + } + } + } + + #[tracing::instrument(skip_all, fields(eoa = %self.eoa, chain_id = %self.chain.chain_id()))] + async fn update_balance_threshold(&self) -> Result<(), EoaExecutorWorkerError> { + let mut health = self.get_eoa_health().await?; + + tracing::info!("Updating balance threshold"); + let balance_threshold = self + .chain + .provider() + .get_balance(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get balance: {}", engine_error), + inner_error: engine_error, + } + })?; + + health.balance_threshold = balance_threshold; + self.store.update_health_data(&health).await?; + Ok(()) + } + + async fn release_eoa_lock(self) { + self.store.release_eoa_lock().await; + } +} diff --git a/executors/src/eoa/worker/send.rs b/executors/src/eoa/worker/send.rs new file mode 100644 index 0000000..eb4c384 --- /dev/null +++ b/executors/src/eoa/worker/send.rs @@ -0,0 +1,402 @@ +use alloy::providers::Provider; +use engine_core::{chain::Chain, error::AlloyRpcErrorToEngineError}; + +use crate::eoa::{ + store::{BorrowedTransaction, PendingTransaction, SubmissionResult}, + worker::{ + EoaExecutorWorker, + error::{ + EoaExecutorWorkerError, SendContext, is_retryable_preparation_error, + should_update_balance_threshold, + }, + }, +}; + +const HEALTH_CHECK_INTERVAL: u64 = 300; // 5 minutes in seconds + +impl EoaExecutorWorker { + // ========== SEND FLOW ========== + #[tracing::instrument(skip_all)] + pub async fn send_flow(&self) -> Result { + // 1. Get EOA health (initializes if needed) and check if we should update balance + let mut health = self.get_eoa_health().await?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Update balance if it's stale + // TODO: refactor this, very ugly + if health.balance <= health.balance_threshold { + if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { + let balance = self + .chain + .provider() + .get_balance(self.eoa) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(&self.chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get balance: {}", engine_error), + inner_error: engine_error, + } + })?; + + health.balance = balance; + health.balance_fetched_at = now; + self.store.update_health_data(&health).await?; + } + + if health.balance <= health.balance_threshold { + tracing::warn!( + "EOA has insufficient balance (<= {} wei), skipping send flow", + health.balance_threshold + ); + return Ok(0); + } + } + + let mut total_sent = 0; + + // 2. Process recycled nonces first + total_sent += self.process_recycled_nonces().await?; + + // 3. Only proceed to new nonces if we successfully used all recycled nonces + let remaining_recycled = self.store.peek_recycled_nonces().await?.len(); + if remaining_recycled == 0 { + let inflight_budget = self.store.get_inflight_budget(self.max_inflight).await?; + if inflight_budget > 0 { + total_sent += self.process_new_transactions(inflight_budget).await?; + } + } else { + tracing::warn!( + "Still have {} recycled nonces, not sending new transactions", + remaining_recycled + ); + } + + Ok(total_sent) + } + + async fn process_recycled_nonces(&self) -> Result { + let mut total_sent: usize = 0; + let mut is_pending_empty = false; + + // Loop to handle preparation failures and refill with new transactions + for _ in 0..10 { + let recycled_nonces = self.store.clean_and_get_recycled_nonces().await?; + + if recycled_nonces.is_empty() { + return Ok(total_sent as u32); + } + + // Get pending transactions to match with recycled nonces + let pending_txs = self + .store + .peek_pending_transactions(recycled_nonces.len() as u64) + .await?; + + // Pair recycled nonces with pending transactions + let mut build_tasks = Vec::new(); + + for (i, nonce) in recycled_nonces.iter().enumerate() { + if let Some(p_tx) = pending_txs.get(i) { + build_tasks + .push(self.build_and_sign_single_transaction_with_retries(p_tx, *nonce)); + } else { + // No more pending transactions for this recycled nonce + is_pending_empty = true; + break; + } + } + + if build_tasks.is_empty() { + break; + } + + // Build and sign all transactions in parallel + let prepared_results = futures::future::join_all(build_tasks).await; + let prepared_results_with_pending = pending_txs + .iter() + .zip(prepared_results.into_iter()) + .collect::>(); + + let cleaned_results = self + .clean_prepration_results(prepared_results_with_pending) + .await?; + + if cleaned_results.is_empty() { + // No successful preparations, try again with more pending transactions + continue; + } + + // Move prepared transactions to borrowed state with recycled nonces + let moved_count = self + .store + .atomic_move_pending_to_borrowed_with_recycled_nonces( + &cleaned_results + .iter() + .map(|borrowed_tx| borrowed_tx.data.clone()) + .collect::>(), + ) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = cleaned_results.len(), + "Moved transactions to borrowed state using recycled nonces" + ); + + // Actually send the transactions to the blockchain + let send_tasks: Vec<_> = cleaned_results + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { + self.chain + .provider() + .send_tx_envelope(signed_tx.into()) + .await + } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // Process send results and update states + let submission_results = send_results + .into_iter() + .zip(cleaned_results.into_iter()) + .map(|(send_result, borrowed_tx)| { + SubmissionResult::from_send_result( + &borrowed_tx, + send_result, + SendContext::InitialBroadcast, + &self.chain, + ) + }) + .collect(); + + // Use batch processing to handle all submission results + let processing_report = self + .store + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); + + total_sent += processing_report.moved_to_submitted; + } + + if is_pending_empty { + let recycled_nonces = self.store.clean_and_get_recycled_nonces().await?; + let mut build_tasks = Vec::new(); + + for nonce in recycled_nonces { + build_tasks.push(self.send_noop_transaction(nonce)); + } + + let send_results = futures::future::join_all(build_tasks).await; + + let successful_sends = send_results + .into_iter() + .filter_map(|result| result.ok()) + .collect::>(); + + self.store + .process_noop_transactions(&successful_sends) + .await?; + } + + Ok(total_sent as u32) + } + + async fn clean_prepration_results( + &self, + results: Vec<( + &PendingTransaction, + Result, + )>, + ) -> Result, EoaExecutorWorkerError> { + let mut cleaned_results = Vec::new(); + let mut balance_threshold_update_needed = false; + + for (pending, result) in results.into_iter() { + match result { + Ok(borrowed_data) => { + cleaned_results.push(borrowed_data); + } + Err(e) => { + // Track balance threshold issues + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, .. + } = &e + { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } + + // For deterministic build failures, fail the transaction immediately + if !is_retryable_preparation_error(&e) { + self.store + .fail_pending_transaction(pending, e, self.webhook_queue.clone()) + .await?; + } + } + } + } + + if balance_threshold_update_needed { + if let Err(e) = self.update_balance_threshold().await { + tracing::error!("Failed to update balance threshold: {}", e); + } + } + + Ok(cleaned_results) + } + + /// Process new transactions with fixed iterations and simple sequential nonces + async fn process_new_transactions(&self, budget: u64) -> Result { + if budget == 0 { + return Ok(0); + } + + let mut total_sent: usize = 0; + let mut remaining_budget = budget; + + // Fixed number of iterations to avoid infinite loops + for iteration in 0..10 { + if remaining_budget == 0 { + break; + } + + // Get pending transactions + let pending_txs = self + .store + .peek_pending_transactions(remaining_budget) + .await?; + + if pending_txs.is_empty() { + break; + } + + let optimistic_nonce = self.store.get_optimistic_transaction_count().await?; + let batch_size = pending_txs.len().min(remaining_budget as usize); + + tracing::debug!( + iteration = iteration, + batch_size = batch_size, + starting_nonce = optimistic_nonce, + remaining_budget = remaining_budget, + "Processing new transaction batch" + ); + + // Build and sign all transactions in parallel with sequential nonces + let build_tasks: Vec<_> = pending_txs + .iter() + .take(batch_size) + .enumerate() + .map(|(i, tx)| { + let expected_nonce = optimistic_nonce + i as u64; + self.build_and_sign_single_transaction_with_retries(tx, expected_nonce) + }) + .collect(); + + let prepared_results = futures::future::join_all(build_tasks).await; + let prepared_results_with_pending = pending_txs + .iter() + .take(batch_size) + .zip(prepared_results.into_iter()) + .collect::>(); + + // Clean preparation results (handles failures and removes bad transactions) + let cleaned_results = self + .clean_prepration_results(prepared_results_with_pending) + .await?; + + if cleaned_results.is_empty() { + // No successful preparations, reduce budget and continue + continue; + } + + // Move prepared transactions to borrowed state with incremented nonces + let moved_count = self + .store + .atomic_move_pending_to_borrowed_with_incremented_nonces( + &cleaned_results + .iter() + .map(|borrowed_tx| borrowed_tx.data.clone()) + .collect::>(), + ) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = cleaned_results.len(), + "Moved transactions to borrowed state using incremented nonces" + ); + + // Send the transactions to the blockchain + let send_tasks: Vec<_> = cleaned_results + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { + self.chain + .provider() + .send_tx_envelope(signed_tx.into()) + .await + } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // Process send results and update states + let submission_results = send_results + .into_iter() + .zip(cleaned_results.into_iter()) + .map(|(send_result, borrowed_tx)| { + SubmissionResult::from_send_result( + &borrowed_tx, + send_result, + SendContext::InitialBroadcast, + &self.chain, + ) + }) + .collect(); + + // Use batch processing to handle all submission results + let processing_report = self + .store + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); + + total_sent += processing_report.moved_to_submitted; + remaining_budget = remaining_budget.saturating_sub(moved_count as u64); + + // If we didn't use all our budget in this iteration, we're likely done + if moved_count < batch_size { + break; + } + } + + Ok(total_sent as u32) + } +} diff --git a/executors/src/eoa/worker/transaction.rs b/executors/src/eoa/worker/transaction.rs new file mode 100644 index 0000000..12cd2e7 --- /dev/null +++ b/executors/src/eoa/worker/transaction.rs @@ -0,0 +1,427 @@ +use std::time::Duration; + +use alloy::{ + consensus::{ + SignableTransaction, Signed, TxEip4844Variant, TxEip4844WithSidecar, TypedTransaction, + }, + network::{TransactionBuilder, TransactionBuilder7702}, + primitives::{Bytes, U256}, + providers::Provider, + rpc::types::TransactionRequest as AlloyTransactionRequest, + signers::Signature, + transports::RpcError, +}; +use engine_core::{ + chain::Chain, + credentials::SigningCredential, + error::AlloyRpcErrorToEngineError, + signer::{AccountSigner, EoaSigningOptions}, + transaction::TransactionTypeData, +}; + +use crate::eoa::{ + EoaTransactionRequest, + store::{ + BorrowedTransaction, BorrowedTransactionData, PendingTransaction, SubmittedNoopTransaction, + TransactionData, + }, + worker::{ + EoaExecutorWorker, + error::{EoaExecutorWorkerError, is_retryable_preparation_error}, + }, +}; + +// Retry constants for preparation phase +const MAX_PREPARATION_RETRIES: u32 = 3; +const PREPARATION_RETRY_DELAY_MS: u64 = 100; + +impl EoaExecutorWorker { + pub async fn build_and_sign_single_transaction_with_retries( + &self, + pending_transaction: &PendingTransaction, + nonce: u64, + ) -> Result { + let mut last_error = None; + + // Internal retry loop for retryable errors + for attempt in 0..=MAX_PREPARATION_RETRIES { + if attempt > 0 { + // Simple exponential backoff + let delay = PREPARATION_RETRY_DELAY_MS * (2_u64.pow(attempt - 1)); + tokio::time::sleep(Duration::from_millis(delay)).await; + + tracing::debug!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + "Retrying transaction preparation" + ); + } + + match self + .build_and_sign_single_transaction(pending_transaction, nonce) + .await + { + Ok(result) => return Ok(result), + Err(error) => { + if is_retryable_preparation_error(&error) && attempt < MAX_PREPARATION_RETRIES { + tracing::warn!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + error = %error, + "Retryable error during transaction preparation, will retry" + ); + last_error = Some(error); + continue; + } else { + // Either deterministic error or exceeded max retries + return Err(error); + } + } + } + } + + // This should never be reached, but just in case + Err( + last_error.unwrap_or_else(|| EoaExecutorWorkerError::InternalError { + message: "Unexpected error in retry loop".to_string(), + }), + ) + } + + pub async fn build_and_sign_single_transaction( + &self, + pending_transaction: &PendingTransaction, + nonce: u64, + ) -> Result { + // Get transaction data + let tx_data = pending_transaction.user_request.clone(); + // Build and sign transaction + let signed_tx = self.build_and_sign_transaction(&tx_data, nonce).await?; + + Ok(BorrowedTransaction { + data: BorrowedTransactionData { + transaction_id: pending_transaction.transaction_id.clone(), + hash: signed_tx.hash().to_string(), + signed_transaction: signed_tx, + borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, + queued_at: pending_transaction.queued_at, + }, + user_request: pending_transaction.user_request.clone(), + }) + } + + pub async fn build_and_sign_noop_transaction( + &self, + nonce: u64, + ) -> Result, EoaExecutorWorkerError> { + // Create a minimal transaction to consume the recycled nonce + // Send 0 ETH to self with minimal gas + + // Build no-op transaction (send 0 to self) + let tx_request = AlloyTransactionRequest::default() + .with_from(self.eoa) + .with_to(self.eoa) // Send to self + .with_value(U256::ZERO) // Send 0 value + .with_input(Bytes::new()) // No data + .with_chain_id(self.chain.chain_id()) + .with_nonce(nonce) + .with_gas_limit(21000); // Minimal gas for basic transfer + + let tx_request = self.estimate_gas_fees(tx_request).await?; + let built_tx = tx_request.build_typed_tx().map_err(|e| { + EoaExecutorWorkerError::TransactionBuildFailed { + message: format!("Failed to build typed transaction for no-op: {e:?}"), + } + })?; + + let tx = self + .sign_transaction(built_tx, &self.noop_signing_credential) + .await?; + + Ok(tx) + } + + pub async fn send_noop_transaction( + &self, + nonce: u64, + ) -> Result { + let tx = self.build_and_sign_noop_transaction(nonce).await?; + + self.chain + .provider() + .send_tx_envelope(tx.into()) + .await + .map_err(|e| EoaExecutorWorkerError::TransactionSendError { + message: format!("Failed to send no-op transaction: {e:?}"), + inner_error: e.to_engine_error(&self.chain), + }) + .map(|pending| SubmittedNoopTransaction { + nonce, + hash: pending.tx_hash().to_string(), + }) + } + + async fn estimate_gas_fees( + &self, + tx: AlloyTransactionRequest, + ) -> Result { + // Check what fees are missing and need to be estimated + + // If we have gas_price set, we're doing legacy - don't estimate EIP-1559 + if tx.gas_price.is_some() { + return Ok(tx); + } + + // If we have both EIP-1559 fees set, don't estimate + if tx.max_fee_per_gas.is_some() && tx.max_priority_fee_per_gas.is_some() { + return Ok(tx); + } + + // Try EIP-1559 fees first, fall back to legacy if unsupported + match self.chain.provider().estimate_eip1559_fees().await { + Ok(eip1559_fees) => { + tracing::debug!( + "Using EIP-1559 fees: max_fee={}, max_priority_fee={}", + eip1559_fees.max_fee_per_gas, + eip1559_fees.max_priority_fee_per_gas + ); + + let mut result = tx; + // Only set fees that are missing + if result.max_fee_per_gas.is_none() { + result = result.with_max_fee_per_gas(eip1559_fees.max_fee_per_gas); + } + if result.max_priority_fee_per_gas.is_none() { + result = + result.with_max_priority_fee_per_gas(eip1559_fees.max_priority_fee_per_gas); + } + + Ok(result) + } + Err(eip1559_error) => { + // Check if this is an "unsupported feature" error + if let RpcError::UnsupportedFeature(_) = &eip1559_error { + tracing::debug!("EIP-1559 not supported, falling back to legacy gas price"); + + // Fall back to legacy gas price only if no gas price is set + if tx.authorization_list().is_none() { + match self.chain.provider().get_gas_price().await { + Ok(gas_price) => { + tracing::debug!("Using legacy gas price: {}", gas_price); + Ok(tx.with_gas_price(gas_price)) + } + Err(legacy_error) => Err(EoaExecutorWorkerError::RpcError { + message: format!( + "Failed to get legacy gas price: {}", + legacy_error + ), + inner_error: legacy_error.to_engine_error(&self.chain), + }), + } + } else { + Err(EoaExecutorWorkerError::TransactionBuildFailed { + message: "EIP7702 transactions not supported on chain".to_string(), + }) + } + } else { + // Other EIP-1559 error + Err(EoaExecutorWorkerError::RpcError { + message: format!("Failed to estimate EIP-1559 fees: {}", eip1559_error), + inner_error: eip1559_error.to_engine_error(&self.chain), + }) + } + } + } + } + + pub async fn build_typed_transaction( + &self, + request: &EoaTransactionRequest, + nonce: u64, + ) -> Result { + // Build transaction request from stored data + let mut tx_request = AlloyTransactionRequest::default() + .with_from(request.from) + .with_value(request.value) + .with_input(request.data.clone()) + .with_chain_id(request.chain_id) + .with_nonce(nonce); + + if let Some(to) = request.to { + tx_request = tx_request.with_to(to); + } + + if let Some(gas_limit) = request.gas_limit { + tx_request = tx_request.with_gas_limit(gas_limit); + } + + // Handle gas fees - either from user settings or estimation + tx_request = if let Some(type_data) = &request.transaction_type_data { + // User provided gas settings - respect them first + match type_data { + TransactionTypeData::Eip1559(data) => { + let mut req = tx_request; + if let Some(max_fee) = data.max_fee_per_gas { + req = req.with_max_fee_per_gas(max_fee); + } + if let Some(max_priority) = data.max_priority_fee_per_gas { + req = req.with_max_priority_fee_per_gas(max_priority); + } + + // if either not set, estimate the other one + if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { + req = self.estimate_gas_fees(req).await?; + } + + req + } + TransactionTypeData::Legacy(data) => { + if let Some(gas_price) = data.gas_price { + tx_request.with_gas_price(gas_price) + } else { + // User didn't provide gas price, estimate it + self.estimate_gas_fees(tx_request).await? + } + } + TransactionTypeData::Eip7702(data) => { + let mut req = tx_request; + if let Some(authorization_list) = &data.authorization_list { + req = req.with_authorization_list(authorization_list.clone()); + } + if let Some(max_fee) = data.max_fee_per_gas { + req = req.with_max_fee_per_gas(max_fee); + } + if let Some(max_priority) = data.max_priority_fee_per_gas { + req = req.with_max_priority_fee_per_gas(max_priority); + } + + // if either not set, estimate the other one + if req.max_fee_per_gas.is_none() || req.max_priority_fee_per_gas.is_none() { + req = self.estimate_gas_fees(req).await?; + } + + req + } + } + } else { + // No user settings - estimate appropriate fees + self.estimate_gas_fees(tx_request).await? + }; + + // Estimate gas if needed + if tx_request.gas.is_none() { + match self.chain.provider().estimate_gas(tx_request.clone()).await { + Ok(gas_limit) => { + tx_request = tx_request.with_gas_limit(gas_limit * 110 / 100); // 10% buffer + } + Err(e) => { + // Check if this is a revert + if let RpcError::ErrorResp(error_payload) = &e { + if let Some(revert_data) = error_payload.as_revert_data() { + // This is a revert - the transaction is fundamentally broken + // This should fail the individual transaction, not the worker + return Err(EoaExecutorWorkerError::TransactionSimulationFailed { + message: format!( + "Transaction reverted during gas estimation: {} (revert: {})", + error_payload.message, + hex::encode(&revert_data) + ), + inner_error: e.to_engine_error(&self.chain), + }); + } + } + + // Not a revert - could be RPC issue, this should nack the worker + let engine_error = e.to_engine_error(&self.chain); + return Err(EoaExecutorWorkerError::RpcError { + message: format!("Gas estimation failed: {}", engine_error), + inner_error: engine_error, + }); + } + } + } + + // Build typed transaction + tx_request + .build_typed_tx() + .map_err(|e| EoaExecutorWorkerError::TransactionBuildFailed { + message: format!("Failed to build typed transaction: {:?}", e), + }) + } + + pub async fn sign_transaction( + &self, + typed_tx: TypedTransaction, + credential: &SigningCredential, + ) -> Result, EoaExecutorWorkerError> { + let signing_options = EoaSigningOptions { + from: self.eoa, + chain_id: Some(self.chain_id), + }; + + let signature = self + .signer + .sign_transaction(signing_options, &typed_tx, credential) + .await + .map_err(|engine_error| EoaExecutorWorkerError::SigningError { + message: format!("Failed to sign transaction: {}", engine_error), + inner_error: engine_error, + })?; + + let signature = signature.parse::().map_err(|e| { + EoaExecutorWorkerError::SignatureParsingFailed { + message: format!("Failed to parse signature: {}", e), + } + })?; + + Ok(typed_tx.into_signed(signature)) + } + + async fn build_and_sign_transaction( + &self, + request: &EoaTransactionRequest, + nonce: u64, + ) -> Result, EoaExecutorWorkerError> { + let typed_tx = self.build_typed_transaction(request, nonce).await?; + self.sign_transaction(typed_tx, &request.signing_credential) + .await + } + + pub fn apply_gas_bump_to_typed_transaction( + &self, + mut typed_tx: TypedTransaction, + bump_multiplier: u32, // e.g., 120 for 20% increase + ) -> TypedTransaction { + match &mut typed_tx { + TypedTransaction::Eip1559(tx) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + TypedTransaction::Legacy(tx) => { + tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; + } + TypedTransaction::Eip2930(tx) => { + tx.gas_price = tx.gas_price * bump_multiplier as u128 / 100; + } + TypedTransaction::Eip7702(tx) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + TypedTransaction::Eip4844(tx) => match tx { + TxEip4844Variant::TxEip4844(tx) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + TxEip4844Variant::TxEip4844WithSidecar(TxEip4844WithSidecar { tx, .. }) => { + tx.max_fee_per_gas = tx.max_fee_per_gas * bump_multiplier as u128 / 100; + tx.max_priority_fee_per_gas = + tx.max_priority_fee_per_gas * bump_multiplier as u128 / 100; + } + }, + } + typed_tx + } +} diff --git a/executors/src/external_bundler/confirm.rs b/executors/src/external_bundler/confirm.rs index 1f6e3b6..c1f69af 100644 --- a/executors/src/external_bundler/confirm.rs +++ b/executors/src/external_bundler/confirm.rs @@ -34,7 +34,7 @@ pub struct UserOpConfirmationJobData { pub user_op_hash: Bytes, pub nonce: U256, pub deployment_lock_acquired: bool, - pub webhook_options: Option>, + pub webhook_options: Vec, pub rpc_credentials: RpcCredentials, } @@ -338,7 +338,7 @@ where } impl HasWebhookOptions for UserOpConfirmationJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } diff --git a/executors/src/external_bundler/send.rs b/executors/src/external_bundler/send.rs index 0f7e4bc..bb533b6 100644 --- a/executors/src/external_bundler/send.rs +++ b/executors/src/external_bundler/send.rs @@ -49,7 +49,8 @@ pub struct ExternalBundlerSendJobData { pub execution_options: Erc4337ExecutionOptions, pub signing_credential: SigningCredential, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, pub rpc_credentials: RpcCredentials, @@ -59,7 +60,7 @@ pub struct ExternalBundlerSendJobData { } impl HasWebhookOptions for ExternalBundlerSendJobData { - fn webhook_options(&self) -> Option> { + fn webhook_options(&self) -> Vec { self.webhook_options.clone() } } @@ -417,10 +418,12 @@ where // 7.1. Calculate custom call gas limit let custom_call_gas_limit = { - let gas_limits: Vec = job_data.transactions.iter() + let gas_limits: Vec = job_data + .transactions + .iter() .filter_map(|tx| tx.gas_limit) .collect(); - + if gas_limits.len() == job_data.transactions.len() { // All transactions have gas limits specified, sum them up let total_gas: u64 = gas_limits.iter().sum(); diff --git a/executors/src/webhook/envelope.rs b/executors/src/webhook/envelope.rs index b4b3e06..d351862 100644 --- a/executors/src/webhook/envelope.rs +++ b/executors/src/webhook/envelope.rs @@ -43,7 +43,7 @@ pub struct WebhookNotificationEnvelope { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] -pub struct BareWebhookNotificationEnvelope { +pub struct BareWebhookNotificationEnvelope { pub transaction_id: String, pub event_type: StageEvent, pub executor_name: String, @@ -104,7 +104,7 @@ pub trait ExecutorStage { // --- Webhook Options Trait --- pub trait HasWebhookOptions { - fn webhook_options(&self) -> Option>; + fn webhook_options(&self) -> Vec; } pub trait HasTransactionMetadata { @@ -131,10 +131,7 @@ pub trait WebhookCapable: DurableExecution + ExecutorStage { Self::JobData: HasWebhookOptions, Self::Output: Serialize + Clone, { - let webhook_options = match job.job.data.webhook_options() { - Some(w) => w, - None => return Ok(()), // No webhook configured, skip silently - }; + let webhook_options = job.job.data.webhook_options(); for w in webhook_options { let envelope = WebhookNotificationEnvelope { @@ -166,10 +163,7 @@ pub trait WebhookCapable: DurableExecution + ExecutorStage { Self::JobData: HasWebhookOptions, Self::ErrorData: Serialize + Clone, { - let webhook_options = match job.job.data.webhook_options() { - Some(w) => w, - None => return Ok(()), // No webhook configured, skip silently - }; + let webhook_options = job.job.data.webhook_options(); for w in webhook_options { let now: u64 = chrono::Utc::now().timestamp().try_into().unwrap(); let next_retry_at = nack_data.delay.map(|delay| now + delay.as_secs()); @@ -207,10 +201,7 @@ pub trait WebhookCapable: DurableExecution + ExecutorStage { Self::JobData: HasWebhookOptions, Self::ErrorData: Serialize + Clone, { - let webhook_options = match job.job.data.webhook_options() { - Some(w) => w, - None => return Ok(()), // No webhook configured, skip silently - }; + let webhook_options = job.job.data.webhook_options(); for w in webhook_options { let envelope = WebhookNotificationEnvelope { notification_id: Uuid::new_v4().to_string(), diff --git a/server/src/execution_router/mod.rs b/server/src/execution_router/mod.rs index 9280dd1..8d35986 100644 --- a/server/src/execution_router/mod.rs +++ b/server/src/execution_router/mod.rs @@ -18,7 +18,9 @@ use engine_executors::{ confirm::Eip7702ConfirmationHandler, send::{Eip7702SendHandler, Eip7702SendJobData}, }, - eoa::{EoaExecutorStore, EoaExecutorWorker, EoaExecutorWorkerJobData, EoaTransactionRequest}, + eoa::{ + EoaExecutorJobHandler, EoaExecutorStore, EoaExecutorWorkerJobData, EoaTransactionRequest, + }, external_bundler::{ confirm::UserOpConfirmationHandler, send::{ExternalBundlerSendHandler, ExternalBundlerSendJobData}, @@ -42,7 +44,7 @@ pub struct ExecutionRouter { pub webhook_queue: Arc>, pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, - pub eoa_executor_queue: Arc>>, + pub eoa_executor_queue: Arc>>, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -234,7 +236,7 @@ impl ExecutionRouter { self.execute_eip7702( &execution_request.execution_options.base, eip7702_execution_options, - &execution_request.webhook_options, + execution_request.webhook_options, &execution_request.params, rpc_credentials, signing_credential, @@ -263,7 +265,7 @@ impl ExecutionRouter { self.execute_eoa( &execution_request.execution_options.base, eoa_execution_options, - &execution_request.webhook_options, + execution_request.webhook_options, &execution_request.params, rpc_credentials, signing_credential, @@ -290,7 +292,7 @@ impl ExecutionRouter { &self, base_execution_options: &BaseExecutionOptions, erc4337_execution_options: &Erc4337ExecutionOptions, - webhook_options: &Option>, + webhook_options: &Vec, transactions: &[InnerTransaction], rpc_credentials: RpcCredentials, signing_credential: SigningCredential, @@ -339,7 +341,7 @@ impl ExecutionRouter { &self, base_execution_options: &BaseExecutionOptions, eip7702_execution_options: &Eip7702ExecutionOptions, - webhook_options: &Option>, + webhook_options: Vec, transactions: &[InnerTransaction], rpc_credentials: RpcCredentials, signing_credential: SigningCredential, @@ -350,7 +352,7 @@ impl ExecutionRouter { transactions: transactions.to_vec(), eoa_address: eip7702_execution_options.from, signing_credential, - webhook_options: webhook_options.clone(), + webhook_options, rpc_credentials, nonce: None, // Let the executor handle nonce generation }; @@ -384,7 +386,7 @@ impl ExecutionRouter { &self, base_execution_options: &BaseExecutionOptions, eoa_execution_options: &EoaExecutionOptions, - webhook_options: &Option>, + webhook_options: Vec, transactions: &[InnerTransaction], rpc_credentials: RpcCredentials, signing_credential: SigningCredential, @@ -404,7 +406,7 @@ impl ExecutionRouter { value: transaction.value, data: transaction.data.clone(), gas_limit: transaction.gas_limit, - webhook_options: webhook_options.clone(), + webhook_options: webhook_options.to_vec(), signing_credential: signing_credential.clone(), rpc_credentials, transaction_type_data: transaction.transaction_type_data.clone(), diff --git a/server/src/http/routes/admin/mod.rs b/server/src/http/routes/admin/mod.rs new file mode 100644 index 0000000..bb7845a --- /dev/null +++ b/server/src/http/routes/admin/mod.rs @@ -0,0 +1 @@ +pub mod queue; \ No newline at end of file diff --git a/server/src/http/routes/admin/queue.rs b/server/src/http/routes/admin/queue.rs new file mode 100644 index 0000000..0026dfc --- /dev/null +++ b/server/src/http/routes/admin/queue.rs @@ -0,0 +1,135 @@ +use axum::{ + debug_handler, + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde::Serialize; +use utoipa::ToSchema; + +use crate::http::{error::ApiEngineError, server::EngineServerState, types::SuccessResponse}; + +// ===== TYPES ===== + +#[derive(Debug, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct EmptyIdempotencySetResponse { + pub queue_name: String, + pub message: String, +} + +// ===== ROUTE HANDLER ===== + +#[utoipa::path( + post, + operation_id = "emptyQueueIdempotencySet", + path = "/admin/queue/{queue_name}/empty-idempotency-set", + tag = "Admin", + responses( + (status = 200, description = "Successfully emptied idempotency set", body = SuccessResponse, content_type = "application/json"), + ), + params( + ("queue_name" = String, Path, description = "Queue name - one of: webhook, external_bundler_send, userop_confirm, eoa_executor, eip7702_send, eip7702_confirm"), + ) +)] +/// Empty Queue Idempotency Set +/// +/// Empty the idempotency set for a specific queue. This removes all job IDs from the deduplication set, +/// allowing duplicate jobs to be submitted again. +#[debug_handler] +pub async fn empty_queue_idempotency_set( + State(state): State, + Path(queue_name): Path, +) -> Result { + tracing::info!( + queue_name = %queue_name, + "Processing empty idempotency set request" + ); + + // Map queue name to the appropriate queue and empty its idempotency set + let result = match queue_name.as_str() { + "webhook" => state.queue_manager.webhook_queue.empty_dedupe_set().await, + "external_bundler_send" => { + state + .queue_manager + .external_bundler_send_queue + .empty_dedupe_set() + .await + } + "userop_confirm" => { + state + .queue_manager + .userop_confirm_queue + .empty_dedupe_set() + .await + } + "eoa_executor" => { + state + .queue_manager + .eoa_executor_queue + .empty_dedupe_set() + .await + } + "eip7702_send" => { + state + .queue_manager + .eip7702_send_queue + .empty_dedupe_set() + .await + } + "eip7702_confirm" => { + state + .queue_manager + .eip7702_confirm_queue + .empty_dedupe_set() + .await + } + _ => { + return Err(ApiEngineError( + engine_core::error::EngineError::ValidationError { + message: format!( + "Invalid queue name '{}'. Valid options are: webhook, external_bundler_send, userop_confirm, eoa_executor, eip7702_send, eip7702_confirm", + queue_name + ), + }, + )); + } + }; + + // Handle the result + match result { + Ok(()) => { + tracing::info!( + queue_name = %queue_name, + "Successfully emptied idempotency set" + ); + + Ok(( + StatusCode::OK, + Json(SuccessResponse::new(EmptyIdempotencySetResponse { + queue_name: queue_name.clone(), + message: format!( + "Successfully emptied idempotency set for queue '{}'", + queue_name + ), + })), + )) + } + Err(e) => { + tracing::error!( + queue_name = %queue_name, + error = %e, + "Failed to empty idempotency set" + ); + + Err(ApiEngineError( + engine_core::error::EngineError::InternalError { + message: format!( + "Failed to empty idempotency set for queue '{}': {}", + queue_name, e + ), + }, + )) + } + } +} diff --git a/server/src/http/routes/contract_write.rs b/server/src/http/routes/contract_write.rs index b0853d8..acd4c41 100644 --- a/server/src/http/routes/contract_write.rs +++ b/server/src/http/routes/contract_write.rs @@ -56,7 +56,8 @@ pub struct WriteContractRequest { /// or as separate transactions if atomic batching is not supported pub params: Vec, - pub webhook_options: Option>, + #[serde(default)] + pub webhook_options: Vec, } // ===== CONVENIENCE METHODS ===== diff --git a/server/src/http/routes/mod.rs b/server/src/http/routes/mod.rs index 6c674db..9f84ca8 100644 --- a/server/src/http/routes/mod.rs +++ b/server/src/http/routes/mod.rs @@ -1,3 +1,4 @@ +pub mod admin; pub mod contract_encode; pub mod contract_read; pub mod contract_write; diff --git a/server/src/http/routes/sign_message.rs b/server/src/http/routes/sign_message.rs index 0306341..2a6605b 100644 --- a/server/src/http/routes/sign_message.rs +++ b/server/src/http/routes/sign_message.rs @@ -120,7 +120,7 @@ async fn sign_single_message( eoa_options.clone(), &message_input.message, message_input.format, - signing_credential.clone(), + &signing_credential, ) .await } diff --git a/server/src/http/routes/sign_typed_data.rs b/server/src/http/routes/sign_typed_data.rs index 427c3ba..a6f8a97 100644 --- a/server/src/http/routes/sign_typed_data.rs +++ b/server/src/http/routes/sign_typed_data.rs @@ -143,7 +143,7 @@ async fn sign_single_typed_data( // Direct EOA signing state .eoa_signer - .sign_typed_data(eoa_options.clone(), typed_data, signing_credential.clone()) + .sign_typed_data(eoa_options.clone(), typed_data, signing_credential) .await } SigningOptions::ERC4337(smart_account_options) => { diff --git a/server/src/http/server.rs b/server/src/http/server.rs index 44f09c7..18cd704 100644 --- a/server/src/http/server.rs +++ b/server/src/http/server.rs @@ -64,6 +64,9 @@ impl EngineServer { .routes(routes!( crate::http::routes::sign_typed_data::sign_typed_data )) + .routes(routes!( + crate::http::routes::admin::queue::empty_queue_idempotency_set + )) .layer(cors) .layer(TraceLayer::new_for_http()) .with_state(state); diff --git a/server/src/queue/manager.rs b/server/src/queue/manager.rs index 20fe806..272ee9b 100644 --- a/server/src/queue/manager.rs +++ b/server/src/queue/manager.rs @@ -5,7 +5,7 @@ use alloy::transports::http::reqwest; use engine_core::error::EngineError; use engine_executors::{ eip7702_executor::{confirm::Eip7702ConfirmationHandler, send::Eip7702SendHandler}, - eoa::EoaExecutorWorker, + eoa::EoaExecutorJobHandler, external_bundler::{ confirm::UserOpConfirmationHandler, deployment::{RedisDeploymentCache, RedisDeploymentLock}, @@ -22,7 +22,7 @@ pub struct QueueManager { pub webhook_queue: Arc>, pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, - pub eoa_executor_queue: Arc>>, + pub eoa_executor_queue: Arc>>, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -206,7 +206,7 @@ impl QueueManager { .arc(); // Create EOA executor queue - let eoa_executor_handler = EoaExecutorWorker { + let eoa_executor_handler = EoaExecutorJobHandler { chain_service: chain_service.clone(), eoa_signer: eoa_signer.clone(), webhook_queue: webhook_queue.clone(), diff --git a/thirdweb-core/src/iaw/mod.rs b/thirdweb-core/src/iaw/mod.rs index 9b07542..84b3617 100644 --- a/thirdweb-core/src/iaw/mod.rs +++ b/thirdweb-core/src/iaw/mod.rs @@ -214,9 +214,9 @@ impl IAWClient { /// Sign a message with an EOA pub async fn sign_message( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, - message: String, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, + message: &str, _from: Address, _chain_id: Option, format: Option, @@ -296,9 +296,9 @@ impl IAWClient { /// Sign a typed data structure with an EOA pub async fn sign_typed_data( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, - typed_data: TypedData, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, + typed_data: &TypedData, _from: Address, ) -> Result { // Get ThirdwebAuth headers for billing/authentication @@ -365,9 +365,9 @@ impl IAWClient { /// Sign a transaction with an EOA pub async fn sign_transaction( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, - transaction: EthereumTypedTransaction, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, + transaction: &EthereumTypedTransaction, ) -> Result { // Get ThirdwebAuth headers for billing/authentication let mut headers = thirdweb_auth.to_header_map()?; @@ -435,10 +435,10 @@ impl IAWClient { /// Sign an authorization with an EOA pub async fn sign_authorization( &self, - auth_token: AuthToken, - thirdweb_auth: ThirdwebAuth, + auth_token: &AuthToken, + thirdweb_auth: &ThirdwebAuth, _from: Address, - authorization: Authorization, + authorization: &Authorization, ) -> Result { // Get ThirdwebAuth headers for billing/authentication let mut headers = thirdweb_auth.to_header_map()?; diff --git a/twmq/src/lib.rs b/twmq/src/lib.rs index 177fc0e..96582bb 100644 --- a/twmq/src/lib.rs +++ b/twmq/src/lib.rs @@ -485,6 +485,7 @@ impl Queue { // Normal polling tick _ = interval.tick() => { let queue_clone = outer_queue_clone.clone(); + let queue_name = queue_clone.name(); // Check available permits for batch size let available_permits = semaphore.available_permits(); @@ -504,7 +505,8 @@ impl Queue { let job_id = job.id().to_string(); let handler_clone = handler_clone.clone(); - tokio::spawn(async move { + tokio::spawn( + async move { // Process job - note we don't pass a context here let result = handler_clone.process(&job).await; @@ -519,7 +521,7 @@ impl Queue { // Release permit when done drop(permit); - }.instrument(tracing::info_span!("twmq_worker", job_id))); + }.instrument(tracing::info_span!("twmq_worker", job_id, queue_name))); } } Err(e) => { From 6b70ab56cde3ae4ed457bbb5bb7ad0b82c19004b Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Tue, 15 Jul 2025 06:35:57 +0530 Subject: [PATCH 09/10] remove dbg, fix underflow --- executors/src/eoa/store/atomic.rs | 1 - executors/src/eoa/store/submitted.rs | 4 ++-- executors/src/eoa/worker/mod.rs | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/executors/src/eoa/store/atomic.rs b/executors/src/eoa/store/atomic.rs index 53707a5..50a74fc 100644 --- a/executors/src/eoa/store/atomic.rs +++ b/executors/src/eoa/store/atomic.rs @@ -577,7 +577,6 @@ impl AtomicEoaExecutorStore { results: Vec, webhook_queue: Arc>, ) -> Result { - dbg!("getting here", &results); self.execute_with_watch_and_retry(&ProcessBorrowedTransactions { results, keys: &self.keys, diff --git a/executors/src/eoa/store/submitted.rs b/executors/src/eoa/store/submitted.rs index 12ef378..82d3d2b 100644 --- a/executors/src/eoa/store/submitted.rs +++ b/executors/src/eoa/store/submitted.rs @@ -462,7 +462,7 @@ impl SafeRedisTransaction for CleanAndGetRecycledNonces<'_> { chain_id: self.keys.chain_id, }); }; - count - 1 + count.saturating_sub(1) } }; @@ -475,7 +475,7 @@ impl SafeRedisTransaction for CleanAndGetRecycledNonces<'_> { .filter(|nonce| *nonce < highest_submitted_nonce) .collect(); - return Ok((highest_submitted_nonce, recycled_nonces)); + Ok((highest_submitted_nonce, recycled_nonces)) } fn operation( diff --git a/executors/src/eoa/worker/mod.rs b/executors/src/eoa/worker/mod.rs index 0d36c16..f73ad36 100644 --- a/executors/src/eoa/worker/mod.rs +++ b/executors/src/eoa/worker/mod.rs @@ -398,8 +398,6 @@ impl EoaExecutorWorker { }) .collect(); - dbg!(&submission_results); - // TODO: Implement post-processing analysis for balance threshold updates and nonce resets // Currently we lose the granular error handling that was in the individual atomic operations. // Consider: From 9a29963745acddd182220c6ef0e51ffaaab95c60 Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Tue, 15 Jul 2025 07:17:05 +0530 Subject: [PATCH 10/10] fix event --- executors/src/eoa/events.rs | 2 +- server/src/http/routes/sign_message.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/executors/src/eoa/events.rs b/executors/src/eoa/events.rs index 5d89542..a8f152b 100644 --- a/executors/src/eoa/events.rs +++ b/executors/src/eoa/events.rs @@ -105,7 +105,7 @@ impl EoaExecutorEvent { BareWebhookNotificationEnvelope { transaction_id: self.transaction_id.clone(), executor_name: EXECUTOR_NAME.to_string(), - stage_name: EoaExecutorStage::Send.to_string(), + stage_name: EoaExecutorStage::Confirmation.to_string(), event_type: StageEvent::Nack, payload: SerializableNackData { error: EoaConfirmationError::TransactionReplaced { diff --git a/server/src/http/routes/sign_message.rs b/server/src/http/routes/sign_message.rs index 2a6605b..6bd1c0a 100644 --- a/server/src/http/routes/sign_message.rs +++ b/server/src/http/routes/sign_message.rs @@ -120,7 +120,7 @@ async fn sign_single_message( eoa_options.clone(), &message_input.message, message_input.format, - &signing_credential, + signing_credential, ) .await }