Add MagicBlockInstruction::ScheduleCommitDiffAndUndelegate to efficiently commit changes in delegated accounts

Author: snawaz

Cargo.lock

@@ -124,9 +124,7 @@ magicblock-config = { path = "./magicblock-config" }
 magicblock-config-helpers = { path = "./magicblock-config-helpers" }
 magicblock-config-macro = { path = "./magicblock-config-macro" }
 magicblock-core = { path = "./magicblock-core" }
-magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "aa1de56d90c", features = [
-  "no-entrypoint",
-] }
+magicblock-delegation-program = { path="../delegation-program", features = ["no-entrypoint"] }
 magicblock-geyser-plugin = { path = "./magicblock-geyser-plugin" }
 magicblock-ledger = { path = "./magicblock-ledger" }
 magicblock-metrics = { path = "./magicblock-metrics" }

Cargo.toml

@@ -357,6 +357,7 @@ impl<C: BaseIntentCommittor> ScheduledCommitsProcessorImpl<C> {
             excluded_pubkeys: intent_meta.excluded_pubkeys,
             feepayers: intent_meta.feepayers,
             requested_undelegation: intent_meta.requested_undelegation,
+            commit_diff: intent_meta.commit_diff,
         }
     }
 }
@@ -424,6 +425,7 @@ struct ScheduledBaseIntentMeta {
     feepayers: HashSet<FeePayerAccount>,
     intent_sent_transaction: Transaction,
     requested_undelegation: bool,
+    commit_diff: bool,
 }
 
 impl ScheduledBaseIntentMeta {
@@ -443,6 +445,7 @@ impl ScheduledBaseIntentMeta {
             feepayers,
             intent_sent_transaction: intent.action_sent_transaction.clone(),
             requested_undelegation: intent.is_undelegate(),
+            commit_diff: intent.is_commit_diff(),
         }
     }
 }

magicblock-accounts/src/scheduled_commits_processor.rs

@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
 
 use crate::slot::advance_slot_and_update_ledger;
 
+// ER chain
 pub fn init_slot_ticker<C: ScheduledCommitsProcessor>(
     bank: &Arc<Bank>,
     committor_processor: &Option<Arc<C>>,
@@ -71,6 +72,7 @@ pub fn init_slot_ticker<C: ScheduledCommitsProcessor>(
     })
 }
 
+// ER chain
 async fn handle_scheduled_commits<C: ScheduledCommitsProcessor>(
     bank: &Arc<Bank>,
     committor_processor: &Arc<C>,

magicblock-api/src/tickers.rs

@@ -1,6 +1,8 @@
 use borsh::{BorshDeserialize, BorshSerialize};
 use solana_pubkey::Pubkey;
 
+// this is deployed on Base chain as user-program
+
 #[derive(BorshSerialize, BorshDeserialize, Debug, Clone)]
 pub enum CommittorInstruction {
     /// Initializes the buffer and [Chunks] accounts which will be used to

magicblock-committor-program/src/instruction.rs

@@ -62,6 +62,8 @@ where
                 Err(err) => err,
             };
 
+            error!("execution_err: {}", execution_err);
+
             // Attempt patching
             let flow = self
                 .patch_strategy(
@@ -79,6 +81,7 @@ where
                     cleanup
                 }
                 ControlFlow::Break(()) => {
+                    info!("base_intent: {:?}", base_intent);
                     error!("Could not patch failed intent: {}", base_intent.id);
                     break (execution_err, transaction_strategy);
                 }

magicblock-committor-service/src/intent_executor/single_stage_executor.rs

@@ -30,6 +30,8 @@ pub struct LookupTables {
     pub released: Vec<Pubkey>,
 }
 
+// this is a part of ER.. and communicates with Base chain
+
 #[derive(Debug)]
 pub enum CommittorMessage {
     ReservePubkeysForCommittee {

magicblock-committor-service/src/service.rs

@@ -1,20 +1,33 @@
-use dlp::args::{CallHandlerArgs, CommitStateArgs};
+use dlp::{
+    args::{CallHandlerArgs, CommitDiffArgs, CommitStateArgs},
+    compute_diff,
+};
+use solana_account::ReadableAccount;
 use solana_pubkey::Pubkey;
-use solana_sdk::instruction::{AccountMeta, Instruction};
+use solana_rpc_client::rpc_client::RpcClient;
+use solana_sdk::{
+    commitment_config::CommitmentConfig,
+    instruction::{AccountMeta, Instruction},
+};
 
 #[cfg(test)]
 use crate::tasks::TaskStrategy;
-use crate::tasks::{
-    buffer_task::{BufferTask, BufferTaskType},
-    visitor::Visitor,
-    BaseActionTask, BaseTask, BaseTaskError, BaseTaskResult, CommitTask,
-    FinalizeTask, PreparationState, TaskType, UndelegateTask,
+use crate::{
+    config::ChainConfig,
+    tasks::{
+        buffer_task::{BufferTask, BufferTaskType},
+        visitor::Visitor,
+        BaseActionTask, BaseTask, BaseTaskError, BaseTaskResult, CommitTask,
+        FinalizeTask, PreparationState, TaskType, UndelegateTask,
+    },
+    ComputeBudgetConfig,
 };
 
 /// Task that will be executed on Base layer via arguments
 #[derive(Clone)]
 pub enum ArgsTaskType {
     Commit(CommitTask),
+    CommitDiff(CommitTask),
     Finalize(FinalizeTask),
     Undelegate(UndelegateTask), // Special action really
     BaseAction(BaseActionTask),
@@ -58,6 +71,55 @@ impl BaseTask for ArgsTask {
                     args,
                 )
             }
+            ArgsTaskType::CommitDiff(value) => {
+                let chain_config =
+                    ChainConfig::local(ComputeBudgetConfig::new(1_000_000));
+
+                let rpc_client = RpcClient::new_with_commitment(
+                    chain_config.rpc_uri.to_string(),
+                    CommitmentConfig {
+                        commitment: chain_config.commitment,
+                    },
+                );
+
+                let account = match rpc_client
+                    .get_account(&value.committed_account.pubkey)
+                {
+                    Ok(account) => account,
+                    Err(e) => {
+                        log::warn!("Fallback to commit_state and send full-bytes, as rpc failed to fetch the delegated-account from base chain: {}", e);
+                        let args = CommitStateArgs {
+                            nonce: value.commit_id,
+                            lamports: value.committed_account.account.lamports,
+                            data: value.committed_account.account.data.clone(),
+                            allow_undelegation: value.allow_undelegation,
+                        };
+                        return dlp::instruction_builder::commit_state(
+                            *validator,
+                            value.committed_account.pubkey,
+                            value.committed_account.account.owner,
+                            args,
+                        );
+                    }
+                };
+
+                let args = CommitDiffArgs {
+                    nonce: value.commit_id,
+                    lamports: value.committed_account.account.lamports,
+                    diff: compute_diff(
+                        account.data(),
+                        value.committed_account.account.data(),
+                    ),
+                    allow_undelegation: value.allow_undelegation,
+                };
+                log::warn!("DIFF computed: {:?}", args.diff);
+                dlp::instruction_builder::commit_diff(
+                    *validator,
+                    value.committed_account.pubkey,
+                    value.committed_account.account.owner,
+                    args,
+                )
+            }
             ArgsTaskType::Finalize(value) => {
                 dlp::instruction_builder::finalize(
                     *validator,
@@ -106,6 +168,9 @@ impl BaseTask for ArgsTask {
                     BufferTaskType::Commit(value),
                 )))
             }
+            ArgsTaskType::CommitDiff(_) => {
+                panic!("ArgsTaskType::CommitDiff not handled")
+            }
             ArgsTaskType::BaseAction(_)
             | ArgsTaskType::Finalize(_)
             | ArgsTaskType::Undelegate(_) => Err(self),
@@ -132,6 +197,7 @@ impl BaseTask for ArgsTask {
     fn compute_units(&self) -> u32 {
         match &self.task_type {
             ArgsTaskType::Commit(_) => 65_000,
+            ArgsTaskType::CommitDiff(_) => 65_000,
             ArgsTaskType::BaseAction(task) => task.action.compute_units,
             ArgsTaskType::Undelegate(_) => 70_000,
             ArgsTaskType::Finalize(_) => 40_000,
@@ -146,6 +212,9 @@ impl BaseTask for ArgsTask {
     fn task_type(&self) -> TaskType {
         match &self.task_type {
             ArgsTaskType::Commit(_) => TaskType::Commit,
+            // TODO (snawaz): What should we use here? Commit (in the sense of "category of task"), or add a
+            // new variant "CommitDiff" to indicate a specific instruction?
+            ArgsTaskType::CommitDiff(_) => TaskType::Commit,
             ArgsTaskType::BaseAction(_) => TaskType::Action,
             ArgsTaskType::Undelegate(_) => TaskType::Undelegate,
             ArgsTaskType::Finalize(_) => TaskType::Finalize,
@@ -158,7 +227,9 @@ impl BaseTask for ArgsTask {
     }
 
     fn reset_commit_id(&mut self, commit_id: u64) {
+        // TODO (snawaz): handle CommitDiff as well?
         let ArgsTaskType::Commit(commit_task) = &mut self.task_type else {
+            log::error!("reset_commit_id");
             return;
         };
 

magicblock-committor-service/src/tasks/args_task.rs

@@ -53,6 +53,8 @@ pub enum TaskStrategy {
 pub trait BaseTask: Send + Sync + DynClone {
     /// Gets all pubkeys that involved in Task's instruction
     fn involved_accounts(&self, validator: &Pubkey) -> Vec<Pubkey> {
+        // TODO (snawaz): rewrite it.
+        // currently it is slow as it discards heavy computations and memory allocations.
         self.instruction(validator)
             .accounts
             .iter()