From db432c21016ed2a1da10399101ad3521731f551e Mon Sep 17 00:00:00 2001 From: curryxbo Date: Mon, 17 Mar 2025 14:44:09 +0800 Subject: [PATCH 1/7] Enhance blob fetching functionality, add ForceGetAllBlobs switch for QA testing --- node/derivation/beacon.go | 30 +++++++++++ node/derivation/derivation.go | 93 ++++++++++++++++++++++++++++++----- 2 files changed, 112 insertions(+), 11 deletions(-) diff --git a/node/derivation/beacon.go b/node/derivation/beacon.go index ab35a373..ec6fd97a 100644 --- a/node/derivation/beacon.go +++ b/node/derivation/beacon.go @@ -241,3 +241,33 @@ func dataAndHashesFromTxs(txs types.Transactions, targetTx *types.Transaction) [ } return hashes } + +// GetBlobSidecarsEnhanced is an enhanced version of GetBlobSidecars method, combining two approaches to fetch blob data +// If the first method fails or returns no blobs, it will try the second method +func (cl *L1BeaconClient) GetBlobSidecarsEnhanced(ctx context.Context, ref L1BlockRef, hashes []IndexedBlobHash) ([]*BlobSidecar, error) { + // First try using the original GetBlobSidecars method + blobSidecars, err := cl.GetBlobSidecars(ctx, ref, hashes) + if err != nil || len(blobSidecars) == 0 { + // If failed or no blobs retrieved, try the second method + slotFn, err := cl.GetTimeToSlotFn(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get timeToSlotFn: %w", err) + } + + slot, err := slotFn(ref.Time) + if err != nil { + return nil, fmt.Errorf("failed to calculate slot: %w", err) + } + + // Build request URL and use apiReq method directly + method := fmt.Sprintf("%s%d", sidecarsMethodPrefix, slot) + var blobResp APIGetBlobSidecarsResponse + if err := cl.apiReq(ctx, &blobResp, method); err != nil { + return nil, fmt.Errorf("failed to request blob sidecars: %w", err) + } + + return blobResp.Data, nil + } + + return blobSidecars, nil +} diff --git a/node/derivation/derivation.go b/node/derivation/derivation.go index a3fc43fb..17ae4ec4 100644 --- a/node/derivation/derivation.go +++ b/node/derivation/derivation.go @@ -13,8 +13,10 @@ import ( "github.com/morph-l2/go-ethereum/accounts/abi" "github.com/morph-l2/go-ethereum/accounts/abi/bind" "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" eth "github.com/morph-l2/go-ethereum/core/types" "github.com/morph-l2/go-ethereum/crypto" + "github.com/morph-l2/go-ethereum/crypto/kzg4844" geth "github.com/morph-l2/go-ethereum/eth" "github.com/morph-l2/go-ethereum/ethclient" "github.com/morph-l2/go-ethereum/ethclient/authclient" @@ -32,6 +34,10 @@ import ( var ( RollupEventTopic = "CommitBatch(uint256,bytes32)" RollupEventTopicHash = crypto.Keccak256Hash([]byte(RollupEventTopic)) + + // ForceGetAllBlobs controls whether to force using the method that gets all blobs + // Set to true for QA testing, false for production + ForceGetAllBlobs = true ) type Derivation struct { @@ -297,26 +303,91 @@ func (d *Derivation) fetchRollupDataByTxHash(txHash common.Hash, blockNumber uin if err != nil { return nil, err } - // query blob - block, err := d.l1Client.BlockByNumber(d.ctx, big.NewInt(int64(blockNumber))) - if err != nil { - return nil, err - } - indexedBlobHashes := dataAndHashesFromTxs(block.Transactions(), tx) + + // Get block header to retrieve timestamp header, err := d.l1Client.HeaderByNumber(d.ctx, big.NewInt(int64(blockNumber))) if err != nil { return nil, err } - var bts eth.BlobTxSidecar - if len(indexedBlobHashes) != 0 { - bts, err = d.l1BeaconClient.GetBlobSidecar(context.Background(), L1BlockRef{ + + // Get transaction blob hashes + blobHashes := tx.BlobHashes() + if len(blobHashes) > 0 { + d.logger.Info("Transaction contains blobs", "txHash", txHash, "blobCount", len(blobHashes)) + + // Initialize indexedBlobHashes as nil + var indexedBlobHashes []IndexedBlobHash + + // Only try to build IndexedBlobHash array if not forcing get all blobs + if !ForceGetAllBlobs { + // Try to get the block to build IndexedBlobHash array + block, err := d.l1Client.BlockByNumber(d.ctx, big.NewInt(int64(blockNumber))) + if err == nil { + // Successfully got the block, now build IndexedBlobHash array + d.logger.Info("Building IndexedBlobHash array from block", "blockNumber", blockNumber) + indexedBlobHashes = dataAndHashesFromTxs(block.Transactions(), tx) + d.logger.Info("Built IndexedBlobHash array", "count", len(indexedBlobHashes)) + } else { + d.logger.Info("Failed to get block, will try fetching all blobs", "blockNumber", blockNumber, "error", err) + } + } else { + d.logger.Info("ForceGetAllBlobs is enabled, fetching all blobs for testing") + } + + // Get all blobs corresponding to this timestamp + blobSidecars, err := d.l1BeaconClient.GetBlobSidecarsEnhanced(d.ctx, L1BlockRef{ Time: header.Time, }, indexedBlobHashes) if err != nil { - return nil, fmt.Errorf("getBlockSidecar error:%v", err) + d.logger.Error("Failed to get blobs, continuing processing", "error", err) + } else if len(blobSidecars) > 0 { + // Create blob sidecar + var blobTxSidecar eth.BlobTxSidecar + matchedCount := 0 + + // Match blobs + for _, sidecar := range blobSidecars { + var commitment kzg4844.Commitment + copy(commitment[:], sidecar.KZGCommitment[:]) + versionedHash := KZGToVersionedHash(commitment) + + for _, expectedHash := range blobHashes { + if bytes.Equal(versionedHash[:], expectedHash[:]) { + matchedCount++ + d.logger.Info("Found matching blob", "index", sidecar.Index, "hash", versionedHash.Hex()) + + // Decode and process blob data + var blob Blob + b, err := hexutil.Decode(sidecar.Blob) + if err != nil { + d.logger.Error("Failed to decode blob data", "error", err) + continue + } + copy(blob[:], b) + + // Verify blob + if err := VerifyBlobProof(&blob, commitment, kzg4844.Proof(sidecar.KZGProof)); err != nil { + d.logger.Error("Blob verification failed", "error", err) + continue + } + + // Add to sidecar + blobTxSidecar.Blobs = append(blobTxSidecar.Blobs, *blob.KZGBlob()) + blobTxSidecar.Commitments = append(blobTxSidecar.Commitments, commitment) + blobTxSidecar.Proofs = append(blobTxSidecar.Proofs, kzg4844.Proof(sidecar.KZGProof)) + break + } + } + } + + d.logger.Info("Blob matching results", "matched", matchedCount, "expected", len(blobHashes)) + if matchedCount > 0 { + batch.Sidecar = blobTxSidecar + } } } - batch.Sidecar = bts + + // Get L2 height l2Height, err := d.l2Client.BlockNumber(d.ctx) if err != nil { return nil, fmt.Errorf("query l2 block number error:%v", err) From 0801d2e2c6df85a734dd8d370f485491d3102be9 Mon Sep 17 00:00:00 2001 From: curryxbo Date: Thu, 20 Mar 2025 14:59:16 +0800 Subject: [PATCH 2/7] add test log --- node/derivation/batch_info.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/derivation/batch_info.go b/node/derivation/batch_info.go index 4c1470aa..dc9c7490 100644 --- a/node/derivation/batch_info.go +++ b/node/derivation/batch_info.go @@ -181,6 +181,10 @@ func (bi *BatchInfo) ParseBatch(batch geth.RPCRollupBatch) error { if err := block.Decode(rawBlockContexts[i*60 : i*60+60]); err != nil { return fmt.Errorf("decode chunk block context error:%v", err) } + fmt.Println("block.", block.Number) + fmt.Println("block.txsNum:", block.txsNum) + fmt.Println("block.l1MsgNum:", block.l1MsgNum) + fmt.Println("len(batch.Sidecar.Blobs):", len(batch.Sidecar.Blobs)) if i == 0 { bi.firstBlockNumber = block.Number } From 6fcea1860db1314f4ffa71476ee0770bdc5319b0 Mon Sep 17 00:00:00 2001 From: curryxbo Date: Thu, 20 Mar 2025 15:44:16 +0800 Subject: [PATCH 3/7] Fix GetBlobSidecarsEnhanced to properly handle ForceGetAllBlobs flag --- node/derivation/beacon.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/node/derivation/beacon.go b/node/derivation/beacon.go index ec6fd97a..cc3d5990 100644 --- a/node/derivation/beacon.go +++ b/node/derivation/beacon.go @@ -242,10 +242,35 @@ func dataAndHashesFromTxs(txs types.Transactions, targetTx *types.Transaction) [ return hashes } +// Note: ForceGetAllBlobs is defined in derivation.go in the same package + // GetBlobSidecarsEnhanced is an enhanced version of GetBlobSidecars method, combining two approaches to fetch blob data // If the first method fails or returns no blobs, it will try the second method func (cl *L1BeaconClient) GetBlobSidecarsEnhanced(ctx context.Context, ref L1BlockRef, hashes []IndexedBlobHash) ([]*BlobSidecar, error) { - // First try using the original GetBlobSidecars method + // Check if ForceGetAllBlobs is true or hashes is nil/empty + if ForceGetAllBlobs || hashes == nil || len(hashes) == 0 { + // Skip the first method and directly use the second method + slotFn, err := cl.GetTimeToSlotFn(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get timeToSlotFn: %w", err) + } + + slot, err := slotFn(ref.Time) + if err != nil { + return nil, fmt.Errorf("failed to calculate slot: %w", err) + } + + // Build request URL and use apiReq method directly + method := fmt.Sprintf("%s%d", sidecarsMethodPrefix, slot) + var blobResp APIGetBlobSidecarsResponse + if err := cl.apiReq(ctx, &blobResp, method); err != nil { + return nil, fmt.Errorf("failed to request blob sidecars: %w", err) + } + + return blobResp.Data, nil + } + + // First try using the original GetBlobSidecars method if ForceGetAllBlobs is false blobSidecars, err := cl.GetBlobSidecars(ctx, ref, hashes) if err != nil || len(blobSidecars) == 0 { // If failed or no blobs retrieved, try the second method From 73118bcab804881ce0327289fea2e98deca5c411 Mon Sep 17 00:00:00 2001 From: curryxbo Date: Thu, 20 Mar 2025 17:36:00 +0800 Subject: [PATCH 4/7] add test log --- node/derivation/batch_info.go | 14 +- node/derivation/beacon.go | 25 +- node/derivation/beacon_test.go | 1 + node/derivation/blob/test/fetch_blob_test.go | 298 +++++++++++++++++++ node/derivation/derivation.go | 7 +- 5 files changed, 311 insertions(+), 34 deletions(-) create mode 100644 node/derivation/beacon_test.go create mode 100644 node/derivation/blob/test/fetch_blob_test.go diff --git a/node/derivation/batch_info.go b/node/derivation/batch_info.go index dc9c7490..95ed86f3 100644 --- a/node/derivation/batch_info.go +++ b/node/derivation/batch_info.go @@ -80,6 +80,9 @@ func (bi *BatchInfo) TxNum() uint64 { // ParseBatch This method is externally referenced for parsing Batch func (bi *BatchInfo) ParseBatch(batch geth.RPCRollupBatch) error { + if len(batch.Sidecar.Blobs) == 0 { + return fmt.Errorf("blobs length can not be zero") + } parentBatchHeader := types.BatchHeaderBytes(batch.ParentBatchHeader) parentBatchIndex, err := parentBatchHeader.BatchIndex() if err != nil { @@ -104,9 +107,6 @@ func (bi *BatchInfo) ParseBatch(batch geth.RPCRollupBatch) error { return fmt.Errorf("decode batch header version error:%v", err) } if parentVersion == 0 { - if len(batch.Sidecar.Blobs) == 0 { - return fmt.Errorf("blobs length can not be zero") - } blobData, err := types.RetrieveBlobBytes(&batch.Sidecar.Blobs[0]) if err != nil { return err @@ -204,11 +204,9 @@ func (bi *BatchInfo) ParseBatch(batch geth.RPCRollupBatch) error { } var txs []*eth.Transaction var err error - if len(batch.Sidecar.Blobs) != 0 { - txs, err = tq.dequeue(int(block.txsNum) - int(block.l1MsgNum)) - if err != nil { - return fmt.Errorf("decode txsPayload error:%v", err) - } + txs, err = tq.dequeue(int(block.txsNum) - int(block.l1MsgNum)) + if err != nil { + return fmt.Errorf("decode txsPayload error:%v", err) } txsNum += uint64(block.txsNum) l1MsgNum += uint64(block.l1MsgNum) diff --git a/node/derivation/beacon.go b/node/derivation/beacon.go index cc3d5990..d94101d6 100644 --- a/node/derivation/beacon.go +++ b/node/derivation/beacon.go @@ -247,30 +247,7 @@ func dataAndHashesFromTxs(txs types.Transactions, targetTx *types.Transaction) [ // GetBlobSidecarsEnhanced is an enhanced version of GetBlobSidecars method, combining two approaches to fetch blob data // If the first method fails or returns no blobs, it will try the second method func (cl *L1BeaconClient) GetBlobSidecarsEnhanced(ctx context.Context, ref L1BlockRef, hashes []IndexedBlobHash) ([]*BlobSidecar, error) { - // Check if ForceGetAllBlobs is true or hashes is nil/empty - if ForceGetAllBlobs || hashes == nil || len(hashes) == 0 { - // Skip the first method and directly use the second method - slotFn, err := cl.GetTimeToSlotFn(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get timeToSlotFn: %w", err) - } - - slot, err := slotFn(ref.Time) - if err != nil { - return nil, fmt.Errorf("failed to calculate slot: %w", err) - } - - // Build request URL and use apiReq method directly - method := fmt.Sprintf("%s%d", sidecarsMethodPrefix, slot) - var blobResp APIGetBlobSidecarsResponse - if err := cl.apiReq(ctx, &blobResp, method); err != nil { - return nil, fmt.Errorf("failed to request blob sidecars: %w", err) - } - - return blobResp.Data, nil - } - - // First try using the original GetBlobSidecars method if ForceGetAllBlobs is false + // First try using the original GetBlobSidecars method blobSidecars, err := cl.GetBlobSidecars(ctx, ref, hashes) if err != nil || len(blobSidecars) == 0 { // If failed or no blobs retrieved, try the second method diff --git a/node/derivation/beacon_test.go b/node/derivation/beacon_test.go new file mode 100644 index 00000000..2f79aa0c --- /dev/null +++ b/node/derivation/beacon_test.go @@ -0,0 +1 @@ +package derivation diff --git a/node/derivation/blob/test/fetch_blob_test.go b/node/derivation/blob/test/fetch_blob_test.go new file mode 100644 index 00000000..25267cdf --- /dev/null +++ b/node/derivation/blob/test/fetch_blob_test.go @@ -0,0 +1,298 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" + "github.com/morph-l2/go-ethereum/ethclient" + "github.com/morph-l2/go-ethereum/params" +) + +const ( + GENESIS_TIMESTAMP = uint64(1695902400) // Holesky 创世区块时间戳 + SECONDS_PER_SLOT = uint64(12) // 每个时隙的秒数 +) + +// HTTP 接口 +type HTTP interface { + Get(ctx context.Context, path string, headers http.Header) (*http.Response, error) +} + +// 基本的HTTP客户端实现 +type BasicHTTPClient struct { + endpoint string + client *http.Client +} + +func NewBasicHTTPClient(endpoint string) *BasicHTTPClient { + // 确保endpoint以斜杠结尾 + trimmedEndpoint := strings.TrimSuffix(endpoint, "/") + "/" + return &BasicHTTPClient{ + endpoint: trimmedEndpoint, + client: &http.Client{Timeout: 30 * time.Second}, + } +} + +func (cl *BasicHTTPClient) Get(ctx context.Context, p string, headers http.Header) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, cl.endpoint+p, nil) + if err != nil { + return nil, fmt.Errorf("%w: 构建请求失败", err) + } + for k, values := range headers { + for _, v := range values { + req.Header.Add(k, v) + } + } + return cl.client.Do(req) +} + +// L1BeaconClient +type L1BeaconClient struct { + cl HTTP + timeToSlot func(timestamp uint64) (uint64, error) +} + +// 创建新的beacon客户端 +func NewL1BeaconClient(cl HTTP) *L1BeaconClient { + // 为Holesky网络设置固定的genesis时间戳 + timeToSlotFn := func(timestamp uint64) (uint64, error) { + if timestamp < GENESIS_TIMESTAMP { + return 0, fmt.Errorf("提供的时间戳(%v)早于创世时间戳(%v)", timestamp, GENESIS_TIMESTAMP) + } + return (timestamp - GENESIS_TIMESTAMP) / SECONDS_PER_SLOT, nil + } + + return &L1BeaconClient{cl: cl, timeToSlot: timeToSlotFn} +} + +func (cl *L1BeaconClient) apiReq(ctx context.Context, dest any, method string) error { + headers := http.Header{} + headers.Add("Accept", "application/json") + resp, err := cl.cl.Get(ctx, method, headers) + if err != nil { + return fmt.Errorf("%w: http Get失败", err) + } + if resp.StatusCode != http.StatusOK { + errMsg, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return fmt.Errorf("请求失败，状态码 %d: %s", resp.StatusCode, string(errMsg)) + } + if err := json.NewDecoder(resp.Body).Decode(dest); err != nil { + _ = resp.Body.Close() + return err + } + if err := resp.Body.Close(); err != nil { + return fmt.Errorf("%w: 关闭响应体失败", err) + } + return nil +} + +// BlobSidecar 结构体 +type BlobSidecar struct { + BlockRoot []byte `json:"block_root"` + Index json.Number `json:"index"` + Slot json.Number `json:"slot"` + Blob string `json:"blob"` + KZGCommitment []byte `json:"kzg_commitment"` + KZGProof []byte `json:"kzg_proof"` +} + +// API响应 +type APIGetBlobSidecarsResponse struct { + Data []*BlobSidecar `json:"data"` +} + +// 索引blob哈希 +type IndexedBlobHash struct { + Index uint64 // 在区块中的绝对索引 + Hash common.Hash // blob的哈希，用于一致性检查 +} + +// KZGToVersionedHash 将KZG承诺转换为带版本的哈希 +func KZGToVersionedHash(commitment []byte) common.Hash { + // EIP-4844规范: + // def kzg_to_versioned_hash(commitment: KZGCommitment) -> VersionedHash: + // return VERSIONED_HASH_VERSION_KZG + sha256(commitment)[1:] + var out common.Hash + h := sha256.New() + h.Write(commitment) + copy(out[:], h.Sum(nil)) + out[0] = params.BlobTxHashVersion + return out +} + +// GetBlobSidecarsEnhanced 获取blob sidecars的增强方法 +func (cl *L1BeaconClient) GetBlobSidecarsEnhanced(ctx context.Context, blockTime uint64, hashes []IndexedBlobHash) ([]*BlobSidecar, error) { + // 计算slot + slot, err := cl.timeToSlot(blockTime) + if err != nil { + return nil, fmt.Errorf("计算slot失败: %w", err) + } + + // 首先尝试使用带索引的URL + if len(hashes) > 0 { + builder := strings.Builder{} + builder.WriteString("eth/v1/beacon/blob_sidecars/") + builder.WriteString(strconv.FormatUint(slot, 10)) + builder.WriteRune('?') + v := url.Values{} + + for i := range hashes { + v.Add("indices", strconv.FormatUint(hashes[i].Index, 10)) + } + builder.WriteString(v.Encode()) + + var resp APIGetBlobSidecarsResponse + if err := cl.apiReq(ctx, &resp, builder.String()); err == nil && len(resp.Data) > 0 { + return resp.Data, nil + } + } + + // 如果第一种方法失败或者没有指定索引，尝试第二种方法 + method := fmt.Sprintf("eth/v1/beacon/blob_sidecars/%d", slot) + var blobResp APIGetBlobSidecarsResponse + if err := cl.apiReq(ctx, &blobResp, method); err != nil { + return nil, fmt.Errorf("请求blob sidecars失败: %w", err) + } + + return blobResp.Data, nil +} + +func main() { + // 设置日志 + logger := log.New(os.Stdout, "[BlobTest] ", log.LstdFlags|log.Lshortfile) + logger.Println("开始增强版blob获取测试") + + // 创建上下文 + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // 节点URL和交易哈希 + nodeUrl := "https://omniscient-serene-sanctuary.ethereum-holesky.quiknode.pro/37083bb58e379efb02453b2f07fe4853faf18d03/" + txHashHex := "0x2ada86d083a7b1af5e59c916327ef2255ccb9068834bbf13efa5527ba411718f" + + // 创建ETH客户端 + ethClient, err := ethclient.Dial(nodeUrl) + if err != nil { + logger.Fatalf("连接ETH客户端失败: %v", err) + } + + // 获取交易 + txHash := common.HexToHash(txHashHex) + tx, isPending, err := ethClient.TransactionByHash(ctx, txHash) + if err != nil { + logger.Fatalf("获取交易失败: %v", err) + } + if isPending { + logger.Fatalf("交易仍处于挂起状态") + } + + // 获取交易收据，确定区块号 + receipt, err := ethClient.TransactionReceipt(ctx, txHash) + if err != nil { + logger.Fatalf("获取交易收据失败: %v", err) + } + + blockNumber := receipt.BlockNumber.Uint64() + logger.Printf("交易位于区块: %d", blockNumber) + + // 获取区块 + block, err := ethClient.BlockByNumber(ctx, receipt.BlockNumber) + if err != nil { + logger.Fatalf("获取区块失败: %v", err) + } + + blockTime := block.Time() + logger.Printf("区块时间戳: %d", blockTime) + + // 获取blob哈希 + blobHashes := tx.BlobHashes() + if len(blobHashes) == 0 { + logger.Fatalf("交易不包含blob") + } + logger.Printf("交易包含 %d 个blob", len(blobHashes)) + + for i, hash := range blobHashes { + logger.Printf("Blob #%d 哈希: %s", i, hash.Hex()) + } + + // 创建beacon客户端 + beaconEndpoint := "https://beacon.holesky.ethpandaops.io" + beaconClient := NewL1BeaconClient(NewBasicHTTPClient(beaconEndpoint)) + + // 构建索引哈希数组 + var indexedHashes []IndexedBlobHash + for i, hash := range blobHashes { + indexedHashes = append(indexedHashes, IndexedBlobHash{ + Index: uint64(i), + Hash: hash, + }) + } + + // 调用GetBlobSidecarsEnhanced + blobSidecars, err := beaconClient.GetBlobSidecarsEnhanced(ctx, blockTime, indexedHashes) + if err != nil { + logger.Fatalf("获取blob sidecars失败: %v", err) + } + + if len(blobSidecars) == 0 { + logger.Fatalf("未返回blob sidecars") + } + + logger.Printf("成功获取 %d 个blob sidecar", len(blobSidecars)) + + // 检查每个blob sidecar + matchedCount := 0 + for i, sidecar := range blobSidecars { + // 从索引获取slot + slotNum, _ := sidecar.Slot.Int64() + indexNum, _ := sidecar.Index.Int64() + + logger.Printf("Blob Sidecar #%d: Slot=%d, Index=%d", i, slotNum, indexNum) + + // 如果有承诺数据，验证与哈希的匹配 + if len(sidecar.KZGCommitment) > 0 { + versionedHash := KZGToVersionedHash(sidecar.KZGCommitment) + logger.Printf(" KZG承诺转换为哈希: %s", versionedHash.Hex()) + + // 检查是否匹配任何期望的哈希 + for _, expectedHash := range blobHashes { + if bytes.Equal(versionedHash[:], expectedHash[:]) { + matchedCount++ + logger.Printf(" ✅ 匹配blob哈希: %s", expectedHash.Hex()) + + // 解码并处理blob数据 + blobData, err := hexutil.Decode(sidecar.Blob) + if err != nil { + logger.Printf(" ❌ 解码blob数据失败: %v", err) + continue + } + + // 打印blob数据的前200字节作为预览 + previewLen := 200 + if len(blobData) < previewLen { + previewLen = len(blobData) + } + logger.Printf(" Blob数据预览 (%d字节): 0x%x...", len(blobData), blobData[:previewLen]) + break + } + } + } + } + + logger.Printf("总结: 找到 %d 个匹配的blob (共期望 %d 个)", matchedCount, len(blobHashes)) + logger.Println("测试完成") +} diff --git a/node/derivation/derivation.go b/node/derivation/derivation.go index 17ae4ec4..07d84681 100644 --- a/node/derivation/derivation.go +++ b/node/derivation/derivation.go @@ -339,8 +339,9 @@ func (d *Derivation) fetchRollupDataByTxHash(txHash common.Hash, blockNumber uin Time: header.Time, }, indexedBlobHashes) if err != nil { - d.logger.Error("Failed to get blobs, continuing processing", "error", err) - } else if len(blobSidecars) > 0 { + return nil, fmt.Errorf("failed to get blobs, continuing processing:%v", err) + } + if len(blobSidecars) > 0 { // Create blob sidecar var blobTxSidecar eth.BlobTxSidecar matchedCount := 0 @@ -384,6 +385,8 @@ func (d *Derivation) fetchRollupDataByTxHash(txHash common.Hash, blockNumber uin if matchedCount > 0 { batch.Sidecar = blobTxSidecar } + } else { + return nil, fmt.Errorf("not matched blob,txHash:%v,blockNumber:%v", txHash, blockNumber) } } From cefecac67cd178776816d65dd0316f63f7c25ef2 Mon Sep 17 00:00:00 2001 From: curryxbo Date: Thu, 20 Mar 2025 17:36:52 +0800 Subject: [PATCH 5/7] remove files --- node/derivation/blob/test/fetch_blob_test.go | 298 ------------------- 1 file changed, 298 deletions(-) delete mode 100644 node/derivation/blob/test/fetch_blob_test.go diff --git a/node/derivation/blob/test/fetch_blob_test.go b/node/derivation/blob/test/fetch_blob_test.go deleted file mode 100644 index 25267cdf..00000000 --- a/node/derivation/blob/test/fetch_blob_test.go +++ /dev/null @@ -1,298 +0,0 @@ -package main - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/morph-l2/go-ethereum/common" - "github.com/morph-l2/go-ethereum/common/hexutil" - "github.com/morph-l2/go-ethereum/ethclient" - "github.com/morph-l2/go-ethereum/params" -) - -const ( - GENESIS_TIMESTAMP = uint64(1695902400) // Holesky 创世区块时间戳 - SECONDS_PER_SLOT = uint64(12) // 每个时隙的秒数 -) - -// HTTP 接口 -type HTTP interface { - Get(ctx context.Context, path string, headers http.Header) (*http.Response, error) -} - -// 基本的HTTP客户端实现 -type BasicHTTPClient struct { - endpoint string - client *http.Client -} - -func NewBasicHTTPClient(endpoint string) *BasicHTTPClient { - // 确保endpoint以斜杠结尾 - trimmedEndpoint := strings.TrimSuffix(endpoint, "/") + "/" - return &BasicHTTPClient{ - endpoint: trimmedEndpoint, - client: &http.Client{Timeout: 30 * time.Second}, - } -} - -func (cl *BasicHTTPClient) Get(ctx context.Context, p string, headers http.Header) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, cl.endpoint+p, nil) - if err != nil { - return nil, fmt.Errorf("%w: 构建请求失败", err) - } - for k, values := range headers { - for _, v := range values { - req.Header.Add(k, v) - } - } - return cl.client.Do(req) -} - -// L1BeaconClient -type L1BeaconClient struct { - cl HTTP - timeToSlot func(timestamp uint64) (uint64, error) -} - -// 创建新的beacon客户端 -func NewL1BeaconClient(cl HTTP) *L1BeaconClient { - // 为Holesky网络设置固定的genesis时间戳 - timeToSlotFn := func(timestamp uint64) (uint64, error) { - if timestamp < GENESIS_TIMESTAMP { - return 0, fmt.Errorf("提供的时间戳(%v)早于创世时间戳(%v)", timestamp, GENESIS_TIMESTAMP) - } - return (timestamp - GENESIS_TIMESTAMP) / SECONDS_PER_SLOT, nil - } - - return &L1BeaconClient{cl: cl, timeToSlot: timeToSlotFn} -} - -func (cl *L1BeaconClient) apiReq(ctx context.Context, dest any, method string) error { - headers := http.Header{} - headers.Add("Accept", "application/json") - resp, err := cl.cl.Get(ctx, method, headers) - if err != nil { - return fmt.Errorf("%w: http Get失败", err) - } - if resp.StatusCode != http.StatusOK { - errMsg, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - return fmt.Errorf("请求失败，状态码 %d: %s", resp.StatusCode, string(errMsg)) - } - if err := json.NewDecoder(resp.Body).Decode(dest); err != nil { - _ = resp.Body.Close() - return err - } - if err := resp.Body.Close(); err != nil { - return fmt.Errorf("%w: 关闭响应体失败", err) - } - return nil -} - -// BlobSidecar 结构体 -type BlobSidecar struct { - BlockRoot []byte `json:"block_root"` - Index json.Number `json:"index"` - Slot json.Number `json:"slot"` - Blob string `json:"blob"` - KZGCommitment []byte `json:"kzg_commitment"` - KZGProof []byte `json:"kzg_proof"` -} - -// API响应 -type APIGetBlobSidecarsResponse struct { - Data []*BlobSidecar `json:"data"` -} - -// 索引blob哈希 -type IndexedBlobHash struct { - Index uint64 // 在区块中的绝对索引 - Hash common.Hash // blob的哈希，用于一致性检查 -} - -// KZGToVersionedHash 将KZG承诺转换为带版本的哈希 -func KZGToVersionedHash(commitment []byte) common.Hash { - // EIP-4844规范: - // def kzg_to_versioned_hash(commitment: KZGCommitment) -> VersionedHash: - // return VERSIONED_HASH_VERSION_KZG + sha256(commitment)[1:] - var out common.Hash - h := sha256.New() - h.Write(commitment) - copy(out[:], h.Sum(nil)) - out[0] = params.BlobTxHashVersion - return out -} - -// GetBlobSidecarsEnhanced 获取blob sidecars的增强方法 -func (cl *L1BeaconClient) GetBlobSidecarsEnhanced(ctx context.Context, blockTime uint64, hashes []IndexedBlobHash) ([]*BlobSidecar, error) { - // 计算slot - slot, err := cl.timeToSlot(blockTime) - if err != nil { - return nil, fmt.Errorf("计算slot失败: %w", err) - } - - // 首先尝试使用带索引的URL - if len(hashes) > 0 { - builder := strings.Builder{} - builder.WriteString("eth/v1/beacon/blob_sidecars/") - builder.WriteString(strconv.FormatUint(slot, 10)) - builder.WriteRune('?') - v := url.Values{} - - for i := range hashes { - v.Add("indices", strconv.FormatUint(hashes[i].Index, 10)) - } - builder.WriteString(v.Encode()) - - var resp APIGetBlobSidecarsResponse - if err := cl.apiReq(ctx, &resp, builder.String()); err == nil && len(resp.Data) > 0 { - return resp.Data, nil - } - } - - // 如果第一种方法失败或者没有指定索引，尝试第二种方法 - method := fmt.Sprintf("eth/v1/beacon/blob_sidecars/%d", slot) - var blobResp APIGetBlobSidecarsResponse - if err := cl.apiReq(ctx, &blobResp, method); err != nil { - return nil, fmt.Errorf("请求blob sidecars失败: %w", err) - } - - return blobResp.Data, nil -} - -func main() { - // 设置日志 - logger := log.New(os.Stdout, "[BlobTest] ", log.LstdFlags|log.Lshortfile) - logger.Println("开始增强版blob获取测试") - - // 创建上下文 - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - // 节点URL和交易哈希 - nodeUrl := "https://omniscient-serene-sanctuary.ethereum-holesky.quiknode.pro/37083bb58e379efb02453b2f07fe4853faf18d03/" - txHashHex := "0x2ada86d083a7b1af5e59c916327ef2255ccb9068834bbf13efa5527ba411718f" - - // 创建ETH客户端 - ethClient, err := ethclient.Dial(nodeUrl) - if err != nil { - logger.Fatalf("连接ETH客户端失败: %v", err) - } - - // 获取交易 - txHash := common.HexToHash(txHashHex) - tx, isPending, err := ethClient.TransactionByHash(ctx, txHash) - if err != nil { - logger.Fatalf("获取交易失败: %v", err) - } - if isPending { - logger.Fatalf("交易仍处于挂起状态") - } - - // 获取交易收据，确定区块号 - receipt, err := ethClient.TransactionReceipt(ctx, txHash) - if err != nil { - logger.Fatalf("获取交易收据失败: %v", err) - } - - blockNumber := receipt.BlockNumber.Uint64() - logger.Printf("交易位于区块: %d", blockNumber) - - // 获取区块 - block, err := ethClient.BlockByNumber(ctx, receipt.BlockNumber) - if err != nil { - logger.Fatalf("获取区块失败: %v", err) - } - - blockTime := block.Time() - logger.Printf("区块时间戳: %d", blockTime) - - // 获取blob哈希 - blobHashes := tx.BlobHashes() - if len(blobHashes) == 0 { - logger.Fatalf("交易不包含blob") - } - logger.Printf("交易包含 %d 个blob", len(blobHashes)) - - for i, hash := range blobHashes { - logger.Printf("Blob #%d 哈希: %s", i, hash.Hex()) - } - - // 创建beacon客户端 - beaconEndpoint := "https://beacon.holesky.ethpandaops.io" - beaconClient := NewL1BeaconClient(NewBasicHTTPClient(beaconEndpoint)) - - // 构建索引哈希数组 - var indexedHashes []IndexedBlobHash - for i, hash := range blobHashes { - indexedHashes = append(indexedHashes, IndexedBlobHash{ - Index: uint64(i), - Hash: hash, - }) - } - - // 调用GetBlobSidecarsEnhanced - blobSidecars, err := beaconClient.GetBlobSidecarsEnhanced(ctx, blockTime, indexedHashes) - if err != nil { - logger.Fatalf("获取blob sidecars失败: %v", err) - } - - if len(blobSidecars) == 0 { - logger.Fatalf("未返回blob sidecars") - } - - logger.Printf("成功获取 %d 个blob sidecar", len(blobSidecars)) - - // 检查每个blob sidecar - matchedCount := 0 - for i, sidecar := range blobSidecars { - // 从索引获取slot - slotNum, _ := sidecar.Slot.Int64() - indexNum, _ := sidecar.Index.Int64() - - logger.Printf("Blob Sidecar #%d: Slot=%d, Index=%d", i, slotNum, indexNum) - - // 如果有承诺数据，验证与哈希的匹配 - if len(sidecar.KZGCommitment) > 0 { - versionedHash := KZGToVersionedHash(sidecar.KZGCommitment) - logger.Printf(" KZG承诺转换为哈希: %s", versionedHash.Hex()) - - // 检查是否匹配任何期望的哈希 - for _, expectedHash := range blobHashes { - if bytes.Equal(versionedHash[:], expectedHash[:]) { - matchedCount++ - logger.Printf(" ✅ 匹配blob哈希: %s", expectedHash.Hex()) - - // 解码并处理blob数据 - blobData, err := hexutil.Decode(sidecar.Blob) - if err != nil { - logger.Printf(" ❌ 解码blob数据失败: %v", err) - continue - } - - // 打印blob数据的前200字节作为预览 - previewLen := 200 - if len(blobData) < previewLen { - previewLen = len(blobData) - } - logger.Printf(" Blob数据预览 (%d字节): 0x%x...", len(blobData), blobData[:previewLen]) - break - } - } - } - } - - logger.Printf("总结: 找到 %d 个匹配的blob (共期望 %d 个)", matchedCount, len(blobHashes)) - logger.Println("测试完成") -} From 61aaf383d2d482db18762763beae4ff432ae5577 Mon Sep 17 00:00:00 2001 From: curryxbo Date: Thu, 10 Apr 2025 14:45:27 +0800 Subject: [PATCH 6/7] clean --- node/derivation/batch_info.go | 4 ---- node/derivation/derivation.go | 24 ++++++++---------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/node/derivation/batch_info.go b/node/derivation/batch_info.go index 95ed86f3..d795092b 100644 --- a/node/derivation/batch_info.go +++ b/node/derivation/batch_info.go @@ -181,10 +181,6 @@ func (bi *BatchInfo) ParseBatch(batch geth.RPCRollupBatch) error { if err := block.Decode(rawBlockContexts[i*60 : i*60+60]); err != nil { return fmt.Errorf("decode chunk block context error:%v", err) } - fmt.Println("block.", block.Number) - fmt.Println("block.txsNum:", block.txsNum) - fmt.Println("block.l1MsgNum:", block.l1MsgNum) - fmt.Println("len(batch.Sidecar.Blobs):", len(batch.Sidecar.Blobs)) if i == 0 { bi.firstBlockNumber = block.Number } diff --git a/node/derivation/derivation.go b/node/derivation/derivation.go index 07d84681..95edf84c 100644 --- a/node/derivation/derivation.go +++ b/node/derivation/derivation.go @@ -34,10 +34,6 @@ import ( var ( RollupEventTopic = "CommitBatch(uint256,bytes32)" RollupEventTopicHash = crypto.Keccak256Hash([]byte(RollupEventTopic)) - - // ForceGetAllBlobs controls whether to force using the method that gets all blobs - // Set to true for QA testing, false for production - ForceGetAllBlobs = true ) type Derivation struct { @@ -319,19 +315,15 @@ func (d *Derivation) fetchRollupDataByTxHash(txHash common.Hash, blockNumber uin var indexedBlobHashes []IndexedBlobHash // Only try to build IndexedBlobHash array if not forcing get all blobs - if !ForceGetAllBlobs { - // Try to get the block to build IndexedBlobHash array - block, err := d.l1Client.BlockByNumber(d.ctx, big.NewInt(int64(blockNumber))) - if err == nil { - // Successfully got the block, now build IndexedBlobHash array - d.logger.Info("Building IndexedBlobHash array from block", "blockNumber", blockNumber) - indexedBlobHashes = dataAndHashesFromTxs(block.Transactions(), tx) - d.logger.Info("Built IndexedBlobHash array", "count", len(indexedBlobHashes)) - } else { - d.logger.Info("Failed to get block, will try fetching all blobs", "blockNumber", blockNumber, "error", err) - } + // Try to get the block to build IndexedBlobHash array + block, err := d.l1Client.BlockByNumber(d.ctx, big.NewInt(int64(blockNumber))) + if err == nil { + // Successfully got the block, now build IndexedBlobHash array + d.logger.Info("Building IndexedBlobHash array from block", "blockNumber", blockNumber) + indexedBlobHashes = dataAndHashesFromTxs(block.Transactions(), tx) + d.logger.Info("Built IndexedBlobHash array", "count", len(indexedBlobHashes)) } else { - d.logger.Info("ForceGetAllBlobs is enabled, fetching all blobs for testing") + d.logger.Info("Failed to get block, will try fetching all blobs", "blockNumber", blockNumber, "error", err) } // Get all blobs corresponding to this timestamp From ecd56c1391ad017714b496da99b4499fcbfe5b7c Mon Sep 17 00:00:00 2001 From: curryxbo Date: Thu, 10 Apr 2025 14:56:10 +0800 Subject: [PATCH 7/7] clean test --- node/derivation/derivation_test.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/node/derivation/derivation_test.go b/node/derivation/derivation_test.go index 653ce96c..69eb750d 100644 --- a/node/derivation/derivation_test.go +++ b/node/derivation/derivation_test.go @@ -34,16 +34,10 @@ func TestUnPackData(t *testing.T) { require.Error(t, err) legacyTxData, err := hexutil.Decode(legacyData) require.NoError(t, err) - legacyBatch, err := d.UnPackData(legacyTxData) - require.NoError(t, err) - LegacyBatchInfo := new(BatchInfo) - err = LegacyBatchInfo.ParseBatch(legacyBatch) + _, err = d.UnPackData(legacyTxData) require.NoError(t, err) beforeMoveBctxTxData, err := hexutil.Decode(beforeMoveBctxData) require.NoError(t, err) - beforeMoveBctxBatch, err := d.UnPackData(beforeMoveBctxTxData) - require.NoError(t, err) - beforeMoveBctxBatchInfo := new(BatchInfo) - err = beforeMoveBctxBatchInfo.ParseBatch(beforeMoveBctxBatch) + _, err = d.UnPackData(beforeMoveBctxTxData) require.NoError(t, err) }