implementation of EIP-4844: Shard Blob Transactions (status-im#1440)

* EIP-4844: add pointEvaluation precompiled contract

* EIP-4844: validate transaction and block header

* EIP-4844: implement DataHash Op Code

* EIP-4844: txPool support excessDataGas calculation

* EIP-4844: make sure tx produce correct txHash

* EIP-4844: node should not automatically broadcast blob tx to it's peers

* EIP-4844: add test cases

* EIP-4844: add EIP-4844 support to t8n tool

* EIP-4844: update nim-eth to branch eip-4844

* fix t8n transaction decoding

* add t8n test data

* EIP-4844: fix blobHash opcode

* disable blobHash test when evmc_enable

nimbus/common/common.nim

@@ -349,6 +349,9 @@ proc isBlockAfterTtd*(com: CommonRef, header: BlockHeader): bool
 func isShanghaiOrLater*(com: CommonRef, t: EthTime): bool =
   com.config.shanghaiTime.isSome and t >= com.config.shanghaiTime.get
 
+func isCancunOrLater*(com: CommonRef, t: EthTime): bool =
+  com.config.cancunTime.isSome and t >= com.config.cancunTime.get
+
 proc consensus*(com: CommonRef, header: BlockHeader): ConsensusType
                 {.gcsafe, raises: [CatchableError].} =
   if com.isBlockAfterTtd(header):

nimbus/constants.nim

@@ -71,4 +71,19 @@ const
 
   DEFAULT_RPC_GAS_CAP* =                    50_000_000.GasInt
 
+  # EIP-4844 constants
+  MAX_CALLDATA_SIZE* = 1 shl 24 # 2^24
+  MAX_ACCESS_LIST_SIZE* = 1 shl 24 # 2^24
+  MAX_ACCESS_LIST_STORAGE_KEYS* = 1 shl 24 # 2^24
+  MAX_VERSIONED_HASHES_LIST_SIZE* = 1 shl 24 # 2^24
+  MAX_TX_WRAP_COMMITMENTS* = 1 shl 12 # 2^12
+  LIMIT_BLOBS_PER_TX* = 1 shl 12 # 2^12
+  BLOB_COMMITMENT_VERSION_KZG* = 0x01.byte
+  FIELD_ELEMENTS_PER_BLOB* = 4096
+  DATA_GAS_PER_BLOB* = (1 shl 17).uint64 # 2^17
+  TARGET_DATA_GAS_PER_BLOCK* = (1 shl 18).uint64 # 2^18
+  MIN_DATA_GASPRICE* = 1'u64
+  DATA_GASPRICE_UPDATE_FRACTION* = 2225652'u64
+  MAX_DATA_GAS_PER_BLOCK* = (1 shl 19).uint64 # 2^19
+
 # End

nimbus/core/eip4844.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2022 Status Research & Development GmbH
+# Copyright (c) 2023 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -9,15 +9,216 @@
 # according to those terms.
 
 import
+  std/[os, strutils],
+  kzg4844/kzg_ex as kzg,
   stew/results,
+  stint,
+  ../constants,
   ../common/common
 
 {.push raises: [].}
 
+type
+  Bytes32 = array[32, byte]
+  Bytes64 = array[64, byte]
+  Bytes48 = array[48, byte]
+
+const
+  BLS_MODULUS_STR = "52435875175126190479447740508185965837690552500527637822603658699938581184513"
+  BLS_MODULUS = parse(BLS_MODULUS_STR, UInt256, 10)
+  PrecompileInputLength = 192
+
+proc pointEvaluationResult(): Bytes64 {.compileTime.} =
+  result[0..<32] = FIELD_ELEMENTS_PER_BLOB.u256.toBytesBE[0..^1]
+  result[32..^1] = BLS_MODULUS.toBytesBE[0..^1]
+
+const
+  PointEvaluationResult* = pointEvaluationResult()
+  POINT_EVALUATION_PRECOMPILE_GAS* = 50000.GasInt
+
+
+# kzgToVersionedHash implements kzg_to_versioned_hash from EIP-4844
+proc kzgToVersionedHash(kzg: kzg.KZGCommitment): VersionedHash =
+  result = keccakHash(kzg)
+  result.data[0] = BLOB_COMMITMENT_VERSION_KZG
+
+# pointEvaluation implements point_evaluation_precompile from EIP-4844
+# return value and gas consumption is handled by pointEvaluation in
+# precompiles.nim
+proc pointEvaluation*(input: openArray[byte]): Result[void, string] =
+  # Verify p(z) = y given commitment that corresponds to the polynomial p(x) and a KZG proof.
+  # Also verify that the provided commitment matches the provided versioned_hash.
+  # The data is encoded as follows: versioned_hash | z | y | commitment | proof |
+
+  if input.len < PrecompileInputLength:
+    return err("invalid input length")
+
+  var
+    versionedHash: Bytes32
+    z: Bytes32
+    y: Bytes32
+    commitment: Bytes48
+    kzgProof: Bytes48
+
+  versionedHash[0..<32] = input[0..<32]
+  z[0..<32] = input[32..<64]
+  y[0..<32] = input[64..<96]
+  commitment[0..<48] = input[96..<144]
+  kzgProof[0..<48]   = input[144..<192]
+
+  # Verify KZG proof
+  let res = kzg.verifyKzgProof(commitment, z, y, kzgProof)
+  if res.isErr:
+    return err(res.error)
+
+  # The actual verify result
+  if not res.get():
+    return err("Failed to verify KZG proof")
+
+  ok()
+
+# calcExcessDataGas implements calc_excess_data_gas from EIP-4844
+proc calcExcessDataGas*(parent: BlockHeader): uint64 =
+  let
+    excessDataGas = parent.excessDataGas.get(0'u64)
+    dataGasUsed = parent.dataGasUsed.get(0'u64)
+
+  if excessDataGas + dataGasUsed < TARGET_DATA_GAS_PER_BLOCK:
+    0'u64
+  else:
+    excessDataGas + dataGasUsed - TARGET_DATA_GAS_PER_BLOCK
+
+# fakeExponential approximates factor * e ** (num / denom) using a taylor expansion
+# as described in the EIP-4844 spec.
+func fakeExponential*(factor, numerator, denominator: uint64): uint64 =
+  var
+    i = 1'u64
+    output = 0'u64
+    numeratorAccum = factor * denominator
+
+  while numeratorAccum > 0'u64:
+    output += numeratorAccum
+    numeratorAccum = (numeratorAccum * numerator) div (denominator * i)
+    i = i + 1'u64
+
+  output div denominator
+
+proc getTotalDataGas*(tx: Transaction): uint64 =
+  DATA_GAS_PER_BLOB * tx.versionedHashes.len.uint64
+
+proc getTotalDataGas*(versionedHashesLen: int): uint64 =
+  DATA_GAS_PER_BLOB * versionedHasheslen.uint64
+
+# getDataGasPrice implements get_data_gas_price from EIP-4844
+func getDataGasprice*(parentExcessDataGas: uint64): uint64 =
+  fakeExponential(
+    MIN_DATA_GASPRICE,
+    parentExcessDataGas,
+    DATA_GASPRICE_UPDATE_FRACTION
+  )
+
+proc calcDataFee*(tx: Transaction,
+                  parentExcessDataGas: Option[uint64]): uint64 =
+  tx.getTotalDataGas *
+    getDataGasprice(parentExcessDataGas.get(0'u64))
+
+proc calcDataFee*(versionedHashesLen: int,
+                  parentExcessDataGas: Option[uint64]): uint64 =
+  getTotalDataGas(versionedHashesLen) *
+    getDataGasprice(parentExcessDataGas.get(0'u64))
+
+func dataGasUsed(txs: openArray[Transaction]): uint64 =
+  for tx in txs:
+    result += tx.getTotalDataGas
+
 # https://eips.ethereum.org/EIPS/eip-4844
 func validateEip4844Header*(
-    com: CommonRef, header: BlockHeader
-): Result[void, string] =
-  if header.excessDataGas.isSome:
-    return err("EIP-4844 not yet implemented")
+    com: CommonRef, header, parentHeader: BlockHeader,
+    txs: openArray[Transaction]): Result[void, string] {.raises: [].} =
+
+  if not com.forkGTE(Cancun):
+    if header.dataGasUsed.isSome:
+      return err("unexpected EIP-4844 dataGasUsed in block header")
+
+    if header.excessDataGas.isSome:
+      return err("unexpected EIP-4844 excessDataGas in block header")
+
+    return ok()
+
+  if header.dataGasUsed.isNone:
+    return err("expect EIP-4844 dataGasUsed in block header")
+
+  if header.excessDataGas.isNone:
+    return err("expect EIP-4844 excessDataGas in block header")
+
+  let
+    headerDataGasUsed = header.dataGasUsed.get()
+    dataGasUsed = dataGasUsed(txs)
+    headerExcessDataGas = header.excessDataGas.get
+    excessDataGas = calcExcessDataGas(parentHeader)
+
+  if dataGasUsed <= MAX_DATA_GAS_PER_BLOCK:
+    return err("dataGasUsed should greater than MAX_DATA_GAS_PER_BLOCK: " & $dataGasUsed)
+
+  if headerDataGasUsed != dataGasUsed:
+    return err("calculated dataGas not equal header.dataGasUsed")
+
+  if headerExcessDataGas != excessDataGas:
+    return err("calculated excessDataGas not equal header.excessDataGas")
+
   return ok()
+
+proc validateBlobTransactionWrapper*(tx: Transaction):
+                                     Result[void, string] {.raises: [].} =
+  if not tx.networkPayload.isNil:
+    return err("tx wrapper is none")
+
+  # note: assert blobs are not malformatted
+  let goodFormatted = tx.versionedHashes.len ==
+                      tx.networkPayload.commitments.len and
+                      tx.versionedHashes.len  ==
+                      tx.networkPayload.blobs.len and
+                      tx.versionedHashes.len ==
+                      tx.networkPayload.proofs.len
+
+  if not goodFormatted:
+    return err("tx wrapper is ill formatted")
+
+  # Verify that commitments match the blobs by checking the KZG proof
+  let res = kzg.verifyBlobKzgProofBatch(tx.networkPayload.blobs,
+              tx.networkPayload.commitments, tx.networkPayload.proofs)
+  if res.isErr:
+    return err(res.error)
+
+  # Actual verification result
+  if not res.get():
+    return err("Failed to verify network payload of a transaction")
+
+  # Now that all commitments have been verified, check that versionedHashes matches the commitments
+  for i in 0 ..< tx.versionedHashes.len:
+    # this additional check also done in tx validation
+    if tx.versionedHashes[i].data[0] != BLOB_COMMITMENT_VERSION_KZG:
+      return err("wrong kzg version in versioned hash at index " & $i)
+
+    if tx.versionedHashes[i] != kzgToVersionedHash(tx.networkPayload.commitments[i]):
+      return err("tx versioned hash not match commitments at index " & $i)
+
+  ok()
+
+proc loadKzgTrustedSetup*(): Result[void, string] =
+  const
+    vendorDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor"
+    trustedSetupDir = vendorDir & "/nim-kzg4844/kzg4844/csources/src"
+
+  const const_preset = "mainnet"
+  const trustedSetup =
+    when const_preset == "mainnet":
+      staticRead trustedSetupDir & "/trusted_setup.txt"
+    elif const_preset == "minimal":
+      staticRead trustedSetupDir & "/trusted_setup_4.txt"
+    else:
+      ""
+  if const_preset == "mainnet" or const_preset == "minimal":
+    Kzg.loadTrustedSetupFromString(trustedSetup)
+  else:
+    ok()

nimbus/core/executor/process_transaction.nim

@@ -78,6 +78,7 @@ proc asyncProcessTransactionImpl(
     baseFee = baseFee256.truncate(GasInt)
     tx = eip1559TxNormalization(tx, baseFee, fork)
     priorityFee = min(tx.maxPriorityFee, tx.maxFee - baseFee)
+    excessDataGas = vmState.parent.excessDataGas.get(0'u64)
 
   # Return failure unless explicitely set `ok()`
   var res: Result[GasInt, string] = err("")
@@ -99,7 +100,7 @@ proc asyncProcessTransactionImpl(
   # before leaving is crucial for some unit tests that us a direct/deep call
   # of the `processTransaction()` function. So there is no `return err()`
   # statement, here.
-  let txRes = roDB.validateTransaction(tx, sender, header.gasLimit, baseFee256, fork)
+  let txRes = roDB.validateTransaction(tx, sender, header.gasLimit, baseFee256, excessDataGas, fork)
   if txRes.isOk:
 
     # EIP-1153

nimbus/core/sealer.nim

@@ -144,7 +144,7 @@ template unsafeQuantityToInt64(q: web3types.Quantity): int64 =
   int64 q
 
 proc toTypedTransaction(tx: Transaction): TypedTransaction =
-  web3types.TypedTransaction(rlp.encode(tx))
+  web3types.TypedTransaction(rlp.encode(tx.removeNetworkPayload))
 
 func toWithdrawal(x: WithdrawalV1): Withdrawal =
   result.index = x.index.uint64

nimbus/core/tx_pool/tx_chain.nim

@@ -52,6 +52,10 @@ type
     profit: UInt256          ## Net reward (w/o PoW specific block rewards)
     txRoot: Hash256          ## `rootHash` after packing
     stateRoot: Hash256       ## `stateRoot` after packing
+    dataGasUsed:
+      Option[uint64]         ## EIP-4844 block dataGasUsed
+    excessDataGas:
+      Option[uint64]         ## EIP-4844 block excessDataGas
 
   TxChainRef* = ref object ##\
     ## State cache of the transaction environment for creating a new\
@@ -139,6 +143,8 @@ proc resetTxEnv(dh: TxChainRef; parent: BlockHeader; fee: Option[UInt256])
 
   dh.txEnv.txRoot = EMPTY_ROOT_HASH
   dh.txEnv.stateRoot = dh.txEnv.vmState.parent.stateRoot
+  dh.txEnv.dataGasUsed = none(uint64)
+  dh.txEnv.excessDataGas = none(uint64)
 
 proc update(dh: TxChainRef; parent: BlockHeader)
     {.gcsafe,raises: [CatchableError].} =
@@ -216,7 +222,9 @@ proc getHeader*(dh: TxChainRef): BlockHeader
     # extraData: Blob       # signing data
     # mixDigest: Hash256    # mining hash for given difficulty
     # nonce:     BlockNonce # mining free vaiable
-    fee:         dh.txEnv.vmState.fee)
+    fee:         dh.txEnv.vmState.fee,
+    dataGasUsed: dh.txEnv.dataGasUsed,
+    excessDataGas: dh.txEnv.excessDataGas)
 
   if dh.com.forkGTE(Shanghai):
     result.withdrawalsRoot = some(calcWithdrawalsRoot(dh.withdrawals))
@@ -369,6 +377,14 @@ proc `txRoot=`*(dh: TxChainRef; val: Hash256) =
 proc `withdrawals=`*(dh: TxChainRef, val: sink seq[Withdrawal]) =
   dh.withdrawals = system.move(val)
 
+proc `excessDataGas=`*(dh: TxChainRef; val: Option[uint64]) =
+  ## Setter
+  dh.txEnv.excessDataGas = val
+
+proc `dataGasUsed=`*(dh: TxChainRef; val: Option[uint64]) =
+  ## Setter
+  dh.txEnv.dataGasUsed = val
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/core/tx_pool/tx_info.nim

@@ -101,6 +101,10 @@ type
       ## Running basic validator failed on current transaction
       "Tx rejected by basic validator"
 
+    txInfoErrInvalidBlob = ##\
+      ## Invalid EIP-4844 kzg validation on blob wrapper
+      "Invalid EIP-4844 blob validation"
+
     # ------ Signature problems ------------------------------------------------
 
     txInfoErrInvalidSender = ##\

nimbus/core/tx_pool/tx_tasks/tx_add.nim

@@ -23,7 +23,8 @@ import
   ./tx_recover,
   chronicles,
   eth/[common, keys],
-  stew/[keyed_queue, sorted_set]
+  stew/[keyed_queue, sorted_set],
+  ../../eip4844
 
 {.push raises: [].}
 
@@ -179,6 +180,14 @@ proc addTxs*(xp: TxPoolRef;
 
   for tx in txs.items:
     var reason: TxInfo
+
+    if tx.txType == TxEip4844:
+      let res = tx.validateBlobTransactionWrapper()
+      if res.isErr:
+        # move item to waste basket     
+        reason = txInfoErrInvalidBlob
+        xp.txDB.reject(tx, reason, txItemPending, res.error)
+        invalidTxMeter(1)
 
     # Create tx item wrapper, preferably recovered from waste basket
     let rcTx = xp.recoverItem(tx, txItemPending, info)

nimbus/core/tx_pool/tx_tasks/tx_classify.nim

@@ -233,8 +233,9 @@ proc classifyValidatePacked*(xp: TxPoolRef;
                else:
                  xp.chain.limits.trgLimit
     tx = item.tx.eip1559TxNormalization(xp.chain.baseFee.GasInt, fork)
+    excessDataGas = vmState.parent.excessDataGas.get(0'u64)
 
-  roDB.validateTransaction(tx, item.sender, gasLimit, baseFee, fork).isOk
+  roDB.validateTransaction(tx, item.sender, gasLimit, baseFee, excessDataGas, fork).isOk
 
 proc classifyPacked*(xp: TxPoolRef; gasBurned, moreBurned: GasInt): bool =
   ## Classifier for *packing* (i.e. adding up `gasUsed` values after executing