increase block gas limit as in sepolia (#720)

* increase block gas limit as in sepolia

* increase block gas limit as in sepolia

* staterecovery: update test data to match new genesis with 2B gas block

* staterecovery: update test data to match new genesis with 2B gas block

* staterecovery: reduce amount of blobs sent to L1

* staterecovery: increase l1 besu memory limit

* staterecovery: disable parallel tests in staterecovery

* staterecovery: donwgrade runner to large

* staterecovery: fix some integration tests

* staterecovery: fix integration tests

.github/workflows/staterecovery-testing.yml

@@ -24,7 +24,7 @@ jobs:
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
       # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     # ~2.5 mins saved vs large
-    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xl
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
     name: Staterecovery tests
     steps:
       - name: Checkout
@@ -37,7 +37,7 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      # Disable cache for pulling docker images as empirically found that this 
+      # Disable cache for pulling docker images as empirically found that this
       # (retrieving cache and loading docker images) actually increased test time-to-completion
       # - name: Restore cached images
       #   id: restore-cached-images

build.gradle

@@ -87,7 +87,7 @@ allprojects {
     systemProperty("L1_RPC_URL", "http://localhost:8445")
     systemProperty("L2_RPC_URL", "http://localhost:8545")
     systemProperty("L1_GENESIS", "docker/config/l1-node/el/genesis.json")
-    systemProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA.json")
+    systemProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA-geth.json")
 
     systemProperties["junit.jupiter.execution.timeout.default"] = "5 m" // 5 minutes
     systemProperties["junit.jupiter.execution.parallel.enabled"] = true

coordinator/ethereum/test-utils/src/main/kotlin/net/consensys/zkevm/ethereum/AccountManager.kt

@@ -229,7 +229,7 @@ object L1AccountManager : AccountManager by WhaleBasedAccountManager(
 
 object L2AccountManager : AccountManager by WhaleBasedAccountManager(
   web3jClient = Web3jClientManager.l2Client,
-  genesisFile = getPathTo(System.getProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA.json")),
+  genesisFile = getPathTo(System.getProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA-geth.json")),
   log = LogManager.getLogger(L2AccountManager::class.java)
 )
 

docker/compose-spec-extra-observability.yml

@@ -51,7 +51,7 @@ services:
     ports:
       - "4000:4000"
     volumes:
-      - ./config/linea-local-dev-genesis-PoA.json:/app/genesis.json:ro
+      - ./config/linea-local-dev-genesis-PoA-geth.json:/app/genesis.json:ro
     networks:
       - linea
 

docker/compose-spec-l1-services.yml

@@ -14,7 +14,7 @@ services:
       retries: 120
     environment:
       LOG4J_CONFIGURATION_FILE: /var/lib/besu/log4j.xml
-      JAVA_OPTS: -Xmx512m -XX:+UnlockExperimentalVMOptions -XX:-UseG1GC -XX:+UseZGC
+      JAVA_OPTS: -Xmx1536m -XX:+UnlockExperimentalVMOptions -XX:-UseG1GC -XX:+UseZGC
     entrypoint:
       - /bin/bash
       - -c

docker/compose-spec-l2-services.yml

@@ -72,7 +72,7 @@ services:
     entrypoint: [ "sh", "/scripts/node.sh", "1337", "/genesis.json", "0x0", "0x1C9C380", "0xF4240" ]
     volumes:
       - ./geth/scripts:/scripts:ro
-      - ./config/linea-local-dev-genesis-PoA.json:/genesis.json:ro
+      - ./config/linea-local-dev-genesis-PoA-geth.json:/genesis.json:ro
       - ./geth/geth-l2-node.key:/boot.key:ro
       - local-dev:/data/
     networks:
@@ -153,7 +153,7 @@ services:
     entrypoint: [ "sh", "/scripts/node.sh", "1337", "/genesis.json", "0xa", "0x1C9C380", "0xF4240" ]
     volumes:
       - ./geth/scripts:/scripts:ro
-      - ./config/linea-local-dev-genesis-PoA.json:/genesis.json:ro
+      - ./config/linea-local-dev-genesis-PoA-geth.json:/genesis.json:ro
       - ../tmp/local/:/data/
     networks:
       linea:

docker/config/linea-besu-sequencer/sequencer.config.toml

@@ -6,7 +6,7 @@ sync-mode="FULL"
 host-allowlist=["*"]
 revert-reason-enabled=true
 
-# target-gas-limit=30000000
+target-gas-limit=2000000000
 min-gas-price=1000000
 tx-pool-min-gas-price=0
 max-peers=10
@@ -60,7 +60,7 @@ plugin-linea-tx-pool-simulation-check-api-enabled=false
 plugin-linea-tx-pool-simulation-check-p2p-enabled=false
 plugin-linea-max-block-calldata-size=109000
 plugin-linea-max-tx-calldata-size=60000
-plugin-linea-max-block-gas=30000000
+plugin-linea-max-block-gas=50000000
 plugin-linea-tx-pool-min-margin="0.8"
 plugin-linea-min-margin="1.0"
 plugin-linea-fixed-gas-cost-wei=30000000

docker/config/linea-local-dev-genesis-PoA-besu.json

@@ -21,7 +21,7 @@
   "coinbase": "0x0000000000000000000000000000000000000000",
   "difficulty": "0x1",
   "extraData": "0x00000000000000000000000000000000000000000000000000000000000000006d976c9b8ceee705d4fe8699b44e5eb58242f4840000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
-  "gasLimit": "0x1C9C380",
+  "gasLimit": "0x77359400",
   "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
   "nonce": "0x0",
   "timestamp": "0x645580D1",

docker/config/linea-local-dev-genesis-PoA.json → docker/config/linea-local-dev-genesis-PoA-geth.json

@@ -20,7 +20,7 @@
   "coinbase": "0x0000000000000000000000000000000000000000",
   "difficulty": "0x1",
   "extraData": "0x00000000000000000000000000000000000000000000000000000000000000006d976c9b8ceee705d4fe8699b44e5eb58242f4840000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
-  "gasLimit": "0x1C9C380",
+  "gasLimit": "0x77359400",
   "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
   "nonce": "0x0",
   "timestamp": "0x645580D1",

e2e/src/config/tests-config/environments/local.ts

@@ -43,7 +43,7 @@ const config: Config = {
       new ethers.JsonRpcProvider(L2_RPC_URL.toString()),
       path.resolve(
         process.env.LOCAL_L2_GENESIS ||
-          path.resolve(__dirname, "../../../../..", "docker/config", "linea-local-dev-genesis-PoA.json"),
+          path.resolve(__dirname, "../../../../..", "docker/config", "linea-local-dev-genesis-PoA-geth.json"),
       ),
     ),
     shomeiEndpoint: SHOMEI_ENDPOINT,

jvm-libs/generic/vertx-helper/src/main/kotlin/net/consensys/linea/vertx/VertxFactory.kt

@@ -51,7 +51,6 @@ object VertxFactory {
       }
       configs.put("metricsOptions", metricsOptions)
     }
-    println(configs)
     return createVertxWithJsonConfigs(configs)
   }
 }

jvm-libs/linea/testing/l1-blob-and-proof-submission/src/main/kotlin/net/consensys/linea/testing/submission/ProverResponsesFileLoader.kt

@@ -62,19 +62,16 @@ fun loadBlobsAndAggregations(
 fun loadBlobsAndAggregationsSortedAndGrouped(
   blobsResponsesDir: String,
   aggregationsResponsesDir: String,
-  ignoreBlobsWithoutAggregation: Boolean = false,
-  numberOfAggregations: Int? = null
+  numberOfAggregations: Int? = null,
+  extraBlobsWithoutAggregation: Int = 0
 ): List<AggregationAndBlobs> {
   var (blobs, aggregations) = loadBlobsAndAggregations(blobsResponsesDir, aggregationsResponsesDir)
 
   if (numberOfAggregations != null) {
     aggregations = aggregations.take(numberOfAggregations)
   }
 
-  return groupBlobsToAggregations(aggregations, blobs)
-    .let {
-      if (ignoreBlobsWithoutAggregation) it.filter { it.aggregation != null } else it
-    }
+  return groupBlobsToAggregations(aggregations, blobs, extraBlobsWithoutAggregation)
 }
 
 data class AggregationAndBlobs(
@@ -84,14 +81,28 @@ data class AggregationAndBlobs(
 
 fun groupBlobsToAggregations(
   aggregations: List<Aggregation>,
-  blobs: List<BlobRecord>
+  blobs: List<BlobRecord>,
+  extraBlobsWithoutAggregation: Int
 ): List<AggregationAndBlobs> {
   val aggBlobs = aggregations.map { agg ->
     AggregationAndBlobs(agg, blobs.filter { it.startBlockNumber in agg.blocksRange })
   }.sortedBy { it.aggregation!!.startBlockNumber }
 
-  val blobsWithoutAgg = blobs.filter { blob ->
-    aggBlobs.none { it.blobs.contains(blob) }
+  return if (extraBlobsWithoutAggregation > 0) {
+    val blobsWithoutAgg = blobs.filter { blob ->
+      aggBlobs.none { it.blobs.contains(blob) }
+    }
+
+    if (blobsWithoutAgg.size < extraBlobsWithoutAggregation) {
+      throw IllegalStateException(
+        "Not enough blobs without aggregation: " +
+          "blobsWithoutAggregation=${blobsWithoutAgg.size} " +
+          "requestedBlobsWithoutAggregation=$extraBlobsWithoutAggregation"
+      )
+    }
+
+    aggBlobs + listOf(AggregationAndBlobs(null, blobsWithoutAgg.take(extraBlobsWithoutAggregation)))
+  } else {
+    aggBlobs
   }
-  return aggBlobs + listOf(AggregationAndBlobs(null, blobsWithoutAgg))
 }

state-recovery/appcore/logic/src/main/kotlin/linea/staterecovery/BlobDecompressorAndDeserializer.kt

@@ -30,7 +30,7 @@ interface BlobDecompressorAndDeserializer {
 
 data class BlockHeaderStaticFields(
   val coinbase: ByteArray,
-  val gasLimit: ULong = 61_000_000UL,
+  val gasLimit: ULong = 2_000_000_000UL,
   val difficulty: ULong = 2UL
 ) {
   companion object {

state-recovery/test-cases/build.gradle

@@ -79,6 +79,8 @@ task integrationTest(type: Test) { test ->
   testClassesDirs = sourceSets.integrationTest.output.classesDirs
 
   dependsOn(":localStackForStateRecoveryComposeUp")
+  maxParallelForks = 1
+  systemProperties["junit.jupiter.execution.parallel.enabled"] = false
   // increase default timeout because this tests submit loads of blobs
   systemProperties["junit.jupiter.execution.timeout.default"] = "10 m" // 10 minutes
 

state-recovery/test-cases/src/integrationTest/kotlin/linea/staterecovery/LineaSubmissionEventsClientIntTest.kt

@@ -58,7 +58,8 @@ class LineaSubmissionEventsClientIntTest {
     aggregationsAndBlobs = loadBlobsAndAggregationsSortedAndGrouped(
       blobsResponsesDir = "$testDataDir/compression/responses",
       aggregationsResponsesDir = "$testDataDir/aggregation/responses",
-      numberOfAggregations = 7
+      numberOfAggregations = 7,
+      extraBlobsWithoutAggregation = 3
     )
     // wait smc deployment finishes
     rollupDeploymentResult = rollupDeploymentFuture.get()

state-recovery/test-cases/src/integrationTest/kotlin/linea/staterecovery/StateRecoveryAppWithFakeExecutionClientIntTest.kt

@@ -40,7 +40,6 @@ import kotlin.time.toJavaDuration
 class StateRecoveryAppWithFakeExecutionClientIntTest {
   private val log = LogManager.getLogger("test.case.StateRecoverAppWithFakeExecutionClientIntTest")
   private lateinit var appConfigs: StateRecoveryApp.Config
-  private lateinit var stateRecoverApp: StateRecoveryApp
   private lateinit var aggregationsAndBlobs: List<AggregationAndBlobs>
   private lateinit var fakeExecutionLayerClient: FakeExecutionLayerClient
   private lateinit var fakeStateManagerClient: FakeStateManagerClient
@@ -65,7 +64,8 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
     aggregationsAndBlobs = loadBlobsAndAggregationsSortedAndGrouped(
       blobsResponsesDir = "$testDataDir/compression/responses",
       aggregationsResponsesDir = "$testDataDir/aggregation/responses",
-      numberOfAggregations = 7
+      numberOfAggregations = 4,
+      extraBlobsWithoutAggregation = 0
     )
     fakeExecutionLayerClient = FakeExecutionLayerClient(
       headBlock = BlockNumberAndHash(number = 0uL, hash = ByteArray(32) { 0 }),
@@ -103,26 +103,24 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
       smartContractErrors = lineaRollupContractErrors
     )
 
-    instantiateStateRecoveryApp()
-
     configureLoggers(
       rootLevel = Level.INFO,
       log.name to Level.DEBUG,
-      "linea.testing.submission" to Level.INFO,
+      "linea.testing.submission" to Level.DEBUG,
       "net.consensys.linea.contract.Web3JContractAsyncHelper" to Level.WARN, // silence noisy gasPrice Caps logs
       "linea.staterecovery.BlobDecompressorToDomainV1" to Level.DEBUG,
       "linea.plugin.staterecovery.clients" to Level.INFO,
       "test.fake.clients.l1.fake-execution-layer" to Level.DEBUG,
       "test.clients.l1.web3j-default" to Level.DEBUG,
-      "test.clients.l1.web3j.receipt-poller" to Level.DEBUG,
+      "test.clients.l1.web3j.receipt-poller" to Level.TRACE,
       "linea.staterecovery.datafetching" to Level.TRACE
     )
   }
 
   fun instantiateStateRecoveryApp(
     debugForceSyncStopBlockNumber: ULong? = null
-  ) {
-    stateRecoverApp = StateRecoveryApp(
+  ): StateRecoveryApp {
+    return StateRecoveryApp(
       vertx = vertx,
       elClient = fakeExecutionLayerClient,
       blobFetcher = appClients.blobScanClient,
@@ -151,8 +149,7 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
       l1Web3jClient = createWeb3jHttpClient(
         rpcUrl = l1RpcUrl,
         log = LogManager.getLogger("test.clients.l1.web3j.receipt-poller")
-      ),
-      log = log
+      )
     )
   }
 
@@ -168,7 +165,7 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
   */
   @Test
   fun `when state recovery disabled and is starting from genesis`() {
-    stateRecoverApp.start().get()
+    instantiateStateRecoveryApp().start().get()
     submitDataToL1ContactAndWaitExecution()
 
     val lastAggregation = aggregationsAndBlobs.findLast { it.aggregation != null }!!.aggregation
@@ -221,7 +218,7 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
 
     val lastFinalizedBlockNumber = finalizationsBeforeCutOff.last().aggregation!!.endBlockNumber
     val expectedStateRecoverStartBlockNumber = lastFinalizedBlockNumber + 1UL
-    stateRecoverApp.start().get()
+    instantiateStateRecoveryApp().start().get()
 
     await()
       .atMost(4.minutes.toJavaDuration())
@@ -293,8 +290,7 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
       number = headBlockNumberAtStart,
       hash = ByteArray(32) { 0 }
     )
-
-    stateRecoverApp.start().get()
+    instantiateStateRecoveryApp().start().get()
     await()
       .atMost(2.minutes.toJavaDuration())
       .pollInterval(1.seconds.toJavaDuration())
@@ -343,6 +339,7 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
       aggregationsAndBlobs[1].aggregation!!.intervalString()
     )
 
+    val stateRecoverApp = instantiateStateRecoveryApp()
     stateRecoverApp.start().get()
     submitDataToL1ContactAndWaitExecution()
 
@@ -359,15 +356,20 @@ class StateRecoveryAppWithFakeExecutionClientIntTest {
   @Test
   fun `should stop synch at forceSyncStopBlockNumber`() {
     val debugForceSyncStopBlockNumber = aggregationsAndBlobs[2].aggregation!!.startBlockNumber
-    instantiateStateRecoveryApp(debugForceSyncStopBlockNumber = debugForceSyncStopBlockNumber)
     log.debug("forceSyncStopBlockNumber={}", fakeStateManagerClient)
-
-    stateRecoverApp.start().get()
+    instantiateStateRecoveryApp(debugForceSyncStopBlockNumber = debugForceSyncStopBlockNumber)
+      .start().get()
     submitDataToL1ContactAndWaitExecution(waitTimeout = 3.minutes)
 
     await()
       .atMost(1.minutes.toJavaDuration())
+      .pollInterval(2.seconds.toJavaDuration())
       .untilAsserted {
+        log.debug(
+          "headBlockNumber={} forceSyncStopBlockNumber={}",
+          fakeExecutionLayerClient.headBlock.number,
+          debugForceSyncStopBlockNumber
+        )
         assertThat(fakeExecutionLayerClient.headBlock.number).isGreaterThanOrEqualTo(debugForceSyncStopBlockNumber)
       }
 

state-recovery/test-cases/src/integrationTest/kotlin/linea/staterecovery/StateRecoveryWithRealBesuAndStateManagerIntTest.kt

@@ -123,7 +123,7 @@ class StateRecoveryWithRealBesuAndStateManagerIntTest {
 
     assertBesuAndShomeiRecoveredAsExpected(
       lastAggregationAndBlobs,
-      timeout = 2.minutes
+      timeout = 5.minutes
     )
   }
 

state-recovery/test-cases/src/integrationTest/kotlin/linea/staterecovery/datafetching/SubmissionsFetchingTaskIntTest.kt

@@ -65,8 +65,8 @@ class SubmissionsFetchingTaskIntTest {
     aggregationsAndBlobs = loadBlobsAndAggregationsSortedAndGrouped(
       blobsResponsesDir = "$testDataDir/compression/responses",
       aggregationsResponsesDir = "$testDataDir/aggregation/responses",
-      ignoreBlobsWithoutAggregation = true,
-      numberOfAggregations = 7
+      numberOfAggregations = 7,
+      extraBlobsWithoutAggregation = 0
     )
     val rollupDeploymentResult = ContractsManager.get()
       .deployLineaRollup(numberOfOperators = 2, contractVersion = LineaContractVersion.V6).get()
@@ -175,7 +175,7 @@ class SubmissionsFetchingTaskIntTest {
     )
   }
 
-  fun assertSubmissionsAreCorrectlyFetched(
+  private fun assertSubmissionsAreCorrectlyFetched(
     l2StartBlockNumber: ULong,
     debugForceSyncStopBlockNumber: ULong? = null
   ) {

testdata/coordinator/prover/v3/aggregation/responses/1-9-25b8078fa37ac651f651d213ea26012ee3aa168c35febdb145bb6b9331bffb5c-getZkAggregatedProof.json

@@ -0,0 +1 @@
+{"finalShnarf":"0x7f3eac7cd2134cb24a1ee3c837f03e4ae09680930169060565c3a39070aac001","parentAggregationFinalShnarf":"0x47452a1b9ebadfe02bdd02f580fa1eba17680d57eec968a591644d05d78ee84f","aggregatedProof":"0x0692b62a9a57cda2e576224a8b5576adf71078407559226d6d9374eb44ffb272213b4bf0cb9b33d751880c1f2c04bea41602f5ea857858e96fd37266e1ea14d016a2bfd0dd874b8578455ac545d5f2e7d2d93e0e937f919937f636877cdf92661c0b8a0b53fccfd0a0a70828428a6401ade96ff704780f155084f4dcbd212ffc2bbad763c1e93c6e687b62ca448b012be83167a53f587a180b3622949ec5b8a7059aeee476a663d232f906ffef61e8cbcb53db208ad73874293cfe6a1f691a972bdad3d2cf955a9aab42ed0a7d9c03cc95c15f1bda076bbd93f382a7caed1ad90bbefc886915aea95ee0cf755349dad5c8e8f30f674fb127a9553fb86780dde62593bfeb9a1609603cf4eced3d3c9edc45e38cd3408d89d0c3a8dd04319b9441220c3750ad7b940c60610e0d7f8c433d351b2ae1aa617cb9fcc44f42ed4c1d600a353ef25c97cbbd4f375f05c265dce3830e23ca697c9fa1f49547f5e14f2d671e0e0c004e72a4baa84d115bce73aa6b7fb8002ae02ca258eb6e2b07d7f257fe1f2a659d0e820fd8d1c001ebf324ddb0400a1db3290c0f61dfa97a8f5850ddb62c377d16b32d83af105224d8e11394cb2f0776dfae9fbfe594f7218eb46418a50948a46a23f9a27fc46d0c4a043a004ff6e9dd0ae6940d7767e6bf96b4dc7e7122b9d132cd8442fcd3cf479cd25d8aa17e1612b68949089a8de1b6b037390f8f2b0be3b4a877363e5d179d725e280dd8dfd3b95a8789c66720222a541222fc0d0ce65aa4cba4afe8f1747068ce9391048289360b2ce42af64a72c30824f2e048063d31aef76859a24c945d1ddfaa0a78cc19119efe5f51fd282cf6ba722fc06029d0ff1eec3a6261d5a9c3edfd7411fa87735f1e5d3c590eff73a426ee6e3bcd0cba0197cdf483a2c3ce547428f4363328cc3e2c2adb5ca368e5ecd4e8d3d9f10c99673d9f7aaa7e91c53d67b8776b4c07eb099bc5dddc024e09ba1a66a703d024797dc6bcf00bfb8239e28fdca360f2d3b8b8623c6b1a7be536682a93d30e0f1c9529f98c13eaf0e8dacd62767b5da20b8ccf004bab6501e73b7a7818d941e41d092acc7b953fcf7b398afdce538d7224e95cd3899e141a8765a8f6a0ac6a6b09a112e48768b99ea846e9770e1910046c7ee476442253c7f4d49451b51cf3e11dc7efed58bb7e714dfb3a1b18c4989d4e2658e7e61be3c0acabfb6129f4b485","aggregatedProverVersion":"3.0.0","aggregatedVerifierIndex":0,"aggregatedProofPublicInput":"0x06af2adb1744be3daccff7031cac960d7ac4ca3e170220b375e75e7a075a816a","dataHashes":["0x011dd44f1325c025b4da60f3f44db7ead68507eb0850635cc9826f2112b3aa9d","0x01e4b19564de0374619316d50032f234313b1d4795d1e113ec267101a1b5ef90","0x0101ecfba4c0945e0d3c8852f93947a15c5cff523633cd8b134e2cec7fff90f2"],"dataParentHash":"0x0000000000000000000000000000000000000000000000000000000000000000","parentStateRootHash":"0x072ead6777750dc20232d1cee8dc9a395c2d350df4bbaa5096c6f59b214dcecd","parentAggregationLastBlockTimestamp":1683325137,"lastFinalizedBlockNumber":0,"finalTimestamp":1740079336,"finalBlockNumber":9,"l1RollingHash":"0x0000000000000000000000000000000000000000000000000000000000000000","l1RollingHashMessageNumber":0,"l2MerkleRoots":[],"l2MerkleTreesDepth":5,"l2MessagingBlocksOffsets":"0x"}