Skip to content

Commit

Permalink
Staterecovery part 13 blockhash fixes (#675)
Browse files Browse the repository at this point in the history
staterecovery: fix blockHash opcode and improve performance of blob fetching from L1

---------

Signed-off-by: Pedro Novais <1478752+jpnovais@users.noreply.github.com>
Co-authored-by: Roman Vaseev <4833306+Filter94@users.noreply.github.com>
  • Loading branch information
jpnovais and Filter94 authored Feb 20, 2025
1 parent eb0088a commit 3caceed
Show file tree
Hide file tree
Showing 238 changed files with 2,245 additions and 1,156 deletions.
52 changes: 26 additions & 26 deletions config/coordinator/coordinator-docker.config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,32 @@ duplicated-logs-debounce-time="PT15S"

eip4844-switch-l2-block-number=0

[conflation]
blocks-limit=3
conflation-deadline="PT6S"
conflation-deadline-check-interval="PT3S"
conflation-deadline-last-block-confirmation-delay="PT2S" # recommended: at least 2 * blockInterval
# This is to prevent inflight trasactions that may change Smart contract state while coordinator is restarted.
# Queries SMC for last finalised block, and keeps polling until this number of blocks observe the same state.
# If state is updated meanwhile, it resets counter and restarts the polling.
consistent-number-of-blocks-on-l1-to-wait=1
fetch-blocks-limit=4000

[blob-compression]
blob-size-limit=102400 # 100KB
handler-polling-interval="PT1S"
# default batches limit is aggregation-proofs-limit -1
# batches-limit must be less than or equal to aggregation-proofs-limit-1
batches-limit=1

[proof-aggregation]
aggregation-proofs-limit=3
aggregation-deadline="PT10S"
aggregation-coordinator-polling-interval="PT2S"
deadline-check-interval="PT8S"
#target-end-blocks=[33, 90, 93]


[prover]
fs-inprogress-request-writing-suffix = ".inprogress_coordinator_writing"
fs-inprogress-proving-suffix-pattern = ".*\\.inprogress\\.prover.*"
Expand All @@ -30,13 +56,6 @@ fs-responses-directory = "/data/prover/v2/aggregation/responses"
#fs-requests-directory = "/data/prover/v3/aggregation/requests"
#fs-responses-directory = "/data/prover/v3/aggregation/responses"

[blob-compression]
blob-size-limit=102400 # 100KB
handler-polling-interval="PT1S"
# default batches limit is aggregation-proofs-limit -1
# batches-limit must be less than or equal to aggregation-proofs-limit-1
batches-limit=1

[zk-traces]
eth-api="http://traces-node:8545"
new-block-polling-interval="PT1S"
Expand Down Expand Up @@ -150,13 +169,6 @@ db-polling-interval="PT1S"
max-aggregations-to-finalize-per-tick=1
proof-submission-delay="PT1S"

[proof-aggregation]
aggregation-proofs-limit=3
aggregation-deadline="PT10S"
aggregation-coordinator-polling-interval="PT2S"
deadline-check-interval="PT8S"
#target-end-blocks=[33, 90, 93]

[finalization-signer]
# Web3j/Web3signer
type="Web3j"
Expand Down Expand Up @@ -265,18 +277,6 @@ num-of-blocks-before-latest=4
[l1-dynamic-gas-price-cap-service.fee-history-storage]
storage-period="PT2M"

[conflation]
blocks-limit=3
conflation-deadline="PT6S"
conflation-deadline-check-interval="PT3S"
conflation-deadline-last-block-confirmation-delay="PT2S" # recommended: at least 2 * blockInterval

# This is to prevent inflight trasactions that may change Smart contract state while coordinator is restarted.
# Queries SMC for last finalised block, and keeps polling until this number of blocks observe the same state.
# If state is updated meanwhile, it resets counter and restarts the polling.
consistent-number-of-blocks-on-l1-to-wait=1
fetch-blocks-limit=4000

[database]
host="postgres"
port="5432"
Expand Down

Large diffs are not rendered by default.

16 changes: 14 additions & 2 deletions contracts/src/_testing/unit/opcodes/OpcodeTester.sol
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,7 @@ contract OpcodeTester {
function storeRollingGlobalVariablesToState() private {
bytes memory fieldsToHashSection1 = abi.encode(
rollingBlockDetailComputations,
blockhash(block.number - 1),
blockhash(block.number),
parentBlocksHashes(),
block.basefee,
block.chainid,
block.coinbase,
Expand All @@ -291,4 +290,17 @@ contract OpcodeTester {
bytes.concat(bytes.concat(fieldsToHashSection1, fieldsToHashSection2), fieldsToHashSection3)
);
}

function parentBlocksHashes() private view returns (bytes32[] memory) {
uint256 endLookBack = 256;
if (block.number < 256) {
endLookBack = block.number;
}
bytes32[] memory blocksHashes = new bytes32[](endLookBack + 1);

for (uint256 i = 0; i <= endLookBack; i++) {
blocksHashes[i] = blockhash(block.number - i);
}
return blocksHashes;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package net.consensys.zkevm.ethereum
import build.linea.contract.l1.LineaContractVersion
import linea.testing.CommandResult
import linea.testing.Runner
import org.hyperledger.besu.datatypes.Address
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.util.regex.Matcher
import java.util.regex.Pattern
Expand Down Expand Up @@ -36,7 +37,13 @@ fun getDeployedAddress(
?.let { addressPattern.matcher(it).also { it.find() } }

return matcher
?.let { DeployedContract(it.group(1), it.group(2).toLong()) }
?.let {
val address = it.group(1)
val deploymentBlockNumber = it.group(2).toLong()
// validated address was correctly parsed
Address.fromHexString(address)
DeployedContract(address, deploymentBlockNumber)
}
?: throw IllegalStateException("Couldn't extract contract address. Expecting pattern: $addressPattern")
}

Expand Down
2 changes: 1 addition & 1 deletion docker/compose-spec-l1-services.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ services:
l1-el-node:
container_name: l1-el-node
hostname: l1-el-node
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-mainnet-f2ff06c}
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-395ec6d}
profiles: [ "l1", "debug", "external-to-monorepo" ]
depends_on:
l1-node-genesis-generator:
Expand Down
17 changes: 10 additions & 7 deletions docker/compose-spec-l2-services.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
sequencer:
hostname: sequencer
container_name: sequencer
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-mainnet-f2ff06c}
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-395ec6d}
profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ]
ports:
- "8545:8545"
Expand Down Expand Up @@ -82,7 +82,7 @@ services:
l2-node-besu:
hostname: l2-node-besu
container_name: l2-node-besu
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-mainnet-f2ff06c}
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-395ec6d}
profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ]
depends_on:
sequencer:
Expand Down Expand Up @@ -162,7 +162,7 @@ services:
traces-node-v2:
hostname: traces-node-v2
container_name: traces-node-v2
image: consensys/linea-besu-package:devnet-b9c010b
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-395ec6d}
profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ]
depends_on:
sequencer:
Expand Down Expand Up @@ -367,7 +367,7 @@ services:
- l1network

zkbesu-shomei:
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-fc27c01}
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-395ec6d}
hostname: zkbesu-shomei
container_name: zkbesu-shomei
profiles: [ "l2", "l2-bc", "external-to-monorepo" ]
Expand Down Expand Up @@ -583,7 +583,7 @@ services:
ipv4_address: 10.10.10.205

zkbesu-shomei-sr:
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-7dfb8e3}
image: consensys/linea-besu-package:${BESU_PACKAGE_TAG:-devnet-395ec6d}
hostname: zkbesu-shomei-sr
container_name: zkbesu-shomei-sr
profiles: [ "external-to-monorepo", "staterecovery" ]
Expand Down Expand Up @@ -614,6 +614,7 @@ services:
- -c
- |
(rm /opt/besu/plugins/linea-staterecovery-besu-plugin-v* || true) && \
(rm /opt/besu/plugins/finalized-tag-updater-v* || true) && \
ls -lh /opt/besu/plugins && \
sed -i '/^CLASSPATH/c\CLASSPATH=/opt/besu/lib/\*\:/opt/besu/plugins/\*' /opt/besu/bin/besu && \
/opt/besu/bin/besu \
Expand All @@ -624,11 +625,13 @@ services:
--Xbonsai-limit-trie-logs-enabled=false \
--plugin-shomei-http-host="11.11.11.117" \
--plugin-shomei-http-port=8888 \
--plugin-staterecovery-l1-rpc-endpoint=http://l1-el-node:8545 \
--plugin-staterecovery-l1-endpoint=http://l1-el-node:8545 \
--plugin-staterecovery-l1-polling-interval=PT1S \
--plugin-staterecovery-l1-earliest-search-block=EARLIEST \
--plugin-staterecovery-l1-highest-search-block=LATEST \
--plugin-staterecovery-shomei-endpoint=http://shomei-sr:8888 \
--plugin-staterecovery-blobscan-endpoint=http://blobscan-api:4001 \
--plugin-staterecovery-linea-sequencer-beneficiary-address=0x6d976c9b8ceee705d4fe8699b44e5eb58242f484 \
--plugin-staterecovery-l1-polling-interval=PT0.01S \
--bootnodes=enode://14408801a444dafc44afbccce2eb755f902aed3b5743fed787b3c790e021fef28b8c827ed896aa4e8fb46e22bd67c39f994a73768b4b382f8597b0d44370e15d@11.11.11.101:30303
volumes:
- ./config/zkbesu-shomei/zkbesu-config.toml:/var/lib/besu/zkbesu-config.toml:ro
Expand Down
1 change: 0 additions & 1 deletion docker/compose-tracing-v2-ci-extension.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ services:
extends:
file: compose-spec-l2-services.yml
service: l2-node-besu
image: consensys/linea-besu-package:devnet-b9c010b
volumes:
- ../config/common/traces-limits-besu-v2.toml:/var/lib/besu/traces-limits.toml:ro

Expand Down
1 change: 0 additions & 1 deletion docker/compose-tracing-v2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ services:
extends:
file: compose-spec-l2-services.yml
service: sequencer
image: consensys/linea-besu-package:devnet-b9c010b
volumes:
- ../config/common/traces-limits-besu-v2.toml:/var/lib/besu/traces-limits.toml:ro

Expand Down
1 change: 1 addition & 0 deletions docker/config/linea-besu-sequencer/sequencer.config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ data-path="/opt/besu/data"
data-storage-format="FOREST"
sync-mode="FULL"
host-allowlist=["*"]
revert-reason-enabled=true

# target-gas-limit=30000000
min-gas-price=1000000
Expand Down
9 changes: 4 additions & 5 deletions docker/config/zkbesu-shomei/log4j-staterecovery.xml
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration level="INFO" monitorInterval="2">
<Properties>
<Property name="root.log.level">INFO</Property>
</Properties>

<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} | %t | %-5level | %c{1} | %msg %n" />
Expand Down Expand Up @@ -46,10 +42,13 @@
<Logger name="linea.plugin.staterecovery.clients.l1" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="linea.plugin.staterecovery.clients.l1.logs-searcher" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="linea.plugin.staterecovery.clients.l1.transaction-details" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Root level="${sys:root.log.level}">
<Root level="INFO">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package linea.kotlin

fun List<ULong>.hasSequentialElements(): Boolean {
if (this.size < 2) return true // A list with less than 2 elements is trivially continuous

for (i in 1 until this.size) {
if (this[i] != this[i - 1] + 1UL) {
return false
}
}
return true
}
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ fun ULongRange.intersection(other: ULongRange): ULongRange {
}
}

fun ULong.minusCoercingUnderflow(other: ULong): ULong {
return if (this > other) {
this - other
} else {
0UL
}
}

fun <T : Comparable<T>> ClosedRange<T>.toIntervalString(): String {
val size = if (start <= endInclusive) {
this.endInclusive.toString().toBigDecimal() - this.start.toString().toBigDecimal() + 1.toBigDecimal()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package linea.kotlin
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test

class ULongExtensionsTest {
@Test
fun `hasSequentialElements should return true for an empty list`() {
val list = emptyList<ULong>()
assertThat(list.hasSequentialElements()).isTrue()
}

@Test
fun `hasSequentialElements should return true for a list with one element`() {
val list = listOf(1UL)
assertThat(list.hasSequentialElements()).isTrue()
}

@Test
fun `hasSequentialElements should return true for a list with sequential elements`() {
val list = listOf(1UL, 2UL, 3UL, 4UL, 5UL)
assertThat(list.hasSequentialElements()).isTrue()
}

@Test
fun `hasSequentialElements should return false for a list with non-sequential elements`() {
val list = listOf(1UL, 3UL, 2UL, 5UL, 4UL)
assertThat(list.hasSequentialElements()).isFalse()
}

@Test
fun `hasSequentialElements should return false for a list with gaps`() {
val list = listOf(1UL, 2UL, 4UL, 5UL)
assertThat(list.hasSequentialElements()).isFalse()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,6 @@ fun loadVertxConfig(): VertxOptions {
// Close the vert.x instance, we don't need it anymore.
val retriever = ConfigRetriever.create(vertx, options)
val parsedOptions = VertxOptions(retriever.config.get())
vertx.close()
vertx.close().get()
return parsedOptions
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package net.consensys.linea.vertx

import io.vertx.core.Vertx
import io.vertx.core.VertxOptions
import io.vertx.core.json.JsonObject
import kotlin.time.Duration
import kotlin.time.Duration.Companion.seconds

object VertxFactory {
fun createVertxWithJsonConfigs(configs: JsonObject): Vertx {
return Vertx.vertx(VertxOptions(configs))
}

fun createVertx(
maxEventLoopExecuteTime: Duration? = 5.seconds,
maxWorkerExecuteTime: Duration? = 30.seconds,
blockedThreadCheckInterval: Duration? = 5.seconds,
warningExceptionTime: Duration? = 60.seconds,
jvmMetricsEnabled: Boolean = true,
prometheusMetricsEnabled: Boolean = true,
preferNativeTransport: Boolean = true
): Vertx {
val configs = JsonObject()
maxEventLoopExecuteTime?.let {
configs.put("maxEventLoopExecuteTime", it.inWholeMilliseconds)
configs.put("maxEventLoopExecuteTimeUnit", "MILLISECONDS")
}
maxWorkerExecuteTime?.let {
configs.put("maxWorkerExecuteTime", it.inWholeMilliseconds)
configs.put("maxWorkerExecuteTimeTimeUnit", "MILLISECONDS")
}
blockedThreadCheckInterval?.let {
configs.put("blockedThreadCheckInterval", it.inWholeMilliseconds)
configs.put("blockedThreadCheckIntervalUnit", "MILLISECONDS")
}
warningExceptionTime?.let {
configs.put("warningExceptionTime", it.inWholeMilliseconds)
configs.put("warningExceptionTimeUnit", "MILLISECONDS")
}
configs.put("preferNativeTransport", preferNativeTransport)

if (jvmMetricsEnabled || prometheusMetricsEnabled) {
val metricsOptions = JsonObject()
metricsOptions.put("enabled", true)
metricsOptions.put("jvmMetricsEnabled", jvmMetricsEnabled)
if (prometheusMetricsEnabled) {
val prometheusOptions = JsonObject()
prometheusOptions.put("enabled", true)
prometheusOptions.put("publishQuantiles", true)
metricsOptions.put("prometheusOptions", prometheusOptions)
}
configs.put("metricsOptions", metricsOptions)
}
println(configs)
return createVertxWithJsonConfigs(configs)
}
}
Loading

0 comments on commit 3caceed

Please sign in to comment.