Merge branch 'dev' into aggregate-attestation

Makefile

@@ -16,7 +16,6 @@ ALL_EXECUTABLE_SPEC_NAMES = \
 
 # A list of fake targets.
 .PHONY: \
-	check_toc     \
 	clean         \
 	coverage      \
 	detect_errors \
@@ -39,7 +38,6 @@ NORM = $(shell tput sgr0)
 
 # Print target descriptions.
 help:
-	@echo "make $(BOLD)check_toc$(NORM)     -- check table of contents"
 	@echo "make $(BOLD)clean$(NORM)         -- delete all untracked files"
 	@echo "make $(BOLD)coverage$(NORM)      -- run pyspec tests with coverage"
 	@echo "make $(BOLD)detect_errors$(NORM) -- detect generator errors"
@@ -85,7 +83,7 @@ $(ETH2SPEC): setup.py | $(VENV)
 
 # Force rebuild/install the eth2spec package.
 eth2spec:
-	$(MAKE) --always-make $(ETH2SPEC)
+	@$(MAKE) --always-make $(ETH2SPEC)
 
 # Create the pyspec for all phases.
 pyspec: $(VENV) setup.py
@@ -99,6 +97,8 @@ pyspec: $(VENV) setup.py
 TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
 
 # Run pyspec tests.
+# Note: for debugging output to show, print to stderr.
+#
 # To run a specific test, append k=<test>, eg:
 #   make test k=test_verify_kzg_proof
 # To run tests for a specific fork, append fork=<fork>, eg:
@@ -117,6 +117,7 @@ test: $(ETH2SPEC) pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
 		-n auto \
+		--capture=no \
 		$(MAYBE_TEST) \
 		$(MAYBE_FORK) \
 		$(PRESET) \
@@ -193,10 +194,6 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-# Check all files and error if any ToC were modified.
-check_toc: $(MARKDOWN_FILES:=.toc)
-	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
-
 # Generate ToC sections & save copy of original if modified.
 %.toc:
 	@cp $* $*.tmp; \
@@ -209,8 +206,12 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 		echo "\033[1;34m See $*.tmp\033[0m"; \
 	fi
 
+# Check all files and error if any ToC were modified.
+_check_toc: $(MARKDOWN_FILES:=.toc)
+	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
+
 # Check for mistakes.
-lint: $(ETH2SPEC) pyspec check_toc
+lint: $(ETH2SPEC) pyspec _check_toc
 	@$(CODESPELL_VENV) . --skip "./.git,$(VENV),$(PYSPEC_DIR)/.mypy_cache" -I .codespell-whitelist
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(PYSPEC_DIR)/eth2spec
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(TEST_GENERATORS_DIR)
@@ -235,17 +236,19 @@ gen_list:
 	done
 
 # Run one generator.
+# This will forcibly rebuild eth2spec just in case.
 # To check modules for a generator, append modcheck=true, eg:
 #   make gen_genesis modcheck=true
 gen_%: MAYBE_MODCHECK := $(if $(filter true,$(modcheck)),--modcheck)
-gen_%: $(ETH2SPEC) pyspec
+gen_%: eth2spec
 	@mkdir -p $(TEST_VECTOR_DIR)
 	@$(PYTHON_VENV) $(GENERATOR_DIR)/$*/main.py \
 		--output $(TEST_VECTOR_DIR) \
 		$(MAYBE_MODCHECK)
 
 # Run all generators then check for errors.
-gen_all: $(GENERATOR_TARGETS) detect_errors
+gen_all: $(GENERATOR_TARGETS)
+	@$(MAKE) detect_errors
 
 # Detect errors in generators.
 detect_errors: $(TEST_VECTOR_DIR)

docker/README.md

@@ -10,7 +10,7 @@ Handy commands:
 
 Ideally manual running of docker containers is for advanced users, we recommend the script based approach described below for most users.
 
-The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
+The `scripts/build_run_docker_tests.sh` script will cover most use cases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
 
 E.g:
 - `./build_run_docker_tests.sh --p mainnet` will run the mainnet preset tests

specs/_features/custody_game/beacon-chain.md

@@ -619,7 +619,7 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed
         for attester_index in attesters:
             if attester_index != custody_slashing.malefactor_index:
                 increase_balance(state, attester_index, whistleblower_reward)
-        # No special whisteblower reward: it is expected to be an attester. Others are free to slash too however.
+        # No special whistleblower reward: it is expected to be an attester. Others are free to slash too however.
     else:
         # The claim was false, the custody bit was correct. Slash the whistleblower that induced this work.
         slash_validator(state, custody_slashing.whistleblower_index)

specs/_features/eip7732/p2p-interface.md

@@ -130,7 +130,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB
 
 There are no new validations for this topic. However, all validations with regards to the `ExecutionPayload` are removed:
 
-- _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer -- i.e. validate that len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK
+- _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer -- i.e. validate that `len(signed_beacon_block.message.body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK`
 - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot
        -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`.
 - If `execution_payload` verification of block's parent by an execution node is *not* complete:

specs/deneb/beacon-chain.md

@@ -20,6 +20,7 @@
     - [`BeaconBlockBody`](#beaconblockbody)
     - [`ExecutionPayload`](#executionpayload)
     - [`ExecutionPayloadHeader`](#executionpayloadheader)
+    - [`BeaconState`](#beaconstate)
 - [Helper functions](#helper-functions)
   - [Misc](#misc)
     - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash)
@@ -171,6 +172,53 @@ class ExecutionPayloadHeader(Container):
     excess_blob_gas: uint64  # [New in Deneb:EIP4844]
 ```
 
+#### `BeaconState`
+
+```python
+class BeaconState(Container):
+    # Versioning
+    genesis_time: uint64
+    genesis_validators_root: Root
+    slot: Slot
+    fork: Fork
+    # History
+    latest_block_header: BeaconBlockHeader
+    block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
+    state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
+    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]
+    # Eth1
+    eth1_data: Eth1Data
+    eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]
+    eth1_deposit_index: uint64
+    # Registry
+    validators: List[Validator, VALIDATOR_REGISTRY_LIMIT]
+    balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT]
+    # Randomness
+    randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR]
+    # Slashings
+    slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR]  # Per-epoch sums of slashed effective balances
+    # Participation
+    previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]
+    current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]
+    # Finality
+    justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH]  # Bit set for every recent justified epoch
+    previous_justified_checkpoint: Checkpoint
+    current_justified_checkpoint: Checkpoint
+    finalized_checkpoint: Checkpoint
+    # Inactivity
+    inactivity_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT]
+    # Sync
+    current_sync_committee: SyncCommittee
+    next_sync_committee: SyncCommittee
+    # Execution
+    latest_execution_payload_header: ExecutionPayloadHeader  # [Modified in Deneb:EIP4844]
+    # Withdrawals
+    next_withdrawal_index: WithdrawalIndex
+    next_withdrawal_validator_index: ValidatorIndex
+    # Deep history valid from Capella onwards
+    historical_summaries: List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT]
+```
+
 ## Helper functions
 
 ### Misc

specs/deneb/p2p-interface.md

@@ -147,7 +147,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB
 New validation:
 
 - _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer --
-  i.e. validate that `len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK`
+  i.e. validate that `len(signed_beacon_block.message.body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK`
 
 ###### `beacon_aggregate_and_proof`
 

specs/electra/p2p-interface.md

@@ -67,7 +67,7 @@ The derivation of the `message-id` remains stable.
 *Updated validation*
 
 - _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer --
-  i.e. validate that `len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_ELECTRA`
+  i.e. validate that `len(signed_beacon_block.message.body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_ELECTRA`
 
 ###### `beacon_aggregate_and_proof`
 

specs/electra/validator.md

@@ -24,6 +24,8 @@
     - [Deposits](#deposits)
     - [Execution payload](#execution-payload)
     - [Execution Requests](#execution-requests)
+  - [Constructing the `BlobSidecar`s](#constructing-the-blobsidecars)
+    - [Sidecar](#sidecar)
 - [Attesting](#attesting)
   - [Construct attestation](#construct-attestation)
 - [Attestation aggregation](#attestation-aggregation)
@@ -241,6 +243,17 @@ def get_execution_requests(execution_requests_list: Sequence[bytes]) -> Executio
     )
 ```
 
+### Constructing the `BlobSidecar`s
+
+#### Sidecar
+
+*[Modified in Electra:EIP7691]*
+
+```python
+def compute_subnet_for_blob_sidecar(blob_index: BlobIndex) -> SubnetID:
+    return SubnetID(blob_index % BLOB_SIDECAR_SUBNET_COUNT_ELECTRA)
+```
+
 ## Attesting
 
 ### Construct attestation

specs/fulu/das-core.md

@@ -105,19 +105,20 @@ class MatrixEntry(Container):
 def get_custody_groups(node_id: NodeID, custody_group_count: uint64) -> Sequence[CustodyIndex]:
     assert custody_group_count <= NUMBER_OF_CUSTODY_GROUPS
 
-    custody_groups: List[uint64] = []
     current_id = uint256(node_id)
+    custody_groups: List[CustodyIndex] = []
     while len(custody_groups) < custody_group_count:
         custody_group = CustodyIndex(
-            bytes_to_uint64(hash(uint_to_bytes(uint256(current_id)))[0:8])
+            bytes_to_uint64(hash(uint_to_bytes(current_id))[0:8])
             % NUMBER_OF_CUSTODY_GROUPS
         )
         if custody_group not in custody_groups:
             custody_groups.append(custody_group)
         if current_id == UINT256_MAX:
             # Overflow prevention
-            current_id = NodeID(0)
-        current_id += 1
+            current_id = uint256(0)
+        else:
+            current_id += 1
 
     assert len(custody_groups) == len(set(custody_groups))
     return sorted(custody_groups)

specs/fulu/p2p-interface.md

@@ -174,7 +174,7 @@ Some gossip meshes are upgraded in the Fulu fork to support upgraded types.
 *Updated validation*
 
 - _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer --
-  i.e. validate that `len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_FULU`
+  i.e. validate that `len(signed_beacon_block.message.body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_FULU`
 
 ##### Blob subnets
 

tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py

@@ -8,7 +8,7 @@
 from typing import Callable
 
 from eth2spec.test.helpers.execution_payload import (
-    compute_el_block_hash,
+    compute_el_block_hash_for_block,
     build_randomized_execution_payload,
 )
 from eth2spec.test.helpers.multi_operations import (
@@ -255,7 +255,7 @@ def random_block_deneb(spec, state, signed_blocks, scenario_state, rng=Random(34
     opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(
         spec, blob_count=rng.randint(0, spec.config.MAX_BLOBS_PER_BLOCK), rng=rng)
     block.body.execution_payload.transactions.append(opaque_tx)
-    block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state)
+    block.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block)
     block.body.blob_kzg_commitments = blob_kzg_commitments
 
     return block
@@ -264,6 +264,7 @@ def random_block_deneb(spec, state, signed_blocks, scenario_state, rng=Random(34
 def random_block_electra(spec, state, signed_blocks, scenario_state, rng=Random(3456)):
     block = random_block_deneb(spec, state, signed_blocks, scenario_state, rng=rng)
     block.body.execution_requests = get_random_execution_requests(spec, state, rng=rng)
+    block.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block)
 
     return block