From 69eef6fff7c85e846f2cd01a3769933af536086c Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 22 Aug 2019 20:32:21 +0600 Subject: [PATCH 01/59] Initial commit of the attestation pool feature --- chain/build.gradle | 1 + .../beacon/chain/pool/AbstractProcessor.java | 18 ++ .../beacon/chain/pool/AbstractVerifier.java | 25 +++ .../beacon/chain/pool/AttestationChurn.java | 10 + .../beacon/chain/pool/AttestationPool.java | 25 +++ .../chain/pool/AttestationProcessor.java | 10 + .../chain/pool/AttestationVerifier.java | 15 ++ .../chain/pool/InMemoryAttestationPool.java | 58 +++++ .../beacon/chain/pool/OffChainAggregates.java | 23 ++ .../chain/pool/ReceivedAttestation.java | 23 ++ .../beacon/chain/pool/UnknownBlockPool.java | 8 + .../chain/pool/basic/KnownAttestations.java | 33 +++ .../chain/pool/basic/LightVerifier.java | 108 ++++++++++ .../pool/churn/AttestationChurnImpl.java | 17 ++ .../beacon/chain/pool/unknown/Queue.java | 165 +++++++++++++++ .../pool/unknown/UnknownBlockPoolImpl.java | 124 +++++++++++ .../verifier/AttestationPoolVerifierImpl.java | 33 +++ .../AttestationSignatureVerifier.java | 116 ++++++++++ .../verifier/AttestationStateVerifier.java | 198 ++++++++++++++++++ .../consensus/spec/BlockProcessing.java | 9 + .../beacon/consensus/spec/HelperFunction.java | 9 + .../org/ethereum/beacon/types/p2p/NodeId.java | 10 + versions.gradle | 1 + 23 files changed, 1039 insertions(+) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/basic/KnownAttestations.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/basic/LightVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/Queue.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationPoolVerifierImpl.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java create mode 100644 types/src/main/java/org/ethereum/beacon/types/p2p/NodeId.java diff --git a/chain/build.gradle b/chain/build.gradle index 8d9e5d254..ad01e4bc4 100644 --- a/chain/build.gradle +++ b/chain/build.gradle @@ -10,6 +10,7 @@ dependencies { implementation 'com.google.guava:guava' implementation 'io.projectreactor:reactor-core' + api "org.apache.commons:commons-collections4" testImplementation 'org.mockito:mockito-core' diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java new file mode 100644 index 000000000..252ba3570 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java @@ -0,0 +1,18 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; + +public abstract class AbstractProcessor implements AttestationProcessor { + protected final SimpleProcessor outbound; + + public AbstractProcessor(Schedulers schedulers, String name) { + this.outbound = new SimpleProcessor<>(schedulers.events(), name + ".outbound"); + } + + @Override + public Publisher out() { + return outbound; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java new file mode 100644 index 000000000..fd6678b49 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java @@ -0,0 +1,25 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; + +public abstract class AbstractVerifier implements AttestationVerifier { + protected final SimpleProcessor valid; + protected final SimpleProcessor invalid; + + public AbstractVerifier(Schedulers schedulers, String name) { + this.valid = new SimpleProcessor<>(schedulers.events(), name + ".valid"); + this.invalid = new SimpleProcessor<>(schedulers.events(), name + ".invalid"); + } + + @Override + public Publisher valid() { + return valid; + } + + @Override + public Publisher invalid() { + return invalid; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java new file mode 100644 index 000000000..d25dd9e7b --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.chain.pool; + +import org.reactivestreams.Publisher; + +public interface AttestationChurn { + + Publisher out(); + + void in(ReceivedAttestation attestation); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java new file mode 100644 index 000000000..f8d5b5f81 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -0,0 +1,25 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.core.types.EpochNumber; +import org.reactivestreams.Publisher; + +public interface AttestationPool { + + int MAX_THREADS = Runtime.getRuntime().availableProcessors(); + + EpochNumber MAX_ATTESTATION_LOOKAHEAD = EpochNumber.of(1); + + int MAX_KNOWN_ATTESTATIONS = 1_000_000; + + int UNKNOWN_POOL_CAPACITY = 100_000; + + Publisher valid(); + + Publisher invalid(); + + Publisher unknownBlock(); + + Publisher aggregates(); + + void start(); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java new file mode 100644 index 000000000..02e2b8f86 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.chain.pool; + +import org.reactivestreams.Publisher; + +public interface AttestationProcessor { + + Publisher out(); + + void in(ReceivedAttestation attestation); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java new file mode 100644 index 000000000..2539b8cb8 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java @@ -0,0 +1,15 @@ +package org.ethereum.beacon.chain.pool; + +import org.reactivestreams.Publisher; + +public interface AttestationVerifier extends AttestationProcessor { + + default Publisher out() { + return valid(); + } + + Publisher valid(); + + Publisher invalid(); + +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java new file mode 100644 index 000000000..5af1dc97c --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -0,0 +1,58 @@ +package org.ethereum.beacon.chain.pool; + +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public class InMemoryAttestationPool implements AttestationPool { + + private final Publisher inbound; + private final AttestationVerifier lightVerifier; + private final AttestationProcessor knownAttestations; + private final UnknownBlockPool unknownBlockPool; + private final AttestationVerifier fullVerifier; + private final AttestationChurn churn; + + public InMemoryAttestationPool( + Publisher inbound, + AttestationVerifier lightVerifier, + AttestationProcessor knownAttestations, + UnknownBlockPool unknownBlockPool, + AttestationVerifier fullVerifier, + AttestationChurn churn) { + this.inbound = inbound; + this.lightVerifier = lightVerifier; + this.knownAttestations = knownAttestations; + this.unknownBlockPool = unknownBlockPool; + this.fullVerifier = fullVerifier; + this.churn = churn; + } + + @Override + public void start() { + Flux.from(inbound).subscribe(lightVerifier::in); + Flux.from(lightVerifier.out()).subscribe(knownAttestations::in); + Flux.from(knownAttestations.out()).subscribe(unknownBlockPool::in); + Flux.from(unknownBlockPool.out()).subscribe(fullVerifier::in); + Flux.from(fullVerifier.out()).subscribe(churn::in); + } + + @Override + public Publisher valid() { + return fullVerifier.valid(); + } + + @Override + public Publisher invalid() { + return Flux.concat(lightVerifier.invalid(), fullVerifier.invalid()); + } + + @Override + public Publisher unknownBlock() { + return unknownBlockPool.unknownBlock(); + } + + @Override + public Publisher aggregates() { + return churn.out(); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java new file mode 100644 index 000000000..92a3b32f7 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java @@ -0,0 +1,23 @@ +package org.ethereum.beacon.chain.pool; + +import java.util.List; +import org.ethereum.beacon.core.operations.Attestation; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class OffChainAggregates { + private final Hash32 blockRoot; + private final List aggregates; + + public OffChainAggregates(Hash32 blockRoot, List aggregates) { + this.blockRoot = blockRoot; + this.aggregates = aggregates; + } + + public Hash32 getBlockRoot() { + return blockRoot; + } + + public List getAggregates() { + return aggregates; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java new file mode 100644 index 000000000..a1f25f196 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java @@ -0,0 +1,23 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.types.p2p.NodeId; + +public class ReceivedAttestation { + + private final NodeId sender; + private final Attestation message; + + public ReceivedAttestation(NodeId sender, Attestation message) { + this.sender = sender; + this.message = message; + } + + public NodeId getSender() { + return sender; + } + + public Attestation getMessage() { + return message; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java new file mode 100644 index 000000000..abdcced6b --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java @@ -0,0 +1,8 @@ +package org.ethereum.beacon.chain.pool; + +import org.reactivestreams.Publisher; + +public interface UnknownBlockPool extends AttestationProcessor { + + Publisher unknownBlock(); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/KnownAttestations.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/KnownAttestations.java new file mode 100644 index 000000000..e5a4d675c --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/KnownAttestations.java @@ -0,0 +1,33 @@ +package org.ethereum.beacon.chain.pool.basic; + +import java.util.function.Function; +import org.apache.commons.collections4.map.LRUMap; +import org.ethereum.beacon.chain.pool.AbstractProcessor; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.AttestationProcessor; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.schedulers.Schedulers; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class KnownAttestations extends AbstractProcessor implements AttestationProcessor { + + private static final Object ENTRY = new Object(); + + private final LRUMap knownAttestationsCache = + new LRUMap<>(AttestationPool.MAX_KNOWN_ATTESTATIONS); + private final Function hasher; + + public KnownAttestations(Schedulers schedulers, Function hasher) { + super(schedulers, "KnownAttestations"); + this.hasher = hasher; + } + + @Override + public void in(ReceivedAttestation attestation) { + Object existed = knownAttestationsCache.put(hasher.apply(attestation.getMessage()), ENTRY); + if (existed == null) { + outbound.onNext(attestation); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/LightVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/LightVerifier.java new file mode 100644 index 000000000..254d44b72 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/LightVerifier.java @@ -0,0 +1,108 @@ +package org.ethereum.beacon.chain.pool.basic; + +import com.google.common.annotations.VisibleForTesting; +import org.ethereum.beacon.chain.pool.AbstractVerifier; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.AttestationVerifier; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public class LightVerifier extends AbstractVerifier implements AttestationVerifier { + + private final BeaconChainSpec spec; + + private Checkpoint finalizedCheckpoint; + private EpochNumber maxAcceptableEpoch; + + public LightVerifier( + Schedulers schedulers, + Publisher finalizedCheckpoint, + Publisher slotClock, + BeaconChainSpec spec) { + super(schedulers, "BasicVerifier"); + + this.spec = spec; + Flux.from(finalizedCheckpoint).subscribe(this::onNewFinalizedCheckpoint); + Flux.from(slotClock).subscribe(this::onNewSlot); + } + + @Override + public void in(ReceivedAttestation attestation) { + if (!isInitialized()) { + return; + } + + if (isValid(attestation)) { + valid.onNext(attestation); + } else { + invalid.onNext(attestation); + } + } + + private boolean isValid(ReceivedAttestation attestation) { + final AttestationData data = attestation.getMessage().getData(); + final Checkpoint finalizedCheckpoint = this.finalizedCheckpoint; + final EpochNumber maxAcceptableEpoch = this.maxAcceptableEpoch; + + // sourceEpoch >= targetEpoch + if (data.getSource().getEpoch().greaterEqual(data.getTarget().getEpoch())) { + return false; + } + + // targetEpoch <= finalizedEpoch + if (data.getTarget().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { + return false; + } + + // sourceEpoch < finalizedEpoch + if (data.getSource().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { + return false; + } + + // targetEpoch > maxAcceptableEpoch + if (data.getTarget().getEpoch().greater(maxAcceptableEpoch)) { + return false; + } + + // finalizedEpoch == sourceEpoch && finalizedRoot != sourceRoot + if (data.getSource().getEpoch().equals(finalizedCheckpoint.getEpoch()) + && !finalizedCheckpoint.getRoot().equals(data.getSource().getRoot())) { + return false; + } + + // crosslinkShard >= SHARD_COUNT + if (data.getCrosslink().getShard().greaterEqual(spec.getConstants().getShardCount())) { + return false; + } + + return true; + } + + @VisibleForTesting + void onNewFinalizedCheckpoint(Checkpoint checkpoint) { + this.finalizedCheckpoint = checkpoint; + } + + @VisibleForTesting + void onNewSlot(SlotNumber slot) { + this.maxAcceptableEpoch = + spec.compute_epoch_of_slot(slot).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); + } + + @VisibleForTesting + boolean isInitialized() { + return finalizedCheckpoint != null && maxAcceptableEpoch != null; + } + + @Override + public Publisher invalid() { + return invalid; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java new file mode 100644 index 000000000..17b9472d0 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java @@ -0,0 +1,17 @@ +package org.ethereum.beacon.chain.pool.churn; + +import org.ethereum.beacon.chain.pool.AttestationChurn; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.reactivestreams.Publisher; + +public class AttestationChurnImpl implements AttestationChurn { + + @Override + public Publisher out() { + return null; + } + + @Override + public void in(ReceivedAttestation attestation) {} +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/Queue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/Queue.java new file mode 100644 index 000000000..5c50a356f --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/Queue.java @@ -0,0 +1,165 @@ +package org.ethereum.beacon.chain.pool.unknown; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.types.EpochNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; + +final class Queue { + private final LinkedList queue = new LinkedList<>(); + private final EpochNumber trackedEpochs; + private final long maxSize; + + private EpochNumber baseLine; + + Queue(EpochNumber trackedEpochs, long maxSize) { + assert maxSize > 0; + assert trackedEpochs.getValue() > 0; + + this.trackedEpochs = trackedEpochs; + this.maxSize = maxSize; + } + + synchronized void moveBaseLine(EpochNumber newBaseLine) { + assert baseLine == null || newBaseLine.greater(baseLine); + + for (long i = 0; i < newBaseLine.minus(baseLine).getValue() && queue.size() > 0; i++) { + queue.removeFirst(); + } + + while (queue.size() < trackedEpochs.getValue()) { + queue.add(new EpochBucket()); + } + + this.baseLine = newBaseLine; + } + + synchronized List evict(Hash32 root) { + if (!isInitialized()) { + return Collections.emptyList(); + } + + List evictedFromAllEpochs = new ArrayList<>(); + for (EpochBucket epoch : queue) { + evictedFromAllEpochs.addAll(epoch.evict(root)); + } + + return evictedFromAllEpochs; + } + + synchronized void add(EpochNumber epoch, Hash32 root, ReceivedAttestation attestation) { + if (!isInitialized() || epoch.less(baseLine)) { + return; + } + + EpochBucket epochBucket = getEpochBucket(epoch); + epochBucket.add(root, attestation); + + purgeQueue(); + } + + private void purgeQueue() { + for (EpochNumber e = baseLine; + computeSize() > maxSize && e.less(baseLine.plus(trackedEpochs)); + e = e.increment()) { + EpochBucket epochBucket = getEpochBucket(e); + while (epochBucket.size() > 0 && computeSize() > maxSize) { + epochBucket.removeOldest(); + } + } + } + + private long computeSize() { + return queue.stream().map(EpochBucket::size).reduce(0L, Long::sum); + } + + private EpochBucket getEpochBucket(EpochNumber epoch) { + assert epoch.greaterEqual(baseLine); + return queue.get(epoch.minus(baseLine).getIntValue()); + } + + private boolean isInitialized() { + return baseLine != null; + } + + static final class EpochBucket { + private final Map> bucket = new HashMap<>(); + private final LinkedList> lruIndex = new LinkedList<>(); + + private long size = 0; + + void add(Hash32 root, ReceivedAttestation attestation) { + LinkedList rootBucket = getOrInsert(root); + rootBucket.add(new RootBucketEntry(System.nanoTime(), attestation)); + updateIndex(); + size += 1; + } + + private LinkedList getOrInsert(Hash32 root) { + LinkedList rootBucket = bucket.get(root); + if (rootBucket == null) { + bucket.put(root, rootBucket = new LinkedList<>()); + lruIndex.add(rootBucket); + } + return rootBucket; + } + + private void updateIndex() { + lruIndex.sort(Comparator.comparing(b -> b.getFirst().timestamp)); + } + + List evict(Hash32 root) { + List evicted = bucket.remove(root); + if (evicted != null) { + size -= evicted.size(); + lruIndex.remove(evicted); + updateIndex(); + } + return evicted != null + ? evicted.stream().map(RootBucketEntry::getAttestation).collect(Collectors.toList()) + : Collections.emptyList(); + } + + ReceivedAttestation removeOldest() { + if (size > 0) { + LinkedList oldestBucket = lruIndex.getFirst(); + RootBucketEntry entry = oldestBucket.removeFirst(); + if (oldestBucket.isEmpty()) { + lruIndex.removeFirst(); + bucket.values().remove(oldestBucket); + } + updateIndex(); + + size -= 1; + return entry.attestation; + } else { + return null; + } + } + + long size() { + return size; + } + } + + static final class RootBucketEntry { + private final long timestamp; + private final ReceivedAttestation attestation; + + public RootBucketEntry(long timestamp, ReceivedAttestation attestation) { + this.timestamp = timestamp; + this.attestation = attestation; + } + + ReceivedAttestation getAttestation() { + return attestation; + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java new file mode 100644 index 000000000..3cbbb3359 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java @@ -0,0 +1,124 @@ +package org.ethereum.beacon.chain.pool.unknown; + +import com.google.common.annotations.VisibleForTesting; +import java.util.List; +import org.ethereum.beacon.chain.pool.AbstractProcessor; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.UnknownBlockPool; +import org.ethereum.beacon.chain.storage.BeaconBlockStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.RunnableEx; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class UnknownBlockPoolImpl extends AbstractProcessor implements UnknownBlockPool { + + /** prev + curr + lookahead */ + private static final EpochNumber TRACKED_EPOCHS = + EpochNumber.of(2).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); + + private final Queue queue = new Queue(TRACKED_EPOCHS, AttestationPool.UNKNOWN_POOL_CAPACITY); + + private final SimpleProcessor unknownBlock; + private final Scheduler executor; + private final BeaconBlockStorage blockStorage; + private final BeaconChainSpec spec; + + private EpochNumber currentBaseLine; + + public UnknownBlockPoolImpl( + Schedulers schedulers, + Publisher slotClock, + Publisher importedBlocks, + Scheduler executor, + BeaconBlockStorage blockStorage, + BeaconChainSpec spec) { + super(schedulers, "UnknownBlockPool"); + + this.blockStorage = blockStorage; + this.spec = spec; + this.executor = executor; + + Flux.from(slotClock).subscribe(this::onNewSlot); + Flux.from(importedBlocks).subscribe(this::onNewImportedBlock); + this.unknownBlock = + new SimpleProcessor<>(schedulers.events(), "UnknownChainPool.unknownChainAttestations"); + } + + @Override + public Publisher out() { + return outbound; + } + + @Override + public void in(ReceivedAttestation attestation) { + if (isInitialized()) { + execute(() -> processAttestation(attestation)); + } + } + + private void processAttestation(ReceivedAttestation attestation) { + AttestationData data = attestation.getMessage().getData(); + + // beacon block has not yet been imported + // it implies that source and target blocks might have not been imported too + if (blockStorage.get(data.getBeaconBlockRoot()).isPresent()) { + outbound.onNext(attestation); + } else { + unknownBlock.onNext(attestation); + queue.add(data.getTarget().getEpoch(), data.getBeaconBlockRoot(), attestation); + } + } + + @VisibleForTesting + void onNewImportedBlock(BeaconBlock block) { + EpochNumber blockEpoch = spec.compute_epoch_of_slot(block.getSlot()); + // blockEpoch < currentBaseLine || blockEpoch >= currentBaseLine + TRACKED_EPOCHS + if (blockEpoch.less(currentBaseLine) + || blockEpoch.greaterEqual(currentBaseLine.plus(TRACKED_EPOCHS))) { + return; + } + + execute( + () -> { + Hash32 blockRoot = spec.hash_tree_root(block); + List attestations = queue.evict(blockRoot); + attestations.forEach(outbound::onNext); + }); + } + + @VisibleForTesting + void onNewSlot(SlotNumber slotNumber) { + EpochNumber currentEpoch = spec.compute_epoch_of_slot(slotNumber); + EpochNumber baseLine = + currentEpoch.equals(spec.getConstants().getGenesisEpoch()) + ? currentEpoch + : currentEpoch.decrement(); + queue.moveBaseLine(baseLine); + + this.currentBaseLine = baseLine; + } + + @VisibleForTesting + boolean isInitialized() { + return currentBaseLine != null; + } + + private void execute(RunnableEx routine) { + executor.execute(routine); + } + + @Override + public Publisher unknownBlock() { + return unknownBlock; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationPoolVerifierImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationPoolVerifierImpl.java new file mode 100644 index 000000000..8e086ed3f --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationPoolVerifierImpl.java @@ -0,0 +1,33 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import org.ethereum.beacon.chain.pool.AttestationVerifier; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public class AttestationPoolVerifierImpl implements AttestationVerifier { + + private final AttestationStateVerifier stateVerifier; + private final AttestationSignatureVerifier signatureVerifier; + + public AttestationPoolVerifierImpl( + AttestationStateVerifier stateVerifier, AttestationSignatureVerifier signatureVerifier) { + this.stateVerifier = stateVerifier; + this.signatureVerifier = signatureVerifier; + } + + @Override + public Publisher valid() { + return signatureVerifier.valid(); + } + + @Override + public void in(ReceivedAttestation attestation) { + stateVerifier.inbound(attestation); + } + + @Override + public Publisher invalid() { + return Flux.concat(stateVerifier.invalid(), signatureVerifier.invalid()); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java new file mode 100644 index 000000000..02adf9e60 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java @@ -0,0 +1,116 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.schedulers.RunnableEx; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.javatuples.Pair; +import org.reactivestreams.Publisher; + +public class AttestationSignatureVerifier { + + private final Queue queue = new Queue(); + private final Scheduler executor; + private final BeaconChainSpec spec; + + private final SimpleProcessor valid; + private final SimpleProcessor invalid; + + public AttestationSignatureVerifier( + Schedulers schedulers, Scheduler executor, BeaconChainSpec spec) { + this.executor = executor; + this.spec = spec; + + this.valid = + new SimpleProcessor<>(schedulers.events(), "AttestationSignatureVerifier.valid"); + this.invalid = + new SimpleProcessor<>(schedulers.events(), "AttestationSignatureVerifier.invalid"); + } + + public Publisher valid() { + return valid; + } + + public void inbound(Pair> attestationTuple) { + queue.add(attestationTuple); + execute(this::nudgeQueue); + } + + private void nudgeQueue() { + while (queue.size() > 0) { + execute( + () -> { + Pair> batch = queue.take(); + process(batch); + }); + } + } + + private void execute(RunnableEx routine) { + executor.execute(routine); + } + + private void process(Pair> attestationTuple) { + if (attestationTuple.getValue1().isEmpty()) { + return; + } + + final BeaconState state = attestationTuple.getValue0(); + for (ReceivedAttestation attestation : attestationTuple.getValue1()) { + IndexedAttestation indexedAttestation = + spec.get_indexed_attestation(state, attestation.getMessage()); + if (spec.is_valid_indexed_attestation(state, indexedAttestation)) { + valid.onNext(attestation); + } else { + invalid.onNext(attestation); + } + } + } + + public Publisher invalid() { + return invalid; + } + + private static final class Queue { + + private final LinkedHashMap>> queue = + new LinkedHashMap<>(); + + synchronized void add(Pair> attestationTuple) { + if (attestationTuple.getValue1().isEmpty()) { + return; + } + + Pair> bucket = + queue.computeIfAbsent( + attestationTuple.getValue1().get(0).getMessage().getData().getTarget(), + key -> Pair.with(attestationTuple.getValue0(), new ArrayList<>())); + bucket.getValue1().addAll(attestationTuple.getValue1()); + } + + synchronized Pair> take() { + Iterator>> it = queue.values().iterator(); + if (it.hasNext()) { + Pair> ret = it.next(); + it.remove(); + return ret; + } else { + return Pair.with(BeaconState.getEmpty(), Collections.emptyList()); + } + } + + int size() { + return queue.size(); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java new file mode 100644 index 000000000..97c1f5e57 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java @@ -0,0 +1,198 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import com.google.common.base.Objects; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.storage.BeaconTupleStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.schedulers.RunnableEx; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.javatuples.Pair; +import org.reactivestreams.Publisher; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class AttestationStateVerifier { + + private final Queue queue = new Queue(); + private final Scheduler executor; + private final BeaconTupleStorage tupleStorage; + private final BeaconChainSpec spec; + private final EmptySlotTransition emptySlotTransition; + + private final SimpleProcessor>> outbound; + private final SimpleProcessor invalid; + + public AttestationStateVerifier( + Schedulers schedulers, + Scheduler executor, + BeaconTupleStorage tupleStorage, + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition) { + this.executor = executor; + this.tupleStorage = tupleStorage; + this.spec = spec; + this.emptySlotTransition = emptySlotTransition; + + this.outbound = new SimpleProcessor<>(schedulers.events(), "AttestationStateVerifier.outbound"); + this.invalid = new SimpleProcessor<>(schedulers.events(), "AttestationStateVerifier.invalid"); + } + + public Publisher>> outbound() { + return outbound; + } + + public void inbound(ReceivedAttestation attestation) { + queue.add(attestation); + execute(this::nudgeQueue); + } + + private void nudgeQueue() { + while (queue.size() > 0) { + execute( + () -> { + List batch = queue.take(); + process(batch); + }); + } + } + + private void execute(RunnableEx routine) { + executor.execute(routine); + } + + private void process(List attestations) { + if (attestations.isEmpty()) { + return; + } + + final Checkpoint target = attestations.get(0).getMessage().getData().getTarget(); + final Hash32 beaconBlockRoot = attestations.get(0).getMessage().getData().getBeaconBlockRoot(); + + Optional rootTuple = tupleStorage.get(beaconBlockRoot); + + // it must be present, otherwise, attestation couldn't be here + // TODO keep assertion for a while, it might be useful to discover bugs + assert rootTuple.isPresent(); + + EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.get().getState().getSlot()); + + // beaconBlockEpoch > targetEpoch + // it must either be equal or less than + if (beaconBlockEpoch.greater(target.getEpoch())) { + attestations.forEach(invalid::onNext); + return; + } + + // beaconBlockEpoch < targetEpoch && targetRoot != beaconBlockRoot + // target checkpoint is built with empty slots upon a block root + // in that case target root and beacon block root must be equal + if (beaconBlockEpoch.less(target.getEpoch()) && !target.getRoot().equals(beaconBlockRoot)) { + attestations.forEach(invalid::onNext); + return; + } + + // compute state, there must be the same state for all attestations + final BeaconState state = computeState(rootTuple.get(), target.getEpoch()); + final List validAttestations = new ArrayList<>(); + for (ReceivedAttestation attestation : attestations) { + // skip signature verification, it's handled by next processor + if (!spec.verify_attestation_impl(state, attestation.getMessage(), false)) { + validAttestations.add(attestation); + } else { + invalid.onNext(attestation); + } + } + + outbound.onNext(Pair.with(state, validAttestations)); + } + + private BeaconState computeState(BeaconTuple rootTuple, EpochNumber targetEpoch) { + EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.getState().getSlot()); + + // block is in the same epoch, no additional state is required to be built + if (beaconBlockEpoch.equals(targetEpoch)) { + return rootTuple.getState(); + } + + // build a state at epoch boundary, it must be enough to proceed + return emptySlotTransition.apply( + rootTuple.getState(), spec.compute_start_slot_of_epoch(targetEpoch)); + } + + public Publisher invalid() { + return invalid; + } + + private static final class StateTuple { + private final Checkpoint target; + private final Hash32 beaconBlockRoot; + + private StateTuple(Checkpoint target, Hash32 beaconBlockRoot) { + this.target = target; + this.beaconBlockRoot = beaconBlockRoot; + } + + static StateTuple from(ReceivedAttestation attestation) { + return new StateTuple( + attestation.getMessage().getData().getTarget(), + attestation.getMessage().getData().getBeaconBlockRoot()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StateTuple that = (StateTuple) o; + return Objects.equal(target, that.target) + && Objects.equal(beaconBlockRoot, that.beaconBlockRoot); + } + + @Override + public int hashCode() { + return Objects.hashCode(target, beaconBlockRoot); + } + } + + private static final class Queue { + + private final LinkedHashMap> queue = + new LinkedHashMap<>(); + + synchronized void add(ReceivedAttestation attestation) { + List bucket = + queue.computeIfAbsent(StateTuple.from(attestation), key -> new ArrayList<>()); + bucket.add(attestation); + } + + synchronized List take() { + Iterator> it = queue.values().iterator(); + if (it.hasNext()) { + List ret = it.next(); + it.remove(); + return ret; + } else { + return Collections.emptyList(); + } + } + + int size() { + return queue.size(); + } + } +} diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java index 6468cb7c7..4254738ec 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java @@ -225,6 +225,11 @@ default void process_attester_slashing(MutableBeaconState state, AttesterSlashin } default boolean verify_attestation(BeaconState state, Attestation attestation) { + return verify_attestation_impl(state, attestation, true); + } + + default boolean verify_attestation_impl(BeaconState state, Attestation attestation, + boolean verify_indexed) { /* data = attestation.data assert data.crosslink.shard < SHARD_COUNT assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) */ @@ -294,6 +299,10 @@ assert len(attestation.aggregation_bits) == len(attestation.custody_bits) == len return false; } + if (!verify_indexed) { + return true; + } + return is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)); } diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java index 15a6fc9ca..7a54c80a2 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java @@ -993,6 +993,11 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe """ */ default boolean is_valid_indexed_attestation(BeaconState state, IndexedAttestation indexed_attestation) { + return is_valid_indexed_attestation_impl(state, indexed_attestation, true); + } + + default boolean is_valid_indexed_attestation_impl(BeaconState state, IndexedAttestation indexed_attestation, + boolean verify_signature) { /* bit_0_indices = indexed_attestation.custody_bit_0_indices bit_1_indices = indexed_attestation.custody_bit_1_indices @@ -1024,6 +1029,10 @@ default boolean is_valid_indexed_attestation(BeaconState state, IndexedAttestati return false; } + if (!verify_signature) { + return true; + } + /* return bls_verify_multiple( pubkeys=[ diff --git a/types/src/main/java/org/ethereum/beacon/types/p2p/NodeId.java b/types/src/main/java/org/ethereum/beacon/types/p2p/NodeId.java new file mode 100644 index 000000000..efba92b44 --- /dev/null +++ b/types/src/main/java/org/ethereum/beacon/types/p2p/NodeId.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.types.p2p; + +import tech.pegasys.artemis.util.bytes.ArrayWrappingBytesValue; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class NodeId extends ArrayWrappingBytesValue implements BytesValue { + public NodeId(byte[] bytes) { + super(bytes); + } +} diff --git a/versions.gradle b/versions.gradle index 6e33972a9..a2afdca12 100644 --- a/versions.gradle +++ b/versions.gradle @@ -10,6 +10,7 @@ dependencyManagement { dependency "org.apache.logging.log4j:log4j-api:${log4j2Version}" dependency "org.apache.logging.log4j:log4j-core:${log4j2Version}" + dependency "org.apache.commons:commons-collections4:4.4" dependency 'org.ethereum:ethereumj-core:1+' dependency 'org.bouncycastle:bcprov-jdk15on:1.60' From d09e87eb54ef87c00d6c62b0e652c090deecfcfd Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 23 Aug 2019 19:47:27 +0600 Subject: [PATCH 02/59] Update signature verification in the pool --- .../beacon/chain/pool/AttestationPool.java | 7 +- .../chain/pool/AttestationProcessor.java | 5 + .../chain/pool/InMemoryAttestationPool.java | 14 +- ...ons.java => ProcessedAttestationPool.java} | 10 +- .../pool/unknown/UnknownBlockPoolImpl.java | 2 +- .../AttestationSignatureVerifier.java | 170 ++++++++++++------ .../verifier/AttestationStateVerifier.java | 166 +++++++---------- ...oolVerifierImpl.java => FullVerifier.java} | 18 +- .../verifier/SignatureVerificationSet.java | 39 ++++ .../beacon/consensus/spec/HelperFunction.java | 33 ++-- .../org/ethereum/beacon/crypto/BLS381.java | 25 +++ .../artemis/util/collections/Bitlist.java | 4 + 12 files changed, 303 insertions(+), 190 deletions(-) rename chain/src/main/java/org/ethereum/beacon/chain/pool/basic/{KnownAttestations.java => ProcessedAttestationPool.java} (67%) rename chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/{AttestationPoolVerifierImpl.java => FullVerifier.java} (62%) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java index f8d5b5f81..1807248c8 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.chain.pool; +import java.time.Duration; import org.ethereum.beacon.core.types.EpochNumber; import org.reactivestreams.Publisher; @@ -11,7 +12,11 @@ public interface AttestationPool { int MAX_KNOWN_ATTESTATIONS = 1_000_000; - int UNKNOWN_POOL_CAPACITY = 100_000; + int UNKNOWN_BLOCK_POOL_SIZE = 100_000; + + int VERIFIER_BUFFER_SIZE = 10_000; + + Duration VERIFIER_INTERVAL = Duration.ofMillis(50); Publisher valid(); diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java index 02e2b8f86..c9b527daa 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.chain.pool; +import java.util.List; import org.reactivestreams.Publisher; public interface AttestationProcessor { @@ -7,4 +8,8 @@ public interface AttestationProcessor { Publisher out(); void in(ReceivedAttestation attestation); + + default void batchIn(List batch) { + batch.forEach(this::in); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 5af1dc97c..e161353b5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -7,7 +7,7 @@ public class InMemoryAttestationPool implements AttestationPool { private final Publisher inbound; private final AttestationVerifier lightVerifier; - private final AttestationProcessor knownAttestations; + private final AttestationProcessor processedAttestationPool; private final UnknownBlockPool unknownBlockPool; private final AttestationVerifier fullVerifier; private final AttestationChurn churn; @@ -15,13 +15,13 @@ public class InMemoryAttestationPool implements AttestationPool { public InMemoryAttestationPool( Publisher inbound, AttestationVerifier lightVerifier, - AttestationProcessor knownAttestations, + AttestationProcessor processedAttestationPool, UnknownBlockPool unknownBlockPool, AttestationVerifier fullVerifier, AttestationChurn churn) { this.inbound = inbound; this.lightVerifier = lightVerifier; - this.knownAttestations = knownAttestations; + this.processedAttestationPool = processedAttestationPool; this.unknownBlockPool = unknownBlockPool; this.fullVerifier = fullVerifier; this.churn = churn; @@ -30,9 +30,11 @@ public InMemoryAttestationPool( @Override public void start() { Flux.from(inbound).subscribe(lightVerifier::in); - Flux.from(lightVerifier.out()).subscribe(knownAttestations::in); - Flux.from(knownAttestations.out()).subscribe(unknownBlockPool::in); - Flux.from(unknownBlockPool.out()).subscribe(fullVerifier::in); + Flux.from(lightVerifier.out()).subscribe(processedAttestationPool::in); + Flux.from(processedAttestationPool.out()).subscribe(unknownBlockPool::in); + Flux.from(unknownBlockPool.out()) + .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL) + .subscribe(fullVerifier::batchIn); Flux.from(fullVerifier.out()).subscribe(churn::in); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/KnownAttestations.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/ProcessedAttestationPool.java similarity index 67% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/basic/KnownAttestations.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/basic/ProcessedAttestationPool.java index e5a4d675c..b6befdc2f 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/KnownAttestations.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/ProcessedAttestationPool.java @@ -10,22 +10,22 @@ import org.ethereum.beacon.schedulers.Schedulers; import tech.pegasys.artemis.ethereum.core.Hash32; -public class KnownAttestations extends AbstractProcessor implements AttestationProcessor { +public class ProcessedAttestationPool extends AbstractProcessor implements AttestationProcessor { private static final Object ENTRY = new Object(); - private final LRUMap knownAttestationsCache = + private final LRUMap processedAttestationHash = new LRUMap<>(AttestationPool.MAX_KNOWN_ATTESTATIONS); private final Function hasher; - public KnownAttestations(Schedulers schedulers, Function hasher) { - super(schedulers, "KnownAttestations"); + public ProcessedAttestationPool(Schedulers schedulers, Function hasher) { + super(schedulers, "ProcessedAttestationPool"); this.hasher = hasher; } @Override public void in(ReceivedAttestation attestation) { - Object existed = knownAttestationsCache.put(hasher.apply(attestation.getMessage()), ENTRY); + Object existed = processedAttestationHash.put(hasher.apply(attestation.getMessage()), ENTRY); if (existed == null) { outbound.onNext(attestation); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java index 3cbbb3359..11c74480f 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java @@ -26,7 +26,7 @@ public class UnknownBlockPoolImpl extends AbstractProcessor implements UnknownBl private static final EpochNumber TRACKED_EPOCHS = EpochNumber.of(2).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); - private final Queue queue = new Queue(TRACKED_EPOCHS, AttestationPool.UNKNOWN_POOL_CAPACITY); + private final Queue queue = new Queue(TRACKED_EPOCHS, AttestationPool.UNKNOWN_BLOCK_POOL_SIZE); private final SimpleProcessor unknownBlock; private final Scheduler executor; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java index 02adf9e60..0906faba8 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java @@ -1,25 +1,30 @@ package org.ethereum.beacon.chain.pool.verifier; import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashMap; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.core.BeaconState; -import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; -import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.AttestationDataAndCustodyBit; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import org.ethereum.beacon.crypto.BLS381.Signature; +import org.ethereum.beacon.crypto.MessageParameters; import org.ethereum.beacon.schedulers.RunnableEx; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; -import org.javatuples.Pair; import org.reactivestreams.Publisher; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.collections.Bitlist; +import tech.pegasys.artemis.util.uint.UInt64; public class AttestationSignatureVerifier { - private final Queue queue = new Queue(); private final Scheduler executor; private final BeaconChainSpec spec; @@ -31,8 +36,7 @@ public AttestationSignatureVerifier( this.executor = executor; this.spec = spec; - this.valid = - new SimpleProcessor<>(schedulers.events(), "AttestationSignatureVerifier.valid"); + this.valid = new SimpleProcessor<>(schedulers.events(), "AttestationSignatureVerifier.valid"); this.invalid = new SimpleProcessor<>(schedulers.events(), "AttestationSignatureVerifier.invalid"); } @@ -41,76 +45,124 @@ public Publisher valid() { return valid; } - public void inbound(Pair> attestationTuple) { - queue.add(attestationTuple); - execute(this::nudgeQueue); + public Publisher invalid() { + return invalid; } - private void nudgeQueue() { - while (queue.size() > 0) { - execute( - () -> { - Pair> batch = queue.take(); - process(batch); - }); - } + public void in(List batch) { + execute( + () -> { + + // validate signature encoding format + List valid = new ArrayList<>(); + for (SignatureVerificationSet set : batch) { + if (BLS381.Signature.validate(set.getAttestation().getMessage().getSignature())) { + valid.add(set); + } else { + invalid.onNext(set.getAttestation()); + } + } + + Map> groupedBySignedMessage = + valid.stream() + .collect( + Collectors.groupingBy(data -> data.getAttestation().getMessage().getData())); + + groupedBySignedMessage.values().forEach(this::process); + }); } - private void execute(RunnableEx routine) { - executor.execute(routine); - } + private void process(List attestations) { + final UInt64 domain = attestations.get(0).getDomain(); + final AttestationData data = attestations.get(0).getAttestation().getMessage().getData(); + + // for aggregation sake, smaller aggregates should go first + attestations.sort( + Comparator.comparing(set -> set.getAttestation().getMessage().getAggregationBits().size())); + + // try to aggregate as much as we can + List aggregates = new ArrayList<>(); + List nonAggregates = new ArrayList<>(); + AggregateVerifier verifier = + new AggregateVerifier(spec.getConstants().getMaxValidatorsPerCommittee().getIntValue()); + for (SignatureVerificationSet set : attestations) { + if (verifier.add(set)) { + aggregates.add(set); + } else { + nonAggregates.add(set); + } + } - private void process(Pair> attestationTuple) { - if (attestationTuple.getValue1().isEmpty()) { - return; + // verify aggregate and fall back to one-by-one verification if it has failed + if (verifier.verify(spec, data, domain)) { + aggregates.forEach(set -> valid.onNext(set.getAttestation())); + } else { + nonAggregates = attestations; } - final BeaconState state = attestationTuple.getValue0(); - for (ReceivedAttestation attestation : attestationTuple.getValue1()) { - IndexedAttestation indexedAttestation = - spec.get_indexed_attestation(state, attestation.getMessage()); - if (spec.is_valid_indexed_attestation(state, indexedAttestation)) { - valid.onNext(attestation); + for (SignatureVerificationSet set : nonAggregates) { + if (verify(spec, set, domain)) { + valid.onNext(set.getAttestation()); } else { - invalid.onNext(attestation); + invalid.onNext(set.getAttestation()); } } } - public Publisher invalid() { - return invalid; + private boolean verify(BeaconChainSpec spec, SignatureVerificationSet set, UInt64 domain) { + AttestationData data = set.getAttestation().getMessage().getData(); + Hash32 bit0Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, false)); + Hash32 bit1Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, true)); + + return BLS381.verifyMultiple( + Arrays.asList( + MessageParameters.create(bit0Hash, domain), MessageParameters.create(bit1Hash, domain)), + Signature.createWithoutValidation(set.getAttestation().getMessage().getSignature()), + Arrays.asList(set.getBit0AggregateKey(), set.getBit1AggregateKey())); } - private static final class Queue { + private static final class AggregateVerifier { + List bit0AggregateKeys = new ArrayList<>(); + List bit1AggregateKeys = new ArrayList<>(); + List signatures = new ArrayList<>(); + Bitlist bits; - private final LinkedHashMap>> queue = - new LinkedHashMap<>(); + AggregateVerifier(int maxValidatorsPerCommittee) { + this.bits = Bitlist.of(maxValidatorsPerCommittee); + } - synchronized void add(Pair> attestationTuple) { - if (attestationTuple.getValue1().isEmpty()) { - return; + boolean add(SignatureVerificationSet set) { + if (!bits.and(set.getAttestation().getMessage().getAggregationBits()).isEmpty()) { + return false; } - Pair> bucket = - queue.computeIfAbsent( - attestationTuple.getValue1().get(0).getMessage().getData().getTarget(), - key -> Pair.with(attestationTuple.getValue0(), new ArrayList<>())); - bucket.getValue1().addAll(attestationTuple.getValue1()); - } + bit0AggregateKeys.add(set.getBit0AggregateKey()); + bit1AggregateKeys.add(set.getBit1AggregateKey()); + signatures.add( + BLS381.Signature.createWithoutValidation( + set.getAttestation().getMessage().getSignature())); - synchronized Pair> take() { - Iterator>> it = queue.values().iterator(); - if (it.hasNext()) { - Pair> ret = it.next(); - it.remove(); - return ret; - } else { - return Pair.with(BeaconState.getEmpty(), Collections.emptyList()); - } + return true; } - int size() { - return queue.size(); + boolean verify(BeaconChainSpec spec, AttestationData data, UInt64 domain) { + PublicKey bit0Key = PublicKey.aggregate(bit0AggregateKeys); + PublicKey bit1Key = PublicKey.aggregate(bit1AggregateKeys); + Signature signature = Signature.aggregate(signatures); + + Hash32 bit0Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, false)); + Hash32 bit1Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, true)); + + return BLS381.verifyMultiple( + Arrays.asList( + MessageParameters.create(bit0Hash, domain), + MessageParameters.create(bit1Hash, domain)), + signature, + Arrays.asList(bit0Key, bit1Key)); } } + + private void execute(RunnableEx routine) { + executor.execute(routine); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java index 97c1f5e57..27aa6b31b 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java @@ -1,19 +1,20 @@ package org.ethereum.beacon.chain.pool.verifier; -import com.google.common.base.Objects; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashMap; +import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; + import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.storage.BeaconTupleStorage; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.transition.EmptySlotTransition; import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.BLSPubkey; import org.ethereum.beacon.core.types.EpochNumber; import org.ethereum.beacon.schedulers.RunnableEx; import org.ethereum.beacon.schedulers.Scheduler; @@ -22,16 +23,16 @@ import org.javatuples.Pair; import org.reactivestreams.Publisher; import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; public class AttestationStateVerifier { - private final Queue queue = new Queue(); private final Scheduler executor; private final BeaconTupleStorage tupleStorage; private final BeaconChainSpec spec; private final EmptySlotTransition emptySlotTransition; - private final SimpleProcessor>> outbound; + private final SimpleProcessor valid; private final SimpleProcessor invalid; public AttestationStateVerifier( @@ -45,41 +46,38 @@ public AttestationStateVerifier( this.spec = spec; this.emptySlotTransition = emptySlotTransition; - this.outbound = new SimpleProcessor<>(schedulers.events(), "AttestationStateVerifier.outbound"); + this.valid = new SimpleProcessor<>(schedulers.events(), "AttestationStateVerifier.valid"); this.invalid = new SimpleProcessor<>(schedulers.events(), "AttestationStateVerifier.invalid"); } - public Publisher>> outbound() { - return outbound; + public Publisher valid() { + return valid; } - public void inbound(ReceivedAttestation attestation) { - queue.add(attestation); - execute(this::nudgeQueue); + public Publisher invalid() { + return invalid; } - private void nudgeQueue() { - while (queue.size() > 0) { - execute( - () -> { - List batch = queue.take(); - process(batch); - }); - } + public void in(List batch) { + execute( + () -> { + Map, List> groupedByState = + batch.stream() + .collect( + Collectors.groupingBy( + attestation -> + Pair.with( + attestation.getMessage().getData().getTarget(), + attestation.getMessage().getData().getBeaconBlockRoot()))); + + groupedByState.forEach((key, value) -> process(key.getValue0(), key.getValue1(), value)); + }); } - private void execute(RunnableEx routine) { - executor.execute(routine); - } - - private void process(List attestations) { - if (attestations.isEmpty()) { - return; - } - - final Checkpoint target = attestations.get(0).getMessage().getData().getTarget(); - final Hash32 beaconBlockRoot = attestations.get(0).getMessage().getData().getBeaconBlockRoot(); - + private void process( + final Checkpoint target, + final Hash32 beaconBlockRoot, + List attestations) { Optional rootTuple = tupleStorage.get(beaconBlockRoot); // it must be present, otherwise, attestation couldn't be here @@ -103,19 +101,42 @@ private void process(List attestations) { return; } - // compute state, there must be the same state for all attestations + // compute state and domain, there must be the same state for all attestations final BeaconState state = computeState(rootTuple.get(), target.getEpoch()); - final List validAttestations = new ArrayList<>(); + final UInt64 domain = spec.get_domain(state, ATTESTATION, target.getEpoch()); for (ReceivedAttestation attestation : attestations) { - // skip signature verification, it's handled by next processor + // skip signature verification, it's passed on the next processor if (!spec.verify_attestation_impl(state, attestation.getMessage(), false)) { - validAttestations.add(attestation); - } else { invalid.onNext(attestation); + continue; + } + + // compute and verify indexed attestation + IndexedAttestation indexedAttestation = + spec.get_indexed_attestation(state, attestation.getMessage()); + if (!spec.is_valid_indexed_attestation_impl(state, indexedAttestation, false)) { + invalid.onNext(attestation); + continue; } - } - outbound.onNext(Pair.with(state, validAttestations)); + // compute data required for signature verification + List bit0Keys = + indexedAttestation.getCustodyBit0Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + List bit1Keys = + indexedAttestation.getCustodyBit1Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + + // send them to signature verifier + valid.onNext( + new SignatureVerificationSet( + spec.bls_aggregate_pubkeys_no_validate(bit0Keys), + spec.bls_aggregate_pubkeys_no_validate(bit1Keys), + domain, + attestation)); + } } private BeaconState computeState(BeaconTuple rootTuple, EpochNumber targetEpoch) { @@ -131,68 +152,7 @@ private BeaconState computeState(BeaconTuple rootTuple, EpochNumber targetEpoch) rootTuple.getState(), spec.compute_start_slot_of_epoch(targetEpoch)); } - public Publisher invalid() { - return invalid; - } - - private static final class StateTuple { - private final Checkpoint target; - private final Hash32 beaconBlockRoot; - - private StateTuple(Checkpoint target, Hash32 beaconBlockRoot) { - this.target = target; - this.beaconBlockRoot = beaconBlockRoot; - } - - static StateTuple from(ReceivedAttestation attestation) { - return new StateTuple( - attestation.getMessage().getData().getTarget(), - attestation.getMessage().getData().getBeaconBlockRoot()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - StateTuple that = (StateTuple) o; - return Objects.equal(target, that.target) - && Objects.equal(beaconBlockRoot, that.beaconBlockRoot); - } - - @Override - public int hashCode() { - return Objects.hashCode(target, beaconBlockRoot); - } - } - - private static final class Queue { - - private final LinkedHashMap> queue = - new LinkedHashMap<>(); - - synchronized void add(ReceivedAttestation attestation) { - List bucket = - queue.computeIfAbsent(StateTuple.from(attestation), key -> new ArrayList<>()); - bucket.add(attestation); - } - - synchronized List take() { - Iterator> it = queue.values().iterator(); - if (it.hasNext()) { - List ret = it.next(); - it.remove(); - return ret; - } else { - return Collections.emptyList(); - } - } - - int size() { - return queue.size(); - } + private void execute(RunnableEx routine) { + executor.execute(routine); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationPoolVerifierImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/FullVerifier.java similarity index 62% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationPoolVerifierImpl.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/FullVerifier.java index 8e086ed3f..2b844e33c 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationPoolVerifierImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/FullVerifier.java @@ -1,19 +1,26 @@ package org.ethereum.beacon.chain.pool.verifier; +import java.util.Collections; +import java.util.List; +import org.ethereum.beacon.chain.pool.AttestationPool; import org.ethereum.beacon.chain.pool.AttestationVerifier; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; -public class AttestationPoolVerifierImpl implements AttestationVerifier { +public class FullVerifier implements AttestationVerifier { private final AttestationStateVerifier stateVerifier; private final AttestationSignatureVerifier signatureVerifier; - public AttestationPoolVerifierImpl( + public FullVerifier( AttestationStateVerifier stateVerifier, AttestationSignatureVerifier signatureVerifier) { this.stateVerifier = stateVerifier; this.signatureVerifier = signatureVerifier; + + Flux.from(stateVerifier.valid()) + .bufferTimeout(AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL) + .subscribe(signatureVerifier::in); } @Override @@ -23,7 +30,12 @@ public Publisher valid() { @Override public void in(ReceivedAttestation attestation) { - stateVerifier.inbound(attestation); + stateVerifier.in(Collections.singletonList(attestation)); + } + + @Override + public void batchIn(List batch) { + stateVerifier.in(batch); } @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java new file mode 100644 index 000000000..57bfc46a4 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java @@ -0,0 +1,39 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import tech.pegasys.artemis.util.uint.UInt64; + +final class SignatureVerificationSet { + private final PublicKey bit0AggregateKey; + private final PublicKey bit1AggregateKey; + private final UInt64 domain; + private final ReceivedAttestation attestation; + + SignatureVerificationSet( + PublicKey bit0AggregateKey, + PublicKey bit1AggregatedKey, + UInt64 domain, + ReceivedAttestation attestation) { + this.bit0AggregateKey = bit0AggregateKey; + this.bit1AggregateKey = bit1AggregatedKey; + this.domain = domain; + this.attestation = attestation; + } + + public PublicKey getBit0AggregateKey() { + return bit0AggregateKey; + } + + public PublicKey getBit1AggregateKey() { + return bit1AggregateKey; + } + + public UInt64 getDomain() { + return domain; + } + + public ReceivedAttestation getAttestation() { + return attestation; + } +} diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java index 7a54c80a2..abea79f0e 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java @@ -1,6 +1,17 @@ package org.ethereum.beacon.consensus.spec; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; + import com.google.common.collect.Ordering; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.BeaconBlockBody; import org.ethereum.beacon.core.BeaconBlockHeader; @@ -37,18 +48,6 @@ import tech.pegasys.artemis.util.uint.UInt64; import tech.pegasys.artemis.util.uint.UInt64s; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Collections.emptyList; -import static java.util.stream.Collectors.toList; -import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; - /** * Helper functions. * @@ -900,6 +899,16 @@ default PublicKey bls_aggregate_pubkeys(List publicKeysBytes) { return PublicKey.aggregate(publicKeys); } + default PublicKey bls_aggregate_pubkeys_no_validate(List publicKeysBytes) { + if (!isBlsVerify()) { + return PublicKey.aggregate(Collections.emptyList()); + } + + List publicKeys = + publicKeysBytes.stream().map(PublicKey::createWithoutValidation).collect(toList()); + return PublicKey.aggregate(publicKeys); + } + /* def get_domain(state: BeaconState, domain_type: DomainType, message_epoch: Epoch=None) -> Domain: """ diff --git a/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java b/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java index 5977574f2..7d0c4d931 100644 --- a/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java +++ b/crypto/src/main/java/org/ethereum/beacon/crypto/BLS381.java @@ -213,6 +213,31 @@ public static Signature create(Bytes96 encoded) { return new Signature(encoded); } + public static Signature createWithoutValidation(Bytes96 encoded) { + return new Signature(encoded); + } + + public static boolean validate(Bytes96 encoded) { + Validator.Result result = Validator.G2.validate(encoded); + if (!result.isValid()) { + return false; + } + + if (!Codec.G2.decode(encoded).isInfinity()) { + ECP2 point = G2.decode(encoded); + if (point.is_infinity()) { + return false; + } + + ECP2 orderCheck = point.mul(ORDER); + if (!orderCheck.is_infinity()) { + return false; + } + } + + return true; + } + /** * Aggregates a list of signatures into a single one. * diff --git a/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java b/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java index 074a98f51..016320c69 100644 --- a/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java +++ b/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java @@ -210,6 +210,10 @@ public int size() { return size; } + public boolean isEmpty() { + return BytesValues.countZeros(this) == size; + } + public long maxSize() { return maxSize; } From 47023dae9d74752289586fa541046de4034b3619 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 28 Aug 2019 01:19:26 +0600 Subject: [PATCH 03/59] Rework reactor logic in attestation pool --- .../beacon/chain/pool/AbstractProcessor.java | 18 -- .../beacon/chain/pool/AbstractVerifier.java | 25 --- .../beacon/chain/pool/AttestationChurn.java | 10 - .../beacon/chain/pool/AttestationPool.java | 11 +- .../chain/pool/AttestationProcessor.java | 15 -- .../chain/pool/AttestationVerifier.java | 15 -- .../beacon/chain/pool/CheckedAttestation.java | 19 ++ .../chain/pool/InMemoryAttestationPool.java | 147 ++++++++++--- .../beacon/chain/pool/StatefulProcessor.java | 6 + .../beacon/chain/pool/UnknownBlockPool.java | 8 - .../pool/basic/ProcessedAttestationPool.java | 33 --- .../pool/checker/AttestationChecker.java | 7 + .../SanityChecker.java} | 54 +---- .../checker/SignatureEncodingChecker.java | 12 + .../pool/churn/AttestationChurnImpl.java | 17 -- .../pool/{ => churn}/OffChainAggregates.java | 2 +- .../reactor/AttestationChurnProcessor.java | 13 ++ .../AttestationVerificationProcessor.java | 35 +++ .../chain/pool/reactor/IdentifyProcessor.java | 47 ++++ .../beacon/chain/pool/reactor/Input.java | 23 ++ .../pool/reactor/SanityCheckProcessor.java | 32 +++ .../pool/registry/AttestationRegistry.java | 7 + .../pool/registry/ProcessedAttestations.java | 23 ++ .../pool/{unknown => registry}/Queue.java | 2 +- .../pool/registry/UnknownAttestationPool.java | 74 +++++++ .../pool/unknown/UnknownBlockPoolImpl.java | 124 ----------- .../verifier/AggregateSignatureVerifier.java | 205 ++++++++++++++++++ .../AttestationSignatureVerifier.java | 168 -------------- .../verifier/AttestationStateVerifier.java | 158 -------------- .../pool/verifier/AttestationVerifier.java | 154 +++++++++++++ .../chain/pool/verifier/BatchVerifier.java | 9 + .../chain/pool/verifier/FullVerifier.java | 45 ---- .../verifier/SignatureVerificationSet.java | 39 ---- .../pool/verifier/SignatureVerifier.java | 10 + .../pool/verifier/VerificationResult.java | 41 ++++ .../beacon/consensus/spec/HelperFunction.java | 14 +- .../stream/AbstractDelegateProcessor.java | 56 +++++ .../org/ethereum/beacon/stream/Fluxes.java | 43 ++++ .../beacon/stream/OutsourcePublisher.java | 21 ++ 39 files changed, 983 insertions(+), 759 deletions(-) delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/basic/ProcessedAttestationPool.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java rename chain/src/main/java/org/ethereum/beacon/chain/pool/{basic/LightVerifier.java => checker/SanityChecker.java} (53%) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java rename chain/src/main/java/org/ethereum/beacon/chain/pool/{ => churn}/OffChainAggregates.java (92%) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java rename chain/src/main/java/org/ethereum/beacon/chain/pool/{unknown => registry}/Queue.java (98%) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/FullVerifier.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java create mode 100644 util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java create mode 100644 util/src/main/java/org/ethereum/beacon/stream/Fluxes.java create mode 100644 util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java deleted file mode 100644 index 252ba3570..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractProcessor.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.SimpleProcessor; -import org.reactivestreams.Publisher; - -public abstract class AbstractProcessor implements AttestationProcessor { - protected final SimpleProcessor outbound; - - public AbstractProcessor(Schedulers schedulers, String name) { - this.outbound = new SimpleProcessor<>(schedulers.events(), name + ".outbound"); - } - - @Override - public Publisher out() { - return outbound; - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java deleted file mode 100644 index fd6678b49..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AbstractVerifier.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.SimpleProcessor; -import org.reactivestreams.Publisher; - -public abstract class AbstractVerifier implements AttestationVerifier { - protected final SimpleProcessor valid; - protected final SimpleProcessor invalid; - - public AbstractVerifier(Schedulers schedulers, String name) { - this.valid = new SimpleProcessor<>(schedulers.events(), name + ".valid"); - this.invalid = new SimpleProcessor<>(schedulers.events(), name + ".invalid"); - } - - @Override - public Publisher valid() { - return valid; - } - - @Override - public Publisher invalid() { - return invalid; - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java deleted file mode 100644 index d25dd9e7b..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationChurn.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -import org.reactivestreams.Publisher; - -public interface AttestationChurn { - - Publisher out(); - - void in(ReceivedAttestation attestation); -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java index 1807248c8..40e9fda3d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -1,12 +1,13 @@ package org.ethereum.beacon.chain.pool; import java.time.Duration; +import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; import org.ethereum.beacon.core.types.EpochNumber; import org.reactivestreams.Publisher; public interface AttestationPool { - int MAX_THREADS = Runtime.getRuntime().availableProcessors(); + int MAX_THREADS = 32; EpochNumber MAX_ATTESTATION_LOOKAHEAD = EpochNumber.of(1); @@ -18,13 +19,13 @@ public interface AttestationPool { Duration VERIFIER_INTERVAL = Duration.ofMillis(50); - Publisher valid(); + Publisher getValid(); - Publisher invalid(); + Publisher getInvalid(); - Publisher unknownBlock(); + Publisher getUnknownAttestations(); - Publisher aggregates(); + Publisher getAggregates(); void start(); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java deleted file mode 100644 index c9b527daa..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationProcessor.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -import java.util.List; -import org.reactivestreams.Publisher; - -public interface AttestationProcessor { - - Publisher out(); - - void in(ReceivedAttestation attestation); - - default void batchIn(List batch) { - batch.forEach(this::in); - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java deleted file mode 100644 index 2539b8cb8..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationVerifier.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -import org.reactivestreams.Publisher; - -public interface AttestationVerifier extends AttestationProcessor { - - default Publisher out() { - return valid(); - } - - Publisher valid(); - - Publisher invalid(); - -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java new file mode 100644 index 000000000..d9e42609e --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java @@ -0,0 +1,19 @@ +package org.ethereum.beacon.chain.pool; + +public class CheckedAttestation { + private final boolean passed; + private final ReceivedAttestation attestation; + + public CheckedAttestation(boolean passed, ReceivedAttestation attestation) { + this.passed = passed; + this.attestation = attestation; + } + + public boolean isPassed() { + return passed; + } + + public ReceivedAttestation getAttestation() { + return attestation; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index e161353b5..c6f4ad5d4 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -1,60 +1,143 @@ package org.ethereum.beacon.chain.pool; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; +import org.ethereum.beacon.chain.pool.reactor.AttestationChurnProcessor; +import org.ethereum.beacon.chain.pool.reactor.AttestationVerificationProcessor; +import org.ethereum.beacon.chain.pool.reactor.IdentifyProcessor; +import org.ethereum.beacon.chain.pool.reactor.Input; +import org.ethereum.beacon.chain.pool.reactor.SanityCheckProcessor; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.Fluxes; +import org.ethereum.beacon.stream.Fluxes.FluxSplit; import org.reactivestreams.Publisher; +import reactor.core.publisher.DirectProcessor; import reactor.core.publisher.Flux; public class InMemoryAttestationPool implements AttestationPool { - private final Publisher inbound; - private final AttestationVerifier lightVerifier; - private final AttestationProcessor processedAttestationPool; - private final UnknownBlockPool unknownBlockPool; - private final AttestationVerifier fullVerifier; - private final AttestationChurn churn; + private final Publisher source; + private final Publisher newSlots; + private final Publisher finalizedCheckpoints; + private final Publisher importedBlocks; + private final Publisher chainHeads; + private final Schedulers schedulers; + + private final SanityCheckProcessor sanityChecker; + private final SignatureEncodingChecker encodingChecker; + private final ProcessedAttestations processedFilter; + private final IdentifyProcessor identifier; + private final AttestationVerificationProcessor verifier; + private final AttestationChurnProcessor churn; + + private final DirectProcessor invalidAttestations = DirectProcessor.create(); + private final DirectProcessor validAttestations = DirectProcessor.create(); public InMemoryAttestationPool( - Publisher inbound, - AttestationVerifier lightVerifier, - AttestationProcessor processedAttestationPool, - UnknownBlockPool unknownBlockPool, - AttestationVerifier fullVerifier, - AttestationChurn churn) { - this.inbound = inbound; - this.lightVerifier = lightVerifier; - this.processedAttestationPool = processedAttestationPool; - this.unknownBlockPool = unknownBlockPool; - this.fullVerifier = fullVerifier; + Publisher source, + Publisher newSlots, + Publisher finalizedCheckpoints, + Publisher importedBlocks, + Publisher chainHeads, + Schedulers schedulers, + SanityCheckProcessor sanityChecker, + SignatureEncodingChecker encodingChecker, + ProcessedAttestations processedFilter, + IdentifyProcessor identifier, + AttestationVerificationProcessor verifier, + AttestationChurnProcessor churn) { + this.source = source; + this.newSlots = newSlots; + this.finalizedCheckpoints = finalizedCheckpoints; + this.importedBlocks = importedBlocks; + this.chainHeads = chainHeads; + this.schedulers = schedulers; + this.sanityChecker = sanityChecker; + this.encodingChecker = encodingChecker; + this.processedFilter = processedFilter; + this.identifier = identifier; + this.verifier = verifier; this.churn = churn; } @Override public void start() { - Flux.from(inbound).subscribe(lightVerifier::in); - Flux.from(lightVerifier.out()).subscribe(processedAttestationPool::in); - Flux.from(processedAttestationPool.out()).subscribe(unknownBlockPool::in); - Flux.from(unknownBlockPool.out()) - .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL) - .subscribe(fullVerifier::batchIn); - Flux.from(fullVerifier.out()).subscribe(churn::in); + Scheduler executor = + schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); + + Flux sourceFx = Flux.from(source).map(Input::wrap).publishOn(executor.toReactor()); + Flux newSlotsFx = Flux.from(newSlots).map(Input::wrap).publishOn(executor.toReactor()); + Flux importedBlocksFx = + Flux.from(importedBlocks).map(Input::wrap).publishOn(executor.toReactor()); + Flux finalizedCheckpointsFx = + Flux.from(finalizedCheckpoints).map(Input::wrap).publishOn(executor.toReactor()); + Flux chainHeadsFx = + Flux.from(chainHeads).map(Input::wrap).publishOn(executor.toReactor()); + + // subscribe sanity checker to its inputs + Flux.merge(sourceFx, newSlotsFx, finalizedCheckpointsFx).subscribe(sanityChecker); + FluxSplit sanityOut = + Fluxes.split(sanityChecker, CheckedAttestation::isPassed); + + // filter already processed attestations + Flux newAttestations = + sanityOut + .getSatisfied() + .map(CheckedAttestation::getAttestation) + .filter(processedFilter::add); + + // check signature encoding + FluxSplit encodingCheckOut = + Fluxes.split(newAttestations, encodingChecker::check); + + // identify attestation target + Flux.merge(encodingCheckOut.getSatisfied().map(Input::wrap), newSlotsFx, importedBlocksFx) + .subscribe(identifier); + + // verify attestations + identifier.bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL).subscribe(verifier); + FluxSplit verifierOut = + Fluxes.split(verifier, CheckedAttestation::isPassed); + + // feed churn + Flux.merge( + verifierOut.getSatisfied().map(CheckedAttestation::getAttestation), + newSlotsFx, + chainHeadsFx); + + // expose invalid attestations + Flux.merge( + sanityOut.getUnsatisfied().map(CheckedAttestation::getAttestation), + encodingCheckOut.getUnsatisfied(), + verifierOut.getUnsatisfied().map(CheckedAttestation::getAttestation)) + .subscribe(invalidAttestations); + + // expose valid attestations + verifierOut.getSatisfied().map(CheckedAttestation::getAttestation).subscribe(validAttestations); } @Override - public Publisher valid() { - return fullVerifier.valid(); + public Publisher getValid() { + return validAttestations; } @Override - public Publisher invalid() { - return Flux.concat(lightVerifier.invalid(), fullVerifier.invalid()); + public Publisher getInvalid() { + return invalidAttestations; } @Override - public Publisher unknownBlock() { - return unknownBlockPool.unknownBlock(); + public Publisher getUnknownAttestations() { + return identifier.getUnknownAttestations(); } @Override - public Publisher aggregates() { - return churn.out(); + public Publisher getAggregates() { + return churn; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java new file mode 100644 index 000000000..86be41cea --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java @@ -0,0 +1,6 @@ +package org.ethereum.beacon.chain.pool; + +public interface StatefulProcessor { + + boolean isStateReady(); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java deleted file mode 100644 index abdcced6b..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/UnknownBlockPool.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -import org.reactivestreams.Publisher; - -public interface UnknownBlockPool extends AttestationProcessor { - - Publisher unknownBlock(); -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/ProcessedAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/ProcessedAttestationPool.java deleted file mode 100644 index b6befdc2f..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/ProcessedAttestationPool.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.ethereum.beacon.chain.pool.basic; - -import java.util.function.Function; -import org.apache.commons.collections4.map.LRUMap; -import org.ethereum.beacon.chain.pool.AbstractProcessor; -import org.ethereum.beacon.chain.pool.AttestationPool; -import org.ethereum.beacon.chain.pool.AttestationProcessor; -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.core.operations.Attestation; -import org.ethereum.beacon.schedulers.Schedulers; -import tech.pegasys.artemis.ethereum.core.Hash32; - -public class ProcessedAttestationPool extends AbstractProcessor implements AttestationProcessor { - - private static final Object ENTRY = new Object(); - - private final LRUMap processedAttestationHash = - new LRUMap<>(AttestationPool.MAX_KNOWN_ATTESTATIONS); - private final Function hasher; - - public ProcessedAttestationPool(Schedulers schedulers, Function hasher) { - super(schedulers, "ProcessedAttestationPool"); - this.hasher = hasher; - } - - @Override - public void in(ReceivedAttestation attestation) { - Object existed = processedAttestationHash.put(hasher.apply(attestation.getMessage()), ENTRY); - if (existed == null) { - outbound.onNext(attestation); - } - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java new file mode 100644 index 000000000..7d95cd0c0 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java @@ -0,0 +1,7 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; + +public interface AttestationChecker { + boolean check(ReceivedAttestation attestation); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/LightVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java similarity index 53% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/basic/LightVerifier.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java index 254d44b72..f3d85e16d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/basic/LightVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java @@ -1,55 +1,30 @@ -package org.ethereum.beacon.chain.pool.basic; +package org.ethereum.beacon.chain.pool.checker; -import com.google.common.annotations.VisibleForTesting; -import org.ethereum.beacon.chain.pool.AbstractVerifier; import org.ethereum.beacon.chain.pool.AttestationPool; -import org.ethereum.beacon.chain.pool.AttestationVerifier; import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.StatefulProcessor; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.EpochNumber; import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.schedulers.Schedulers; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -public class LightVerifier extends AbstractVerifier implements AttestationVerifier { +public class SanityChecker implements AttestationChecker, StatefulProcessor { private final BeaconChainSpec spec; private Checkpoint finalizedCheckpoint; private EpochNumber maxAcceptableEpoch; - public LightVerifier( - Schedulers schedulers, - Publisher finalizedCheckpoint, - Publisher slotClock, - BeaconChainSpec spec) { - super(schedulers, "BasicVerifier"); - + public SanityChecker(BeaconChainSpec spec) { this.spec = spec; - Flux.from(finalizedCheckpoint).subscribe(this::onNewFinalizedCheckpoint); - Flux.from(slotClock).subscribe(this::onNewSlot); } @Override - public void in(ReceivedAttestation attestation) { - if (!isInitialized()) { - return; - } - - if (isValid(attestation)) { - valid.onNext(attestation); - } else { - invalid.onNext(attestation); - } - } + public boolean check(ReceivedAttestation attestation) { + assert isStateReady(); - private boolean isValid(ReceivedAttestation attestation) { final AttestationData data = attestation.getMessage().getData(); - final Checkpoint finalizedCheckpoint = this.finalizedCheckpoint; - final EpochNumber maxAcceptableEpoch = this.maxAcceptableEpoch; // sourceEpoch >= targetEpoch if (data.getSource().getEpoch().greaterEqual(data.getTarget().getEpoch())) { @@ -85,24 +60,17 @@ private boolean isValid(ReceivedAttestation attestation) { return true; } - @VisibleForTesting - void onNewFinalizedCheckpoint(Checkpoint checkpoint) { + public void feedFinalizedCheckpoint(Checkpoint checkpoint) { this.finalizedCheckpoint = checkpoint; } - @VisibleForTesting - void onNewSlot(SlotNumber slot) { + public void feedNewSlot(SlotNumber newSlot) { this.maxAcceptableEpoch = - spec.compute_epoch_of_slot(slot).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); - } - - @VisibleForTesting - boolean isInitialized() { - return finalizedCheckpoint != null && maxAcceptableEpoch != null; + spec.compute_epoch_of_slot(newSlot).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); } @Override - public Publisher invalid() { - return invalid; + public boolean isStateReady() { + return finalizedCheckpoint != null && maxAcceptableEpoch != null; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java new file mode 100644 index 000000000..6306dfa43 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java @@ -0,0 +1,12 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.crypto.BLS381; + +public class SignatureEncodingChecker implements AttestationChecker { + + @Override + public boolean check(ReceivedAttestation attestation) { + return BLS381.Signature.validate(attestation.getMessage().getSignature()); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java deleted file mode 100644 index 17b9472d0..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.ethereum.beacon.chain.pool.churn; - -import org.ethereum.beacon.chain.pool.AttestationChurn; -import org.ethereum.beacon.chain.pool.OffChainAggregates; -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.reactivestreams.Publisher; - -public class AttestationChurnImpl implements AttestationChurn { - - @Override - public Publisher out() { - return null; - } - - @Override - public void in(ReceivedAttestation attestation) {} -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java similarity index 92% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java index 92a3b32f7..55607e568 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.chain.pool; +package org.ethereum.beacon.chain.pool.churn; import java.util.List; import org.ethereum.beacon.core.operations.Attestation; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java new file mode 100644 index 000000000..0cd99a70a --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java @@ -0,0 +1,13 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; +import org.ethereum.beacon.stream.AbstractDelegateProcessor; + +public class AttestationChurnProcessor + extends AbstractDelegateProcessor { + + @Override + protected void hookOnNext(Input value) { + // TODO implement + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java new file mode 100644 index 000000000..e9a9ba54e --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java @@ -0,0 +1,35 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import java.util.List; +import java.util.stream.Stream; +import org.ethereum.beacon.chain.pool.CheckedAttestation; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; +import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; +import org.ethereum.beacon.chain.pool.verifier.VerificationResult; +import org.ethereum.beacon.chain.storage.BeaconTupleStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.stream.AbstractDelegateProcessor; + +public class AttestationVerificationProcessor + extends AbstractDelegateProcessor, CheckedAttestation> { + + private final BatchVerifier verifier; + + public AttestationVerificationProcessor( + BeaconTupleStorage tupleStorage, + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition) { + this.verifier = new AttestationVerifier(tupleStorage, spec, emptySlotTransition); + } + + @Override + protected void hookOnNext(List batch) { + VerificationResult result = verifier.verify(batch); + Stream.concat( + result.getValid().stream().map(att -> new CheckedAttestation(true, att)), + result.getInvalid().stream().map(att -> new CheckedAttestation(false, att))) + .forEach(this::publishOut); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java new file mode 100644 index 000000000..69f8fb531 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java @@ -0,0 +1,47 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.storage.BeaconBlockStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.stream.AbstractDelegateProcessor; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.FluxSink; + +public class IdentifyProcessor extends AbstractDelegateProcessor { + + private final UnknownAttestationPool pool; + private final DirectProcessor unknownAttestations = DirectProcessor.create(); + private final FluxSink unknownOut = unknownAttestations.sink(); + + public IdentifyProcessor(BeaconBlockStorage blockStorage, BeaconChainSpec spec) { + this.pool = new UnknownAttestationPool(blockStorage, spec); + } + + @Override + protected void hookOnNext(Input value) { + if (value.getType().equals(BeaconBlock.class)) { + // forward attestations identified with a new block + pool.feedNewImportedBlock(value.unbox()).forEach(this::publishOut); + } else if (value.getType().equals(SlotNumber.class)) { + pool.feedNewSlot(value.unbox()); + } else if (value.getType().equals(ReceivedAttestation.class)) { + if (!pool.add(value.unbox())) { + // forward attestations not added to the pool + publishOut(value.unbox()); + } else { + // expose not yet identified attestations + unknownOut.next(value.unbox()); + } + } else { + throw new IllegalArgumentException( + "Unsupported input type: " + value.getType().getSimpleName()); + } + } + + public DirectProcessor getUnknownAttestations() { + return unknownAttestations; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java new file mode 100644 index 000000000..c94d41b4b --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java @@ -0,0 +1,23 @@ +package org.ethereum.beacon.chain.pool.reactor; + +public class Input { + + private final Object value; + + private Input(Object value) { + this.value = value; + } + + public static Input wrap(Object value) { + return new Input(value); + } + + public Class getType() { + return value.getClass(); + } + + @SuppressWarnings("unchecked") + T unbox() { + return (T) value; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java new file mode 100644 index 000000000..f512f8f1a --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java @@ -0,0 +1,32 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.CheckedAttestation; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.stream.AbstractDelegateProcessor; + +public class SanityCheckProcessor extends AbstractDelegateProcessor { + + private final SanityChecker checker; + + public SanityCheckProcessor(BeaconChainSpec spec) { + this.checker = new SanityChecker(spec); + } + + @Override + protected void hookOnNext(Input value) { + if (value.getType().equals(Checkpoint.class)) { + checker.feedFinalizedCheckpoint(value.unbox()); + } else if (value.getType().equals(SlotNumber.class)) { + checker.feedNewSlot(value.unbox()); + } else if (value.getType().equals(ReceivedAttestation.class)) { + publishOut(new CheckedAttestation(checker.check(value.unbox()), value.unbox())); + } else { + throw new IllegalArgumentException( + "Unsupported input type: " + value.getType().getSimpleName()); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java new file mode 100644 index 000000000..1495d343a --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java @@ -0,0 +1,7 @@ +package org.ethereum.beacon.chain.pool.registry; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; + +public interface AttestationRegistry { + boolean add(ReceivedAttestation attestation); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java new file mode 100644 index 000000000..924d5ff48 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java @@ -0,0 +1,23 @@ +package org.ethereum.beacon.chain.pool.registry; + +import java.util.function.Function; +import org.apache.commons.collections4.map.LRUMap; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.operations.Attestation; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class ProcessedAttestations implements AttestationRegistry { + private static final Object ENTRY = new Object(); + private final LRUMap cache = new LRUMap<>(AttestationPool.MAX_KNOWN_ATTESTATIONS); + private final Function hasher; + + public ProcessedAttestations(Function hasher) { + this.hasher = hasher; + } + + @Override + public boolean add(ReceivedAttestation attestation) { + return null == cache.put(hasher.apply(attestation.getMessage()), ENTRY); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/Queue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java similarity index 98% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/Queue.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java index 5c50a356f..7274a6c7c 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/Queue.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.chain.pool.unknown; +package org.ethereum.beacon.chain.pool.registry; import java.util.ArrayList; import java.util.Collections; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java new file mode 100644 index 000000000..352463a4f --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -0,0 +1,74 @@ +package org.ethereum.beacon.chain.pool.registry; + +import java.util.Collections; +import java.util.List; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.chain.storage.BeaconBlockStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class UnknownAttestationPool implements AttestationRegistry, StatefulProcessor { + + /** prev + curr + lookahead */ + private static final EpochNumber TRACKED_EPOCHS = + EpochNumber.of(2).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); + + private final Queue queue = new Queue(TRACKED_EPOCHS, AttestationPool.UNKNOWN_BLOCK_POOL_SIZE); + + private final BeaconBlockStorage blockStorage; + private final BeaconChainSpec spec; + + private EpochNumber currentBaseLine; + + public UnknownAttestationPool(BeaconBlockStorage blockStorage, BeaconChainSpec spec) { + this.blockStorage = blockStorage; + this.spec = spec; + } + + @Override + public boolean add(ReceivedAttestation attestation) { + AttestationData data = attestation.getMessage().getData(); + + // beacon block has not yet been imported + // it implies that source and target blocks might have not been imported too + if (blockStorage.get(data.getBeaconBlockRoot()).isPresent()) { + return false; + } else { + queue.add(data.getTarget().getEpoch(), data.getBeaconBlockRoot(), attestation); + return true; + } + } + + public List feedNewImportedBlock(BeaconBlock block) { + EpochNumber blockEpoch = spec.compute_epoch_of_slot(block.getSlot()); + // blockEpoch < currentBaseLine || blockEpoch >= currentBaseLine + TRACKED_EPOCHS + if (blockEpoch.less(currentBaseLine) + || blockEpoch.greaterEqual(currentBaseLine.plus(TRACKED_EPOCHS))) { + return Collections.emptyList(); + } + + Hash32 blockRoot = spec.hash_tree_root(block); + return queue.evict(blockRoot); + } + + public void feedNewSlot(SlotNumber slotNumber) { + EpochNumber currentEpoch = spec.compute_epoch_of_slot(slotNumber); + EpochNumber baseLine = + currentEpoch.equals(spec.getConstants().getGenesisEpoch()) + ? currentEpoch + : currentEpoch.decrement(); + queue.moveBaseLine(baseLine); + + this.currentBaseLine = baseLine; + } + + public boolean isStateReady() { + return currentBaseLine != null; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java deleted file mode 100644 index 11c74480f..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/unknown/UnknownBlockPoolImpl.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.ethereum.beacon.chain.pool.unknown; - -import com.google.common.annotations.VisibleForTesting; -import java.util.List; -import org.ethereum.beacon.chain.pool.AbstractProcessor; -import org.ethereum.beacon.chain.pool.AttestationPool; -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.chain.pool.UnknownBlockPool; -import org.ethereum.beacon.chain.storage.BeaconBlockStorage; -import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.core.BeaconBlock; -import org.ethereum.beacon.core.operations.attestation.AttestationData; -import org.ethereum.beacon.core.types.EpochNumber; -import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.schedulers.RunnableEx; -import org.ethereum.beacon.schedulers.Scheduler; -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.SimpleProcessor; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import tech.pegasys.artemis.ethereum.core.Hash32; - -public class UnknownBlockPoolImpl extends AbstractProcessor implements UnknownBlockPool { - - /** prev + curr + lookahead */ - private static final EpochNumber TRACKED_EPOCHS = - EpochNumber.of(2).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); - - private final Queue queue = new Queue(TRACKED_EPOCHS, AttestationPool.UNKNOWN_BLOCK_POOL_SIZE); - - private final SimpleProcessor unknownBlock; - private final Scheduler executor; - private final BeaconBlockStorage blockStorage; - private final BeaconChainSpec spec; - - private EpochNumber currentBaseLine; - - public UnknownBlockPoolImpl( - Schedulers schedulers, - Publisher slotClock, - Publisher importedBlocks, - Scheduler executor, - BeaconBlockStorage blockStorage, - BeaconChainSpec spec) { - super(schedulers, "UnknownBlockPool"); - - this.blockStorage = blockStorage; - this.spec = spec; - this.executor = executor; - - Flux.from(slotClock).subscribe(this::onNewSlot); - Flux.from(importedBlocks).subscribe(this::onNewImportedBlock); - this.unknownBlock = - new SimpleProcessor<>(schedulers.events(), "UnknownChainPool.unknownChainAttestations"); - } - - @Override - public Publisher out() { - return outbound; - } - - @Override - public void in(ReceivedAttestation attestation) { - if (isInitialized()) { - execute(() -> processAttestation(attestation)); - } - } - - private void processAttestation(ReceivedAttestation attestation) { - AttestationData data = attestation.getMessage().getData(); - - // beacon block has not yet been imported - // it implies that source and target blocks might have not been imported too - if (blockStorage.get(data.getBeaconBlockRoot()).isPresent()) { - outbound.onNext(attestation); - } else { - unknownBlock.onNext(attestation); - queue.add(data.getTarget().getEpoch(), data.getBeaconBlockRoot(), attestation); - } - } - - @VisibleForTesting - void onNewImportedBlock(BeaconBlock block) { - EpochNumber blockEpoch = spec.compute_epoch_of_slot(block.getSlot()); - // blockEpoch < currentBaseLine || blockEpoch >= currentBaseLine + TRACKED_EPOCHS - if (blockEpoch.less(currentBaseLine) - || blockEpoch.greaterEqual(currentBaseLine.plus(TRACKED_EPOCHS))) { - return; - } - - execute( - () -> { - Hash32 blockRoot = spec.hash_tree_root(block); - List attestations = queue.evict(blockRoot); - attestations.forEach(outbound::onNext); - }); - } - - @VisibleForTesting - void onNewSlot(SlotNumber slotNumber) { - EpochNumber currentEpoch = spec.compute_epoch_of_slot(slotNumber); - EpochNumber baseLine = - currentEpoch.equals(spec.getConstants().getGenesisEpoch()) - ? currentEpoch - : currentEpoch.decrement(); - queue.moveBaseLine(baseLine); - - this.currentBaseLine = baseLine; - } - - @VisibleForTesting - boolean isInitialized() { - return currentBaseLine != null; - } - - private void execute(RunnableEx routine) { - executor.execute(routine); - } - - @Override - public Publisher unknownBlock() { - return unknownBlock; - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java new file mode 100644 index 000000000..42b6c35d8 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java @@ -0,0 +1,205 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.AttestationDataAndCustodyBit; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.types.BLSPubkey; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import org.ethereum.beacon.crypto.BLS381.Signature; +import org.ethereum.beacon.crypto.MessageParameters; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.collections.Bitlist; +import tech.pegasys.artemis.util.uint.UInt64; + +public class AggregateSignatureVerifier implements SignatureVerifier { + private final BeaconChainSpec spec; + private final UInt64 domain; + private final List attestations = new ArrayList<>(); + + public AggregateSignatureVerifier(BeaconChainSpec spec, UInt64 domain) { + this.spec = spec; + this.domain = domain; + } + + @Override + public void feed(BeaconState state, IndexedAttestation indexed, ReceivedAttestation attestation) { + attestations.add(VerifiableAttestation.create(spec, state, indexed, attestation)); + } + + @Override + public VerificationResult verify() { + Map> signedMessageGroups = + attestations.stream().collect(Collectors.groupingBy(VerifiableAttestation::getData)); + + return signedMessageGroups.entrySet().stream() + .map(e -> verifyGroup(e.getKey(), e.getValue())) + .reduce(VerificationResult.EMPTY, VerificationResult::merge); + } + + private VerificationResult verifyGroup(AttestationData data, List group) { + final List valid = new ArrayList<>(); + final List invalid = new ArrayList<>(); + + // for aggregation sake, smaller aggregates should go first + group.sort(Comparator.comparing(attestation -> attestation.getAggregationBits().size())); + + // try to aggregate as much as we can + List aggregated = new ArrayList<>(); + List notAggregated = new ArrayList<>(); + AggregateSignature aggregate = new AggregateSignature(); + for (VerifiableAttestation attestation : group) { + if (aggregate.add( + attestation.aggregationBits, + attestation.bit0Key, + attestation.bit1Key, + attestation.signature)) { + aggregated.add(attestation); + } else { + notAggregated.add(attestation); + } + } + + // verify aggregate and fall back to one-by-one verification if it has failed + if (verifySignature(data, aggregate.getKey0(), aggregate.getKey1(), aggregate.getSignature())) { + aggregated.stream().map(VerifiableAttestation::getAttestation).forEach(valid::add); + } else { + notAggregated = group; + } + + for (VerifiableAttestation attestation : notAggregated) { + if (verifySignature(data, attestation.bit0Key, attestation.bit1Key, attestation.signature)) { + valid.add(attestation.attestation); + } else { + invalid.add(attestation.attestation); + } + } + + return new VerificationResult(valid, invalid); + } + + private boolean verifySignature( + AttestationData data, PublicKey bit0Key, PublicKey bit1Key, Signature signature) { + Hash32 bit0Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, false)); + Hash32 bit1Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, true)); + + return BLS381.verifyMultiple( + Arrays.asList( + MessageParameters.create(bit0Hash, domain), MessageParameters.create(bit1Hash, domain)), + signature, + Arrays.asList(bit0Key, bit1Key)); + } + + private static final class AggregateSignature { + private Bitlist bits; + private List key0s = new ArrayList<>(); + private List key1s = new ArrayList<>(); + private List sigs = new ArrayList<>(); + + boolean add(Bitlist bits, PublicKey key0, PublicKey key1, BLS381.Signature sig) { + // if bits has intersection it's not possible to get a viable aggregate + if (this.bits != null && !this.bits.and(bits).isEmpty()) { + return false; + } + + if (this.bits == null) { + this.bits = bits; + } else { + this.bits = this.bits.or(bits); + } + + key0s.add(key0); + key1s.add(key1); + sigs.add(sig); + + return true; + } + + PublicKey getKey0() { + return PublicKey.aggregate(key0s); + } + + PublicKey getKey1() { + return PublicKey.aggregate(key1s); + } + + Signature getSignature() { + return Signature.aggregate(sigs); + } + } + + private static final class VerifiableAttestation { + + static VerifiableAttestation create( + BeaconChainSpec spec, + BeaconState state, + IndexedAttestation indexed, + ReceivedAttestation attestation) { + + List bit0Keys = + indexed.getCustodyBit0Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + List bit1Keys = + indexed.getCustodyBit1Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + + // pre-process aggregated pubkeys + PublicKey bit0AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit0Keys); + PublicKey bit1AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit1Keys); + + return new VerifiableAttestation( + attestation, + attestation.getMessage().getData(), + attestation.getMessage().getAggregationBits(), + bit0AggregateKey, + bit1AggregateKey, + BLS381.Signature.createWithoutValidation(attestation.getMessage().getSignature())); + } + + private final ReceivedAttestation attestation; + + private final AttestationData data; + private final Bitlist aggregationBits; + private final PublicKey bit0Key; + private final PublicKey bit1Key; + private final BLS381.Signature signature; + + public VerifiableAttestation( + ReceivedAttestation attestation, + AttestationData data, + Bitlist aggregationBits, + PublicKey bit0Key, + PublicKey bit1Key, + Signature signature) { + this.attestation = attestation; + this.data = data; + this.aggregationBits = aggregationBits; + this.bit0Key = bit0Key; + this.bit1Key = bit1Key; + this.signature = signature; + } + + public AttestationData getData() { + return data; + } + + public Bitlist getAggregationBits() { + return aggregationBits; + } + + public ReceivedAttestation getAttestation() { + return attestation; + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java deleted file mode 100644 index 0906faba8..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationSignatureVerifier.java +++ /dev/null @@ -1,168 +0,0 @@ -package org.ethereum.beacon.chain.pool.verifier; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.core.operations.attestation.AttestationData; -import org.ethereum.beacon.core.operations.attestation.AttestationDataAndCustodyBit; -import org.ethereum.beacon.crypto.BLS381; -import org.ethereum.beacon.crypto.BLS381.PublicKey; -import org.ethereum.beacon.crypto.BLS381.Signature; -import org.ethereum.beacon.crypto.MessageParameters; -import org.ethereum.beacon.schedulers.RunnableEx; -import org.ethereum.beacon.schedulers.Scheduler; -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.SimpleProcessor; -import org.reactivestreams.Publisher; -import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.collections.Bitlist; -import tech.pegasys.artemis.util.uint.UInt64; - -public class AttestationSignatureVerifier { - - private final Scheduler executor; - private final BeaconChainSpec spec; - - private final SimpleProcessor valid; - private final SimpleProcessor invalid; - - public AttestationSignatureVerifier( - Schedulers schedulers, Scheduler executor, BeaconChainSpec spec) { - this.executor = executor; - this.spec = spec; - - this.valid = new SimpleProcessor<>(schedulers.events(), "AttestationSignatureVerifier.valid"); - this.invalid = - new SimpleProcessor<>(schedulers.events(), "AttestationSignatureVerifier.invalid"); - } - - public Publisher valid() { - return valid; - } - - public Publisher invalid() { - return invalid; - } - - public void in(List batch) { - execute( - () -> { - - // validate signature encoding format - List valid = new ArrayList<>(); - for (SignatureVerificationSet set : batch) { - if (BLS381.Signature.validate(set.getAttestation().getMessage().getSignature())) { - valid.add(set); - } else { - invalid.onNext(set.getAttestation()); - } - } - - Map> groupedBySignedMessage = - valid.stream() - .collect( - Collectors.groupingBy(data -> data.getAttestation().getMessage().getData())); - - groupedBySignedMessage.values().forEach(this::process); - }); - } - - private void process(List attestations) { - final UInt64 domain = attestations.get(0).getDomain(); - final AttestationData data = attestations.get(0).getAttestation().getMessage().getData(); - - // for aggregation sake, smaller aggregates should go first - attestations.sort( - Comparator.comparing(set -> set.getAttestation().getMessage().getAggregationBits().size())); - - // try to aggregate as much as we can - List aggregates = new ArrayList<>(); - List nonAggregates = new ArrayList<>(); - AggregateVerifier verifier = - new AggregateVerifier(spec.getConstants().getMaxValidatorsPerCommittee().getIntValue()); - for (SignatureVerificationSet set : attestations) { - if (verifier.add(set)) { - aggregates.add(set); - } else { - nonAggregates.add(set); - } - } - - // verify aggregate and fall back to one-by-one verification if it has failed - if (verifier.verify(spec, data, domain)) { - aggregates.forEach(set -> valid.onNext(set.getAttestation())); - } else { - nonAggregates = attestations; - } - - for (SignatureVerificationSet set : nonAggregates) { - if (verify(spec, set, domain)) { - valid.onNext(set.getAttestation()); - } else { - invalid.onNext(set.getAttestation()); - } - } - } - - private boolean verify(BeaconChainSpec spec, SignatureVerificationSet set, UInt64 domain) { - AttestationData data = set.getAttestation().getMessage().getData(); - Hash32 bit0Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, false)); - Hash32 bit1Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, true)); - - return BLS381.verifyMultiple( - Arrays.asList( - MessageParameters.create(bit0Hash, domain), MessageParameters.create(bit1Hash, domain)), - Signature.createWithoutValidation(set.getAttestation().getMessage().getSignature()), - Arrays.asList(set.getBit0AggregateKey(), set.getBit1AggregateKey())); - } - - private static final class AggregateVerifier { - List bit0AggregateKeys = new ArrayList<>(); - List bit1AggregateKeys = new ArrayList<>(); - List signatures = new ArrayList<>(); - Bitlist bits; - - AggregateVerifier(int maxValidatorsPerCommittee) { - this.bits = Bitlist.of(maxValidatorsPerCommittee); - } - - boolean add(SignatureVerificationSet set) { - if (!bits.and(set.getAttestation().getMessage().getAggregationBits()).isEmpty()) { - return false; - } - - bit0AggregateKeys.add(set.getBit0AggregateKey()); - bit1AggregateKeys.add(set.getBit1AggregateKey()); - signatures.add( - BLS381.Signature.createWithoutValidation( - set.getAttestation().getMessage().getSignature())); - - return true; - } - - boolean verify(BeaconChainSpec spec, AttestationData data, UInt64 domain) { - PublicKey bit0Key = PublicKey.aggregate(bit0AggregateKeys); - PublicKey bit1Key = PublicKey.aggregate(bit1AggregateKeys); - Signature signature = Signature.aggregate(signatures); - - Hash32 bit0Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, false)); - Hash32 bit1Hash = spec.hash_tree_root(new AttestationDataAndCustodyBit(data, true)); - - return BLS381.verifyMultiple( - Arrays.asList( - MessageParameters.create(bit0Hash, domain), - MessageParameters.create(bit1Hash, domain)), - signature, - Arrays.asList(bit0Key, bit1Key)); - } - } - - private void execute(RunnableEx routine) { - executor.execute(routine); - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java deleted file mode 100644 index 27aa6b31b..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationStateVerifier.java +++ /dev/null @@ -1,158 +0,0 @@ -package org.ethereum.beacon.chain.pool.verifier; - -import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.ethereum.beacon.chain.BeaconTuple; -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.chain.storage.BeaconTupleStorage; -import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.consensus.transition.EmptySlotTransition; -import org.ethereum.beacon.core.BeaconState; -import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; -import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.types.BLSPubkey; -import org.ethereum.beacon.core.types.EpochNumber; -import org.ethereum.beacon.schedulers.RunnableEx; -import org.ethereum.beacon.schedulers.Scheduler; -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.SimpleProcessor; -import org.javatuples.Pair; -import org.reactivestreams.Publisher; -import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.uint.UInt64; - -public class AttestationStateVerifier { - - private final Scheduler executor; - private final BeaconTupleStorage tupleStorage; - private final BeaconChainSpec spec; - private final EmptySlotTransition emptySlotTransition; - - private final SimpleProcessor valid; - private final SimpleProcessor invalid; - - public AttestationStateVerifier( - Schedulers schedulers, - Scheduler executor, - BeaconTupleStorage tupleStorage, - BeaconChainSpec spec, - EmptySlotTransition emptySlotTransition) { - this.executor = executor; - this.tupleStorage = tupleStorage; - this.spec = spec; - this.emptySlotTransition = emptySlotTransition; - - this.valid = new SimpleProcessor<>(schedulers.events(), "AttestationStateVerifier.valid"); - this.invalid = new SimpleProcessor<>(schedulers.events(), "AttestationStateVerifier.invalid"); - } - - public Publisher valid() { - return valid; - } - - public Publisher invalid() { - return invalid; - } - - public void in(List batch) { - execute( - () -> { - Map, List> groupedByState = - batch.stream() - .collect( - Collectors.groupingBy( - attestation -> - Pair.with( - attestation.getMessage().getData().getTarget(), - attestation.getMessage().getData().getBeaconBlockRoot()))); - - groupedByState.forEach((key, value) -> process(key.getValue0(), key.getValue1(), value)); - }); - } - - private void process( - final Checkpoint target, - final Hash32 beaconBlockRoot, - List attestations) { - Optional rootTuple = tupleStorage.get(beaconBlockRoot); - - // it must be present, otherwise, attestation couldn't be here - // TODO keep assertion for a while, it might be useful to discover bugs - assert rootTuple.isPresent(); - - EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.get().getState().getSlot()); - - // beaconBlockEpoch > targetEpoch - // it must either be equal or less than - if (beaconBlockEpoch.greater(target.getEpoch())) { - attestations.forEach(invalid::onNext); - return; - } - - // beaconBlockEpoch < targetEpoch && targetRoot != beaconBlockRoot - // target checkpoint is built with empty slots upon a block root - // in that case target root and beacon block root must be equal - if (beaconBlockEpoch.less(target.getEpoch()) && !target.getRoot().equals(beaconBlockRoot)) { - attestations.forEach(invalid::onNext); - return; - } - - // compute state and domain, there must be the same state for all attestations - final BeaconState state = computeState(rootTuple.get(), target.getEpoch()); - final UInt64 domain = spec.get_domain(state, ATTESTATION, target.getEpoch()); - for (ReceivedAttestation attestation : attestations) { - // skip signature verification, it's passed on the next processor - if (!spec.verify_attestation_impl(state, attestation.getMessage(), false)) { - invalid.onNext(attestation); - continue; - } - - // compute and verify indexed attestation - IndexedAttestation indexedAttestation = - spec.get_indexed_attestation(state, attestation.getMessage()); - if (!spec.is_valid_indexed_attestation_impl(state, indexedAttestation, false)) { - invalid.onNext(attestation); - continue; - } - - // compute data required for signature verification - List bit0Keys = - indexedAttestation.getCustodyBit0Indices().stream() - .map(i -> state.getValidators().get(i).getPubKey()) - .collect(Collectors.toList()); - List bit1Keys = - indexedAttestation.getCustodyBit1Indices().stream() - .map(i -> state.getValidators().get(i).getPubKey()) - .collect(Collectors.toList()); - - // send them to signature verifier - valid.onNext( - new SignatureVerificationSet( - spec.bls_aggregate_pubkeys_no_validate(bit0Keys), - spec.bls_aggregate_pubkeys_no_validate(bit1Keys), - domain, - attestation)); - } - } - - private BeaconState computeState(BeaconTuple rootTuple, EpochNumber targetEpoch) { - EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.getState().getSlot()); - - // block is in the same epoch, no additional state is required to be built - if (beaconBlockEpoch.equals(targetEpoch)) { - return rootTuple.getState(); - } - - // build a state at epoch boundary, it must be enough to proceed - return emptySlotTransition.apply( - rootTuple.getState(), spec.compute_start_slot_of_epoch(targetEpoch)); - } - - private void execute(RunnableEx routine) { - executor.execute(routine); - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java new file mode 100644 index 000000000..af88a9c06 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java @@ -0,0 +1,154 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import static org.ethereum.beacon.core.spec.SignatureDomains.ATTESTATION; + +import com.google.common.base.Objects; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.storage.BeaconTupleStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +public class AttestationVerifier implements BatchVerifier { + + private final BeaconTupleStorage tupleStorage; + private final BeaconChainSpec spec; + private final EmptySlotTransition emptySlotTransition; + + public AttestationVerifier( + BeaconTupleStorage tupleStorage, + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition) { + this.tupleStorage = tupleStorage; + this.spec = spec; + this.emptySlotTransition = emptySlotTransition; + } + + @Override + public VerificationResult verify(List batch) { + Map> targetGroups = + batch.stream().collect(Collectors.groupingBy(AttestingTarget::from)); + + return targetGroups.entrySet().stream() + .map(e -> verifyGroup(e.getKey(), e.getValue())) + .reduce(VerificationResult.EMPTY, VerificationResult::merge); + } + + private VerificationResult verifyGroup(AttestingTarget target, List group) { + Optional rootTuple = tupleStorage.get(target.blockRoot); + + // it must be present, otherwise, attestation couldn't be here + // TODO keep assertion for a while, it might be useful to discover bugs + assert rootTuple.isPresent(); + + EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.get().getState().getSlot()); + + // beaconBlockEpoch > targetEpoch + // it must either be equal or less than + if (beaconBlockEpoch.greater(target.checkpoint.getEpoch())) { + return VerificationResult.allInvalid(group); + } + + // beaconBlockEpoch < targetEpoch && targetRoot != beaconBlockRoot + // target checkpoint is built with empty slots upon a block root + // in that case target root and beacon block root must be equal + if (beaconBlockEpoch.less(target.checkpoint.getEpoch()) + && !target.checkpoint.getRoot().equals(target.blockRoot)) { + return VerificationResult.allInvalid(group); + } + + // compute state and domain, there must be the same state for all attestations + final BeaconState state = computeState(rootTuple.get(), target.checkpoint.getEpoch()); + final UInt64 domain = spec.get_domain(state, ATTESTATION, target.checkpoint.getEpoch()); + final AggregateSignatureVerifier signatureVerifier = + new AggregateSignatureVerifier(spec, domain); + final List invalid = new ArrayList<>(); + + for (ReceivedAttestation attestation : group) { + Optional result = verifyIndexed(state, attestation.getMessage()); + if (result.isPresent()) { + signatureVerifier.feed(state, result.get(), attestation); + } else { + invalid.add(attestation); + } + } + + VerificationResult signatureResult = signatureVerifier.verify(); + return VerificationResult.allInvalid(invalid).merge(signatureResult); + } + + private Optional verifyIndexed(BeaconState state, Attestation attestation) { + // skip indexed attestation verification, it's explicitly done in the next step + if (!spec.verify_attestation_impl(state, attestation, false)) { + return Optional.empty(); + } + + // compute and verify indexed attestation + // skip signature verification + IndexedAttestation indexedAttestation = spec.get_indexed_attestation(state, attestation); + if (!spec.is_valid_indexed_attestation_impl(state, indexedAttestation, false)) { + return Optional.empty(); + } + + return Optional.of(indexedAttestation); + } + + private BeaconState computeState(BeaconTuple rootTuple, EpochNumber targetEpoch) { + EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.getState().getSlot()); + + // block is in the same epoch, no additional state is required to be built + if (beaconBlockEpoch.equals(targetEpoch)) { + return rootTuple.getState(); + } + + // build a state at epoch boundary, it must be enough to proceed + return emptySlotTransition.apply( + rootTuple.getState(), spec.compute_start_slot_of_epoch(targetEpoch)); + } + + private static final class AttestingTarget { + + static AttestingTarget from(ReceivedAttestation attestation) { + AttestationData data = attestation.getMessage().getData(); + return new AttestingTarget(data.getTarget(), data.getBeaconBlockRoot()); + } + + private final Checkpoint checkpoint; + private final Hash32 blockRoot; + + private AttestingTarget(Checkpoint checkpoint, Hash32 blockRoot) { + this.checkpoint = checkpoint; + this.blockRoot = blockRoot; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AttestingTarget that = (AttestingTarget) o; + return Objects.equal(checkpoint, that.checkpoint) && Objects.equal(blockRoot, that.blockRoot); + } + + @Override + public int hashCode() { + return Objects.hashCode(checkpoint, blockRoot); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java new file mode 100644 index 000000000..9f07d2459 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java @@ -0,0 +1,9 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.List; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; + +public interface BatchVerifier { + + VerificationResult verify(List batch); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/FullVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/FullVerifier.java deleted file mode 100644 index 2b844e33c..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/FullVerifier.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.ethereum.beacon.chain.pool.verifier; - -import java.util.Collections; -import java.util.List; -import org.ethereum.beacon.chain.pool.AttestationPool; -import org.ethereum.beacon.chain.pool.AttestationVerifier; -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -public class FullVerifier implements AttestationVerifier { - - private final AttestationStateVerifier stateVerifier; - private final AttestationSignatureVerifier signatureVerifier; - - public FullVerifier( - AttestationStateVerifier stateVerifier, AttestationSignatureVerifier signatureVerifier) { - this.stateVerifier = stateVerifier; - this.signatureVerifier = signatureVerifier; - - Flux.from(stateVerifier.valid()) - .bufferTimeout(AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL) - .subscribe(signatureVerifier::in); - } - - @Override - public Publisher valid() { - return signatureVerifier.valid(); - } - - @Override - public void in(ReceivedAttestation attestation) { - stateVerifier.in(Collections.singletonList(attestation)); - } - - @Override - public void batchIn(List batch) { - stateVerifier.in(batch); - } - - @Override - public Publisher invalid() { - return Flux.concat(stateVerifier.invalid(), signatureVerifier.invalid()); - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java deleted file mode 100644 index 57bfc46a4..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerificationSet.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.ethereum.beacon.chain.pool.verifier; - -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.crypto.BLS381.PublicKey; -import tech.pegasys.artemis.util.uint.UInt64; - -final class SignatureVerificationSet { - private final PublicKey bit0AggregateKey; - private final PublicKey bit1AggregateKey; - private final UInt64 domain; - private final ReceivedAttestation attestation; - - SignatureVerificationSet( - PublicKey bit0AggregateKey, - PublicKey bit1AggregatedKey, - UInt64 domain, - ReceivedAttestation attestation) { - this.bit0AggregateKey = bit0AggregateKey; - this.bit1AggregateKey = bit1AggregatedKey; - this.domain = domain; - this.attestation = attestation; - } - - public PublicKey getBit0AggregateKey() { - return bit0AggregateKey; - } - - public PublicKey getBit1AggregateKey() { - return bit1AggregateKey; - } - - public UInt64 getDomain() { - return domain; - } - - public ReceivedAttestation getAttestation() { - return attestation; - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java new file mode 100644 index 000000000..67ff4df68 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; + +public interface SignatureVerifier { + void feed(BeaconState state, IndexedAttestation indexed, ReceivedAttestation attestation); + VerificationResult verify(); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java new file mode 100644 index 000000000..495e51156 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java @@ -0,0 +1,41 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; + +public final class VerificationResult { + + static VerificationResult allInvalid(List attestations) { + return new VerificationResult(Collections.emptyList(), attestations); + } + + public static final VerificationResult EMPTY = + new VerificationResult(Collections.emptyList(), Collections.emptyList()); + + private final List valid; + private final List invalid; + + VerificationResult(List valid, List invalid) { + this.valid = valid; + this.invalid = invalid; + } + + public List getValid() { + return valid; + } + + public List getInvalid() { + return invalid; + } + + public VerificationResult merge(VerificationResult other) { + List valid = new ArrayList<>(this.valid); + List invalid = new ArrayList<>(this.invalid); + valid.addAll(other.valid); + invalid.addAll(other.invalid); + + return new VerificationResult(valid, invalid); + } +} diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java index abea79f0e..d597fa55b 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/HelperFunction.java @@ -896,7 +896,12 @@ default PublicKey bls_aggregate_pubkeys(List publicKeysBytes) { } List publicKeys = publicKeysBytes.stream().map(PublicKey::create).collect(toList()); - return PublicKey.aggregate(publicKeys); + + if (publicKeys.size() == 1) { + return publicKeys.get(0); + } else { + return PublicKey.aggregate(publicKeys); + } } default PublicKey bls_aggregate_pubkeys_no_validate(List publicKeysBytes) { @@ -906,7 +911,12 @@ default PublicKey bls_aggregate_pubkeys_no_validate(List publicKeysBy List publicKeys = publicKeysBytes.stream().map(PublicKey::createWithoutValidation).collect(toList()); - return PublicKey.aggregate(publicKeys); + + if (publicKeys.size() == 1) { + return publicKeys.get(0); + } else { + return PublicKey.aggregate(publicKeys); + } } /* diff --git a/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java b/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java new file mode 100644 index 000000000..ee31eb226 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java @@ -0,0 +1,56 @@ +package org.ethereum.beacon.stream; + +import javax.annotation.Nonnull; +import org.reactivestreams.Subscription; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.BaseSubscriber; +import reactor.core.publisher.FluxProcessor; + +public abstract class AbstractDelegateProcessor extends FluxProcessor { + + private final Subscriber subscriber; + private final OutsourcePublisher publisher; + + public AbstractDelegateProcessor() { + this.subscriber = new Subscriber(); + this.publisher = new OutsourcePublisher<>(); + } + + @Override + public void onSubscribe(Subscription s) { + subscriber.onSubscribe(s); + } + + @Override + public void onNext(IN in) { + subscriber.onNext(in); + } + + @Override + public void onError(Throwable t) { + subscriber.onError(t); + } + + @Override + public void onComplete() { + subscriber.onComplete(); + } + + @Override + public void subscribe(@Nonnull CoreSubscriber actual) { + publisher.subscribe(actual); + } + + private final class Subscriber extends BaseSubscriber { + @Override + protected void hookOnNext(IN value) { + AbstractDelegateProcessor.this.hookOnNext(value); + } + } + + protected abstract void hookOnNext(IN value); + + protected void publishOut(OUT value) { + publisher.publishOut(value); + } +} diff --git a/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java b/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java new file mode 100644 index 000000000..55a1212c3 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java @@ -0,0 +1,43 @@ +package org.ethereum.beacon.stream; + +import java.util.function.Predicate; +import org.javatuples.Pair; +import org.reactivestreams.Publisher; +import reactor.core.Disposable; +import reactor.core.publisher.ConnectableFlux; +import reactor.core.publisher.Flux; + +public abstract class Fluxes { + private Fluxes() {} + + public static FluxSplit split(Publisher source, Predicate predicate) { + return new FluxSplit<>(source, predicate); + } + + public static final class FluxSplit { + private final Flux satisfied; + private final Flux unsatisfied; + private final Disposable disposable; + + FluxSplit(Publisher source, Predicate predicate) { + ConnectableFlux> splitter = + Flux.from(source).map(value -> Pair.with(predicate.test(value), value)).publish(); + + this.satisfied = splitter.filter(Pair::getValue0).map(Pair::getValue1); + this.unsatisfied = splitter.filter(pair -> !pair.getValue0()).map(Pair::getValue1); + this.disposable = splitter.connect(); + } + + public Flux getSatisfied() { + return satisfied; + } + + public Flux getUnsatisfied() { + return unsatisfied; + } + + public Disposable getDisposable() { + return disposable; + } + } +} diff --git a/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java b/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java new file mode 100644 index 000000000..736ef03df --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java @@ -0,0 +1,21 @@ +package org.ethereum.beacon.stream; + +import org.reactivestreams.Publisher; +import org.reactivestreams.Subscriber; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.FluxSink; + +public class OutsourcePublisher implements Publisher { + + private final DirectProcessor delegate = DirectProcessor.create(); + private final FluxSink out = delegate.sink(); + + public void publishOut(T value) { + out.next(value); + } + + @Override + public void subscribe(Subscriber s) { + delegate.subscribe(s); + } +} From 771fe79dc69204b28ef06834fa667f6565aa3391 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 28 Aug 2019 01:36:43 +0600 Subject: [PATCH 04/59] Instantiate attestation pool --- .../beacon/chain/pool/AttestationPool.java | 53 ++++++++++++++++++- .../chain/pool/InMemoryAttestationPool.java | 18 ++++--- .../AttestationVerificationProcessor.java | 11 +--- .../chain/pool/reactor/IdentifyProcessor.java | 6 +-- .../pool/reactor/SanityCheckProcessor.java | 5 +- .../pool/registry/ProcessedAttestations.java | 5 +- .../pool/registry/UnknownAttestationPool.java | 13 ++--- 7 files changed, 78 insertions(+), 33 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java index 40e9fda3d..22ea46329 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -1,8 +1,21 @@ package org.ethereum.beacon.chain.pool; import java.time.Duration; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; +import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; import org.reactivestreams.Publisher; public interface AttestationPool { @@ -13,7 +26,7 @@ public interface AttestationPool { int MAX_KNOWN_ATTESTATIONS = 1_000_000; - int UNKNOWN_BLOCK_POOL_SIZE = 100_000; + int UNKNOWN_ATTESTATION_POOL_SIZE = 100_000; int VERIFIER_BUFFER_SIZE = 10_000; @@ -28,4 +41,42 @@ public interface AttestationPool { Publisher getAggregates(); void start(); + + static AttestationPool create( + Publisher source, + Publisher newSlots, + Publisher finalizedCheckpoints, + Publisher importedBlocks, + Publisher chainHeads, + Schedulers schedulers, + BeaconChainSpec spec, + BeaconChainStorage storage, + EmptySlotTransition emptySlotTransition) { + + SanityChecker sanityChecker = new SanityChecker(spec); + SignatureEncodingChecker encodingChecker = new SignatureEncodingChecker(); + ProcessedAttestations processedFilter = + new ProcessedAttestations(spec::hash_tree_root, MAX_KNOWN_ATTESTATIONS); + UnknownAttestationPool unknownAttestationPool = + new UnknownAttestationPool( + storage.getBlockStorage(), + spec, + MAX_ATTESTATION_LOOKAHEAD, + UNKNOWN_ATTESTATION_POOL_SIZE); + BatchVerifier batchVerifier = + new AttestationVerifier(storage.getTupleStorage(), spec, emptySlotTransition); + + return new InMemoryAttestationPool( + source, + newSlots, + finalizedCheckpoints, + importedBlocks, + chainHeads, + schedulers, + sanityChecker, + encodingChecker, + processedFilter, + unknownAttestationPool, + batchVerifier); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index c6f4ad5d4..26089eba5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.chain.pool; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; import org.ethereum.beacon.chain.pool.reactor.AttestationChurnProcessor; @@ -8,6 +9,8 @@ import org.ethereum.beacon.chain.pool.reactor.Input; import org.ethereum.beacon.chain.pool.reactor.SanityCheckProcessor; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; @@ -45,24 +48,23 @@ public InMemoryAttestationPool( Publisher importedBlocks, Publisher chainHeads, Schedulers schedulers, - SanityCheckProcessor sanityChecker, + SanityChecker sanityChecker, SignatureEncodingChecker encodingChecker, ProcessedAttestations processedFilter, - IdentifyProcessor identifier, - AttestationVerificationProcessor verifier, - AttestationChurnProcessor churn) { + UnknownAttestationPool unknownAttestationPool, + BatchVerifier batchVerifier) { this.source = source; this.newSlots = newSlots; this.finalizedCheckpoints = finalizedCheckpoints; this.importedBlocks = importedBlocks; this.chainHeads = chainHeads; this.schedulers = schedulers; - this.sanityChecker = sanityChecker; + this.sanityChecker = new SanityCheckProcessor(sanityChecker); this.encodingChecker = encodingChecker; this.processedFilter = processedFilter; - this.identifier = identifier; - this.verifier = verifier; - this.churn = churn; + this.identifier = new IdentifyProcessor(unknownAttestationPool); + this.verifier = new AttestationVerificationProcessor(batchVerifier); + this.churn = new AttestationChurnProcessor(); } @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java index e9a9ba54e..48f72b76b 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java @@ -4,12 +4,8 @@ import java.util.stream.Stream; import org.ethereum.beacon.chain.pool.CheckedAttestation; import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; import org.ethereum.beacon.chain.pool.verifier.VerificationResult; -import org.ethereum.beacon.chain.storage.BeaconTupleStorage; -import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.consensus.transition.EmptySlotTransition; import org.ethereum.beacon.stream.AbstractDelegateProcessor; public class AttestationVerificationProcessor @@ -17,11 +13,8 @@ public class AttestationVerificationProcessor private final BatchVerifier verifier; - public AttestationVerificationProcessor( - BeaconTupleStorage tupleStorage, - BeaconChainSpec spec, - EmptySlotTransition emptySlotTransition) { - this.verifier = new AttestationVerifier(tupleStorage, spec, emptySlotTransition); + public AttestationVerificationProcessor(BatchVerifier verifier) { + this.verifier = verifier; } @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java index 69f8fb531..7aba8751a 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java @@ -2,8 +2,6 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; -import org.ethereum.beacon.chain.storage.BeaconBlockStorage; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.stream.AbstractDelegateProcessor; @@ -16,8 +14,8 @@ public class IdentifyProcessor extends AbstractDelegateProcessor unknownAttestations = DirectProcessor.create(); private final FluxSink unknownOut = unknownAttestations.sink(); - public IdentifyProcessor(BeaconBlockStorage blockStorage, BeaconChainSpec spec) { - this.pool = new UnknownAttestationPool(blockStorage, spec); + public IdentifyProcessor(UnknownAttestationPool pool) { + this.pool = pool; } @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java index f512f8f1a..830a4a616 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java @@ -3,7 +3,6 @@ import org.ethereum.beacon.chain.pool.CheckedAttestation; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.SanityChecker; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.stream.AbstractDelegateProcessor; @@ -12,8 +11,8 @@ public class SanityCheckProcessor extends AbstractDelegateProcessor cache = new LRUMap<>(AttestationPool.MAX_KNOWN_ATTESTATIONS); + private final LRUMap cache; private final Function hasher; - public ProcessedAttestations(Function hasher) { + public ProcessedAttestations(Function hasher, int size) { this.hasher = hasher; + this.cache = new LRUMap<>(size); } @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java index 352463a4f..5d73861ee 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -2,7 +2,6 @@ import java.util.Collections; import java.util.List; -import org.ethereum.beacon.chain.pool.AttestationPool; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.StatefulProcessor; import org.ethereum.beacon.chain.storage.BeaconBlockStorage; @@ -16,19 +15,21 @@ public class UnknownAttestationPool implements AttestationRegistry, StatefulProcessor { /** prev + curr + lookahead */ - private static final EpochNumber TRACKED_EPOCHS = - EpochNumber.of(2).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); + private final EpochNumber trackedEpochs; - private final Queue queue = new Queue(TRACKED_EPOCHS, AttestationPool.UNKNOWN_BLOCK_POOL_SIZE); + private final Queue queue; private final BeaconBlockStorage blockStorage; private final BeaconChainSpec spec; private EpochNumber currentBaseLine; - public UnknownAttestationPool(BeaconBlockStorage blockStorage, BeaconChainSpec spec) { + public UnknownAttestationPool( + BeaconBlockStorage blockStorage, BeaconChainSpec spec, EpochNumber lookahead, long size) { this.blockStorage = blockStorage; this.spec = spec; + this.trackedEpochs = EpochNumber.of(2).plus(lookahead); + this.queue = new Queue(trackedEpochs, size); } @Override @@ -49,7 +50,7 @@ public List feedNewImportedBlock(BeaconBlock block) { EpochNumber blockEpoch = spec.compute_epoch_of_slot(block.getSlot()); // blockEpoch < currentBaseLine || blockEpoch >= currentBaseLine + TRACKED_EPOCHS if (blockEpoch.less(currentBaseLine) - || blockEpoch.greaterEqual(currentBaseLine.plus(TRACKED_EPOCHS))) { + || blockEpoch.greaterEqual(currentBaseLine.plus(trackedEpochs))) { return Collections.emptyList(); } From 4a039bba7925238cbb3bf68e69ac34d55e4a801f Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 28 Aug 2019 12:32:38 +0600 Subject: [PATCH 05/59] Check isInitialized in attestation processors if needed --- .../beacon/chain/pool/StatefulProcessor.java | 2 +- .../beacon/chain/pool/checker/SanityChecker.java | 4 ++-- .../chain/pool/reactor/IdentifyProcessor.java | 14 ++++++++------ .../chain/pool/reactor/SanityCheckProcessor.java | 4 +++- .../pool/registry/UnknownAttestationPool.java | 5 ++++- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java index 86be41cea..64a88d86d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java @@ -2,5 +2,5 @@ public interface StatefulProcessor { - boolean isStateReady(); + boolean isInitialized(); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java index f3d85e16d..3d2638ed9 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java @@ -22,7 +22,7 @@ public SanityChecker(BeaconChainSpec spec) { @Override public boolean check(ReceivedAttestation attestation) { - assert isStateReady(); + assert isInitialized(); final AttestationData data = attestation.getMessage().getData(); @@ -70,7 +70,7 @@ public void feedNewSlot(SlotNumber newSlot) { } @Override - public boolean isStateReady() { + public boolean isInitialized() { return finalizedCheckpoint != null && maxAcceptableEpoch != null; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java index 7aba8751a..3d5bb60db 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java @@ -26,12 +26,14 @@ protected void hookOnNext(Input value) { } else if (value.getType().equals(SlotNumber.class)) { pool.feedNewSlot(value.unbox()); } else if (value.getType().equals(ReceivedAttestation.class)) { - if (!pool.add(value.unbox())) { - // forward attestations not added to the pool - publishOut(value.unbox()); - } else { - // expose not yet identified attestations - unknownOut.next(value.unbox()); + if (pool.isInitialized()) { + if (!pool.add(value.unbox())) { + // forward attestations not added to the pool + publishOut(value.unbox()); + } else { + // expose not yet identified attestations + unknownOut.next(value.unbox()); + } } } else { throw new IllegalArgumentException( diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java index 830a4a616..3172871de 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java @@ -22,7 +22,9 @@ protected void hookOnNext(Input value) { } else if (value.getType().equals(SlotNumber.class)) { checker.feedNewSlot(value.unbox()); } else if (value.getType().equals(ReceivedAttestation.class)) { - publishOut(new CheckedAttestation(checker.check(value.unbox()), value.unbox())); + if (checker.isInitialized()) { + publishOut(new CheckedAttestation(checker.check(value.unbox()), value.unbox())); + } } else { throw new IllegalArgumentException( "Unsupported input type: " + value.getType().getSimpleName()); diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java index 5d73861ee..358807c5a 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -34,6 +34,8 @@ public UnknownAttestationPool( @Override public boolean add(ReceivedAttestation attestation) { + assert isInitialized(); + AttestationData data = attestation.getMessage().getData(); // beacon block has not yet been imported @@ -69,7 +71,8 @@ public void feedNewSlot(SlotNumber slotNumber) { this.currentBaseLine = baseLine; } - public boolean isStateReady() { + @Override + public boolean isInitialized() { return currentBaseLine != null; } } From da9fb0efa6194d55a38570896a8c0fb44f7b8d38 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 28 Aug 2019 21:05:31 +0600 Subject: [PATCH 06/59] Add javadocs and polish attestation pool impl --- .../beacon/chain/pool/AttestationPool.java | 67 ++++++++++++-- .../chain/pool/InMemoryAttestationPool.java | 17 +++- .../beacon/chain/pool/StatefulProcessor.java | 18 ++++ .../pool/checker/AttestationChecker.java | 14 +++ .../chain/pool/checker/SanityChecker.java | 44 ++++------ .../checker/SignatureEncodingChecker.java | 12 +++ .../chain/pool/checker/TimeFrameFilter.java | 87 +++++++++++++++++++ .../chain/pool/churn/OffChainAggregates.java | 5 ++ .../chain/pool/reactor/IdentifyProcessor.java | 16 ++-- .../pool/reactor/SanityCheckProcessor.java | 3 - .../pool/reactor/TimeFrameProcessor.java | 33 +++++++ .../pool/registry/AttestationRegistry.java | 14 +++ .../pool/registry/ProcessedAttestations.java | 15 +++- .../beacon/chain/pool/registry/Queue.java | 85 ++++++++++++++++-- .../pool/registry/UnknownAttestationPool.java | 42 ++++++++- 15 files changed, 416 insertions(+), 56 deletions(-) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java index 22ea46329..f5581953f 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -1,8 +1,10 @@ package org.ethereum.beacon.chain.pool; import java.time.Duration; +import org.ethereum.beacon.chain.BeaconChain; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; @@ -18,28 +20,80 @@ import org.ethereum.beacon.schedulers.Schedulers; import org.reactivestreams.Publisher; +/** + * Attestation pool API. + * + *

Along with {@link BeaconChain} attestation pool is one of the central components of the Beacon + * chain client. Its main responsibilities are to verify attestation coming from the wire and + * accumulate those of them which are not yet included on chain. + * + *

A list of attestation pool clients: + * + *

    + *
  • Wire - valid attestations should be further propagated; peers sent attestations that are + * eventually invalid must be dropped. + *
  • Fork choice - LMD GHOST is driven by attestations received from the wire even if they are + * not yet on chain. + *
  • Validator - attestations not yet included on chain should be included into a new block + * produced by proposer. + *
+ */ public interface AttestationPool { + /** A number of threads in the executor processing attestation pool. */ int MAX_THREADS = 32; + /** Discard attestations with target epoch greater than current epoch plus this number. */ EpochNumber MAX_ATTESTATION_LOOKAHEAD = EpochNumber.of(1); - int MAX_KNOWN_ATTESTATIONS = 1_000_000; + /** + * Max number of attestations kept by processed attestations registry. An entry of this registry + * should be represented by a hash of registered attestation. + */ + int MAX_PROCESSED_ATTESTATIONS = 1_000_000; - int UNKNOWN_ATTESTATION_POOL_SIZE = 100_000; + /** Max number of attestations made to not yet known block that could be kept in memory. */ + int MAX_UNKNOWN_ATTESTATIONS = 100_000; + /** Max size of a buffer that collects attestations before passing them on the main verifier. */ int VERIFIER_BUFFER_SIZE = 10_000; + /** A throttling interval for verifier buffer. */ Duration VERIFIER_INTERVAL = Duration.ofMillis(50); + /** + * Valid attestations publisher. + * + * @return a publisher. + */ Publisher getValid(); + /** + * Invalid attestations publisher. + * + * @return a publisher. + */ Publisher getInvalid(); + /** + * Publishes attestations which block is not yet a known block. + * + *

These attestations should be passed to a wire module in order to request a block. + * + * @return a publisher. + */ Publisher getUnknownAttestations(); + /** + * Publishes aggregated attestations that are not yet included on chain. + * + *

It should be a source of attestations for block proposer. + * + * @return a publisher. + */ Publisher getAggregates(); + /** Launches the pool. */ void start(); static AttestationPool create( @@ -53,16 +107,14 @@ static AttestationPool create( BeaconChainStorage storage, EmptySlotTransition emptySlotTransition) { + TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); SanityChecker sanityChecker = new SanityChecker(spec); SignatureEncodingChecker encodingChecker = new SignatureEncodingChecker(); ProcessedAttestations processedFilter = - new ProcessedAttestations(spec::hash_tree_root, MAX_KNOWN_ATTESTATIONS); + new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); UnknownAttestationPool unknownAttestationPool = new UnknownAttestationPool( - storage.getBlockStorage(), - spec, - MAX_ATTESTATION_LOOKAHEAD, - UNKNOWN_ATTESTATION_POOL_SIZE); + storage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); BatchVerifier batchVerifier = new AttestationVerifier(storage.getTupleStorage(), spec, emptySlotTransition); @@ -73,6 +125,7 @@ static AttestationPool create( importedBlocks, chainHeads, schedulers, + timeFrameFilter, sanityChecker, encodingChecker, processedFilter, diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 26089eba5..5f1924b9a 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -2,12 +2,14 @@ import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; import org.ethereum.beacon.chain.pool.reactor.AttestationChurnProcessor; import org.ethereum.beacon.chain.pool.reactor.AttestationVerificationProcessor; import org.ethereum.beacon.chain.pool.reactor.IdentifyProcessor; import org.ethereum.beacon.chain.pool.reactor.Input; import org.ethereum.beacon.chain.pool.reactor.SanityCheckProcessor; +import org.ethereum.beacon.chain.pool.reactor.TimeFrameProcessor; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; @@ -31,6 +33,7 @@ public class InMemoryAttestationPool implements AttestationPool { private final Publisher chainHeads; private final Schedulers schedulers; + private final TimeFrameProcessor timeFrameProcessor; private final SanityCheckProcessor sanityChecker; private final SignatureEncodingChecker encodingChecker; private final ProcessedAttestations processedFilter; @@ -48,6 +51,7 @@ public InMemoryAttestationPool( Publisher importedBlocks, Publisher chainHeads, Schedulers schedulers, + TimeFrameFilter timeFrameFilter, SanityChecker sanityChecker, SignatureEncodingChecker encodingChecker, ProcessedAttestations processedFilter, @@ -59,6 +63,7 @@ public InMemoryAttestationPool( this.importedBlocks = importedBlocks; this.chainHeads = chainHeads; this.schedulers = schedulers; + this.timeFrameProcessor = new TimeFrameProcessor(timeFrameFilter); this.sanityChecker = new SanityCheckProcessor(sanityChecker); this.encodingChecker = encodingChecker; this.processedFilter = processedFilter; @@ -81,8 +86,16 @@ public void start() { Flux chainHeadsFx = Flux.from(chainHeads).map(Input::wrap).publishOn(executor.toReactor()); - // subscribe sanity checker to its inputs - Flux.merge(sourceFx, newSlotsFx, finalizedCheckpointsFx).subscribe(sanityChecker); + // start from time frame processor + Flux.merge(sourceFx, newSlotsFx, finalizedCheckpointsFx).subscribe(timeFrameProcessor); + FluxSplit timeFrameOut = + Fluxes.split(timeFrameProcessor, CheckedAttestation::isPassed); + + // subscribe sanity checker + Flux.merge( + timeFrameOut.getSatisfied().map(att -> Input.wrap(att.getAttestation())), + finalizedCheckpointsFx) + .subscribe(sanityChecker); FluxSplit sanityOut = Fluxes.split(sanityChecker, CheckedAttestation::isPassed); diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java index 64a88d86d..ea0d4d592 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/StatefulProcessor.java @@ -1,6 +1,24 @@ package org.ethereum.beacon.chain.pool; +/** + * Stateful processor. + * + *

A processor that requires particular inner state to be initialized before it could be safely + * called by its clients. + * + *

{@link #isInitialized()} method indicates whether processor has already been initialized or + * not. It's a client responsibility to check {@link #isInitialized()} result before calling to the + * instance of this processor. + * + *

Implementor MAY throw an {@link AssertionError} if it's been called before inner state has + * been initialised. + */ public interface StatefulProcessor { + /** + * Checks whether processor state is initialized or not. + * + * @return {@code true} if processor is ready to work, {@link false}, otherwise. + */ boolean isInitialized(); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java index 7d95cd0c0..228acccfa 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/AttestationChecker.java @@ -2,6 +2,20 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; +/** + * An interface of light weight attestation checker. + * + *

Implementations of this interface SHOULD NOT execute I/O operations or run checks that are in + * high demand to CPU resources. Usually, implementation of this interface runs quick checks that + * could be done with the attestation itself without involving any other data. + */ public interface AttestationChecker { + + /** + * Given attestation runs a check. + * + * @param attestation an attestation to check. + * @return {@code true} if checks are passed successfully, {@code false} otherwise. + */ boolean check(ReceivedAttestation attestation); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java index 3d2638ed9..042b0fe91 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java @@ -1,20 +1,25 @@ package org.ethereum.beacon.chain.pool.checker; -import org.ethereum.beacon.chain.pool.AttestationPool; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.StatefulProcessor; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.types.EpochNumber; -import org.ethereum.beacon.core.types.SlotNumber; +/** + * Given attestation runs a number of sanity checks against it. + * + *

This is one of the first processors in attestation pool pipeline. Attestations that are not + * passing these checks SHOULD be considered invalid. + * + *

Note: this implementation is not thread-safe. + */ public class SanityChecker implements AttestationChecker, StatefulProcessor { + /** A beacon chain spec. */ private final BeaconChainSpec spec; - + /** Most recent finalized checkpoint. */ private Checkpoint finalizedCheckpoint; - private EpochNumber maxAcceptableEpoch; public SanityChecker(BeaconChainSpec spec) { this.spec = spec; @@ -31,21 +36,6 @@ public boolean check(ReceivedAttestation attestation) { return false; } - // targetEpoch <= finalizedEpoch - if (data.getTarget().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { - return false; - } - - // sourceEpoch < finalizedEpoch - if (data.getSource().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { - return false; - } - - // targetEpoch > maxAcceptableEpoch - if (data.getTarget().getEpoch().greater(maxAcceptableEpoch)) { - return false; - } - // finalizedEpoch == sourceEpoch && finalizedRoot != sourceRoot if (data.getSource().getEpoch().equals(finalizedCheckpoint.getEpoch()) && !finalizedCheckpoint.getRoot().equals(data.getSource().getRoot())) { @@ -60,17 +50,19 @@ public boolean check(ReceivedAttestation attestation) { return true; } + /** + * Update the most recent finalized checkpoint. + * + *

This method should be called each time new finalized checkpoint appears. + * + * @param checkpoint finalized checkpoint. + */ public void feedFinalizedCheckpoint(Checkpoint checkpoint) { this.finalizedCheckpoint = checkpoint; } - public void feedNewSlot(SlotNumber newSlot) { - this.maxAcceptableEpoch = - spec.compute_epoch_of_slot(newSlot).plus(AttestationPool.MAX_ATTESTATION_LOOKAHEAD); - } - @Override public boolean isInitialized() { - return finalizedCheckpoint != null && maxAcceptableEpoch != null; + return finalizedCheckpoint != null; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java index 6306dfa43..32daf0095 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingChecker.java @@ -1,8 +1,20 @@ package org.ethereum.beacon.chain.pool.checker; import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.crypto.BLS381; +/** + * Checks signature encoding format. + * + *

Attestations with invalid signature encoding SHOULD be considered as invalid. + * + *

This is relatively heavy check in terms of CPU cycles as it involves a few operations on a + * field numbers and one point multiplication. It's recommended to put this checker after {@link + * ProcessedAttestations} registry. + * + *

Note: this implementation is not thread-safe. + */ public class SignatureEncodingChecker implements AttestationChecker { @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java new file mode 100644 index 000000000..01da5887a --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java @@ -0,0 +1,87 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; + +/** + * Filters attestations by time frame of its target and source. + * + *

Attestations not passing these checks SHOULD NOT be considered as invalid, they rather SHOULD + * be considered as those which client is not interested in. Filtered out attestations SHOULD merely + * be discarded. + * + *

This is the first filter in attestation processors pipeline. + * + *

Note: this implementation is not thread-safe. + */ +public class TimeFrameFilter implements AttestationChecker, StatefulProcessor { + + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + /** Accept attestations not older than current epoch plus this number. */ + private final EpochNumber maxAttestationLookahead; + + /** Most recent finalized checkpoint. */ + private Checkpoint finalizedCheckpoint; + /** Upper time frame boundary. */ + private EpochNumber maxAcceptableEpoch; + + public TimeFrameFilter(BeaconChainSpec spec, EpochNumber maxAttestationLookahead) { + this.spec = spec; + this.maxAttestationLookahead = maxAttestationLookahead; + } + + @Override + public boolean isInitialized() { + return finalizedCheckpoint != null && maxAcceptableEpoch != null; + } + + @Override + public boolean check(ReceivedAttestation attestation) { + assert isInitialized(); + + final AttestationData data = attestation.getMessage().getData(); + + // targetEpoch <= finalizedEpoch + if (data.getTarget().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { + return false; + } + + // sourceEpoch < finalizedEpoch + if (data.getSource().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { + return false; + } + + // targetEpoch > maxAcceptableEpoch + if (data.getTarget().getEpoch().greater(maxAcceptableEpoch)) { + return false; + } + + return false; + } + + /** + * Update the most recent finalized checkpoint. + * + *

This method should be called each time new finalized checkpoint appears. + * + * @param checkpoint finalized checkpoint. + */ + public void feedFinalizedCheckpoint(Checkpoint checkpoint) { + this.finalizedCheckpoint = checkpoint; + } + + /** + * This method should be called on each new slot. + * + * @param newSlot a new slot. + */ + public void feedNewSlot(SlotNumber newSlot) { + this.maxAcceptableEpoch = spec.compute_epoch_of_slot(newSlot).plus(maxAttestationLookahead); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java index 55607e568..67d98ef00 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java @@ -4,6 +4,11 @@ import org.ethereum.beacon.core.operations.Attestation; import tech.pegasys.artemis.ethereum.core.Hash32; +/** + * A DTO for aggregated attestations that are not yet included on chain. + * + *

Beacon block proposer should be fed with this data. + */ public class OffChainAggregates { private final Hash32 blockRoot; private final List aggregates; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java index 3d5bb60db..41d652ff7 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java @@ -27,13 +27,15 @@ protected void hookOnNext(Input value) { pool.feedNewSlot(value.unbox()); } else if (value.getType().equals(ReceivedAttestation.class)) { if (pool.isInitialized()) { - if (!pool.add(value.unbox())) { - // forward attestations not added to the pool - publishOut(value.unbox()); - } else { - // expose not yet identified attestations - unknownOut.next(value.unbox()); - } + return; + } + + if (!pool.add(value.unbox())) { + // forward attestations not added to the pool + publishOut(value.unbox()); + } else { + // expose not yet identified attestations + unknownOut.next(value.unbox()); } } else { throw new IllegalArgumentException( diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java index 3172871de..9c1787656 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java @@ -4,7 +4,6 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.stream.AbstractDelegateProcessor; public class SanityCheckProcessor extends AbstractDelegateProcessor { @@ -19,8 +18,6 @@ public SanityCheckProcessor(SanityChecker checker) { protected void hookOnNext(Input value) { if (value.getType().equals(Checkpoint.class)) { checker.feedFinalizedCheckpoint(value.unbox()); - } else if (value.getType().equals(SlotNumber.class)) { - checker.feedNewSlot(value.unbox()); } else if (value.getType().equals(ReceivedAttestation.class)) { if (checker.isInitialized()) { publishOut(new CheckedAttestation(checker.check(value.unbox()), value.unbox())); diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java new file mode 100644 index 000000000..120b3fde0 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java @@ -0,0 +1,33 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.CheckedAttestation; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.stream.AbstractDelegateProcessor; + +public class TimeFrameProcessor extends AbstractDelegateProcessor { + + private final TimeFrameFilter filter; + + public TimeFrameProcessor(TimeFrameFilter filter) { + this.filter = filter; + } + + @Override + protected void hookOnNext(Input value) { + if (value.getType().equals(Checkpoint.class)) { + filter.feedFinalizedCheckpoint(value.unbox()); + } else if (value.getType().equals(SlotNumber.class)) { + filter.feedNewSlot(value.unbox()); + } else if (value.getType().equals(ReceivedAttestation.class)) { + if (filter.isInitialized()) { + publishOut(new CheckedAttestation(filter.check(value.unbox()), value.unbox())); + } + } else { + throw new IllegalArgumentException( + "Unsupported input type: " + value.getType().getSimpleName()); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java index 1495d343a..7122fd8c5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/AttestationRegistry.java @@ -2,6 +2,20 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; +/** + * An attestation registry interface. + * + *

Usually, an implementation of this interface tracks a set of attestation or identities of + * attestations passed on it. + */ public interface AttestationRegistry { + + /** + * Adds attestation to the registry. + * + * @param attestation an attestation to be registered. + * @return {@link true} if an attestation is new to the registry, {@link false} if the attestation + * has been already added. + */ boolean add(ReceivedAttestation attestation); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java index 913d5ba4e..82e0a4edb 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/ProcessedAttestations.java @@ -2,14 +2,27 @@ import java.util.function.Function; import org.apache.commons.collections4.map.LRUMap; -import org.ethereum.beacon.chain.pool.AttestationPool; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.core.operations.Attestation; import tech.pegasys.artemis.ethereum.core.Hash32; +/** + * A registry that stores relatively big number of attestation hashes. + * + *

This particular implementation is based on LRU map. + * + *

It's recommended to place this registry prior to highly resource demanded processors in order + * to prevent double work. + * + *

Note: this implementation is not thread-safe. + */ public class ProcessedAttestations implements AttestationRegistry { + + /** An entry of the map. */ private static final Object ENTRY = new Object(); + /** LRU attestation cache. */ private final LRUMap cache; + /** A function that given attestation returns its hash. */ private final Function hasher; public ProcessedAttestations(Function hasher, int size) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java index 7274a6c7c..8cc468134 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java @@ -12,11 +12,24 @@ import org.ethereum.beacon.core.types.EpochNumber; import tech.pegasys.artemis.ethereum.core.Hash32; +/** + * Maintains attestations parked by {@link UnknownAttestationPool}. + * + *

In terms of a number of parked attestations this queue holds LRU contract. Given a new + * attestation if overall attestation number exceeds {@link #maxSize} the earliest added attestation + * will be purged from the queue. + * + *

Note: this implementation is not thread-safe. + */ final class Queue { + + /** A queue is a list of epoch buckets. See {@link EpochBucket}. */ private final LinkedList queue = new LinkedList<>(); + /** A number of epochs held by the {@link #queue}. */ private final EpochNumber trackedEpochs; + /** Max number of overall parked attestations. */ private final long maxSize; - + /** A lower time frame boundary of attestation queue. */ private EpochNumber baseLine; Queue(EpochNumber trackedEpochs, long maxSize) { @@ -27,7 +40,14 @@ final class Queue { this.maxSize = maxSize; } - synchronized void moveBaseLine(EpochNumber newBaseLine) { + /** + * Moves base line forward. + * + *

Removes attestations made to epochs standing behind new base line. + * + * @param newBaseLine a new base line. + */ + void moveBaseLine(EpochNumber newBaseLine) { assert baseLine == null || newBaseLine.greater(baseLine); for (long i = 0; i < newBaseLine.minus(baseLine).getValue() && queue.size() > 0; i++) { @@ -41,7 +61,13 @@ synchronized void moveBaseLine(EpochNumber newBaseLine) { this.baseLine = newBaseLine; } - synchronized List evict(Hash32 root) { + /** + * Given a block hash evicts a list of attestations made to that block. + * + * @param root a block root. + * @return evicted attestations. + */ + List evict(Hash32 root) { if (!isInitialized()) { return Collections.emptyList(); } @@ -54,7 +80,14 @@ synchronized List evict(Hash32 root) { return evictedFromAllEpochs; } - synchronized void add(EpochNumber epoch, Hash32 root, ReceivedAttestation attestation) { + /** + * Queues attestation. + * + * @param epoch target epoch. + * @param root beacon block root. + * @param attestation an attestation object. + */ + void add(EpochNumber epoch, Hash32 root, ReceivedAttestation attestation) { if (!isInitialized() || epoch.less(baseLine)) { return; } @@ -65,36 +98,60 @@ synchronized void add(EpochNumber epoch, Hash32 root, ReceivedAttestation attest purgeQueue(); } + /** Purges queue by its {@link #maxSize}. */ private void purgeQueue() { for (EpochNumber e = baseLine; computeSize() > maxSize && e.less(baseLine.plus(trackedEpochs)); e = e.increment()) { EpochBucket epochBucket = getEpochBucket(e); while (epochBucket.size() > 0 && computeSize() > maxSize) { - epochBucket.removeOldest(); + epochBucket.removeEarliest(); } } } + /** @return a number of attestations in the queue. */ private long computeSize() { return queue.stream().map(EpochBucket::size).reduce(0L, Long::sum); } + /** + * Returns an epoch bucket. + * + * @param epoch an epoch number. + * @return a bucket corresponding to the epoch. + */ private EpochBucket getEpochBucket(EpochNumber epoch) { assert epoch.greaterEqual(baseLine); return queue.get(epoch.minus(baseLine).getIntValue()); } + /** @return {@code true} if {@link #baseLine} is defined, {@code false} otherwise. */ private boolean isInitialized() { return baseLine != null; } + /** + * An epoch bucket. + * + *

Holds attestation with the same target epoch. + */ static final class EpochBucket { + + /** A map of beacon block root on a list of attestations made to that root. */ private final Map> bucket = new HashMap<>(); + /** An LRU index for the attestations. */ private final LinkedList> lruIndex = new LinkedList<>(); + /** A number of attestations in the bucket. */ private long size = 0; + /** + * Adds attestation to the bucket. + * + * @param root beacon block root. + * @param attestation an attestation. + */ void add(Hash32 root, ReceivedAttestation attestation) { LinkedList rootBucket = getOrInsert(root); rootBucket.add(new RootBucketEntry(System.nanoTime(), attestation)); @@ -115,6 +172,12 @@ private void updateIndex() { lruIndex.sort(Comparator.comparing(b -> b.getFirst().timestamp)); } + /** + * Given beacon block root evicts attestations made to that root. + * + * @param root beacon block root. + * @return a list of evicted attestations. + */ List evict(Hash32 root) { List evicted = bucket.remove(root); if (evicted != null) { @@ -127,7 +190,12 @@ List evict(Hash32 root) { : Collections.emptyList(); } - ReceivedAttestation removeOldest() { + /** + * Removes an earliest attestation from the bucket. + * + * @return removed attestation. + */ + ReceivedAttestation removeEarliest() { if (size > 0) { LinkedList oldestBucket = lruIndex.getFirst(); RootBucketEntry entry = oldestBucket.removeFirst(); @@ -144,13 +212,18 @@ ReceivedAttestation removeOldest() { } } + /** @return size of a bucket. */ long size() { return size; } } + /** Entry that stores LRU timestamp along with attestation. */ static final class RootBucketEntry { + + /** Timestamp identifying a moment in time of when attestation was added to the queue. */ private final long timestamp; + /** An attestation itself. */ private final ReceivedAttestation attestation; public RootBucketEntry(long timestamp, ReceivedAttestation attestation) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java index 358807c5a..688df756f 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -12,16 +12,39 @@ import org.ethereum.beacon.core.types.SlotNumber; import tech.pegasys.artemis.ethereum.core.Hash32; +/** + * Registers and manages attestation that were made to not yet known block. + * + *

There are two main use cases: + * + *

    + *
  • Pass attestation on {@link #add(ReceivedAttestation)} method. This method checks whether + * attestation block exists or not. If it exists then attestation is not added to the registry + * and the call will return {@code false}, otherwise, attestation will be added to the + * registry and {@link true} will be returned. + *
  • When new imported block comes attestations are checked against it. If there are + * attestations made to that block they are evicted from pool and should be forwarded to + * upstream processor. + *
+ * + *

Attestations that haven't been identified for a certain perido of time are purged from the + * pool. This part of the logic is based on {@link #feedNewSlot(SlotNumber)} calls. Effectively, + * pool contains attestation which target epoch lays between {@code previous_epoch} and {@code + * current_epoch + epoch_lookahead}. + * + *

Note: this implementation is not thread-safe. + */ public class UnknownAttestationPool implements AttestationRegistry, StatefulProcessor { - /** prev + curr + lookahead */ + /** A number of tracked epochs: previous_epoch + current_epoch + epoch_lookahead. */ private final EpochNumber trackedEpochs; - + /** A queue that maintains pooled attestations. */ private final Queue queue; - + /** A block storage. */ private final BeaconBlockStorage blockStorage; + /** A beacon chain spec. */ private final BeaconChainSpec spec; - + /** A lower time frame boundary of attestation queue. */ private EpochNumber currentBaseLine; public UnknownAttestationPool( @@ -48,6 +71,12 @@ public boolean add(ReceivedAttestation attestation) { } } + /** + * Processes recently imported block. + * + * @param block a block. + * @return a list of attestations that have been identified by the block. + */ public List feedNewImportedBlock(BeaconBlock block) { EpochNumber blockEpoch = spec.compute_epoch_of_slot(block.getSlot()); // blockEpoch < currentBaseLine || blockEpoch >= currentBaseLine + TRACKED_EPOCHS @@ -60,6 +89,11 @@ public List feedNewImportedBlock(BeaconBlock block) { return queue.evict(blockRoot); } + /** + * Processes new slot. + * + * @param slotNumber a slot number. + */ public void feedNewSlot(SlotNumber slotNumber) { EpochNumber currentEpoch = spec.compute_epoch_of_slot(slotNumber); EpochNumber baseLine = From 548625c09e3782a2bafcb4ab5accc7fe1422e450 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 29 Aug 2019 00:02:58 +0600 Subject: [PATCH 07/59] Continue to polish and document attestation pool --- .../beacon/chain/pool/CheckedAttestation.java | 1 + .../chain/pool/ReceivedAttestation.java | 3 + .../AttestationVerificationProcessor.java | 15 ++ .../chain/pool/reactor/IdentifyProcessor.java | 18 ++ .../beacon/chain/pool/reactor/Input.java | 7 + .../pool/reactor/SanityCheckProcessor.java | 16 ++ .../pool/reactor/TimeFrameProcessor.java | 17 ++ .../pool/registry/UnknownAttestationPool.java | 8 +- .../pool/verifier/AggregateSignature.java | 72 +++++++ .../verifier/AggregateSignatureVerifier.java | 177 ++++++------------ .../pool/verifier/AttestationVerifier.java | 80 +++++++- .../chain/pool/verifier/BatchVerifier.java | 20 ++ .../pool/verifier/SignatureVerifier.java | 10 - .../pool/verifier/VerifiableAttestation.java | 108 +++++++++++ .../pool/verifier/VerificationResult.java | 1 + .../stream/AbstractDelegateProcessor.java | 23 +++ .../org/ethereum/beacon/stream/Fluxes.java | 25 +++ .../beacon/stream/OutsourcePublisher.java | 15 ++ 18 files changed, 477 insertions(+), 139 deletions(-) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignature.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerifiableAttestation.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java index d9e42609e..e3c4fbf76 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.chain.pool; +/** A simple DTO that carries an attestation and a result of some check run against it. */ public class CheckedAttestation { private final boolean passed; private final ReceivedAttestation attestation; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java index a1f25f196..b74c9d4d5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java @@ -3,9 +3,12 @@ import org.ethereum.beacon.core.operations.Attestation; import org.ethereum.beacon.types.p2p.NodeId; +/** An attestation received from the wire. */ public class ReceivedAttestation { + /** An id of a node sent this attestation. */ private final NodeId sender; + /** An attestation message itself. */ private final Attestation message; public ReceivedAttestation(NodeId sender, Attestation message) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java index 48f72b76b..6d795995d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java @@ -8,6 +8,21 @@ import org.ethereum.beacon.chain.pool.verifier.VerificationResult; import org.ethereum.beacon.stream.AbstractDelegateProcessor; +/** + * A processor that throttles attestations through {@link BatchVerifier}. + * + *

Input: + * + *

    + *
  • a list of {@link ReceivedAttestation} + *
+ * + *

Output: + * + *

    + *
  • attestations tagged with verification flag + *
+ */ public class AttestationVerificationProcessor extends AbstractDelegateProcessor, CheckedAttestation> { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java index 41d652ff7..976c5edc7 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java @@ -8,6 +8,24 @@ import reactor.core.publisher.DirectProcessor; import reactor.core.publisher.FluxSink; +/** + * Processor throttling attestations through {@link UnknownAttestationPool}. + * + *

Input: + * + *

    + *
  • newly imported blocks + *
  • new slots + *
  • attestations + *
+ * + *

Output: + * + *

    + *
  • instantly identified attestations + *
  • attestations identified upon a new block come + *
+ */ public class IdentifyProcessor extends AbstractDelegateProcessor { private final UnknownAttestationPool pool; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java index c94d41b4b..c07ec5c5b 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java @@ -1,7 +1,14 @@ package org.ethereum.beacon.chain.pool.reactor; +/** + * Facilitate an ability heterogeneous + * + */ public class Input { + /** + * Wrapped value + */ private final Object value; private Input(Object value) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java index 9c1787656..29b697411 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java @@ -6,6 +6,22 @@ import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.stream.AbstractDelegateProcessor; +/** + * Processor throttling attestations through a {@link SanityChecker}. + * + *

Input: + * + *

    + *
  • recently finalized checkpoints. + *
  • attestations. + *
+ * + *

Output: + * + *

    + *
  • attestations tagged with the check flag. + *
+ */ public class SanityCheckProcessor extends AbstractDelegateProcessor { private final SanityChecker checker; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java index 120b3fde0..fda721fea 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java @@ -7,6 +7,23 @@ import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.stream.AbstractDelegateProcessor; +/** + * Processor throttling attestations through a {@link TimeFrameFilter}. + * + *

Input: + * + *

    + *
  • recently finalized checkpoints. + *
  • new slots. + *
  • attestations. + *
+ * + *

Output: + * + *

    + *
  • attestations tagged with the check flag. + *
+ */ public class TimeFrameProcessor extends AbstractDelegateProcessor { private final TimeFrameFilter filter; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java index 688df756f..fd26e0f5b 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -27,10 +27,10 @@ * upstream processor. * * - *

Attestations that haven't been identified for a certain perido of time are purged from the - * pool. This part of the logic is based on {@link #feedNewSlot(SlotNumber)} calls. Effectively, - * pool contains attestation which target epoch lays between {@code previous_epoch} and {@code - * current_epoch + epoch_lookahead}. + *

Attestations that haven't been identified for a certain period of time are purged from the + * queue and eventually discarded. This part of the logic is based on {@link + * #feedNewSlot(SlotNumber)} calls. Effectively, queue contains attestation which target epoch lays + * between {@code previous_epoch} and {@code current_epoch + epoch_lookahead}. * *

Note: this implementation is not thread-safe. */ diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignature.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignature.java new file mode 100644 index 000000000..e72daf888 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignature.java @@ -0,0 +1,72 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.ArrayList; +import java.util.List; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import org.ethereum.beacon.crypto.BLS381.Signature; +import tech.pegasys.artemis.util.collections.Bitlist; + +/** A helper class aiding signature aggregation in context of beacon chain attestations. */ +final class AggregateSignature { + private Bitlist bits; + private List key0s = new ArrayList<>(); + private List key1s = new ArrayList<>(); + private List sigs = new ArrayList<>(); + + /** + * Adds yet another attestation to the aggregation churn. + * + * @param bits an aggregate bitfield. + * @param key0 bit0 key. + * @param key1 bit1 key. + * @param sig a signature. + * @return {@code true} if it could be aggregated successfully, {@code false} if given bitfield + * has intersection with an accumulated one. + */ + boolean add(Bitlist bits, PublicKey key0, PublicKey key1, BLS381.Signature sig) { + // if bits has intersection it's not possible to get a viable aggregate + if (this.bits != null && !this.bits.and(bits).isEmpty()) { + return false; + } + + if (this.bits == null) { + this.bits = bits; + } else { + this.bits = this.bits.or(bits); + } + + key0s.add(key0); + key1s.add(key1); + sigs.add(sig); + + return true; + } + + /** + * Computes and returns aggregate bit0 public key. + * + * @return a public key. + */ + PublicKey getKey0() { + return PublicKey.aggregate(key0s); + } + + /** + * Computes and returns aggregate bit1 public key. + * + * @return a public key. + */ + PublicKey getKey1() { + return PublicKey.aggregate(key1s); + } + + /** + * Computes and returns aggregate signature. + * + * @return a signature. + */ + Signature getSignature() { + return Signature.aggregate(sigs); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java index 42b6c35d8..7b1b20556 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java @@ -12,31 +12,59 @@ import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.operations.attestation.AttestationDataAndCustodyBit; import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; -import org.ethereum.beacon.core.types.BLSPubkey; import org.ethereum.beacon.crypto.BLS381; import org.ethereum.beacon.crypto.BLS381.PublicKey; import org.ethereum.beacon.crypto.BLS381.Signature; import org.ethereum.beacon.crypto.MessageParameters; import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; -public class AggregateSignatureVerifier implements SignatureVerifier { +/** + * An implementation of aggregate-then-verify strategy. + * + *

In a few words this strategy looks as follows: + * + *

    + *
  1. Aggregate as much attestation as we can. + *
  2. Verify aggregate signature. + *
  3. Verify signatures of attestations that aren't aggregated in a one-by-one fashion. + *
+ * + *

If second step fails verification falls back to one-by-one strategy. + */ +public class AggregateSignatureVerifier { + + /** A beacon chain spec. */ private final BeaconChainSpec spec; + /** A domain which attestation signatures has been created with. */ private final UInt64 domain; + /** Verification churn. */ private final List attestations = new ArrayList<>(); - public AggregateSignatureVerifier(BeaconChainSpec spec, UInt64 domain) { + AggregateSignatureVerifier(BeaconChainSpec spec, UInt64 domain) { this.spec = spec; this.domain = domain; } - @Override - public void feed(BeaconState state, IndexedAttestation indexed, ReceivedAttestation attestation) { + /** + * Adds an attestation to the verification churn. + * + * @param state a state attestation built upon. + * @param indexed an instance of corresponding {@link IndexedAttestation}. + * @param attestation an attestation itself. + */ + void add(BeaconState state, IndexedAttestation indexed, ReceivedAttestation attestation) { attestations.add(VerifiableAttestation.create(spec, state, indexed, attestation)); } - @Override + /** + * Verifies previously added attestation. + * + *

First, attestations are grouped by {@link AttestationData} and then each group is passed + * onto {@link #verifyGroup(AttestationData, List)}. + * + * @return a result of singature verification. + */ public VerificationResult verify() { Map> signedMessageGroups = attestations.stream().collect(Collectors.groupingBy(VerifiableAttestation::getData)); @@ -46,6 +74,13 @@ public VerificationResult verify() { .reduce(VerificationResult.EMPTY, VerificationResult::merge); } + /** + * Verifies a group of attestations signing the same {@link AttestationData}. + * + * @param data attestation data. + * @param group a group. + * @return a result of verification. + */ private VerificationResult verifyGroup(AttestationData data, List group) { final List valid = new ArrayList<>(); final List invalid = new ArrayList<>(); @@ -59,10 +94,10 @@ private VerificationResult verifyGroup(AttestationData data, List key0s = new ArrayList<>(); - private List key1s = new ArrayList<>(); - private List sigs = new ArrayList<>(); - - boolean add(Bitlist bits, PublicKey key0, PublicKey key1, BLS381.Signature sig) { - // if bits has intersection it's not possible to get a viable aggregate - if (this.bits != null && !this.bits.and(bits).isEmpty()) { - return false; - } - - if (this.bits == null) { - this.bits = bits; - } else { - this.bits = this.bits.or(bits); - } - - key0s.add(key0); - key1s.add(key1); - sigs.add(sig); - - return true; - } - - PublicKey getKey0() { - return PublicKey.aggregate(key0s); - } - - PublicKey getKey1() { - return PublicKey.aggregate(key1s); - } - - Signature getSignature() { - return Signature.aggregate(sigs); - } - } - - private static final class VerifiableAttestation { - - static VerifiableAttestation create( - BeaconChainSpec spec, - BeaconState state, - IndexedAttestation indexed, - ReceivedAttestation attestation) { - - List bit0Keys = - indexed.getCustodyBit0Indices().stream() - .map(i -> state.getValidators().get(i).getPubKey()) - .collect(Collectors.toList()); - List bit1Keys = - indexed.getCustodyBit1Indices().stream() - .map(i -> state.getValidators().get(i).getPubKey()) - .collect(Collectors.toList()); - - // pre-process aggregated pubkeys - PublicKey bit0AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit0Keys); - PublicKey bit1AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit1Keys); - - return new VerifiableAttestation( - attestation, - attestation.getMessage().getData(), - attestation.getMessage().getAggregationBits(), - bit0AggregateKey, - bit1AggregateKey, - BLS381.Signature.createWithoutValidation(attestation.getMessage().getSignature())); - } - - private final ReceivedAttestation attestation; - - private final AttestationData data; - private final Bitlist aggregationBits; - private final PublicKey bit0Key; - private final PublicKey bit1Key; - private final BLS381.Signature signature; - - public VerifiableAttestation( - ReceivedAttestation attestation, - AttestationData data, - Bitlist aggregationBits, - PublicKey bit0Key, - PublicKey bit1Key, - Signature signature) { - this.attestation = attestation; - this.data = data; - this.aggregationBits = aggregationBits; - this.bit0Key = bit0Key; - this.bit1Key = bit1Key; - this.signature = signature; - } - - public AttestationData getData() { - return data; - } - - public Bitlist getAggregationBits() { - return aggregationBits; - } - - public ReceivedAttestation getAttestation() { - return attestation; - } - } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java index af88a9c06..f2fdf3b2c 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java @@ -12,6 +12,7 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.storage.BeaconTupleStorage; import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.BeaconStateEx; import org.ethereum.beacon.consensus.transition.EmptySlotTransition; import org.ethereum.beacon.core.BeaconState; import org.ethereum.beacon.core.operations.Attestation; @@ -22,10 +23,41 @@ import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.uint.UInt64; +/** + * An implementation of {@link BatchVerifier}. + * + *

There are three steps of batch verification: + * + *

    + *
  • Group batch by beacon block root and target. + *
  • Calculate state for each group and run checks against this state. + *
  • Pass group onto signature verifier. + *
+ * + *

Current implementation relies on {@link AggregateSignatureVerifier} which is pretty efficient. + * {@link AggregateSignatureVerifier} tries to first aggregate attestations and then verify a + * signature of that aggregate in a single operation instead of verifying signature of each + * standalone attestation. A group succeeded with state verification is next passed onto signature + * verifier. A nice part of it is that aggregatable attestation groups are subsets of groups made + * against attestation target. + * + *

Verification hierarchy can be represented with a diagram: + * + * 

+ *                        attestation_batch
+ *                       /                 \
+ * state:               target_1 ... target_N
+ *                     /        \
+ * signature:  aggregate_1 ... aggregate_N
+ *
+ */ public class AttestationVerifier implements BatchVerifier { + /** A beacon tuple storage. */ private final BeaconTupleStorage tupleStorage; + /** A beacon chain spec. */ private final BeaconChainSpec spec; + /** An empty slot transition. */ private final EmptySlotTransition emptySlotTransition; public AttestationVerifier( @@ -47,6 +79,13 @@ public VerificationResult verify(List batch) { .reduce(VerificationResult.EMPTY, VerificationResult::merge); } + /** + * Verifies a group of attestations with the same target. + * + * @param target a target. + * @param group a group. + * @return result of verification. + */ private VerificationResult verifyGroup(AttestingTarget target, List group) { Optional rootTuple = tupleStorage.get(target.blockRoot); @@ -71,7 +110,8 @@ private VerificationResult verifyGroup(AttestingTarget target, List result = verifyIndexed(state, attestation.getMessage()); if (result.isPresent()) { - signatureVerifier.feed(state, result.get(), attestation); + signatureVerifier.add(state, result.get(), attestation); } else { invalid.add(attestation); } @@ -90,6 +130,21 @@ private VerificationResult verifyGroup(AttestingTarget target, List + *
  • Runs main checks defined in the spec; these checks verifies attestation against a state + * it's been made upon. + *
  • Computes {@link IndexedAttestation} and runs checks against it omitting signature + * verification. + * + * + * @param state a state attestation built upon. + * @param attestation an attestation. + * @return an optional filled with {@link IndexedAttestation} instance if verification passed + * successfully, empty optional box is returned otherwise. + */ private Optional verifyIndexed(BeaconState state, Attestation attestation) { // skip indexed attestation verification, it's explicitly done in the next step if (!spec.verify_attestation_impl(state, attestation, false)) { @@ -106,19 +161,30 @@ private Optional verifyIndexed(BeaconState state, Attestatio return Optional.of(indexedAttestation); } - private BeaconState computeState(BeaconTuple rootTuple, EpochNumber targetEpoch) { - EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(rootTuple.getState().getSlot()); + /** + * Given epoch and beacon state computes a state that attestation built upon. + * + * @param state a state after attestation beacon block has been imported. + * @param targetEpoch target epoch of attestation. + * @return computed state. + */ + private BeaconState computeState(BeaconStateEx state, EpochNumber targetEpoch) { + EpochNumber beaconBlockEpoch = spec.compute_epoch_of_slot(state.getSlot()); // block is in the same epoch, no additional state is required to be built if (beaconBlockEpoch.equals(targetEpoch)) { - return rootTuple.getState(); + return state; } // build a state at epoch boundary, it must be enough to proceed - return emptySlotTransition.apply( - rootTuple.getState(), spec.compute_start_slot_of_epoch(targetEpoch)); + return emptySlotTransition.apply(state, spec.compute_start_slot_of_epoch(targetEpoch)); } + /** + * A wrapper for attestation target checkpoint and beacon block root. + * + *

    This is the entity which initial verification groups are built around. + */ private static final class AttestingTarget { static AttestingTarget from(ReceivedAttestation attestation) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java index 9f07d2459..854b9b5d4 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/BatchVerifier.java @@ -2,8 +2,28 @@ import java.util.List; import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.AttestationChecker; +/** + * An interface of attestation verifier that processes attestations in a batch. + * + *

    Opposed to {@link AttestationChecker}, verifier involves I/O operations and checks highly + * demanded to CPU resources. + * + *

    Verifying attestations in a batch aids resource saving by grouping them by beacon block root + * and target they are attesting to. A state calculated for each of such groups is reused across the + * group. It saves a lot of resources as state calculation is pretty heavy operation. + * + *

    It is highly recommended to place this verifier to the end of verification pipeline due to its + * high demand to computational resources. + */ public interface BatchVerifier { + /** + * Verifies a batch of attestations. + * + * @param batch a batch. + * @return result of verification. + */ VerificationResult verify(List batch); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java deleted file mode 100644 index 67ff4df68..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/SignatureVerifier.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.ethereum.beacon.chain.pool.verifier; - -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.core.BeaconState; -import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; - -public interface SignatureVerifier { - void feed(BeaconState state, IndexedAttestation indexed, ReceivedAttestation attestation); - VerificationResult verify(); -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerifiableAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerifiableAttestation.java new file mode 100644 index 000000000..a3377ea6b --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerifiableAttestation.java @@ -0,0 +1,108 @@ +package org.ethereum.beacon.chain.pool.verifier; + +import java.util.List; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.types.BLSPubkey; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.BLS381.PublicKey; +import org.ethereum.beacon.crypto.BLS381.Signature; +import tech.pegasys.artemis.util.collections.Bitlist; + +/** + * An artificial entity exclusively related to signature verification process. + * + *

    Being created from {@link IndexedAttestation}, {@link BeaconState} and {@link Attestation} + * itself contains all the information required to proceed with signature verification: + * + *

    + * + *

      + *
    • bit0 and bit1 aggregate public keys + *
    • signature + *
    • aggregation bits + *
    • attestation data + *
    + */ +final class VerifiableAttestation { + + static VerifiableAttestation create( + BeaconChainSpec spec, + BeaconState state, + IndexedAttestation indexed, + ReceivedAttestation attestation) { + + List bit0Keys = + indexed.getCustodyBit0Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + List bit1Keys = + indexed.getCustodyBit1Indices().stream() + .map(i -> state.getValidators().get(i).getPubKey()) + .collect(Collectors.toList()); + + // pre-process aggregated pubkeys + PublicKey bit0AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit0Keys); + PublicKey bit1AggregateKey = spec.bls_aggregate_pubkeys_no_validate(bit1Keys); + + return new VerifiableAttestation( + attestation, + attestation.getMessage().getData(), + attestation.getMessage().getAggregationBits(), + bit0AggregateKey, + bit1AggregateKey, + BLS381.Signature.createWithoutValidation(attestation.getMessage().getSignature())); + } + + private final ReceivedAttestation attestation; + + private final AttestationData data; + private final Bitlist aggregationBits; + private final PublicKey bit0Key; + private final PublicKey bit1Key; + private final BLS381.Signature signature; + + public VerifiableAttestation( + ReceivedAttestation attestation, + AttestationData data, + Bitlist aggregationBits, + PublicKey bit0Key, + PublicKey bit1Key, + Signature signature) { + this.attestation = attestation; + this.data = data; + this.aggregationBits = aggregationBits; + this.bit0Key = bit0Key; + this.bit1Key = bit1Key; + this.signature = signature; + } + + public AttestationData getData() { + return data; + } + + public Bitlist getAggregationBits() { + return aggregationBits; + } + + public ReceivedAttestation getAttestation() { + return attestation; + } + + public PublicKey getBit0Key() { + return bit0Key; + } + + public PublicKey getBit1Key() { + return bit1Key; + } + + public Signature getSignature() { + return signature; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java index 495e51156..3ff21a78e 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/VerificationResult.java @@ -5,6 +5,7 @@ import java.util.List; import org.ethereum.beacon.chain.pool.ReceivedAttestation; +/** Result of attestation batch verification. Contains a list of valid and invalid attestations. */ public final class VerificationResult { static VerificationResult allInvalid(List attestations) { diff --git a/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java b/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java index ee31eb226..a1f71b6bd 100644 --- a/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java +++ b/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java @@ -4,8 +4,21 @@ import org.reactivestreams.Subscription; import reactor.core.CoreSubscriber; import reactor.core.publisher.BaseSubscriber; +import reactor.core.publisher.DirectProcessor; import reactor.core.publisher.FluxProcessor; +/** + * An abstract processor built atop of reactor abstractions. + * + *

    Delegates to {@link OutsourcePublisher} which it its turn delegates to {@link + * DirectProcessor}. + * + *

    Note: DirectProcessor does not coordinate backpressure between its Subscribers and the + * upstream, but consumes its upstream in an * unbounded manner. + * + * @param a kind of input data. + * @param a kind of output data. + */ public abstract class AbstractDelegateProcessor extends FluxProcessor { private final Subscriber subscriber; @@ -48,8 +61,18 @@ protected void hookOnNext(IN value) { } } + /** + * Called when there is a new input value. + * + * @param value a value. + */ protected abstract void hookOnNext(IN value); + /** + * Should be called in order to publish a new value. + * + * @param value a value. + */ protected void publishOut(OUT value) { publisher.publishOut(value); } diff --git a/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java b/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java index 55a1212c3..2078182f6 100644 --- a/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java +++ b/util/src/main/java/org/ethereum/beacon/stream/Fluxes.java @@ -7,18 +7,40 @@ import reactor.core.publisher.ConnectableFlux; import reactor.core.publisher.Flux; +/** Various utility methods to work with {@link Flux}. */ public abstract class Fluxes { private Fluxes() {} + /** + * Given predicate creates a flux split. + * + * @param source a source. + * @param predicate a predicate. + * @param a kind of source data. + * @return a flux split. + */ public static FluxSplit split(Publisher source, Predicate predicate) { return new FluxSplit<>(source, predicate); } + /** + * A split of some publisher made upon a predicate. + * + *

    Built atop of {@link ConnectableFlux}. + * + * @param a kind of source data. + */ public static final class FluxSplit { private final Flux satisfied; private final Flux unsatisfied; private final Disposable disposable; + /** + * Given source and predicate creates a split. + * + * @param source a source publisher. + * @param predicate a predicate. + */ FluxSplit(Publisher source, Predicate predicate) { ConnectableFlux> splitter = Flux.from(source).map(value -> Pair.with(predicate.test(value), value)).publish(); @@ -28,14 +50,17 @@ public static final class FluxSplit { this.disposable = splitter.connect(); } + /** @return a flux of data items which predicates with {@code true}. */ public Flux getSatisfied() { return satisfied; } + /** @return a flux of data items which predicates with {@code false}. */ public Flux getUnsatisfied() { return unsatisfied; } + /** @return a disposable for connection between source and outcomes. */ public Disposable getDisposable() { return disposable; } diff --git a/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java b/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java index 736ef03df..b6d49c060 100644 --- a/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java +++ b/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java @@ -5,11 +5,26 @@ import reactor.core.publisher.DirectProcessor; import reactor.core.publisher.FluxSink; +/** + * An implementation of publisher which could be manually fed with data as a source. + * + *

    Delegates to {@link DirectProcessor} and uses its sink to feed the data. + * + *

    Note: DirectProcessor does not coordinate backpressure between its Subscribers and the + * upstream, but consumes its upstream in an * unbounded manner. + * + * @param a kind of data. + */ public class OutsourcePublisher implements Publisher { private final DirectProcessor delegate = DirectProcessor.create(); private final FluxSink out = delegate.sink(); + /** + * Publishes a new value. + * + * @param value a value. + */ public void publishOut(T value) { out.next(value); } From 19b740383df8b18cf6f6cc8ff48cb21e30f9255c Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 29 Aug 2019 00:14:08 +0600 Subject: [PATCH 08/59] Get rid of redundant Input class --- .../chain/pool/InMemoryAttestationPool.java | 20 +++++-------- .../reactor/AttestationChurnProcessor.java | 4 +-- .../chain/pool/reactor/IdentifyProcessor.java | 24 ++++++++------- .../beacon/chain/pool/reactor/Input.java | 30 ------------------- .../pool/reactor/SanityCheckProcessor.java | 15 +++++----- .../pool/reactor/TimeFrameProcessor.java | 19 ++++++------ 6 files changed, 41 insertions(+), 71 deletions(-) delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 5f1924b9a..3822ec157 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -7,7 +7,6 @@ import org.ethereum.beacon.chain.pool.reactor.AttestationChurnProcessor; import org.ethereum.beacon.chain.pool.reactor.AttestationVerificationProcessor; import org.ethereum.beacon.chain.pool.reactor.IdentifyProcessor; -import org.ethereum.beacon.chain.pool.reactor.Input; import org.ethereum.beacon.chain.pool.reactor.SanityCheckProcessor; import org.ethereum.beacon.chain.pool.reactor.TimeFrameProcessor; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; @@ -77,14 +76,12 @@ public void start() { Scheduler executor = schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); - Flux sourceFx = Flux.from(source).map(Input::wrap).publishOn(executor.toReactor()); - Flux newSlotsFx = Flux.from(newSlots).map(Input::wrap).publishOn(executor.toReactor()); - Flux importedBlocksFx = - Flux.from(importedBlocks).map(Input::wrap).publishOn(executor.toReactor()); - Flux finalizedCheckpointsFx = - Flux.from(finalizedCheckpoints).map(Input::wrap).publishOn(executor.toReactor()); - Flux chainHeadsFx = - Flux.from(chainHeads).map(Input::wrap).publishOn(executor.toReactor()); + Flux sourceFx = Flux.from(source).publishOn(executor.toReactor()); + Flux newSlotsFx = Flux.from(newSlots).publishOn(executor.toReactor()); + Flux importedBlocksFx = Flux.from(importedBlocks).publishOn(executor.toReactor()); + Flux finalizedCheckpointsFx = + Flux.from(finalizedCheckpoints).publishOn(executor.toReactor()); + Flux chainHeadsFx = Flux.from(chainHeads).publishOn(executor.toReactor()); // start from time frame processor Flux.merge(sourceFx, newSlotsFx, finalizedCheckpointsFx).subscribe(timeFrameProcessor); @@ -93,7 +90,7 @@ public void start() { // subscribe sanity checker Flux.merge( - timeFrameOut.getSatisfied().map(att -> Input.wrap(att.getAttestation())), + timeFrameOut.getSatisfied().map(CheckedAttestation::getAttestation), finalizedCheckpointsFx) .subscribe(sanityChecker); FluxSplit sanityOut = @@ -111,8 +108,7 @@ public void start() { Fluxes.split(newAttestations, encodingChecker::check); // identify attestation target - Flux.merge(encodingCheckOut.getSatisfied().map(Input::wrap), newSlotsFx, importedBlocksFx) - .subscribe(identifier); + Flux.merge(encodingCheckOut.getSatisfied(), newSlotsFx, importedBlocksFx).subscribe(identifier); // verify attestations identifier.bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL).subscribe(verifier); diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java index 0cd99a70a..ad3ac8103 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java @@ -4,10 +4,10 @@ import org.ethereum.beacon.stream.AbstractDelegateProcessor; public class AttestationChurnProcessor - extends AbstractDelegateProcessor { + extends AbstractDelegateProcessor { @Override - protected void hookOnNext(Input value) { + protected void hookOnNext(Object value) { // TODO implement } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java index 976c5edc7..aab87b428 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java @@ -26,7 +26,7 @@ *

  • attestations identified upon a new block come * */ -public class IdentifyProcessor extends AbstractDelegateProcessor { +public class IdentifyProcessor extends AbstractDelegateProcessor { private final UnknownAttestationPool pool; private final DirectProcessor unknownAttestations = DirectProcessor.create(); @@ -37,27 +37,29 @@ public IdentifyProcessor(UnknownAttestationPool pool) { } @Override - protected void hookOnNext(Input value) { - if (value.getType().equals(BeaconBlock.class)) { + protected void hookOnNext(Object value) { + if (value.getClass().equals(BeaconBlock.class)) { // forward attestations identified with a new block - pool.feedNewImportedBlock(value.unbox()).forEach(this::publishOut); - } else if (value.getType().equals(SlotNumber.class)) { - pool.feedNewSlot(value.unbox()); - } else if (value.getType().equals(ReceivedAttestation.class)) { + pool.feedNewImportedBlock((BeaconBlock) value).forEach(this::publishOut); + } else if (value.getClass().equals(SlotNumber.class)) { + pool.feedNewSlot((SlotNumber) value); + } else if (value.getClass().equals(ReceivedAttestation.class)) { if (pool.isInitialized()) { return; } - if (!pool.add(value.unbox())) { + ReceivedAttestation attestation = (ReceivedAttestation) value; + + if (!pool.add(attestation)) { // forward attestations not added to the pool - publishOut(value.unbox()); + publishOut(attestation); } else { // expose not yet identified attestations - unknownOut.next(value.unbox()); + unknownOut.next(attestation); } } else { throw new IllegalArgumentException( - "Unsupported input type: " + value.getType().getSimpleName()); + "Unsupported input type: " + value.getClass().getSimpleName()); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java deleted file mode 100644 index c07ec5c5b..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/Input.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.ethereum.beacon.chain.pool.reactor; - -/** - * Facilitate an ability heterogeneous - * - */ -public class Input { - - /** - * Wrapped value - */ - private final Object value; - - private Input(Object value) { - this.value = value; - } - - public static Input wrap(Object value) { - return new Input(value); - } - - public Class getType() { - return value.getClass(); - } - - @SuppressWarnings("unchecked") - T unbox() { - return (T) value; - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java index 29b697411..a9a156946 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java @@ -22,7 +22,7 @@ *
  • attestations tagged with the check flag. * */ -public class SanityCheckProcessor extends AbstractDelegateProcessor { +public class SanityCheckProcessor extends AbstractDelegateProcessor { private final SanityChecker checker; @@ -31,16 +31,17 @@ public SanityCheckProcessor(SanityChecker checker) { } @Override - protected void hookOnNext(Input value) { - if (value.getType().equals(Checkpoint.class)) { - checker.feedFinalizedCheckpoint(value.unbox()); - } else if (value.getType().equals(ReceivedAttestation.class)) { + protected void hookOnNext(Object value) { + if (value.getClass().equals(Checkpoint.class)) { + checker.feedFinalizedCheckpoint((Checkpoint) value); + } else if (value.getClass().equals(ReceivedAttestation.class)) { if (checker.isInitialized()) { - publishOut(new CheckedAttestation(checker.check(value.unbox()), value.unbox())); + ReceivedAttestation attestation = (ReceivedAttestation) value; + publishOut(new CheckedAttestation(checker.check(attestation), attestation)); } } else { throw new IllegalArgumentException( - "Unsupported input type: " + value.getType().getSimpleName()); + "Unsupported input type: " + value.getClass().getSimpleName()); } } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java index fda721fea..cd8a7c180 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java @@ -24,7 +24,7 @@ *
  • attestations tagged with the check flag. * */ -public class TimeFrameProcessor extends AbstractDelegateProcessor { +public class TimeFrameProcessor extends AbstractDelegateProcessor { private final TimeFrameFilter filter; @@ -33,18 +33,19 @@ public TimeFrameProcessor(TimeFrameFilter filter) { } @Override - protected void hookOnNext(Input value) { - if (value.getType().equals(Checkpoint.class)) { - filter.feedFinalizedCheckpoint(value.unbox()); - } else if (value.getType().equals(SlotNumber.class)) { - filter.feedNewSlot(value.unbox()); - } else if (value.getType().equals(ReceivedAttestation.class)) { + protected void hookOnNext(Object value) { + if (value.getClass().equals(Checkpoint.class)) { + filter.feedFinalizedCheckpoint((Checkpoint) value); + } else if (value.getClass().equals(SlotNumber.class)) { + filter.feedNewSlot((SlotNumber) value); + } else if (value.getClass().equals(ReceivedAttestation.class)) { if (filter.isInitialized()) { - publishOut(new CheckedAttestation(filter.check(value.unbox()), value.unbox())); + ReceivedAttestation attestation = (ReceivedAttestation) value; + publishOut(new CheckedAttestation(filter.check(attestation), attestation)); } } else { throw new IllegalArgumentException( - "Unsupported input type: " + value.getType().getSimpleName()); + "Unsupported input type: " + value.getClass().getSimpleName()); } } } From 4494576d7cbf33e29a478e85652d658bf992470c Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 29 Aug 2019 00:15:45 +0600 Subject: [PATCH 09/59] Simplify name of processors in attestaiton pool --- .../chain/pool/InMemoryAttestationPool.java | 34 +++++++++---------- ...hurnProcessor.java => ChurnProcessor.java} | 2 +- ...ssor.java => IdentificationProcessor.java} | 4 +-- ...eckProcessor.java => SanityProcessor.java} | 4 +-- ...FrameProcessor.java => TimeProcessor.java} | 4 +-- ...cessor.java => VerificationProcessor.java} | 4 +-- 6 files changed, 26 insertions(+), 26 deletions(-) rename chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/{AttestationChurnProcessor.java => ChurnProcessor.java} (89%) rename chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/{IdentifyProcessor.java => IdentificationProcessor.java} (92%) rename chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/{SanityCheckProcessor.java => SanityProcessor.java} (89%) rename chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/{TimeFrameProcessor.java => TimeProcessor.java} (90%) rename chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/{AttestationVerificationProcessor.java => VerificationProcessor.java} (91%) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 3822ec157..4a24c6fdc 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -4,11 +4,11 @@ import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; -import org.ethereum.beacon.chain.pool.reactor.AttestationChurnProcessor; -import org.ethereum.beacon.chain.pool.reactor.AttestationVerificationProcessor; -import org.ethereum.beacon.chain.pool.reactor.IdentifyProcessor; -import org.ethereum.beacon.chain.pool.reactor.SanityCheckProcessor; -import org.ethereum.beacon.chain.pool.reactor.TimeFrameProcessor; +import org.ethereum.beacon.chain.pool.reactor.ChurnProcessor; +import org.ethereum.beacon.chain.pool.reactor.VerificationProcessor; +import org.ethereum.beacon.chain.pool.reactor.IdentificationProcessor; +import org.ethereum.beacon.chain.pool.reactor.SanityProcessor; +import org.ethereum.beacon.chain.pool.reactor.TimeProcessor; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; @@ -32,13 +32,13 @@ public class InMemoryAttestationPool implements AttestationPool { private final Publisher chainHeads; private final Schedulers schedulers; - private final TimeFrameProcessor timeFrameProcessor; - private final SanityCheckProcessor sanityChecker; + private final TimeProcessor timeProcessor; + private final SanityProcessor sanityChecker; private final SignatureEncodingChecker encodingChecker; private final ProcessedAttestations processedFilter; - private final IdentifyProcessor identifier; - private final AttestationVerificationProcessor verifier; - private final AttestationChurnProcessor churn; + private final IdentificationProcessor identifier; + private final VerificationProcessor verifier; + private final ChurnProcessor churn; private final DirectProcessor invalidAttestations = DirectProcessor.create(); private final DirectProcessor validAttestations = DirectProcessor.create(); @@ -62,13 +62,13 @@ public InMemoryAttestationPool( this.importedBlocks = importedBlocks; this.chainHeads = chainHeads; this.schedulers = schedulers; - this.timeFrameProcessor = new TimeFrameProcessor(timeFrameFilter); - this.sanityChecker = new SanityCheckProcessor(sanityChecker); + this.timeProcessor = new TimeProcessor(timeFrameFilter); + this.sanityChecker = new SanityProcessor(sanityChecker); this.encodingChecker = encodingChecker; this.processedFilter = processedFilter; - this.identifier = new IdentifyProcessor(unknownAttestationPool); - this.verifier = new AttestationVerificationProcessor(batchVerifier); - this.churn = new AttestationChurnProcessor(); + this.identifier = new IdentificationProcessor(unknownAttestationPool); + this.verifier = new VerificationProcessor(batchVerifier); + this.churn = new ChurnProcessor(); } @Override @@ -84,9 +84,9 @@ public void start() { Flux chainHeadsFx = Flux.from(chainHeads).publishOn(executor.toReactor()); // start from time frame processor - Flux.merge(sourceFx, newSlotsFx, finalizedCheckpointsFx).subscribe(timeFrameProcessor); + Flux.merge(sourceFx, newSlotsFx, finalizedCheckpointsFx).subscribe(timeProcessor); FluxSplit timeFrameOut = - Fluxes.split(timeFrameProcessor, CheckedAttestation::isPassed); + Fluxes.split(timeProcessor, CheckedAttestation::isPassed); // subscribe sanity checker Flux.merge( diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java similarity index 89% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java index ad3ac8103..5202a5f10 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationChurnProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java @@ -3,7 +3,7 @@ import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; import org.ethereum.beacon.stream.AbstractDelegateProcessor; -public class AttestationChurnProcessor +public class ChurnProcessor extends AbstractDelegateProcessor { @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java similarity index 92% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java index aab87b428..3fbe53a4e 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentifyProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java @@ -26,13 +26,13 @@ *
  • attestations identified upon a new block come * */ -public class IdentifyProcessor extends AbstractDelegateProcessor { +public class IdentificationProcessor extends AbstractDelegateProcessor { private final UnknownAttestationPool pool; private final DirectProcessor unknownAttestations = DirectProcessor.create(); private final FluxSink unknownOut = unknownAttestations.sink(); - public IdentifyProcessor(UnknownAttestationPool pool) { + public IdentificationProcessor(UnknownAttestationPool pool) { this.pool = pool; } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java similarity index 89% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java index a9a156946..cd7bf7629 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityCheckProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java @@ -22,11 +22,11 @@ *
  • attestations tagged with the check flag. * */ -public class SanityCheckProcessor extends AbstractDelegateProcessor { +public class SanityProcessor extends AbstractDelegateProcessor { private final SanityChecker checker; - public SanityCheckProcessor(SanityChecker checker) { + public SanityProcessor(SanityChecker checker) { this.checker = checker; } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java similarity index 90% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java index cd8a7c180..f0d260bd2 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeFrameProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java @@ -24,11 +24,11 @@ *
  • attestations tagged with the check flag. * */ -public class TimeFrameProcessor extends AbstractDelegateProcessor { +public class TimeProcessor extends AbstractDelegateProcessor { private final TimeFrameFilter filter; - public TimeFrameProcessor(TimeFrameFilter filter) { + public TimeProcessor(TimeFrameFilter filter) { this.filter = filter; } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java similarity index 91% rename from chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java rename to chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java index 6d795995d..bec6b5224 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/AttestationVerificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java @@ -23,12 +23,12 @@ *
  • attestations tagged with verification flag * */ -public class AttestationVerificationProcessor +public class VerificationProcessor extends AbstractDelegateProcessor, CheckedAttestation> { private final BatchVerifier verifier; - public AttestationVerificationProcessor(BatchVerifier verifier) { + public VerificationProcessor(BatchVerifier verifier) { this.verifier = verifier; } From 817abddc5c0d4228fb56dd9fb719a9e796b5f263 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 29 Aug 2019 01:15:23 +0600 Subject: [PATCH 10/59] Add a javadoc to InMemoryAttestationPool class --- .../ethereum/beacon/chain/pool/InMemoryAttestationPool.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 4a24c6fdc..2b46f4362 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -5,10 +5,10 @@ import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; import org.ethereum.beacon.chain.pool.reactor.ChurnProcessor; -import org.ethereum.beacon.chain.pool.reactor.VerificationProcessor; import org.ethereum.beacon.chain.pool.reactor.IdentificationProcessor; import org.ethereum.beacon.chain.pool.reactor.SanityProcessor; import org.ethereum.beacon.chain.pool.reactor.TimeProcessor; +import org.ethereum.beacon.chain.pool.reactor.VerificationProcessor; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; @@ -23,6 +23,10 @@ import reactor.core.publisher.DirectProcessor; import reactor.core.publisher.Flux; +/** + * An implementation of attestation pool based on Reactor + * library, one of the implementation of reactive streams. + */ public class InMemoryAttestationPool implements AttestationPool { private final Publisher source; From faf766b52bd512d584924adecf9a8d73e6221e70 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 29 Aug 2019 01:25:05 +0600 Subject: [PATCH 11/59] Add attestation pool diagram --- chain/src/main/resources/doc-files/attestation-pool.svg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 chain/src/main/resources/doc-files/attestation-pool.svg diff --git a/chain/src/main/resources/doc-files/attestation-pool.svg b/chain/src/main/resources/doc-files/attestation-pool.svg new file mode 100644 index 000000000..0bc0d9f24 --- /dev/null +++ b/chain/src/main/resources/doc-files/attestation-pool.svg @@ -0,0 +1,2 @@ + +
    TimeFrameChecker

    target_epoch < finalized_epoch
    AND
    target_epoch > curr_epoch + lookahead
    [Not supported by viewer]
    Discard
    Discard
    SanityChecker
    SanityChecker
    Invalid
    Invalid
    drop peer
    drop peer
    ProcessedAttestation
    is new?
    [Not supported by viewer]
    Discard
    Discard
    SignatureEncodingChecker
    is encoding valid?
    [Not supported by viewer]
    UnknownAttestationPool
    is attested block imported?
    [Not supported by viewer]
    Finalized checkpoint
    Finalized checkpoint<br>
    Newly imported block
    Newly imported block
    New slot
    New slot
    AttestationVerifier
    valid?
    [Not supported by viewer]
    timeout buffer
    timeout buffer
    Discard by timeout
    [Not supported by viewer]
    Valid
    Valid
    Propagate
    Propagate
    AttestationChurn
    proved to be onchain?
    [Not supported by viewer]
    proposer
    proposer
    LMD GHOST
    LMD GHOST
    Head
    Head
    Discard by timeout
    Discard by timeout     \ No newline at end of file From e9182d09772c960da556f39109804d3846083fa9 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 29 Aug 2019 01:31:24 +0600 Subject: [PATCH 12/59] Update pool diagram --- chain/src/main/resources/doc-files/attestation-pool.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/src/main/resources/doc-files/attestation-pool.svg b/chain/src/main/resources/doc-files/attestation-pool.svg index 0bc0d9f24..89decf7ca 100644 --- a/chain/src/main/resources/doc-files/attestation-pool.svg +++ b/chain/src/main/resources/doc-files/attestation-pool.svg @@ -1,2 +1,2 @@ -
    TimeFrameChecker

    target_epoch < finalized_epoch
    AND
    target_epoch > curr_epoch + lookahead
    [Not supported by viewer]
    Discard
    Discard
    SanityChecker
    SanityChecker
    Invalid
    Invalid
    drop peer
    drop peer
    ProcessedAttestation
    is new?
    [Not supported by viewer]
    Discard
    Discard
    SignatureEncodingChecker
    is encoding valid?
    [Not supported by viewer]
    UnknownAttestationPool
    is attested block imported?
    [Not supported by viewer]
    Finalized checkpoint
    Finalized checkpoint<br>
    Newly imported block
    Newly imported block
    New slot
    New slot
    AttestationVerifier
    valid?
    [Not supported by viewer]
    timeout buffer
    timeout buffer
    Discard by timeout
    [Not supported by viewer]
    Valid
    Valid
    Propagate
    Propagate
    AttestationChurn
    proved to be onchain?
    [Not supported by viewer]
    proposer
    proposer
    LMD GHOST
    LMD GHOST
    Head
    Head
    Discard by timeout
    Discard by timeout     \ No newline at end of file +
    TimeFrameChecker

    target_epoch < finalized_epoch
    AND
    target_epoch > curr_epoch + lookahead
    [Not supported by viewer]
    Discard
    Discard
    SanityChecker
    SanityChecker
    Invalid
    Invalid
    drop peer
    drop peer
    ProcessedAttestation
    is new?
    [Not supported by viewer]
    Discard
    Discard
    SignatureEncodingChecker
    is encoding valid?
    [Not supported by viewer]
    UnknownAttestationPool
    is attested block imported?
    [Not supported by viewer]
    Finalized checkpoint
    Finalized checkpoint<br>
    Newly imported block
    Newly imported block
    New slot
    New slot
    AttestationVerifier
    valid?
    [Not supported by viewer]
    timeout buffer
    timeout buffer
    Discard by timeout
    [Not supported by viewer]
    Valid
    Valid
    Propagate
    Propagate
    AttestationChurn
    proved to be onchain?
    [Not supported by viewer]
    proposer
    proposer
    LMD GHOST
    LMD GHOST
    Head
    Head
    Discard by timeout
    Discard by timeout
    Wire
    Wire     \ No newline at end of file From cb4130ed7e0556e98d505197bfe26dce241fdad6 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 30 Aug 2019 20:08:17 +0600 Subject: [PATCH 13/59] Rework attestation pool processors --- .../chain/pool/InMemoryAttestationPool.java | 127 +++++++++--------- .../chain/pool/reactor/ChurnProcessor.java | 24 +++- .../pool/reactor/DoubleWorkProcessor.java | 51 +++++++ .../pool/reactor/IdentificationProcessor.java | 66 ++++----- .../chain/pool/reactor/SanityProcessor.java | 57 +++++--- .../reactor/SignatureEncodingProcessor.java | 52 +++++++ .../chain/pool/reactor/TimeProcessor.java | 54 ++++---- .../pool/reactor/VerificationProcessor.java | 39 +++--- .../pool/registry/UnknownAttestationPool.java | 4 + .../beacon/stream/OutsourcePublisher.java | 10 +- 10 files changed, 325 insertions(+), 159 deletions(-) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 2b46f4362..9f94dbd70 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -1,12 +1,15 @@ package org.ethereum.beacon.chain.pool; +import java.util.List; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; import org.ethereum.beacon.chain.pool.reactor.ChurnProcessor; +import org.ethereum.beacon.chain.pool.reactor.DoubleWorkProcessor; import org.ethereum.beacon.chain.pool.reactor.IdentificationProcessor; import org.ethereum.beacon.chain.pool.reactor.SanityProcessor; +import org.ethereum.beacon.chain.pool.reactor.SignatureEncodingProcessor; import org.ethereum.beacon.chain.pool.reactor.TimeProcessor; import org.ethereum.beacon.chain.pool.reactor.VerificationProcessor; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; @@ -15,13 +18,11 @@ import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.Fluxes; -import org.ethereum.beacon.stream.Fluxes.FluxSplit; import org.reactivestreams.Publisher; import reactor.core.publisher.DirectProcessor; import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; /** * An implementation of attestation pool based on Reactor @@ -36,16 +37,17 @@ public class InMemoryAttestationPool implements AttestationPool { private final Publisher chainHeads; private final Schedulers schedulers; - private final TimeProcessor timeProcessor; - private final SanityProcessor sanityChecker; + private final TimeFrameFilter timeFrameFilter; + private final SanityChecker sanityChecker; private final SignatureEncodingChecker encodingChecker; private final ProcessedAttestations processedFilter; - private final IdentificationProcessor identifier; - private final VerificationProcessor verifier; - private final ChurnProcessor churn; + private final UnknownAttestationPool unknownPool; + private final BatchVerifier verifier; private final DirectProcessor invalidAttestations = DirectProcessor.create(); private final DirectProcessor validAttestations = DirectProcessor.create(); + private final DirectProcessor unknownAttestations = DirectProcessor.create(); + private final DirectProcessor offChainAggregates = DirectProcessor.create(); public InMemoryAttestationPool( Publisher source, @@ -58,7 +60,7 @@ public InMemoryAttestationPool( SanityChecker sanityChecker, SignatureEncodingChecker encodingChecker, ProcessedAttestations processedFilter, - UnknownAttestationPool unknownAttestationPool, + UnknownAttestationPool unknownPool, BatchVerifier batchVerifier) { this.source = source; this.newSlots = newSlots; @@ -66,74 +68,77 @@ public InMemoryAttestationPool( this.importedBlocks = importedBlocks; this.chainHeads = chainHeads; this.schedulers = schedulers; - this.timeProcessor = new TimeProcessor(timeFrameFilter); - this.sanityChecker = new SanityProcessor(sanityChecker); + this.timeFrameFilter = timeFrameFilter; + this.sanityChecker = sanityChecker; this.encodingChecker = encodingChecker; this.processedFilter = processedFilter; - this.identifier = new IdentificationProcessor(unknownAttestationPool); - this.verifier = new VerificationProcessor(batchVerifier); - this.churn = new ChurnProcessor(); + this.unknownPool = unknownPool; + this.verifier = batchVerifier; } @Override public void start() { - Scheduler executor = - schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); - - Flux sourceFx = Flux.from(source).publishOn(executor.toReactor()); - Flux newSlotsFx = Flux.from(newSlots).publishOn(executor.toReactor()); - Flux importedBlocksFx = Flux.from(importedBlocks).publishOn(executor.toReactor()); - Flux finalizedCheckpointsFx = - Flux.from(finalizedCheckpoints).publishOn(executor.toReactor()); - Flux chainHeadsFx = Flux.from(chainHeads).publishOn(executor.toReactor()); - - // start from time frame processor - Flux.merge(sourceFx, newSlotsFx, finalizedCheckpointsFx).subscribe(timeProcessor); - FluxSplit timeFrameOut = - Fluxes.split(timeProcessor, CheckedAttestation::isPassed); - - // subscribe sanity checker - Flux.merge( - timeFrameOut.getSatisfied().map(CheckedAttestation::getAttestation), - finalizedCheckpointsFx) - .subscribe(sanityChecker); - FluxSplit sanityOut = - Fluxes.split(sanityChecker, CheckedAttestation::isPassed); - - // filter already processed attestations - Flux newAttestations = - sanityOut - .getSatisfied() - .map(CheckedAttestation::getAttestation) - .filter(processedFilter::add); + Scheduler parallelExecutor = + schedulers + .newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS) + .toReactor(); + + // create sources + Flux sourceFx = Flux.from(source); + Flux newSlotsFx = Flux.from(newSlots); + Flux finalizedCheckpointsFx = Flux.from(finalizedCheckpoints); + Flux importedBlocksFx = Flux.from(importedBlocks); + Flux chainHeadsFx = Flux.from(chainHeads); + + // check time frames + TimeProcessor timeProcessor = + new TimeProcessor( + timeFrameFilter, schedulers, sourceFx, finalizedCheckpointsFx, newSlotsFx); + + // run sanity check + SanityProcessor sanityProcessor = + new SanityProcessor(sanityChecker, schedulers, timeProcessor, finalizedCheckpointsFx); + + // discard already processed attestations + DoubleWorkProcessor doubleWorkProcessor = + new DoubleWorkProcessor(processedFilter, schedulers, sanityProcessor.getValid()); // check signature encoding - FluxSplit encodingCheckOut = - Fluxes.split(newAttestations, encodingChecker::check); + SignatureEncodingProcessor encodingProcessor = + new SignatureEncodingProcessor( + encodingChecker, doubleWorkProcessor.publishOn(parallelExecutor)); // identify attestation target - Flux.merge(encodingCheckOut.getSatisfied(), newSlotsFx, importedBlocksFx).subscribe(identifier); + IdentificationProcessor identificationProcessor = + new IdentificationProcessor( + unknownPool, schedulers, encodingProcessor.getValid(), newSlotsFx, importedBlocksFx); // verify attestations - identifier.bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL).subscribe(verifier); - FluxSplit verifierOut = - Fluxes.split(verifier, CheckedAttestation::isPassed); + Flux> verificationThrottle = + identificationProcessor + .getIdentified() + .publishOn(parallelExecutor) + .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL); + VerificationProcessor verificationProcessor = + new VerificationProcessor(verifier, verificationThrottle); // feed churn - Flux.merge( - verifierOut.getSatisfied().map(CheckedAttestation::getAttestation), - newSlotsFx, - chainHeadsFx); + ChurnProcessor churnProcessor = new ChurnProcessor(schedulers, chainHeadsFx, newSlotsFx); + Scheduler outScheduler = schedulers.events().toReactor(); + // expose valid attestations + verificationProcessor.getValid().publishOn(outScheduler).subscribe(validAttestations); + // expose not yet identified + identificationProcessor.getUnknown().publishOn(outScheduler).subscribe(unknownAttestations); + // expose aggregates + churnProcessor.publishOn(outScheduler).subscribe(offChainAggregates); // expose invalid attestations Flux.merge( - sanityOut.getUnsatisfied().map(CheckedAttestation::getAttestation), - encodingCheckOut.getUnsatisfied(), - verifierOut.getUnsatisfied().map(CheckedAttestation::getAttestation)) + sanityProcessor.getInvalid(), + encodingProcessor.getInvalid(), + verificationProcessor.getInvalid()) + .publishOn(outScheduler) .subscribe(invalidAttestations); - - // expose valid attestations - verifierOut.getSatisfied().map(CheckedAttestation::getAttestation).subscribe(validAttestations); } @Override @@ -148,11 +153,11 @@ public Publisher getInvalid() { @Override public Publisher getUnknownAttestations() { - return identifier.getUnknownAttestations(); + return unknownAttestations; } @Override public Publisher getAggregates() { - return churn; + return offChainAggregates; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java index 5202a5f10..cefb36bb5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java @@ -1,13 +1,27 @@ package org.ethereum.beacon.chain.pool.reactor; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; -import org.ethereum.beacon.stream.AbstractDelegateProcessor; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.OutsourcePublisher; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.Flux; -public class ChurnProcessor - extends AbstractDelegateProcessor { +public class ChurnProcessor extends Flux { - @Override - protected void hookOnNext(Object value) { + private final OutsourcePublisher out = new OutsourcePublisher<>(); + + public ChurnProcessor( + Schedulers schedulers, Flux chainHeads, Flux newSlots) {} + + protected void hookOnNext(ReceivedAttestation value) { // TODO implement } + + @Override + public void subscribe(CoreSubscriber actual) { + out.subscribe(actual); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java new file mode 100644 index 000000000..9edfe607d --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java @@ -0,0 +1,51 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.OutsourcePublisher; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +/** + * Passes attestations through {@link ProcessedAttestations} filter. + * + *

    Input: + * + *

      + *
    • attestations + *
    + * + *

    Output: + * + *

      + *
    • Not yet processed attestations + *
    + */ +public class DoubleWorkProcessor extends Flux { + + private final ProcessedAttestations processedAttestations; + private final OutsourcePublisher out = new OutsourcePublisher<>(); + + public DoubleWorkProcessor( + ProcessedAttestations processedAttestations, + Schedulers schedulers, + Flux source) { + this.processedAttestations = processedAttestations; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-double-work").toReactor(); + source.publishOn(scheduler).subscribe(this::hookOnNext); + } + + private void hookOnNext(ReceivedAttestation attestation) { + if (!processedAttestations.add(attestation)) { + out.publishOut(attestation); + } + } + + @Override + public void subscribe(CoreSubscriber actual) { + out.subscribe(actual); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java index 3fbe53a4e..066d1f141 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java @@ -4,12 +4,13 @@ import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.stream.AbstractDelegateProcessor; -import reactor.core.publisher.DirectProcessor; -import reactor.core.publisher.FluxSink; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.OutsourcePublisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; /** - * Processor throttling attestations through {@link UnknownAttestationPool}. + * Passes attestations through {@link UnknownAttestationPool}. * *

    Input: * @@ -24,46 +25,49 @@ *

      *
    • instantly identified attestations *
    • attestations identified upon a new block come + *
    • attestations with not yet imported block *
    */ -public class IdentificationProcessor extends AbstractDelegateProcessor { +public class IdentificationProcessor { private final UnknownAttestationPool pool; - private final DirectProcessor unknownAttestations = DirectProcessor.create(); - private final FluxSink unknownOut = unknownAttestations.sink(); + private final OutsourcePublisher identified = new OutsourcePublisher<>(); + private final OutsourcePublisher unknown = new OutsourcePublisher<>(); - public IdentificationProcessor(UnknownAttestationPool pool) { + public IdentificationProcessor( + UnknownAttestationPool pool, + Schedulers schedulers, + Flux source, + Flux newSlots, + Flux newImportedBlocks) { this.pool = pool; - } - @Override - protected void hookOnNext(Object value) { - if (value.getClass().equals(BeaconBlock.class)) { - // forward attestations identified with a new block - pool.feedNewImportedBlock((BeaconBlock) value).forEach(this::publishOut); - } else if (value.getClass().equals(SlotNumber.class)) { - pool.feedNewSlot((SlotNumber) value); - } else if (value.getClass().equals(ReceivedAttestation.class)) { - if (pool.isInitialized()) { - return; - } + Scheduler scheduler = + schedulers.newSingleThreadDaemon("pool-attestation-identifier").toReactor(); + newSlots.publishOn(scheduler).subscribe(this.pool::feedNewSlot); + newImportedBlocks.publishOn(scheduler).subscribe(this::hookOnNext); + source.publishOn(scheduler).subscribe(this::hookOnNext); + } - ReceivedAttestation attestation = (ReceivedAttestation) value; + private void hookOnNext(BeaconBlock block) { + pool.feedNewImportedBlock(block).forEach(identified::publishOut); + } - if (!pool.add(attestation)) { - // forward attestations not added to the pool - publishOut(attestation); + private void hookOnNext(ReceivedAttestation attestation) { + if (pool.isInitialized()) { + if (pool.add(attestation)) { + unknown.publishOut(attestation); } else { - // expose not yet identified attestations - unknownOut.next(attestation); + identified.publishOut(attestation); } - } else { - throw new IllegalArgumentException( - "Unsupported input type: " + value.getClass().getSimpleName()); } } - public DirectProcessor getUnknownAttestations() { - return unknownAttestations; + public OutsourcePublisher getIdentified() { + return identified; + } + + public OutsourcePublisher getUnknown() { + return unknown; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java index cd7bf7629..22a4e15af 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java @@ -1,47 +1,66 @@ package org.ethereum.beacon.chain.pool.reactor; -import org.ethereum.beacon.chain.pool.CheckedAttestation; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.stream.AbstractDelegateProcessor; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.OutsourcePublisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; /** - * Processor throttling attestations through a {@link SanityChecker}. + * Passes attestations through a {@link SanityChecker}. * *

    Input: * *

      - *
    • recently finalized checkpoints. - *
    • attestations. + *
    • recently finalized checkpoints + *
    • attestations *
    * *

    Output: * *

      - *
    • attestations tagged with the check flag. + *
    • attestations successfully passed sanity checks + *
    • invalid attestations *
    */ -public class SanityProcessor extends AbstractDelegateProcessor { +public class SanityProcessor { private final SanityChecker checker; - public SanityProcessor(SanityChecker checker) { + private final OutsourcePublisher valid = new OutsourcePublisher<>(); + private final OutsourcePublisher invalid = new OutsourcePublisher<>(); + + public SanityProcessor( + SanityChecker checker, + Schedulers schedulers, + Flux source, + Flux finalizedCheckpoints) { this.checker = checker; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-sanity-checker").toReactor(); + finalizedCheckpoints.publishOn(scheduler).subscribe(this.checker::feedFinalizedCheckpoint); + source.publishOn(scheduler).subscribe(this::hookOnNext); } - @Override - protected void hookOnNext(Object value) { - if (value.getClass().equals(Checkpoint.class)) { - checker.feedFinalizedCheckpoint((Checkpoint) value); - } else if (value.getClass().equals(ReceivedAttestation.class)) { - if (checker.isInitialized()) { - ReceivedAttestation attestation = (ReceivedAttestation) value; - publishOut(new CheckedAttestation(checker.check(attestation), attestation)); - } + private void hookOnNext(ReceivedAttestation attestation) { + if (!checker.isInitialized()) { + return; + } + + if (checker.check(attestation)) { + valid.publishOut(attestation); } else { - throw new IllegalArgumentException( - "Unsupported input type: " + value.getClass().getSimpleName()); + invalid.publishOut(attestation); } } + + public Flux getValid() { + return valid; + } + + public Flux getInvalid() { + return invalid; + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java new file mode 100644 index 000000000..d51f5fc3f --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java @@ -0,0 +1,52 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.stream.OutsourcePublisher; +import reactor.core.publisher.Flux; + +/** + * Passes attestations through {@link SignatureEncodingChecker}. + * + *

    Input: + * + *

      + *
    • attestations + *
    + * + *

    Output: + * + *

      + *
    • attestations with valid signature encoding + *
    • invalid attestations + *
    + */ +public class SignatureEncodingProcessor { + + private final SignatureEncodingChecker checker; + private final OutsourcePublisher valid = new OutsourcePublisher<>(); + private final OutsourcePublisher invalid = new OutsourcePublisher<>(); + + public SignatureEncodingProcessor( + SignatureEncodingChecker checker, Flux source) { + this.checker = checker; + + source.subscribe(this::hookOnNext); + } + + private void hookOnNext(ReceivedAttestation attestation) { + if (checker.check(attestation)) { + valid.publishOut(attestation); + } else { + invalid.publishOut(attestation); + } + } + + public Flux getValid() { + return valid; + } + + public Flux getInvalid() { + return invalid; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java index f0d260bd2..62f89cea5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java @@ -1,51 +1,59 @@ package org.ethereum.beacon.chain.pool.reactor; -import org.ethereum.beacon.chain.pool.CheckedAttestation; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.stream.AbstractDelegateProcessor; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.OutsourcePublisher; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; /** - * Processor throttling attestations through a {@link TimeFrameFilter}. + * Passes attestations through a {@link TimeFrameFilter}. * *

    Input: * *

      - *
    • recently finalized checkpoints. - *
    • new slots. - *
    • attestations. + *
    • recently finalized checkpoints + *
    • new slots + *
    • attestations *
    * *

    Output: * *

      - *
    • attestations tagged with the check flag. + *
    • attestations satisfying time frames *
    */ -public class TimeProcessor extends AbstractDelegateProcessor { +public class TimeProcessor extends Flux { private final TimeFrameFilter filter; + private final OutsourcePublisher out = new OutsourcePublisher<>(); - public TimeProcessor(TimeFrameFilter filter) { + public TimeProcessor( + TimeFrameFilter filter, + Schedulers schedulers, + Flux source, + Flux finalizedCheckpoints, + Flux newSlots) { this.filter = filter; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-time-frame-filter").toReactor(); + source.publishOn(scheduler).subscribe(this::hookOnNext); + finalizedCheckpoints.publishOn(scheduler).subscribe(this.filter::feedFinalizedCheckpoint); + newSlots.publishOn(scheduler).subscribe(this.filter::feedNewSlot); } - @Override - protected void hookOnNext(Object value) { - if (value.getClass().equals(Checkpoint.class)) { - filter.feedFinalizedCheckpoint((Checkpoint) value); - } else if (value.getClass().equals(SlotNumber.class)) { - filter.feedNewSlot((SlotNumber) value); - } else if (value.getClass().equals(ReceivedAttestation.class)) { - if (filter.isInitialized()) { - ReceivedAttestation attestation = (ReceivedAttestation) value; - publishOut(new CheckedAttestation(filter.check(attestation), attestation)); - } - } else { - throw new IllegalArgumentException( - "Unsupported input type: " + value.getClass().getSimpleName()); + private void hookOnNext(ReceivedAttestation attestation) { + if (filter.isInitialized() && filter.check(attestation)) { + out.publishOut(attestation); } } + + @Override + public void subscribe(CoreSubscriber actual) { + out.subscribe(actual); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java index bec6b5224..396f923fa 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java @@ -1,43 +1,52 @@ package org.ethereum.beacon.chain.pool.reactor; import java.util.List; -import java.util.stream.Stream; -import org.ethereum.beacon.chain.pool.CheckedAttestation; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; import org.ethereum.beacon.chain.pool.verifier.VerificationResult; -import org.ethereum.beacon.stream.AbstractDelegateProcessor; +import org.ethereum.beacon.stream.OutsourcePublisher; +import reactor.core.publisher.Flux; /** - * A processor that throttles attestations through {@link BatchVerifier}. + * Passes attestations through {@link BatchVerifier}. * *

    Input: * *

      - *
    • a list of {@link ReceivedAttestation} + *
    • batches of {@link ReceivedAttestation} *
    * *

    Output: * *

      - *
    • attestations tagged with verification flag + *
    • attestations successfully passed verification + *
    • invalid attestations *
    */ -public class VerificationProcessor - extends AbstractDelegateProcessor, CheckedAttestation> { +public class VerificationProcessor { private final BatchVerifier verifier; - public VerificationProcessor(BatchVerifier verifier) { + private final OutsourcePublisher valid = new OutsourcePublisher<>(); + private final OutsourcePublisher invalid = new OutsourcePublisher<>(); + + public VerificationProcessor(BatchVerifier verifier, Flux> source) { this.verifier = verifier; + + source.subscribe(this::hookOnNext); } - @Override - protected void hookOnNext(List batch) { + private void hookOnNext(List batch) { VerificationResult result = verifier.verify(batch); - Stream.concat( - result.getValid().stream().map(att -> new CheckedAttestation(true, att)), - result.getInvalid().stream().map(att -> new CheckedAttestation(false, att))) - .forEach(this::publishOut); + result.getValid().forEach(valid::publishOut); + result.getInvalid().forEach(invalid::publishOut); + } + + public Flux getValid() { + return valid; + } + + public Flux getInvalid() { + return invalid; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java index fd26e0f5b..b8cd1ba5e 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -78,6 +78,10 @@ public boolean add(ReceivedAttestation attestation) { * @return a list of attestations that have been identified by the block. */ public List feedNewImportedBlock(BeaconBlock block) { + if (!isInitialized()) { + return Collections.emptyList(); + } + EpochNumber blockEpoch = spec.compute_epoch_of_slot(block.getSlot()); // blockEpoch < currentBaseLine || blockEpoch >= currentBaseLine + TRACKED_EPOCHS if (blockEpoch.less(currentBaseLine) diff --git a/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java b/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java index b6d49c060..66d78bae6 100644 --- a/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java +++ b/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.stream; -import org.reactivestreams.Publisher; -import org.reactivestreams.Subscriber; +import reactor.core.CoreSubscriber; import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; /** @@ -15,7 +15,7 @@ * * @param a kind of data. */ -public class OutsourcePublisher implements Publisher { +public class OutsourcePublisher extends Flux { private final DirectProcessor delegate = DirectProcessor.create(); private final FluxSink out = delegate.sink(); @@ -30,7 +30,7 @@ public void publishOut(T value) { } @Override - public void subscribe(Subscriber s) { - delegate.subscribe(s); + public void subscribe(CoreSubscriber actual) { + delegate.subscribe(actual); } } From df528e515d067676230b875acc142b5e6b0a6ca6 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 16:12:26 +0300 Subject: [PATCH 14/59] JUNIT5: add. Add test draft --- chain/build.gradle | 8 ++++ .../beacon/chain/DefaultBeaconChainTest.java | 40 ++++++------------- .../beacon/chain/SlotTickerTests.java | 23 +++++------ .../ethereum/beacon/chain/StreamTests.java | 12 +++--- .../ObservableStateProcessorTest.java | 37 ++++++++--------- .../pool/InMemoryAttestationPoolTest.java | 22 ++++++++++ test/src/test/resources/eth2.0-spec-tests | 2 +- 7 files changed, 75 insertions(+), 69 deletions(-) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java diff --git a/chain/build.gradle b/chain/build.gradle index ad01e4bc4..14b59ddad 100644 --- a/chain/build.gradle +++ b/chain/build.gradle @@ -17,4 +17,12 @@ dependencies { // Gradle does not import test sources alongside with main sources // use a workaround until better solution will be found testImplementation project(':consensus').sourceSets.test.output + + testImplementation 'org.junit.jupiter:junit-jupiter:5.5.1' + + test { + useJUnitPlatform { + excludeTags 'FIX' + } + } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java b/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java index 7108d1010..efa39cf42 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java @@ -1,38 +1,24 @@ package org.ethereum.beacon.chain; -import java.util.Collections; -import java.util.stream.IntStream; import org.ethereum.beacon.chain.MutableBeaconChain.ImportResult; import org.ethereum.beacon.chain.storage.BeaconChainStorage; -import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; -import org.ethereum.beacon.chain.storage.impl.SerializerFactory; -import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.consensus.BeaconStateEx; -import org.ethereum.beacon.consensus.BlockTransition; -import org.ethereum.beacon.consensus.ChainStart; -import org.ethereum.beacon.consensus.StateTransition; -import org.ethereum.beacon.consensus.transition.BeaconStateExImpl; -import org.ethereum.beacon.consensus.transition.EmptySlotTransition; -import org.ethereum.beacon.consensus.transition.ExtendedSlotTransition; -import org.ethereum.beacon.consensus.transition.InitialStateTransition; -import org.ethereum.beacon.consensus.transition.PerEpochTransition; +import org.ethereum.beacon.chain.storage.impl.*; +import org.ethereum.beacon.consensus.*; +import org.ethereum.beacon.consensus.transition.*; import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; -import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; -import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; -import org.ethereum.beacon.consensus.verifier.VerificationResult; -import org.ethereum.beacon.core.BeaconBlock; -import org.ethereum.beacon.core.BeaconBlockBody; -import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.consensus.verifier.*; +import org.ethereum.beacon.core.*; import org.ethereum.beacon.core.state.Eth1Data; -import org.ethereum.beacon.core.types.BLSSignature; -import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.core.types.*; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.schedulers.Schedulers; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.*; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.Collections; +import java.util.stream.IntStream; + public class DefaultBeaconChainTest { @Test @@ -50,7 +36,7 @@ public void insertAChain() { beaconChain.init(); BeaconTuple initialTuple = beaconChain.getRecentlyProcessed(); - Assert.assertEquals( + Assertions.assertEquals( spec.getConstants().getGenesisSlot(), initialTuple.getBlock().getSlot()); IntStream.range(0, 10) @@ -59,8 +45,8 @@ public void insertAChain() { BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); BeaconBlock aBlock = createBlock(recentlyProcessed, spec, schedulers.getCurrentTime(), perSlotTransition); - Assert.assertEquals(ImportResult.OK, beaconChain.insert(aBlock)); - Assert.assertEquals(aBlock, beaconChain.getRecentlyProcessed().getBlock()); + Assertions.assertEquals(ImportResult.OK, beaconChain.insert(aBlock)); + Assertions.assertEquals(aBlock, beaconChain.getRecentlyProcessed().getBlock()); System.out.println("Inserted block: " + (idx + 1)); }); diff --git a/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java b/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java index 9ae310644..ba9e45239 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/SlotTickerTests.java @@ -1,19 +1,14 @@ package org.ethereum.beacon.chain; import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.core.BeaconState; -import org.ethereum.beacon.core.MutableBeaconState; +import org.ethereum.beacon.core.*; import org.ethereum.beacon.core.spec.SpecConstants; -import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.core.types.*; import org.ethereum.beacon.schedulers.Schedulers; -import org.junit.Test; +import org.junit.jupiter.api.*; import reactor.core.publisher.Flux; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import java.util.concurrent.*; public class SlotTickerTests { public static final int MILLIS_IN_SECOND = 1000; @@ -56,16 +51,16 @@ public void testSlotTicker() throws Exception { .subscribe( slotNumber -> { if (previousTick.greater(SlotNumber.ZERO)) { - assertEquals(previousTick.increment(), slotNumber); + Assertions.assertEquals(previousTick.increment(), slotNumber); bothAssertsRun.countDown(); } else { - assertTrue(slotNumber.greater(genesisSlot)); // first tracked tick + Assertions.assertTrue(slotNumber.greater(genesisSlot)); // first tracked tick bothAssertsRun.countDown(); } previousTick = slotNumber; }); - assertTrue( - String.format("%s assertion(s) was not correct or not tested", bothAssertsRun.getCount()), - bothAssertsRun.await(4, TimeUnit.SECONDS)); + Assertions.assertTrue( + (bothAssertsRun.await(4, TimeUnit.SECONDS)), + String.format("%s assertion(s) was not correct or not tested", bothAssertsRun.getCount())); } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java b/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java index b1b551d21..e81f379c7 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/StreamTests.java @@ -1,15 +1,13 @@ package org.ethereum.beacon.chain; -import java.time.Duration; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.*; import org.reactivestreams.Publisher; import reactor.core.Disposable; -import reactor.core.publisher.Flux; -import reactor.core.publisher.FluxSink; -import reactor.core.publisher.ReplayProcessor; +import reactor.core.publisher.*; import reactor.core.scheduler.Schedulers; +import java.time.Duration; + public class StreamTests { FluxSink sink; @@ -44,7 +42,7 @@ public void test1() throws InterruptedException { } @Test - @Ignore + @Disabled public void intervalTest1() throws InterruptedException { long initDelay = (System.currentTimeMillis() / 10000 + 1) * 10000 - System.currentTimeMillis(); diff --git a/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java index 5862bdc89..3ca2186b5 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorTest.java @@ -1,17 +1,14 @@ package org.ethereum.beacon.chain.observer; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; import org.ethereum.beacon.chain.util.SampleObservableState; import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.schedulers.ControlledSchedulers; -import org.ethereum.beacon.schedulers.Schedulers; -import org.junit.Assert; -import org.junit.Test; +import org.ethereum.beacon.schedulers.*; +import org.junit.jupiter.api.*; import reactor.core.publisher.Flux; +import java.time.Duration; +import java.util.*; + public class ObservableStateProcessorTest { @Test @@ -34,19 +31,19 @@ public void test1() throws Exception { System.out.println(states); System.out.println(slots); - Assert.assertEquals(1, states.size()); - Assert.assertEquals(0, slots.size()); + Assertions.assertEquals(1, states.size()); + Assertions.assertEquals(0, slots.size()); schedulers.addTime(Duration.ofSeconds(10)); System.out.println(states); System.out.println(slots); - Assert.assertEquals(2, states.size()); - Assert.assertEquals(1, slots.size()); + Assertions.assertEquals(2, states.size()); + Assertions.assertEquals(1, slots.size()); - Assert.assertEquals(genesisSlot.getValue() + 1, slots.get(0).getValue()); - Assert.assertEquals(genesisSlot.increment(), states.get(1).getLatestSlotState().getSlot()); + Assertions.assertEquals(genesisSlot.getValue() + 1, slots.get(0).getValue()); + Assertions.assertEquals(genesisSlot.increment(), states.get(1).getLatestSlotState().getSlot()); } @Test @@ -66,15 +63,15 @@ public void test2() throws Exception { List slots = new ArrayList<>(); Flux.from(sample.slotTicker.getTickerStream()).subscribe(slots::add); - Assert.assertEquals(1, states.size()); - Assert.assertEquals(0, slots.size()); + Assertions.assertEquals(1, states.size()); + Assertions.assertEquals(0, slots.size()); schedulers.addTime(Duration.ofSeconds(10)); - Assert.assertEquals(2, states.size()); - Assert.assertEquals(1, slots.size()); + Assertions.assertEquals(2, states.size()); + Assertions.assertEquals(1, slots.size()); - Assert.assertEquals(genesisSlot.getValue() + 61, slots.get(0).getValue()); - Assert.assertEquals(genesisSlot.plus(61), states.get(1).getLatestSlotState().getSlot()); + Assertions.assertEquals(genesisSlot.getValue() + 61, slots.get(0).getValue()); + Assertions.assertEquals(genesisSlot.plus(61), states.get(1).getLatestSlotState().getSlot()); } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java new file mode 100644 index 000000000..2a5d72509 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -0,0 +1,22 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.junit.jupiter.api.Test; +import org.reactivestreams.Publisher; + +class InMemoryAttestationPoolTest { + + private Publisher source = new DefaultTestPublisher<>(); + private Publisher newSlots = new DefaultTestPublisher<>(); + private Publisher finalizedCheckpoints = new DefaultTestPublisher<>(); + private Publisher importedBlocks = new DefaultTestPublisher<>(); + private Publisher chainHeads = new DefaultTestPublisher<>(); + + @Test + void integrationTest() { + final AttestationPool pool = new InMemoryAttestationPool() + } + +} diff --git a/test/src/test/resources/eth2.0-spec-tests b/test/src/test/resources/eth2.0-spec-tests index ae6dd9011..aaa1673f5 160000 --- a/test/src/test/resources/eth2.0-spec-tests +++ b/test/src/test/resources/eth2.0-spec-tests @@ -1 +1 @@ -Subproject commit ae6dd9011df05fab8c7e651c09cf9c940973bf81 +Subproject commit aaa1673f508103e11304833e0456e4149f880065 From a1219bad94d233eb52882668929d0ea2d4fb4059 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 16:45:58 +0300 Subject: [PATCH 15/59] Create attestation-pool at test --- .../pool/InMemoryAttestationPoolTest.java | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 2a5d72509..6878dbaa0 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -1,22 +1,66 @@ package org.ethereum.beacon.chain.pool; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.chain.storage.impl.*; +import org.ethereum.beacon.consensus.*; +import org.ethereum.beacon.consensus.transition.*; import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.*; +import org.ethereum.beacon.db.InMemoryDatabase; +import org.ethereum.beacon.schedulers.Schedulers; import org.junit.jupiter.api.Test; import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; class InMemoryAttestationPoolTest { - private Publisher source = new DefaultTestPublisher<>(); - private Publisher newSlots = new DefaultTestPublisher<>(); - private Publisher finalizedCheckpoints = new DefaultTestPublisher<>(); - private Publisher importedBlocks = new DefaultTestPublisher<>(); - private Publisher chainHeads = new DefaultTestPublisher<>(); + private Publisher source = Flux.empty(); + private Publisher newSlots = Flux.empty(); + private Publisher finalizedCheckpoints = Flux.empty(); + private Publisher importedBlocks = Flux.empty(); + private Publisher chainHeads = Flux.empty(); + private Schedulers schedulers = Schedulers.createDefault(); + + private SpecConstants specConstants = + new SpecConstants() { + @Override + public SlotNumber getGenesisSlot() { + return SlotNumber.of(12345); + } + + @Override + public Time getSecondsPerSlot() { + return Time.of(1); + } + }; + private BeaconChainSpec spec = BeaconChainSpec.createWithDefaultHasher(specConstants); + private InMemoryDatabase db = new InMemoryDatabase(); + private BeaconChainStorage beaconChainStorage = + new SSZBeaconChainStorageFactory( + spec.getObjectHasher(), SerializerFactory.createSSZ(specConstants)) + .create(db); + + private PerEpochTransition perEpochTransition = new PerEpochTransition(spec); + private StateTransition perSlotTransition = new PerSlotTransition(spec); + private EmptySlotTransition slotTransition = + new EmptySlotTransition( + new ExtendedSlotTransition(perEpochTransition, perSlotTransition, spec)); @Test void integrationTest() { - final AttestationPool pool = new InMemoryAttestationPool() + final AttestationPool pool = AttestationPool.create( + source, + newSlots, + finalizedCheckpoints, + importedBlocks, + chainHeads, + schedulers, + spec, + beaconChainStorage, + slotTransition + ); } } From d394a3bddaa5728c8ca43dc28cb6bce9932b034e Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 17:05:38 +0300 Subject: [PATCH 16/59] add reactor tests --- chain/build.gradle | 1 + .../ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/chain/build.gradle b/chain/build.gradle index 14b59ddad..b57657432 100644 --- a/chain/build.gradle +++ b/chain/build.gradle @@ -13,6 +13,7 @@ dependencies { api "org.apache.commons:commons-collections4" testImplementation 'org.mockito:mockito-core' + testCompile 'io.projectreactor:reactor-test' // Gradle does not import test sources alongside with main sources // use a workaround until better solution will be found diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 6878dbaa0..c801d885e 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -61,6 +61,8 @@ void integrationTest() { beaconChainStorage, slotTransition ); + + pool.start(); } } From 23f74e2f330a379da0f52d96f95dda3a09564dcd Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 19:49:23 +0300 Subject: [PATCH 17/59] pool-test: add verify blockchain creation --- .../pool/InMemoryAttestationPoolTest.java | 101 ++++++++++++++++-- 1 file changed, 94 insertions(+), 7 deletions(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index c801d885e..b369b98df 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -1,26 +1,32 @@ package org.ethereum.beacon.chain.pool; +import org.ethereum.beacon.chain.*; import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.impl.*; import org.ethereum.beacon.consensus.*; import org.ethereum.beacon.consensus.transition.*; -import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; +import org.ethereum.beacon.consensus.verifier.*; +import org.ethereum.beacon.core.*; import org.ethereum.beacon.core.spec.SpecConstants; -import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.state.*; import org.ethereum.beacon.core.types.*; -import org.ethereum.beacon.db.InMemoryDatabase; +import org.ethereum.beacon.db.*; import org.ethereum.beacon.schedulers.Schedulers; import org.junit.jupiter.api.Test; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.Collections; class InMemoryAttestationPoolTest { private Publisher source = Flux.empty(); private Publisher newSlots = Flux.empty(); private Publisher finalizedCheckpoints = Flux.empty(); - private Publisher importedBlocks = Flux.empty(); - private Publisher chainHeads = Flux.empty(); private Schedulers schedulers = Schedulers.createDefault(); private SpecConstants specConstants = @@ -35,7 +41,25 @@ public Time getSecondsPerSlot() { return Time.of(1); } }; - private BeaconChainSpec spec = BeaconChainSpec.createWithDefaultHasher(specConstants); + private BeaconChainSpec spec = BeaconChainSpec.Builder.createWithDefaultParams() + .withConstants(new SpecConstants() { + @Override + public ShardNumber getShardCount() { + return ShardNumber.of(16); + } + + @Override + public SlotNumber.EpochLength getSlotsPerEpoch() { + return new SlotNumber.EpochLength(UInt64.valueOf(4)); + } + }) + .withComputableGenesisTime(false) + .withVerifyDepositProof(false) + .withBlsVerifyProofOfPossession(false) + .withBlsVerify(false) + .withCache(true) + .build(); + private InMemoryDatabase db = new InMemoryDatabase(); private BeaconChainStorage beaconChainStorage = new SSZBeaconChainStorageFactory( @@ -50,6 +74,21 @@ public Time getSecondsPerSlot() { @Test void integrationTest() { + final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); + final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); + final BeaconBlock aBlock = createBlock(recentlyProcessed, spec, + schedulers.getCurrentTime(), perSlotTransition); + + final Publisher importedBlocks = Flux.just(aBlock); + StepVerifier.create(importedBlocks) + .expectNext(aBlock) + .verifyComplete(); + + final Publisher chainHeads = Flux.just(aBlock); + StepVerifier.create(chainHeads) + .expectNext(aBlock) + .verifyComplete(); + final AttestationPool pool = AttestationPool.create( source, newSlots, @@ -61,8 +100,56 @@ void integrationTest() { beaconChainStorage, slotTransition ); + } + + private MutableBeaconChain createBeaconChain( + BeaconChainSpec spec, StateTransition perSlotTransition, Schedulers schedulers) { + Time start = Time.castFrom(UInt64.valueOf(schedulers.getCurrentTime() / 1000)); + ChainStart chainStart = new ChainStart(start, Eth1Data.EMPTY, Collections.emptyList()); + BlockTransition initialTransition = + new InitialStateTransition(chainStart, spec); + BlockTransition perBlockTransition = + StateTransitionTestUtil.createPerBlockTransition(); + StateTransition perEpochTransition = + StateTransitionTestUtil.createStateWithNoTransition(); - pool.start(); + BeaconBlockVerifier blockVerifier = (block, state) -> VerificationResult.PASSED; + BeaconStateVerifier stateVerifier = (block, state) -> VerificationResult.PASSED; + Database database = Database.inMemoryDB(); + BeaconChainStorage chainStorage = new SSZBeaconChainStorageFactory( + spec.getObjectHasher(), SerializerFactory.createSSZ(spec.getConstants())) + .create(database); + + return new DefaultBeaconChain( + spec, + initialTransition, + new EmptySlotTransition( + new ExtendedSlotTransition(new PerEpochTransition(spec) { + @Override + public BeaconStateEx apply(BeaconStateEx stateEx) { + return perEpochTransition.apply(stateEx); + } + }, perSlotTransition, spec)), + perBlockTransition, + blockVerifier, + stateVerifier, + chainStorage, + schedulers); } + private BeaconBlock createBlock( + BeaconTuple parent, + BeaconChainSpec spec, long currentTime, + StateTransition perSlotTransition) { + BeaconBlock block = + new BeaconBlock( + spec.get_current_slot(parent.getState(), currentTime), + spec.signing_root(parent.getBlock()), + Hash32.ZERO, + BeaconBlockBody.getEmpty(spec.getConstants()), + BLSSignature.ZERO); + BeaconState state = perSlotTransition.apply(new BeaconStateExImpl(parent.getState())); + + return block.withStateRoot(spec.hash_tree_root(state)); + } } From 5a471373ccb242dadabb6349806631deec994c36 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 20:05:42 +0300 Subject: [PATCH 18/59] pool-test: add verify ReceivedAttestation creation --- .../pool/InMemoryAttestationPoolTest.java | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index b369b98df..7d41f3b38 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -8,23 +8,28 @@ import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; import org.ethereum.beacon.consensus.verifier.*; import org.ethereum.beacon.core.*; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.*; import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.core.state.*; import org.ethereum.beacon.core.types.*; +import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.db.*; import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.types.p2p.NodeId; import org.junit.jupiter.api.Test; import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.test.StepVerifier; import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.*; +import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; import java.util.Collections; class InMemoryAttestationPoolTest { - private Publisher source = Flux.empty(); private Publisher newSlots = Flux.empty(); private Publisher finalizedCheckpoints = Flux.empty(); private Schedulers schedulers = Schedulers.createDefault(); @@ -89,6 +94,14 @@ void integrationTest() { .expectNext(aBlock) .verifyComplete(); + final NodeId sender = new NodeId(new byte[0]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + final Publisher source = Flux.just(attestation); + StepVerifier.create(source) + .expectNext(attestation) + .verifyComplete(); + final AttestationPool pool = AttestationPool.create( source, newSlots, @@ -100,6 +113,8 @@ void integrationTest() { beaconChainStorage, slotTransition ); + + pool.start(); } private MutableBeaconChain createBeaconChain( @@ -152,4 +167,28 @@ private BeaconBlock createBlock( return block.withStateRoot(spec.hash_tree_root(state)); } + + private Attestation createAttestation(BytesValue someValue) { + final AttestationData attestationData = createAttestationData(); + final Attestation attestation = + new Attestation( + Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), + attestationData, + Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), + BLSSignature.wrap(Bytes96.fromHexString("cc")), + specConstants); + + return attestation; + } + + public AttestationData createAttestationData() { + final AttestationData expected = + new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.ZERO, Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(123), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + + return expected; + } } From 1ea3c89acd98430581ee2364656b9079d91bc369 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 20:16:56 +0300 Subject: [PATCH 19/59] pool-test: verify SlotsNumber creation --- .../beacon/chain/pool/InMemoryAttestationPoolTest.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 7d41f3b38..3e66ac5fd 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -30,7 +30,6 @@ class InMemoryAttestationPoolTest { - private Publisher newSlots = Flux.empty(); private Publisher finalizedCheckpoints = Flux.empty(); private Schedulers schedulers = Schedulers.createDefault(); @@ -102,6 +101,13 @@ void integrationTest() { .expectNext(attestation) .verifyComplete(); + + final SlotNumber slotNumber = SlotNumber.of(100L); + final Publisher newSlots = Flux.just(slotNumber); + StepVerifier.create(newSlots) + .expectNext(slotNumber) + .verifyComplete(); + final AttestationPool pool = AttestationPool.create( source, newSlots, From 037ba11d7c724abf57979eb9aa33c802dcf34471 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 20:57:39 +0300 Subject: [PATCH 20/59] pool-test: a little refactroing --- .../pool/InMemoryAttestationPoolTest.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 3e66ac5fd..824b3b631 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -83,17 +83,7 @@ void integrationTest() { final BeaconBlock aBlock = createBlock(recentlyProcessed, spec, schedulers.getCurrentTime(), perSlotTransition); - final Publisher importedBlocks = Flux.just(aBlock); - StepVerifier.create(importedBlocks) - .expectNext(aBlock) - .verifyComplete(); - - final Publisher chainHeads = Flux.just(aBlock); - StepVerifier.create(chainHeads) - .expectNext(aBlock) - .verifyComplete(); - - final NodeId sender = new NodeId(new byte[0]); + final NodeId sender = new NodeId(new byte[100]); final Attestation message = createAttestation(BytesValue.fromHexString("aa")); final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); final Publisher source = Flux.just(attestation); @@ -101,13 +91,22 @@ void integrationTest() { .expectNext(attestation) .verifyComplete(); - final SlotNumber slotNumber = SlotNumber.of(100L); final Publisher newSlots = Flux.just(slotNumber); StepVerifier.create(newSlots) .expectNext(slotNumber) .verifyComplete(); + final Publisher importedBlocks = Flux.just(aBlock); + StepVerifier.create(importedBlocks) + .expectNext(aBlock) + .verifyComplete(); + + final Publisher chainHeads = Flux.just(aBlock); + StepVerifier.create(chainHeads) + .expectNext(aBlock) + .verifyComplete(); + final AttestationPool pool = AttestationPool.create( source, newSlots, From 0b1e0f1381fefdb53ca65bc2365bb871b0ff3a6d Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 21:24:53 +0300 Subject: [PATCH 21/59] pool-test: fix parent block chain --- .../ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 824b3b631..2ef765ba1 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -79,6 +79,7 @@ public SlotNumber.EpochLength getSlotsPerEpoch() { @Test void integrationTest() { final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); + beaconChain.init(); final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); final BeaconBlock aBlock = createBlock(recentlyProcessed, spec, schedulers.getCurrentTime(), perSlotTransition); From 0aff0ce54cc2a7e6f4a72b304d118810065335c5 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Mon, 2 Sep 2019 21:33:25 +0300 Subject: [PATCH 22/59] pool-test: add empty checkpoint publisher --- .../beacon/chain/pool/InMemoryAttestationPoolTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 2ef765ba1..8987155d0 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -30,7 +30,6 @@ class InMemoryAttestationPoolTest { - private Publisher finalizedCheckpoints = Flux.empty(); private Schedulers schedulers = Schedulers.createDefault(); private SpecConstants specConstants = @@ -98,6 +97,12 @@ void integrationTest() { .expectNext(slotNumber) .verifyComplete(); + final Checkpoint checkpoint = Checkpoint.EMPTY; + final Publisher finalizedCheckpoints = Flux.just(checkpoint); + StepVerifier.create(finalizedCheckpoints) + .expectNext(checkpoint) + .verifyComplete(); + final Publisher importedBlocks = Flux.just(aBlock); StepVerifier.create(importedBlocks) .expectNext(aBlock) From 7138a49a33712422dcee5382e9b9b9711861c593 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Tue, 3 Sep 2019 18:00:09 +0300 Subject: [PATCH 23/59] test: fix NPE to create UnknownAttestationPool --- .../ethereum/beacon/chain/pool/registry/Queue.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java index 8cc468134..ba75c82a5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java @@ -1,17 +1,12 @@ package org.ethereum.beacon.chain.pool.registry; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.core.types.EpochNumber; import tech.pegasys.artemis.ethereum.core.Hash32; +import java.util.*; +import java.util.stream.Collectors; + /** * Maintains attestations parked by {@link UnknownAttestationPool}. * @@ -38,6 +33,7 @@ final class Queue { this.trackedEpochs = trackedEpochs; this.maxSize = maxSize; + this.baseLine = EpochNumber.ZERO; } /** From 520859951c628ddc6ffa8564bb4148e5e5d91113 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Wed, 4 Sep 2019 13:26:12 +0300 Subject: [PATCH 24/59] test: correct verification --- .../chain/pool/InMemoryAttestationPoolTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 8987155d0..e533b50cc 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -89,29 +89,34 @@ void integrationTest() { final Publisher source = Flux.just(attestation); StepVerifier.create(source) .expectNext(attestation) - .verifyComplete(); + .expectComplete() + .verify(); final SlotNumber slotNumber = SlotNumber.of(100L); final Publisher newSlots = Flux.just(slotNumber); StepVerifier.create(newSlots) .expectNext(slotNumber) - .verifyComplete(); + .expectComplete() + .verify(); final Checkpoint checkpoint = Checkpoint.EMPTY; final Publisher finalizedCheckpoints = Flux.just(checkpoint); StepVerifier.create(finalizedCheckpoints) .expectNext(checkpoint) - .verifyComplete(); + .expectComplete() + .verify(); final Publisher importedBlocks = Flux.just(aBlock); StepVerifier.create(importedBlocks) .expectNext(aBlock) - .verifyComplete(); + .expectComplete() + .verify(); final Publisher chainHeads = Flux.just(aBlock); StepVerifier.create(chainHeads) .expectNext(aBlock) - .verifyComplete(); + .expectComplete() + .verify(); final AttestationPool pool = AttestationPool.create( source, From b65b8cfc34af5d29cc33e684839e61283774ecc6 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 29 Aug 2019 12:59:46 +0300 Subject: [PATCH 25/59] Switch to rocksDB JNI 6.2.2 since it includes compression codecs on Windows --- versions.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.gradle b/versions.gradle index a2afdca12..7bdbad651 100644 --- a/versions.gradle +++ b/versions.gradle @@ -39,7 +39,7 @@ dependencyManagement { dependency "info.picocli:picocli:3.9.4" dependency "io.netty:netty-all:4.1.36.Final" - dependency "org.rocksdb:rocksdbjni:5.5.1" + dependency "org.rocksdb:rocksdbjni:6.2.2" dependency "com.googlecode.concurrent-locks:concurrent-locks:1.0.0" } } From 5339f80c2c196f3855ace3d7f919a027935905d0 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 29 Aug 2019 13:21:50 +0300 Subject: [PATCH 26/59] Configuration of eth1_block_hash value for EmulatorContract added. The default interop value is b'0x42' * 32. --- .../config/main/conract/EmulatorContract.java | 25 ++++++++++++++++++- .../org/ethereum/beacon/node/ConfigUtils.java | 11 ++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java index 96a9350be..ea47f54b0 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java @@ -1,9 +1,12 @@ package org.ethereum.beacon.emulator.config.main.conract; import com.fasterxml.jackson.annotation.JsonFormat; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys; +import tech.pegasys.artemis.util.bytes.Bytes32; + +import java.util.Arrays; import java.util.Date; import java.util.List; -import org.ethereum.beacon.emulator.config.main.ValidatorKeys; public class EmulatorContract extends Contract { @@ -12,6 +15,18 @@ public class EmulatorContract extends Contract { private Integer balance; private List keys; + private String eth1BlockHash = createInteropEth1BlockHash(); + + /** + * @return HEX String representation of a hash32 consisting of 32 0x42 bytes, which is the interop + * default. + */ + private static String createInteropEth1BlockHash() { + byte ch = 0x42; + byte[] res = new byte[32]; + Arrays.fill(res, ch); + return Bytes32.wrap(res).toString(); + } public Date getGenesisTime() { return genesisTime; @@ -36,4 +51,12 @@ public List getKeys() { public void setKeys(List keys) { this.keys = keys; } + + public String getEth1BlockHash() { + return eth1BlockHash; + } + + public void setEth1BlockHash(String eth1BlockHash) { + this.eth1BlockHash = eth1BlockHash; + } } diff --git a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java index d0705756b..bb7335212 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java @@ -97,15 +97,16 @@ public static DepositContract createDepositContract(Contract config, BeaconChain deposits.stream().map(Deposit::getData).collect(Collectors.toList()), Integer::new, 1L << spec.getConstants().getDepositContractTreeDepth().getIntValue()); + Hash32 blockHash = + eConfig.getEth1BlockHash() == null || eConfig.getEth1BlockHash().length() == 0 + ? Hash32.random(random) + : Hash32.wrap(Bytes32.fromHexString(eConfig.getEth1BlockHash())); Eth1Data eth1Data = new Eth1Data( - spec.hash_tree_root(depositDataList), - UInt64.valueOf(deposits.size()), - Hash32.random(random)); + spec.hash_tree_root(depositDataList), UInt64.valueOf(deposits.size()), blockHash); ChainStart chainStart = new ChainStart(Time.of(eConfig.getGenesisTime().getTime() / 1000), eth1Data, deposits); - SimpleDepositContract depositContract = new SimpleDepositContract(chainStart); - return depositContract; + return new SimpleDepositContract(chainStart); } else { throw new IllegalArgumentException( "This config class is not yet supported: " + config.getClass()); From 8bbaacd102bc07f45343daf101443a1faeef3902 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 29 Aug 2019 14:05:51 +0300 Subject: [PATCH 27/59] Mocked start key pairs generation implemented. --- .../util/SimulationKeyPairGenerator.java | 55 +++++++++++++++++++ .../emulator/config/main/ValidatorKeys.java | 13 +++++ .../org/ethereum/beacon/node/ConfigUtils.java | 17 +++--- 3 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java new file mode 100644 index 000000000..d3b19d549 --- /dev/null +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java @@ -0,0 +1,55 @@ +package org.ethereum.beacon.start.common.util; + +import com.google.common.primitives.Bytes; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.Hashes; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +/** + * Key pair generation utilities for simulation purposes. + */ +public class SimulationKeyPairGenerator { + private static BigInteger CURVE_ORDER = new BigInteger("52435875175126190479447740508185965837690552500527637822603658699938581184513"); + + /** + * Generate public/private key pairs according to mocked start spec. + * @see + * Pubkey/privkey generation + * + * @param count - amount of key parirs to generate + * @return the generated key pairs + */ + public static List generateInteropKeys(int count) { + List ret = new ArrayList<>(); + for (int i = 0; i < count; i++) { + byte[] res = BigInteger.valueOf(i).toByteArray(); + Bytes.reverse(res); + BytesValue wrap = BytesValue.wrap(Bytes.ensureCapacity(res, 32, 0)); + byte[] res2 = Hashes.sha256(wrap).extractArray(); + Bytes.reverse(res2); + + BigInteger privKey = new BigInteger(1, res2).mod(CURVE_ORDER); + ret.add(BLS381.KeyPair.create(BLS381.PrivateKey.create(privKey))); + } + return ret; + } + + public static List generateRandomKeys(int seed, int start, int count) { + Random random = new Random(seed); + for (int i = 0; i < start; i++) { + Bytes32.random(random); + } + List ret = new ArrayList<>(); + for (int i = 0; i < count; i++) { + ret.add(BLS381.KeyPair.create(BLS381.PrivateKey.create(Bytes32.random(random)))); + } + return ret; + } +} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/ValidatorKeys.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/ValidatorKeys.java index ee1de7a5e..eb86a2b26 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/ValidatorKeys.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/ValidatorKeys.java @@ -9,6 +9,7 @@ @JsonSubTypes.Type(value = ValidatorKeys.Generate.class, name = "generate"), @JsonSubTypes.Type(value = ValidatorKeys.Private.class, name = "private"), @JsonSubTypes.Type(value = ValidatorKeys.Public.class, name = "public"), + @JsonSubTypes.Type(value = ValidatorKeys.InteropKeys.class, name = "interop"), }) public abstract class ValidatorKeys { @@ -42,6 +43,18 @@ public void setStartIndex(int startIndex) { } } + public static class InteropKeys extends ValidatorKeys { + private int count; + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } + } + public static class ExplicitKeys extends ValidatorKeys { private List keys; diff --git a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java index bb7335212..ec1381be2 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java @@ -1,6 +1,5 @@ package org.ethereum.beacon.node; -import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.stream.Collectors; @@ -17,6 +16,7 @@ import org.ethereum.beacon.emulator.config.main.Signer.Insecure; import org.ethereum.beacon.emulator.config.main.ValidatorKeys; import org.ethereum.beacon.emulator.config.main.ValidatorKeys.Generate; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys.InteropKeys; import org.ethereum.beacon.emulator.config.main.ValidatorKeys.Private; import org.ethereum.beacon.emulator.config.main.ValidatorKeys.Public; import org.ethereum.beacon.emulator.config.main.conract.Contract; @@ -24,6 +24,7 @@ import org.ethereum.beacon.pow.DepositContract; import org.ethereum.beacon.start.common.util.SimpleDepositContract; import org.ethereum.beacon.start.common.util.SimulateUtils; +import org.ethereum.beacon.start.common.util.SimulationKeyPairGenerator; import org.ethereum.beacon.validator.crypto.BLS381Credentials; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; @@ -66,15 +67,11 @@ public static List createKeyPairs(ValidatorKeys keys) { .collect(Collectors.toList()); } else if (keys instanceof Generate) { Generate genKeys = (Generate) keys; - Random random = new Random(genKeys.getSeed()); - for (int i = 0; i < genKeys.getStartIndex(); i++) { - Bytes32.random(random); - } - List ret = new ArrayList<>(); - for (int i = 0; i < genKeys.getCount(); i++) { - ret.add(KeyPair.create(PrivateKey.create(Bytes32.random(random)))); - } - return ret; + return SimulationKeyPairGenerator.generateRandomKeys( + genKeys.getSeed(), genKeys.getStartIndex(), genKeys.getCount()); + } else if (keys instanceof InteropKeys) { + InteropKeys interopKeys = (InteropKeys) keys; + return SimulationKeyPairGenerator.generateInteropKeys(interopKeys.getCount()); } else { throw new IllegalArgumentException("Unknown ValidatorKeys subclass: " + keys.getClass()); } From 5020f61da3801bd1d45fc763a31bcbf6a9d8c453 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 29 Aug 2019 15:30:14 +0300 Subject: [PATCH 28/59] Withdrawal credentials for mocked start generation implemented. --- .../start/common/util/SimulateUtils.java | 88 ++++++++++++++++--- .../config/main/conract/EmulatorContract.java | 9 ++ .../org/ethereum/beacon/node/ConfigUtils.java | 22 ++++- 3 files changed, 105 insertions(+), 14 deletions(-) diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulateUtils.java b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulateUtils.java index 8619f21b5..d875290e5 100644 --- a/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulateUtils.java +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulateUtils.java @@ -9,13 +9,12 @@ import org.ethereum.beacon.core.types.Gwei; import org.ethereum.beacon.crypto.BLS381; import org.ethereum.beacon.crypto.BLS381.PrivateKey; +import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.crypto.MessageParameters; import org.javatuples.Pair; +import org.jetbrains.annotations.NotNull; import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes4; -import tech.pegasys.artemis.util.bytes.Bytes48; -import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.*; import tech.pegasys.artemis.util.uint.UInt64; import java.util.ArrayList; @@ -53,7 +52,20 @@ public static Deposit getDepositForKeyPair(Random rnd, public static synchronized Deposit getDepositForKeyPair(Random rnd, BLS381.KeyPair keyPair, BeaconChainSpec spec, Gwei initBalance, boolean isProofVerifyEnabled) { - Hash32 withdrawalCredentials = Hash32.random(rnd); + Hash32 credentials = Hash32.random(rnd); + List depositProof = Collections.singletonList(Hash32.random(rnd)); + return getDepositForKeyPair( + keyPair, credentials, initBalance, depositProof, spec, isProofVerifyEnabled); + } + + @NotNull + private static Deposit getDepositForKeyPair( + BLS381.KeyPair keyPair, + Hash32 withdrawalCredentials, + Gwei initBalance, + List depositProof, + BeaconChainSpec spec, + boolean isProofVerifyEnabled) { DepositData depositDataWithoutSignature = new DepositData( BLSPubkey.wrap(Bytes48.leftPad(keyPair.getPublic().getEncodedBytes())), @@ -73,11 +85,11 @@ public static synchronized Deposit getDepositForKeyPair(Random rnd, Deposit deposit = Deposit.create( - Collections.singletonList(Hash32.random(rnd)), + depositProof, new DepositData( - BLSPubkey.wrap(Bytes48.leftPad(keyPair.getPublic().getEncodedBytes())), - withdrawalCredentials, - spec.getConstants().getMaxEffectiveBalance(), + depositDataWithoutSignature.getPubKey(), + depositDataWithoutSignature.getWithdrawalCredentials(), + depositDataWithoutSignature.getAmount(), signature)); return deposit; } @@ -104,16 +116,70 @@ public static synchronized List getDepositsForKeyPairs( BeaconChainSpec spec, Gwei initBalance, boolean isProofVerifyEnabled) { + List withdrawalCredentials = generateRandomWithdrawalCredentials(rnd, keyPairs); + List> depositProofs = generateRandomDepositProofs(rnd, keyPairs); + return getDepositsForKeyPairs( + keyPairs, withdrawalCredentials, initBalance, depositProofs, spec, isProofVerifyEnabled); + } + @NotNull + public static List getDepositsForKeyPairs( + List keyPairs, + List withdrawalCredentials, + Gwei initBalance, + List> depositProofs, + BeaconChainSpec spec, + boolean isProofVerifyEnabled) { List deposits = new ArrayList<>(); - for (BLS381.KeyPair keyPair : keyPairs) { - deposits.add(getDepositForKeyPair(rnd, keyPair, spec, initBalance, isProofVerifyEnabled)); + for (int i = 0; i < keyPairs.size(); i++) { + BLS381.KeyPair keyPair = keyPairs.get(i); + Hash32 credentials = withdrawalCredentials.get(i); + List depositProof = depositProofs.get(i); + deposits.add(getDepositForKeyPair( + keyPair, credentials, initBalance, depositProof, spec, isProofVerifyEnabled)); } return deposits; } + @NotNull + public static List> generateRandomDepositProofs( + Random rnd, List keyPairs) { + List> depositProofs = new ArrayList<>(); + for (BLS381.KeyPair keyPair : keyPairs) { + List depositProof = Collections.singletonList(Hash32.random(rnd)); + depositProofs.add(depositProof); + } + return depositProofs; + } + + @NotNull + public static List generateRandomWithdrawalCredentials( + Random rnd, List keyPairs) { + List withdrawalCredentials = new ArrayList<>(); + for (BLS381.KeyPair keyPair : keyPairs) { + withdrawalCredentials.add(Hash32.random(rnd)); + } + return withdrawalCredentials; + } + + /** + * Generate withdrawal credentials according to mocked start spec. + * @See + */ + public static List generateInteropCredentials( + UInt64 blsWithdrawalPrefix, List keyPairs) { + List withdrawalCredentials = new ArrayList<>(); + for (BLS381.KeyPair keyPair : keyPairs) { + MutableBytes32 credentials = + Hashes.sha256(keyPair.getPublic().getEncodedBytes()).mutableCopy(); + credentials.set(0, blsWithdrawalPrefix.toBytes8().get(0)); + withdrawalCredentials.add(Hash32.wrap(credentials.copy())); + } + return withdrawalCredentials; + } + private static synchronized Pair, List> generateRandomDeposits( UInt64 startIndex, Random rnd, BeaconChainSpec spec, int count, boolean isProofVerifyEnabled) { List validatorsKeys = new ArrayList<>(); diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java index ea47f54b0..e4463e340 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java @@ -16,6 +16,7 @@ public class EmulatorContract extends Contract { private Integer balance; private List keys; private String eth1BlockHash = createInteropEth1BlockHash(); + private boolean interopCredentials = true; /** * @return HEX String representation of a hash32 consisting of 32 0x42 bytes, which is the interop @@ -59,4 +60,12 @@ public String getEth1BlockHash() { public void setEth1BlockHash(String eth1BlockHash) { this.eth1BlockHash = eth1BlockHash; } + + public boolean isInteropCredentials() { + return interopCredentials; + } + + public void setInteropCredentials(boolean interopCredentials) { + this.interopCredentials = interopCredentials; + } } diff --git a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java index ec1381be2..657f446f2 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java @@ -86,9 +86,25 @@ public static DepositContract createDepositContract(Contract config, BeaconChain eConfig.getBalance() != null ? Gwei.ofEthers(eConfig.getBalance()) : spec.getConstants().getMaxEffectiveBalance(); - List deposits = - SimulateUtils.getDepositsForKeyPairs( - UInt64.ZERO, random, keyPairs, spec, amount, verifyProof); + + List deposits; + if (eConfig.isInteropCredentials()) { + // depostis with mocked start credentials for interop + List withdrawalCredentials = + SimulateUtils.generateInteropCredentials( + spec.getConstants().getBlsWithdrawalPrefix(), keyPairs); + // keep random proofs for now + List> depositProofs = + SimulateUtils.generateRandomDepositProofs(random, keyPairs); + deposits = + SimulateUtils.getDepositsForKeyPairs( + keyPairs, withdrawalCredentials, amount, depositProofs, spec, verifyProof); + } else { + deposits = + SimulateUtils.getDepositsForKeyPairs( + UInt64.ZERO, random, keyPairs, spec, amount, verifyProof); + } + ReadList depositDataList = ReadList.wrap( deposits.stream().map(Deposit::getData).collect(Collectors.toList()), From 11b7521a711174345bf091cc7fd5db62f3da9d4d Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 29 Aug 2019 17:08:21 +0300 Subject: [PATCH 29/59] Added a command line option to specify yaml spec constants file (simple, non-hierarchical format). --- .../chainspec}/SpecConstantsDataMerged.java | 5 ++- .../java/org/ethereum/beacon/node/Node.java | 11 +++++ .../beacon/node/NodeCommandLauncher.java | 40 +++++++++++++++++++ .../org/ethereum/beacon/test/TestUtils.java | 2 +- 4 files changed, 55 insertions(+), 3 deletions(-) rename {test/src/test/java/org/ethereum/beacon/test/type => start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec}/SpecConstantsDataMerged.java (95%) diff --git a/test/src/test/java/org/ethereum/beacon/test/type/SpecConstantsDataMerged.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecConstantsDataMerged.java similarity index 95% rename from test/src/test/java/org/ethereum/beacon/test/type/SpecConstantsDataMerged.java rename to start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecConstantsDataMerged.java index 686cd332e..c47ef7407 100644 --- a/test/src/test/java/org/ethereum/beacon/test/type/SpecConstantsDataMerged.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecConstantsDataMerged.java @@ -1,7 +1,8 @@ -package org.ethereum.beacon.test.type; +package org.ethereum.beacon.emulator.config.chainspec; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonUnwrapped; +import org.ethereum.beacon.emulator.config.Config; import org.ethereum.beacon.emulator.config.chainspec.DepositContractParametersData; import org.ethereum.beacon.emulator.config.chainspec.GweiValuesData; import org.ethereum.beacon.emulator.config.chainspec.HonestValidatorParametersData; @@ -14,7 +15,7 @@ import org.ethereum.beacon.emulator.config.chainspec.TimeParametersData; @JsonIgnoreProperties(ignoreUnknown = false) -public class SpecConstantsDataMerged extends SpecConstantsData { +public class SpecConstantsDataMerged extends SpecConstantsData implements Config { @JsonUnwrapped private DepositContractParametersData depositContractParameters; @JsonUnwrapped private HonestValidatorParametersData honestValidatorParameters; @JsonUnwrapped private InitialValuesData initialValues; diff --git a/start/node/src/main/java/org/ethereum/beacon/node/Node.java b/start/node/src/main/java/org/ethereum/beacon/node/Node.java index 7ef7a62ef..5ba769354 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/Node.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/Node.java @@ -91,6 +91,13 @@ public class Node implements Runnable { ) private String genesisTime; + @CommandLine.Option( + names = "--spec-constants", + paramLabel = "spec-constants", + description = "Path to a spec constants file in yaml format (flat format)" + ) + private String specConstantsFile; + public String getName() { return name; } @@ -111,6 +118,10 @@ public String getGenesisTime() { return genesisTime; } + public String getSpecConstantsFile() { + return specConstantsFile; + } + public static void main(String[] args) { try { CommandLine commandLine = new CommandLine(new Node()); diff --git a/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java b/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java index 9430172f7..8eabcc51b 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java @@ -32,6 +32,8 @@ import org.ethereum.beacon.emulator.config.ConfigBuilder; import org.ethereum.beacon.emulator.config.ConfigException; import org.ethereum.beacon.emulator.config.chainspec.SpecBuilder; +import org.ethereum.beacon.emulator.config.chainspec.SpecConstantsData; +import org.ethereum.beacon.emulator.config.chainspec.SpecConstantsDataMerged; import org.ethereum.beacon.emulator.config.chainspec.SpecData; import org.ethereum.beacon.emulator.config.main.MainConfig; import org.ethereum.beacon.emulator.config.main.Signer.Insecure; @@ -45,10 +47,15 @@ import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.start.common.NodeLauncher; import org.ethereum.beacon.start.common.util.MDCControlledSchedulers; +import org.ethereum.beacon.util.Objects; import org.ethereum.beacon.validator.crypto.BLS381Credentials; import org.ethereum.beacon.wire.net.ConnectionManager; import org.ethereum.beacon.wire.net.netty.NettyClient; import org.ethereum.beacon.wire.net.netty.NettyServer; +import org.jetbrains.annotations.NotNull; + +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Paths; public class NodeCommandLauncher implements Runnable { private static final Logger logger = LogManager.getLogger("node"); @@ -206,6 +213,18 @@ public NodeCommandLauncher build() { ConfigBuilder specConfigBuilder = new ConfigBuilder<>(SpecData.class).addYamlConfigFromResources("/config/spec-constants.yml"); + + if (cliOptions != null + && cliOptions.getSpecConstantsFile() != null + && cliOptions.getSpecConstantsFile().length() > 0) { + SpecData chainSpec = config.getChainSpec(); + SpecConstantsDataMerged specConstantsYaml = + loadSpecConstantsDataMerged(cliOptions.getSpecConstantsFile()); + SpecConstantsData mergedConstants = + mergeSpecConstantsData(chainSpec.getSpecConstants(), specConstantsYaml); + chainSpec.setSpecConstants(mergedConstants); + config.setChainSpec(chainSpec); + } if (config.getChainSpec().isDefined()) { specConfigBuilder.addConfig(config.getChainSpec()); } @@ -333,6 +352,27 @@ public NodeCommandLauncher build() { logLevel); } + @NotNull + private static SpecConstantsData mergeSpecConstantsData(SpecConstantsData specConsts, SpecConstantsDataMerged specConstsYaml) { + if (specConsts == null) { + return specConstsYaml; + } else { + try { + return Objects.copyProperties(specConsts, specConstsYaml); + } catch (IllegalAccessException| InvocationTargetException e) { + throw new RuntimeException( + String.format("Failed to merge config %s into main config", specConsts), e); + } + } + } + + private static SpecConstantsDataMerged loadSpecConstantsDataMerged(String specConstants) { + ConfigBuilder specConstsBuilder = + new ConfigBuilder<>(SpecConstantsDataMerged.class); + specConstsBuilder.addYamlConfig(Paths.get(specConstants).toFile()); + return specConstsBuilder.build(); + } + public Builder withConfigFromFile(File file) { this.config = new ConfigBuilder<>(MainConfig.class).addYamlConfig(file).build(); return this; diff --git a/test/src/test/java/org/ethereum/beacon/test/TestUtils.java b/test/src/test/java/org/ethereum/beacon/test/TestUtils.java index 4d789c4e4..62c5fc733 100644 --- a/test/src/test/java/org/ethereum/beacon/test/TestUtils.java +++ b/test/src/test/java/org/ethereum/beacon/test/TestUtils.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.core.spec.SpecConstantsResolver; import org.ethereum.beacon.emulator.config.chainspec.SpecBuilder; import org.ethereum.beacon.emulator.config.chainspec.SpecConstantsData; +import org.ethereum.beacon.emulator.config.chainspec.SpecConstantsDataMerged; import org.ethereum.beacon.emulator.config.chainspec.SpecData; import org.ethereum.beacon.emulator.config.chainspec.SpecDataUtils; import org.ethereum.beacon.emulator.config.chainspec.SpecHelpersData; @@ -17,7 +18,6 @@ import org.ethereum.beacon.ssz.SSZBuilder; import org.ethereum.beacon.ssz.SSZSerializer; import org.ethereum.beacon.test.type.DataMapperTestCase; -import org.ethereum.beacon.test.type.SpecConstantsDataMerged; import org.ethereum.beacon.test.type.TestCase; import org.ethereum.beacon.test.type.ssz.SszGenericCase; import org.ethereum.beacon.test.type.ssz.SszStaticCase; From bfabbea007261d7a332832ee24e0abacad5fff66 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 29 Aug 2019 17:17:07 +0300 Subject: [PATCH 30/59] Modified default config to match interop settings. --- .../src/main/resources/config/default-node-config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/start/node/src/main/resources/config/default-node-config.yml b/start/node/src/main/resources/config/default-node-config.yml index fb3625c0c..e021dfc13 100644 --- a/start/node/src/main/resources/config/default-node-config.yml +++ b/start/node/src/main/resources/config/default-node-config.yml @@ -7,9 +7,11 @@ config: contract: !emulator balance: 55 keys: - - !generate + - !interop count: 16 - seed: 0 + # seed: 0 + interopCredentials: true + # eth1BlockHash: 0x4242424242424242424242424242424242424242424242424242424242424242 # signer: !insecure # keys: # - !generate @@ -40,4 +42,6 @@ chainSpec: blsVerifyProofOfPossession: false blsSign: false enableCache: false + # for interop, genesis time should be overridden with the specified value, so ignore computation + computableGenesisTime: true From 6c2e407469572635558216eb2caada45837795ed Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 29 Aug 2019 17:42:39 +0300 Subject: [PATCH 31/59] For interop, MAX_EFFECTIVE_BALANCE should be used as a balance value. Will be set so by default, if missing. --- start/node/src/main/resources/config/default-node-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start/node/src/main/resources/config/default-node-config.yml b/start/node/src/main/resources/config/default-node-config.yml index e021dfc13..dad4809a9 100644 --- a/start/node/src/main/resources/config/default-node-config.yml +++ b/start/node/src/main/resources/config/default-node-config.yml @@ -5,7 +5,7 @@ config: validator: contract: !emulator - balance: 55 + # balance: 55 keys: - !interop count: 16 From 9a8762026d84aee7c7b10156a92644946622902e Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 30 Aug 2019 20:55:11 +0600 Subject: [PATCH 32/59] Use BCParameters.ORDER to initialize SimulationKeyPairGenerator#CURVE_ORDER --- .../beacon/start/common/util/SimulationKeyPairGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java index d3b19d549..ee9114e2b 100644 --- a/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulationKeyPairGenerator.java @@ -3,6 +3,7 @@ import com.google.common.primitives.Bytes; import org.ethereum.beacon.crypto.BLS381; import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.crypto.bls.bc.BCParameters; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -15,7 +16,7 @@ * Key pair generation utilities for simulation purposes. */ public class SimulationKeyPairGenerator { - private static BigInteger CURVE_ORDER = new BigInteger("52435875175126190479447740508185965837690552500527637822603658699938581184513"); + private static BigInteger CURVE_ORDER = BCParameters.ORDER; /** * Generate public/private key pairs according to mocked start spec. From b84de20e592cf910254fc5407b2a20285461c84e Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Thu, 5 Sep 2019 12:30:00 +0300 Subject: [PATCH 33/59] test: draft commit --- .../beacon/chain/pool/InMemoryAttestationPoolTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index e533b50cc..4cc45fd04 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -30,7 +30,7 @@ class InMemoryAttestationPoolTest { - private Schedulers schedulers = Schedulers.createDefault(); + private Schedulers schedulers = Schedulers.createControlled(); private SpecConstants specConstants = new SpecConstants() { @@ -207,4 +207,9 @@ public AttestationData createAttestationData() { return expected; } + + @Test + void testValidAttestation() { + + } } From e1f2b1e205d188ed83e32897b941c397bf97e7d2 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Thu, 5 Sep 2019 13:25:50 +0300 Subject: [PATCH 34/59] test: fix check TimeFrameFilter --- .../chain/pool/checker/TimeFrameFilter.java | 8 +- .../pool/InMemoryAttestationPoolTest.java | 84 ++++++++++++++++++- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java index 01da5887a..dfac57df7 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java @@ -1,12 +1,10 @@ package org.ethereum.beacon.chain.pool.checker; -import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.chain.pool.*; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.types.EpochNumber; -import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.*; /** * Filters attestations by time frame of its target and source. @@ -62,7 +60,7 @@ public boolean check(ReceivedAttestation attestation) { return false; } - return false; + return true; } /** diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 4cc45fd04..dcd40e37c 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -1,11 +1,15 @@ package org.ethereum.beacon.chain.pool; import org.ethereum.beacon.chain.*; +import org.ethereum.beacon.chain.pool.checker.*; +import org.ethereum.beacon.chain.pool.registry.*; +import org.ethereum.beacon.chain.pool.verifier.*; import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.impl.*; import org.ethereum.beacon.consensus.*; import org.ethereum.beacon.consensus.transition.*; import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; +import org.ethereum.beacon.consensus.verifier.VerificationResult; import org.ethereum.beacon.consensus.verifier.*; import org.ethereum.beacon.core.*; import org.ethereum.beacon.core.operations.Attestation; @@ -28,6 +32,8 @@ import java.util.Collections; +import static org.ethereum.beacon.chain.pool.AttestationPool.*; + class InMemoryAttestationPoolTest { private Schedulers schedulers = Schedulers.createControlled(); @@ -133,6 +139,80 @@ void integrationTest() { pool.start(); } + @Test + void integrationTest2() { + final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); + beaconChain.init(); + final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); + final BeaconBlock aBlock = createBlock(recentlyProcessed, spec, + schedulers.getCurrentTime(), perSlotTransition); + + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + final Publisher source = Flux.just(attestation); + StepVerifier.create(source) + .expectNext(attestation) + .expectComplete() + .verify(); + + final SlotNumber slotNumber = SlotNumber.of(100L); + final Publisher newSlots = Flux.just(slotNumber); + StepVerifier.create(newSlots) + .expectNext(slotNumber) + .expectComplete() + .verify(); + + final Checkpoint checkpoint = Checkpoint.EMPTY; + final Publisher finalizedCheckpoints = Flux.just(checkpoint); + StepVerifier.create(finalizedCheckpoints) + .expectNext(checkpoint) + .expectComplete() + .verify(); + + final Publisher importedBlocks = Flux.just(aBlock); + StepVerifier.create(importedBlocks) + .expectNext(aBlock) + .expectComplete() + .verify(); + + final Publisher chainHeads = Flux.just(aBlock); + StepVerifier.create(chainHeads) + .expectNext(aBlock) + .expectComplete() + .verify(); + + final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); + timeFrameFilter.feedFinalizedCheckpoint(checkpoint); + timeFrameFilter.feedNewSlot(slotNumber); + + final SanityChecker sanityChecker = new SanityChecker(spec); + final SignatureEncodingChecker encodingChecker = new SignatureEncodingChecker(); + final ProcessedAttestations processedFilter = + new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); + final UnknownAttestationPool unknownAttestationPool = + new UnknownAttestationPool( + beaconChainStorage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); + final BatchVerifier batchVerifier = + new AttestationVerifier(beaconChainStorage.getTupleStorage(), spec, slotTransition); + + final AttestationPool pool = new InMemoryAttestationPool( + source, + newSlots, + finalizedCheckpoints, + importedBlocks, + chainHeads, + schedulers, + timeFrameFilter, + sanityChecker, + encodingChecker, + processedFilter, + unknownAttestationPool, + batchVerifier); + + pool.start(); + } + private MutableBeaconChain createBeaconChain( BeaconChainSpec spec, StateTransition perSlotTransition, Schedulers schedulers) { Time start = Time.castFrom(UInt64.valueOf(schedulers.getCurrentTime() / 1000)); @@ -201,8 +281,8 @@ public AttestationData createAttestationData() { final AttestationData expected = new AttestationData( Hashes.sha256(BytesValue.fromHexString("aa")), - new Checkpoint(EpochNumber.ZERO, Hashes.sha256(BytesValue.fromHexString("bb"))), - new Checkpoint(EpochNumber.of(123), Hashes.sha256(BytesValue.fromHexString("cc"))), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(1), Hashes.sha256(BytesValue.fromHexString("cc"))), Crosslink.EMPTY); return expected; From ba01970553a1e3168dc6dce05dca4021a1e11b0d Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 5 Sep 2019 17:52:31 +0600 Subject: [PATCH 35/59] Implement simple suboptimal attestation churn --- .../chain/pool/AttestationAggregate.java | 76 +++++++++ .../beacon/chain/pool/AttestationPool.java | 15 +- .../beacon/chain/pool/CheckedAttestation.java | 20 --- .../chain/pool/InMemoryAttestationPool.java | 27 +++- .../beacon/chain/pool/OffChainAggregates.java | 35 ++++ .../chain/pool/churn/AttestationChurn.java | 24 +++ .../pool/churn/AttestationChurnImpl.java | 150 ++++++++++++++++++ .../beacon/chain/pool/churn/ChurnQueue.java | 95 +++++++++++ .../chain/pool/reactor/ChurnProcessor.java | 36 ++++- .../pool/InMemoryAttestationPoolTest.java | 106 +++++++++---- 10 files changed, 520 insertions(+), 64 deletions(-) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java new file mode 100644 index 000000000..4b51e6a86 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java @@ -0,0 +1,76 @@ +package org.ethereum.beacon.chain.pool; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.crypto.BLS381; +import tech.pegasys.artemis.util.collections.Bitlist; + +public class AttestationAggregate { + + public static AttestationAggregate create(Attestation attestation) { + List signatures = new ArrayList<>(); + signatures.add(attestation.getSignature()); + return new AttestationAggregate( + attestation.getAggregationBits(), + attestation.getCustodyBits(), + attestation.getData(), + signatures); + } + + private final Bitlist aggregationBits; + private final AttestationData data; + private final Bitlist custodyBits; + private final List signatures; + + public AttestationAggregate( + Bitlist aggregationBits, + Bitlist custodyBits, + AttestationData data, + List signatures) { + this.aggregationBits = aggregationBits; + this.custodyBits = custodyBits; + this.data = data; + this.signatures = signatures; + } + + public boolean add(Attestation attestation) { + if (isAggregatable(attestation)) { + aggregationBits.or(attestation.getAggregationBits()); + custodyBits.or(attestation.getCustodyBits()); + signatures.add(attestation.getSignature()); + + return true; + } else { + return false; + } + } + + private boolean isAggregatable(Attestation attestation) { + if (!data.equals(attestation.getData())) { + return false; + } + + if (!aggregationBits.and(attestation.getAggregationBits()).isEmpty()) { + return false; + } + + return true; + } + + public Attestation getAggregate(SpecConstants specConstants) { + BLSSignature signature = + BLSSignature.wrap( + BLS381.Signature.aggregate( + signatures.stream() + .map(BLS381.Signature::createWithoutValidation) + .collect(Collectors.toList())) + .getEncoded()); + + return new Attestation(aggregationBits, data, custodyBits, signature, specConstants); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java index f5581953f..b50888ba1 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -2,10 +2,12 @@ import java.time.Duration; import org.ethereum.beacon.chain.BeaconChain; +import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; -import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.chain.pool.churn.AttestationChurnImpl; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; @@ -61,6 +63,9 @@ public interface AttestationPool { /** A throttling interval for verifier buffer. */ Duration VERIFIER_INTERVAL = Duration.ofMillis(50); + /** Max number of attestations held by {@link AttestationChurn}. */ + int ATTESTATION_CHURN_SIZE = 10_000; + /** * Valid attestations publisher. * @@ -99,9 +104,10 @@ public interface AttestationPool { static AttestationPool create( Publisher source, Publisher newSlots, + Publisher justifiedCheckpoints, Publisher finalizedCheckpoints, Publisher importedBlocks, - Publisher chainHeads, + Publisher chainHeads, Schedulers schedulers, BeaconChainSpec spec, BeaconChainStorage storage, @@ -117,10 +123,12 @@ static AttestationPool create( storage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); BatchVerifier batchVerifier = new AttestationVerifier(storage.getTupleStorage(), spec, emptySlotTransition); + AttestationChurn attestationChurn = new AttestationChurnImpl(spec, ATTESTATION_CHURN_SIZE); return new InMemoryAttestationPool( source, newSlots, + justifiedCheckpoints, finalizedCheckpoints, importedBlocks, chainHeads, @@ -130,6 +138,7 @@ static AttestationPool create( encodingChecker, processedFilter, unknownAttestationPool, - batchVerifier); + batchVerifier, + attestationChurn); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java deleted file mode 100644 index e3c4fbf76..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/CheckedAttestation.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -/** A simple DTO that carries an attestation and a result of some check run against it. */ -public class CheckedAttestation { - private final boolean passed; - private final ReceivedAttestation attestation; - - public CheckedAttestation(boolean passed, ReceivedAttestation attestation) { - this.passed = passed; - this.attestation = attestation; - } - - public boolean isPassed() { - return passed; - } - - public ReceivedAttestation getAttestation() { - return attestation; - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 9f94dbd70..0b261bee6 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -1,10 +1,11 @@ package org.ethereum.beacon.chain.pool; import java.util.List; +import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; -import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; import org.ethereum.beacon.chain.pool.reactor.ChurnProcessor; import org.ethereum.beacon.chain.pool.reactor.DoubleWorkProcessor; import org.ethereum.beacon.chain.pool.reactor.IdentificationProcessor; @@ -32,9 +33,10 @@ public class InMemoryAttestationPool implements AttestationPool { private final Publisher source; private final Publisher newSlots; + private final Publisher justifiedCheckpoints; private final Publisher finalizedCheckpoints; private final Publisher importedBlocks; - private final Publisher chainHeads; + private final Publisher chainHeads; private final Schedulers schedulers; private final TimeFrameFilter timeFrameFilter; @@ -43,6 +45,7 @@ public class InMemoryAttestationPool implements AttestationPool { private final ProcessedAttestations processedFilter; private final UnknownAttestationPool unknownPool; private final BatchVerifier verifier; + private final AttestationChurn attestationChurn; private final DirectProcessor invalidAttestations = DirectProcessor.create(); private final DirectProcessor validAttestations = DirectProcessor.create(); @@ -52,18 +55,21 @@ public class InMemoryAttestationPool implements AttestationPool { public InMemoryAttestationPool( Publisher source, Publisher newSlots, + Publisher justifiedCheckpoints, Publisher finalizedCheckpoints, Publisher importedBlocks, - Publisher chainHeads, + Publisher chainHeads, Schedulers schedulers, TimeFrameFilter timeFrameFilter, SanityChecker sanityChecker, SignatureEncodingChecker encodingChecker, ProcessedAttestations processedFilter, UnknownAttestationPool unknownPool, - BatchVerifier batchVerifier) { + BatchVerifier batchVerifier, + AttestationChurn attestationChurn) { this.source = source; this.newSlots = newSlots; + this.justifiedCheckpoints = justifiedCheckpoints; this.finalizedCheckpoints = finalizedCheckpoints; this.importedBlocks = importedBlocks; this.chainHeads = chainHeads; @@ -74,6 +80,7 @@ public InMemoryAttestationPool( this.processedFilter = processedFilter; this.unknownPool = unknownPool; this.verifier = batchVerifier; + this.attestationChurn = attestationChurn; } @Override @@ -86,9 +93,10 @@ public void start() { // create sources Flux sourceFx = Flux.from(source); Flux newSlotsFx = Flux.from(newSlots); + Flux justifiedCheckpointsFx = Flux.from(justifiedCheckpoints); Flux finalizedCheckpointsFx = Flux.from(finalizedCheckpoints); Flux importedBlocksFx = Flux.from(importedBlocks); - Flux chainHeadsFx = Flux.from(chainHeads); + Flux chainHeadsFx = Flux.from(chainHeads); // check time frames TimeProcessor timeProcessor = @@ -123,7 +131,14 @@ public void start() { new VerificationProcessor(verifier, verificationThrottle); // feed churn - ChurnProcessor churnProcessor = new ChurnProcessor(schedulers, chainHeadsFx, newSlotsFx); + ChurnProcessor churnProcessor = + new ChurnProcessor( + attestationChurn, + schedulers, + chainHeadsFx, + justifiedCheckpointsFx, + finalizedCheckpointsFx, + verificationProcessor.getValid()); Scheduler outScheduler = schedulers.events().toReactor(); // expose valid attestations diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java new file mode 100644 index 000000000..4676d6c9e --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java @@ -0,0 +1,35 @@ +package org.ethereum.beacon.chain.pool; + +import java.util.List; +import org.ethereum.beacon.core.types.SlotNumber; +import tech.pegasys.artemis.ethereum.core.Hash32; + +/** + * A DTO for aggregated attestations that are not yet included on chain. + * + *

    Beacon block proposer should be fed with this data. + */ +public class OffChainAggregates { + private final Hash32 blockRoot; + private final SlotNumber slot; + private final List aggregates; + + public OffChainAggregates( + Hash32 blockRoot, SlotNumber slot, List aggregates) { + this.blockRoot = blockRoot; + this.slot = slot; + this.aggregates = aggregates; + } + + public Hash32 getBlockRoot() { + return blockRoot; + } + + public List getAggregates() { + return aggregates; + } + + public SlotNumber getEpoch() { + return slot; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java new file mode 100644 index 000000000..7dd590859 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java @@ -0,0 +1,24 @@ +package org.ethereum.beacon.chain.pool.churn; + +import java.util.List; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.chain.pool.StatefulProcessor; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; + +public interface AttestationChurn extends StatefulProcessor { + + OffChainAggregates compute(BeaconTuple tuple); + + void add(List attestation); + + void feedFinalizedCheckpoint(Checkpoint checkpoint); + + void feedJustifiedCheckpoint(Checkpoint checkpoint); + + static AttestationChurn create(BeaconChainSpec spec, long size) { + return new AttestationChurnImpl(spec, size); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java new file mode 100644 index 000000000..8ac2876d4 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java @@ -0,0 +1,150 @@ +package org.ethereum.beacon.chain.pool.churn; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.AttestationAggregate; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import tech.pegasys.artemis.util.collections.Bitlist; +import tech.pegasys.artemis.util.uint.UInt64s; + +public class AttestationChurnImpl implements AttestationChurn { + + /** A queue that maintains pooled attestations. */ + private final ChurnQueue queue; + /** A beacon chain spec. */ + private final BeaconChainSpec spec; + + private Checkpoint justifiedCheckpoint; + private EpochNumber lowerEpoch; + private EpochNumber upperEpoch; + + public AttestationChurnImpl(BeaconChainSpec spec, long size) { + this.spec = spec; + this.queue = new ChurnQueue(size); + } + + @Override + public OffChainAggregates compute(BeaconTuple tuple) { + assert isInitialized(); + + // update epoch boundaries + updateEpochBoundaries( + spec.get_previous_epoch(tuple.getState()), spec.get_current_epoch(tuple.getState())); + + // compute coverage + Map coverage = computeCoverage(tuple.getState()); + + // filter attestations by coverage + List offChainAttestations = + queue.stream() + .filter( + attestation -> { + Bitlist bits = coverage.get(attestation.getData()); + if (bits == null) { + return true; + } + + return bits.and(attestation.getAggregationBits()).isEmpty(); + }) + .collect(Collectors.toList()); + + // compute aggregates + List aggregates = computeAggregates(offChainAttestations); + + return new OffChainAggregates( + spec.signing_root(tuple.getBlock()), tuple.getState().getSlot(), aggregates); + } + + @Override + public void feedFinalizedCheckpoint(Checkpoint checkpoint) { + // finalized checkpoint takes precedence + updateJustifiedCheckpoint(checkpoint); + } + + @Override + public void feedJustifiedCheckpoint(Checkpoint checkpoint) { + // discard forks if justified checkpoint is updated + if (checkpoint.getEpoch().greater(justifiedCheckpoint.getEpoch())) { + updateJustifiedCheckpoint(checkpoint); + } + } + + private void updateJustifiedCheckpoint(Checkpoint checkpoint) { + updateEpochBoundaries(checkpoint.getEpoch(), UInt64s.max(checkpoint.getEpoch(), upperEpoch)); + this.justifiedCheckpoint = checkpoint; + } + + private void updateEpochBoundaries(EpochNumber lowerEpoch, EpochNumber upperEpoch) { + assert lowerEpoch.lessEqual(upperEpoch); + + // return if there is nothing to update + // newLower <= lower || newUpper <= upper + if (lowerEpoch.lessEqual(this.lowerEpoch) || upperEpoch.lessEqual(this.upperEpoch)) { + return; + } + + queue.updateEpochBoundaries(lowerEpoch, upperEpoch); + + this.lowerEpoch = lowerEpoch; + this.upperEpoch = upperEpoch; + } + + @Override + public boolean isInitialized() { + return true; + } + + @Override + public void add(List attestations) { + queue.add(attestations); + } + + private Map computeCoverage(BeaconState state) { + Map coverage = new HashMap<>(); + + Stream.concat( + state.getPreviousEpochAttestations().stream(), + state.getCurrentEpochAttestations().stream()) + .forEach( + pending -> { + Bitlist bitlist = coverage.get(pending.getData()); + if (bitlist == null) { + coverage.put(pending.getData(), pending.getAggregationBits()); + } else { + coverage.put(pending.getData(), bitlist.or(pending.getAggregationBits())); + } + }); + + return coverage; + } + + private List computeAggregates(List attestations) { + if (attestations.isEmpty()) { + return Collections.emptyList(); + } + + List aggregates = new ArrayList<>(); + AttestationAggregate current = null; + + for (Attestation attestation : attestations) { + if (current == null || !current.add(attestation)) { + current = AttestationAggregate.create(attestation); + aggregates.add(current); + } + } + + return aggregates; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java new file mode 100644 index 000000000..fa5e76c10 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java @@ -0,0 +1,95 @@ +package org.ethereum.beacon.chain.pool.churn; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.types.EpochNumber; + +final class ChurnQueue { + + private final long maxSize; + private final LinkedList buckets = new LinkedList<>(); + + private EpochNumber lowerEpoch; + private EpochNumber upperEpoch; + private long size = 0; + + public ChurnQueue(long maxSize) { + this.maxSize = maxSize; + } + + Stream stream() { + return buckets.stream().map(Bucket::getAttestations).flatMap(List::stream); + } + + void add(List attestation) { + attestation.forEach(this::addImpl); + // heavy operation, don't wanna run it after each added attestation + purgeQueue(); + } + + private void addImpl(Attestation attestation) { + EpochNumber epoch = attestation.getData().getTarget().getEpoch(); + if (epoch.greaterEqual(lowerEpoch) && epoch.lessEqual(upperEpoch)) { + Bucket bucket = getOrCreateBucket(epoch); + bucket.attestations.add(attestation); + + size += 1; + } + } + + void updateEpochBoundaries(EpochNumber lower, EpochNumber upper) { + assert lower.lessEqual(upper); + + while (buckets.size() > 0 && buckets.getFirst().epoch.less(lower)) { + buckets.removeFirst(); + } + + while (buckets.size() > 0 && buckets.getLast().epoch.greater(upper)) { + buckets.removeLast(); + } + + this.lowerEpoch = lower; + this.upperEpoch = upper; + } + + Bucket getOrCreateBucket(EpochNumber epoch) { + for (Bucket bucket : buckets) { + if (bucket.epoch.equals(epoch)) { + return bucket; + } + } + + Bucket newBucket = new Bucket(epoch); + buckets.add(new Bucket(epoch)); + buckets.sort(Comparator.comparing(Bucket::getEpoch)); + + return newBucket; + } + + void purgeQueue() { + if (maxSize > size) { + // TODO calculate weights and sieve the attestations + } + } + + private static final class Bucket { + private final EpochNumber epoch; + private final List attestations = new ArrayList<>(); + + Bucket(EpochNumber epoch) { + this.epoch = epoch; + } + + EpochNumber getEpoch() { + return epoch; + } + + List getAttestations() { + return attestations; + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java index cefb36bb5..95cebdf20 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java @@ -1,23 +1,47 @@ package org.ethereum.beacon.chain.pool.reactor; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.OffChainAggregates; import org.ethereum.beacon.chain.pool.ReceivedAttestation; -import org.ethereum.beacon.chain.pool.churn.OffChainAggregates; -import org.ethereum.beacon.core.BeaconBlock; -import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.OutsourcePublisher; import reactor.core.CoreSubscriber; import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; public class ChurnProcessor extends Flux { + private final AttestationChurn churn; private final OutsourcePublisher out = new OutsourcePublisher<>(); public ChurnProcessor( - Schedulers schedulers, Flux chainHeads, Flux newSlots) {} + AttestationChurn churn, + Schedulers schedulers, + Flux chainHeads, + Flux justifiedCheckpoints, + Flux finalizedCheckpoints, + Flux source) { + this.churn = churn; - protected void hookOnNext(ReceivedAttestation value) { - // TODO implement + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-attestation-churn").toReactor(); + chainHeads.publishOn(scheduler).subscribe(this::hookOnNext); + justifiedCheckpoints.publishOn(scheduler).subscribe(this.churn::feedJustifiedCheckpoint); + finalizedCheckpoints.publishOn(scheduler).subscribe(this.churn::feedFinalizedCheckpoint); + source + .publishOn(scheduler) + .map(ReceivedAttestation::getMessage) + .bufferTimeout(AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL) + .subscribe(this.churn::add); + } + + private void hookOnNext(BeaconTuple tuple) { + if (churn.isInitialized()) { + OffChainAggregates aggregates = churn.compute(tuple); + out.publishOut(aggregates); + } } @Override diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index dcd40e37c..048639af5 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -1,24 +1,57 @@ package org.ethereum.beacon.chain.pool; -import org.ethereum.beacon.chain.*; -import org.ethereum.beacon.chain.pool.checker.*; -import org.ethereum.beacon.chain.pool.registry.*; -import org.ethereum.beacon.chain.pool.verifier.*; +import static org.ethereum.beacon.chain.pool.AttestationPool.ATTESTATION_CHURN_SIZE; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; + +import java.util.Collections; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.DefaultBeaconChain; +import org.ethereum.beacon.chain.MutableBeaconChain; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; +import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; import org.ethereum.beacon.chain.storage.BeaconChainStorage; -import org.ethereum.beacon.chain.storage.impl.*; -import org.ethereum.beacon.consensus.*; -import org.ethereum.beacon.consensus.transition.*; +import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.BeaconStateEx; +import org.ethereum.beacon.consensus.BlockTransition; +import org.ethereum.beacon.consensus.ChainStart; +import org.ethereum.beacon.consensus.StateTransition; +import org.ethereum.beacon.consensus.transition.BeaconStateExImpl; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.consensus.transition.ExtendedSlotTransition; +import org.ethereum.beacon.consensus.transition.InitialStateTransition; +import org.ethereum.beacon.consensus.transition.PerEpochTransition; +import org.ethereum.beacon.consensus.transition.PerSlotTransition; import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; +import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; +import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; import org.ethereum.beacon.consensus.verifier.VerificationResult; -import org.ethereum.beacon.consensus.verifier.*; -import org.ethereum.beacon.core.*; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconBlockBody; +import org.ethereum.beacon.core.BeaconState; import org.ethereum.beacon.core.operations.Attestation; -import org.ethereum.beacon.core.operations.attestation.*; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; import org.ethereum.beacon.core.spec.SpecConstants; -import org.ethereum.beacon.core.state.*; -import org.ethereum.beacon.core.types.*; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.ShardNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.Time; import org.ethereum.beacon.crypto.Hashes; -import org.ethereum.beacon.db.*; +import org.ethereum.beacon.db.Database; +import org.ethereum.beacon.db.InMemoryDatabase; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.types.p2p.NodeId; import org.junit.jupiter.api.Test; @@ -26,14 +59,11 @@ import reactor.core.publisher.Flux; import reactor.test.StepVerifier; import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.bytes.*; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; -import java.util.Collections; - -import static org.ethereum.beacon.chain.pool.AttestationPool.*; - class InMemoryAttestationPoolTest { private Schedulers schedulers = Schedulers.createControlled(); @@ -86,8 +116,9 @@ void integrationTest() { final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); beaconChain.init(); final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); - final BeaconBlock aBlock = createBlock(recentlyProcessed, spec, - schedulers.getCurrentTime(), perSlotTransition); + final BeaconTuple aTuple = createBlock(recentlyProcessed, spec, + schedulers.getCurrentTime(), perSlotTransition); + final BeaconBlock aBlock = aTuple.getBlock(); final NodeId sender = new NodeId(new byte[100]); final Attestation message = createAttestation(BytesValue.fromHexString("aa")); @@ -111,6 +142,11 @@ void integrationTest() { .expectNext(checkpoint) .expectComplete() .verify(); + final Publisher justifiedCheckpoints = Flux.just(checkpoint); + StepVerifier.create(justifiedCheckpoints) + .expectNext(checkpoint) + .expectComplete() + .verify(); final Publisher importedBlocks = Flux.just(aBlock); StepVerifier.create(importedBlocks) @@ -118,15 +154,16 @@ void integrationTest() { .expectComplete() .verify(); - final Publisher chainHeads = Flux.just(aBlock); + final Publisher chainHeads = Flux.just(recentlyProcessed); StepVerifier.create(chainHeads) - .expectNext(aBlock) + .expectNext(recentlyProcessed) .expectComplete() .verify(); final AttestationPool pool = AttestationPool.create( source, newSlots, + justifiedCheckpoints, finalizedCheckpoints, importedBlocks, chainHeads, @@ -144,8 +181,9 @@ void integrationTest2() { final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); beaconChain.init(); final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); - final BeaconBlock aBlock = createBlock(recentlyProcessed, spec, - schedulers.getCurrentTime(), perSlotTransition); + final BeaconTuple aTuple = createBlock(recentlyProcessed, spec, + schedulers.getCurrentTime(), perSlotTransition); + final BeaconBlock aBlock = aTuple.getBlock(); final NodeId sender = new NodeId(new byte[100]); final Attestation message = createAttestation(BytesValue.fromHexString("aa")); @@ -170,15 +208,21 @@ void integrationTest2() { .expectComplete() .verify(); + final Publisher justifiedCheckpoints = Flux.just(checkpoint); + StepVerifier.create(justifiedCheckpoints) + .expectNext(checkpoint) + .expectComplete() + .verify(); + final Publisher importedBlocks = Flux.just(aBlock); StepVerifier.create(importedBlocks) .expectNext(aBlock) .expectComplete() .verify(); - final Publisher chainHeads = Flux.just(aBlock); + final Publisher chainHeads = Flux.just(aTuple); StepVerifier.create(chainHeads) - .expectNext(aBlock) + .expectNext(aTuple) .expectComplete() .verify(); @@ -195,10 +239,12 @@ void integrationTest2() { beaconChainStorage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); final BatchVerifier batchVerifier = new AttestationVerifier(beaconChainStorage.getTupleStorage(), spec, slotTransition); + final AttestationChurn attestationChurn = AttestationChurn.create(spec, ATTESTATION_CHURN_SIZE); final AttestationPool pool = new InMemoryAttestationPool( source, newSlots, + justifiedCheckpoints, finalizedCheckpoints, importedBlocks, chainHeads, @@ -208,7 +254,8 @@ void integrationTest2() { encodingChecker, processedFilter, unknownAttestationPool, - batchVerifier); + batchVerifier, + attestationChurn); pool.start(); } @@ -248,7 +295,7 @@ public BeaconStateEx apply(BeaconStateEx stateEx) { schedulers); } - private BeaconBlock createBlock( + private BeaconTuple createBlock( BeaconTuple parent, BeaconChainSpec spec, long currentTime, StateTransition perSlotTransition) { @@ -261,7 +308,8 @@ private BeaconBlock createBlock( BLSSignature.ZERO); BeaconState state = perSlotTransition.apply(new BeaconStateExImpl(parent.getState())); - return block.withStateRoot(spec.hash_tree_root(state)); + return BeaconTuple.of( + block.withStateRoot(spec.hash_tree_root(state)), new BeaconStateExImpl(state)); } private Attestation createAttestation(BytesValue someValue) { From 2db4a973459b3cb186bcbc5d0a8af25a0cad32da Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 5 Sep 2019 18:06:59 +0600 Subject: [PATCH 36/59] Verify attestations against state before aggregation --- .../beacon/chain/pool/churn/AttestationChurnImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java index 8ac2876d4..5b2a68c47 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java @@ -46,7 +46,7 @@ public OffChainAggregates compute(BeaconTuple tuple) { // compute coverage Map coverage = computeCoverage(tuple.getState()); - // filter attestations by coverage + // check attestations against coverage and state List offChainAttestations = queue.stream() .filter( @@ -58,6 +58,8 @@ public OffChainAggregates compute(BeaconTuple tuple) { return bits.and(attestation.getAggregationBits()).isEmpty(); }) + .filter( + attestation -> spec.verify_attestation_impl(tuple.getState(), attestation, false)) .collect(Collectors.toList()); // compute aggregates From f9edbd273491ae0a079e42fca2e1f7b62a16799e Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Thu, 5 Sep 2019 19:15:41 +0300 Subject: [PATCH 37/59] test: change out to SimpleProcessor --- .../chain/pool/reactor/TimeProcessor.java | 7 +- .../chain/pool/reactor/TimeProcessorTest.java | 122 ++++++++++++++++++ 2 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java index 62f89cea5..b39c82983 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java @@ -5,7 +5,7 @@ import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.OutsourcePublisher; +import org.ethereum.beacon.stream.SimpleProcessor; import reactor.core.CoreSubscriber; import reactor.core.publisher.Flux; import reactor.core.scheduler.Scheduler; @@ -30,7 +30,7 @@ public class TimeProcessor extends Flux { private final TimeFrameFilter filter; - private final OutsourcePublisher out = new OutsourcePublisher<>(); + private final SimpleProcessor out; public TimeProcessor( TimeFrameFilter filter, @@ -41,6 +41,7 @@ public TimeProcessor( this.filter = filter; Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-time-frame-filter").toReactor(); + out = new SimpleProcessor<>(scheduler, "pool-time-simple-processor"); source.publishOn(scheduler).subscribe(this::hookOnNext); finalizedCheckpoints.publishOn(scheduler).subscribe(this.filter::feedFinalizedCheckpoint); newSlots.publishOn(scheduler).subscribe(this.filter::feedNewSlot); @@ -48,7 +49,7 @@ public TimeProcessor( private void hookOnNext(ReceivedAttestation attestation) { if (filter.isInitialized() && filter.check(attestation)) { - out.publishOut(attestation); + out.onNext(attestation); } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java new file mode 100644 index 000000000..dc3f2e267 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java @@ -0,0 +1,122 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.ShardNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.schedulers.ControlledSchedulers; +import org.ethereum.beacon.schedulers.Schedulers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.test.StepVerifier; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.Bitlist; +import tech.pegasys.artemis.util.uint.UInt64; + +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; + +class TimeProcessorTest { + + private final SpecConstants specConstants = + new SpecConstants() { + @Override + public SlotNumber getGenesisSlot() { + return SlotNumber.of(12345); + } + + @Override + public Time getSecondsPerSlot() { + return Time.of(1); + } + }; + + private final BeaconChainSpec spec = BeaconChainSpec.Builder.createWithDefaultParams() + .withConstants(new SpecConstants() { + @Override + public ShardNumber getShardCount() { + return ShardNumber.of(16); + } + + @Override + public SlotNumber.EpochLength getSlotsPerEpoch() { + return new SlotNumber.EpochLength(UInt64.valueOf(4)); + } + }) + .withComputableGenesisTime(false) + .withVerifyDepositProof(false) + .withBlsVerifyProofOfPossession(false) + .withBlsVerify(false) + .withCache(true) + .build(); + + private final Checkpoint checkpoint = Checkpoint.EMPTY; + private final SlotNumber slotNumber = SlotNumber.of(100L); + private final ControlledSchedulers schedulers = Schedulers.createControlled(); + + private TimeProcessor timeProcessor; + private DirectProcessor source = DirectProcessor.create(); + final FluxSink sourceSink = source.sink(); + + private Flux finalizedCheckpoints = Flux.empty(); + private Flux newSlots = Flux.empty(); + + @BeforeEach + void setUp() { + + final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); + timeFrameFilter.feedFinalizedCheckpoint(checkpoint); + timeFrameFilter.feedNewSlot(slotNumber); + + + timeProcessor = new TimeProcessor(timeFrameFilter, schedulers, source, finalizedCheckpoints, newSlots); + StepVerifier.create(timeProcessor) + .verifyComplete(); + +// timeProcessor.blockFirst() +// sourceSink.next(); + +// schedulers.addTime(1); + + } + + + @Test + void testValidAttestation() { + + } + + private Attestation createAttestation(BytesValue someValue) { + final AttestationData attestationData = createAttestationData(); + + return new Attestation( + Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), + attestationData, + Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), + BLSSignature.wrap(Bytes96.fromHexString("cc")), + specConstants); + } + + private AttestationData createAttestationData() { + + return new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(1), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + } + +} From e669476c87f43c8e6f121f23803c7781434fe1bd Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Sat, 7 Sep 2019 15:49:42 +0600 Subject: [PATCH 38/59] Update attestation pool with a bunch of fixes --- .../ethereum/beacon/chain/BeaconChain.java | 18 +++++++ .../beacon/chain/DefaultBeaconChain.java | 45 ++++++++++++++++- .../chain/pool/AttestationAggregate.java | 8 ++-- .../chain/pool/InMemoryAttestationPool.java | 3 +- .../beacon/chain/pool/OffChainAggregates.java | 2 +- .../chain/pool/ReceivedAttestation.java | 4 ++ .../chain/pool/checker/SanityChecker.java | 9 ++-- .../chain/pool/checker/TimeFrameFilter.java | 13 +++-- .../chain/pool/churn/AttestationChurn.java | 3 ++ .../pool/churn/AttestationChurnImpl.java | 48 ++++++++++++++----- .../beacon/chain/pool/churn/ChurnQueue.java | 12 +++-- .../chain/pool/reactor/ChurnProcessor.java | 6 ++- .../pool/reactor/DoubleWorkProcessor.java | 2 +- .../beacon/chain/pool/registry/Queue.java | 3 +- .../pool/registry/UnknownAttestationPool.java | 2 +- .../pool/verifier/AttestationVerifier.java | 5 -- .../beacon/start/common/Launcher.java | 4 ++ .../artemis/util/collections/Bitlist.java | 2 +- 18 files changed, 148 insertions(+), 41 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java index 1dc675350..3fee37a00 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/BeaconChain.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.chain; +import org.ethereum.beacon.core.state.Checkpoint; import org.reactivestreams.Publisher; public interface BeaconChain { @@ -15,5 +16,22 @@ public interface BeaconChain { */ BeaconTuple getRecentlyProcessed(); + /** + * Returns the most recent justified checkpoint. + * + * @return a checkpoint. + */ + Publisher getJustifiedCheckpoints(); + + /** + * Returns the most recent finalized checkpoint. + * + *

    Note: finalized checkpoints are published by {@link #getJustifiedCheckpoints()} + * either. + * + * @return a checkpoint. + */ + Publisher getFinalizedCheckpoints(); + void init(); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java index 3390e6106..e075bed96 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java @@ -8,9 +8,9 @@ import org.apache.logging.log4j.Logger; import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.BeaconTupleStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.BeaconStateEx; import org.ethereum.beacon.consensus.BlockTransition; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.transition.EmptySlotTransition; import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; @@ -38,6 +38,8 @@ public class DefaultBeaconChain implements MutableBeaconChain { private final BeaconTupleStorage tupleStorage; private final SimpleProcessor blockStream; + private final SimpleProcessor justifiedCheckpointStream; + private final SimpleProcessor finalizedCheckpointStream; private final Schedulers schedulers; private BeaconTuple recentlyProcessed; @@ -62,6 +64,10 @@ public DefaultBeaconChain( this.schedulers = schedulers; blockStream = new SimpleProcessor<>(schedulers.events(), "DefaultBeaconChain.block"); + justifiedCheckpointStream = + new SimpleProcessor<>(schedulers.events(), "DefaultBeaconChain.justifiedCheckpoint"); + finalizedCheckpointStream = + new SimpleProcessor<>(schedulers.events(), "DefaultBeaconChain.finalizedCheckpoint"); } @Override @@ -70,6 +76,8 @@ public void init() { initializeStorage(); } this.recentlyProcessed = fetchRecentTuple(); + justifiedCheckpointStream.onNext(fetchJustifiedCheckpoint()); + finalizedCheckpointStream.onNext(fetchFinalizedCheckpoint()); blockStream.onNext(new BeaconTupleDetails(recentlyProcessed)); } @@ -168,12 +176,35 @@ public BeaconTuple getRecentlyProcessed() { private void updateFinality(BeaconState previous, BeaconState current) { if (!previous.getFinalizedCheckpoint().equals(current.getFinalizedCheckpoint())) { chainStorage.getFinalizedStorage().set(current.getFinalizedCheckpoint()); + finalizedCheckpointStream.onNext(current.getFinalizedCheckpoint()); } if (!previous.getCurrentJustifiedCheckpoint().equals(current.getCurrentJustifiedCheckpoint())) { - chainStorage.getJustifiedStorage().set(current.getCurrentJustifiedCheckpoint()); + // store new justified checkpoint if its epoch greater than previous one + if (current + .getCurrentJustifiedCheckpoint() + .getEpoch() + .greater(fetchJustifiedCheckpoint().getEpoch())) { + chainStorage.getJustifiedStorage().set(current.getCurrentJustifiedCheckpoint()); + } + + justifiedCheckpointStream.onNext(current.getFinalizedCheckpoint()); } } + private Checkpoint fetchJustifiedCheckpoint() { + return chainStorage + .getJustifiedStorage() + .get() + .orElseThrow(() -> new RuntimeException("Justified checkpoint not found")); + } + + private Checkpoint fetchFinalizedCheckpoint() { + return chainStorage + .getFinalizedStorage() + .get() + .orElseThrow(() -> new RuntimeException("Finalized checkpoint not found")); + } + private BeaconStateEx pullParentState(BeaconBlock block) { Optional parent = tupleStorage.get(block.getParentRoot()); checkArgument(parent.isPresent(), "No parent for block %s", block); @@ -208,4 +239,14 @@ private boolean rejectedByTime(BeaconBlock block) { public Publisher getBlockStatesStream() { return blockStream; } + + @Override + public Publisher getJustifiedCheckpoints() { + return justifiedCheckpointStream; + } + + @Override + public Publisher getFinalizedCheckpoints() { + return finalizedCheckpointStream; + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java index 4b51e6a86..15810e988 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationAggregate.java @@ -22,9 +22,9 @@ public static AttestationAggregate create(Attestation attestation) { signatures); } - private final Bitlist aggregationBits; + private Bitlist aggregationBits; + private Bitlist custodyBits; private final AttestationData data; - private final Bitlist custodyBits; private final List signatures; public AttestationAggregate( @@ -40,8 +40,8 @@ public AttestationAggregate( public boolean add(Attestation attestation) { if (isAggregatable(attestation)) { - aggregationBits.or(attestation.getAggregationBits()); - custodyBits.or(attestation.getCustodyBits()); + aggregationBits = aggregationBits.or(attestation.getAggregationBits()); + custodyBits = custodyBits.or(attestation.getCustodyBits()); signatures.add(attestation.getSignature()); return true; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 0b261bee6..697171942 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -126,7 +126,7 @@ public void start() { identificationProcessor .getIdentified() .publishOn(parallelExecutor) - .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL); + .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL, parallelExecutor); VerificationProcessor verificationProcessor = new VerificationProcessor(verifier, verificationThrottle); @@ -135,6 +135,7 @@ public void start() { new ChurnProcessor( attestationChurn, schedulers, + newSlotsFx, chainHeadsFx, justifiedCheckpointsFx, finalizedCheckpointsFx, diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java index 4676d6c9e..567eae21d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/OffChainAggregates.java @@ -29,7 +29,7 @@ public List getAggregates() { return aggregates; } - public SlotNumber getEpoch() { + public SlotNumber getSlot() { return slot; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java index b74c9d4d5..371b8ee57 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/ReceivedAttestation.java @@ -16,6 +16,10 @@ public ReceivedAttestation(NodeId sender, Attestation message) { this.message = message; } + public ReceivedAttestation(Attestation message) { + this(null, message); + } + public NodeId getSender() { return sender; } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java index 042b0fe91..e884ea9cd 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/SanityChecker.java @@ -5,6 +5,7 @@ import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.state.Checkpoint; +import tech.pegasys.artemis.ethereum.core.Hash32; /** * Given attestation runs a number of sanity checks against it. @@ -31,13 +32,15 @@ public boolean check(ReceivedAttestation attestation) { final AttestationData data = attestation.getMessage().getData(); - // sourceEpoch >= targetEpoch - if (data.getSource().getEpoch().greaterEqual(data.getTarget().getEpoch())) { + // sourceEpoch > targetEpoch + if (data.getSource().getEpoch().greater(data.getTarget().getEpoch())) { return false; } // finalizedEpoch == sourceEpoch && finalizedRoot != sourceRoot - if (data.getSource().getEpoch().equals(finalizedCheckpoint.getEpoch()) + // do not run this check for sourceRoot == ZERO + if (!data.getSource().getRoot().equals(Hash32.ZERO) + && data.getSource().getEpoch().equals(finalizedCheckpoint.getEpoch()) && !finalizedCheckpoint.getRoot().equals(data.getSource().getRoot())) { return false; } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java index dfac57df7..2336552d1 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/checker/TimeFrameFilter.java @@ -1,10 +1,12 @@ package org.ethereum.beacon.chain.pool.checker; -import org.ethereum.beacon.chain.pool.*; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.StatefulProcessor; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.types.*; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; /** * Filters attestations by time frame of its target and source. @@ -45,13 +47,13 @@ public boolean check(ReceivedAttestation attestation) { final AttestationData data = attestation.getMessage().getData(); - // targetEpoch <= finalizedEpoch - if (data.getTarget().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { + // targetEpoch < finalizedEpoch + if (data.getTarget().getEpoch().less(finalizedCheckpoint.getEpoch())) { return false; } // sourceEpoch < finalizedEpoch - if (data.getSource().getEpoch().lessEqual(finalizedCheckpoint.getEpoch())) { + if (data.getSource().getEpoch().less(finalizedCheckpoint.getEpoch())) { return false; } @@ -72,6 +74,7 @@ public boolean check(ReceivedAttestation attestation) { */ public void feedFinalizedCheckpoint(Checkpoint checkpoint) { this.finalizedCheckpoint = checkpoint; + this.maxAcceptableEpoch = checkpoint.getEpoch().plus(maxAttestationLookahead); } /** diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java index 7dd590859..77500384c 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurn.java @@ -7,6 +7,7 @@ import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.Attestation; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; public interface AttestationChurn extends StatefulProcessor { @@ -18,6 +19,8 @@ public interface AttestationChurn extends StatefulProcessor { void feedJustifiedCheckpoint(Checkpoint checkpoint); + void feedNewSlot(SlotNumber slotNumber); + static AttestationChurn create(BeaconChainSpec spec, long size) { return new AttestationChurnImpl(spec, size); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java index 5b2a68c47..3c0d4fb95 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/AttestationChurnImpl.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -12,10 +13,12 @@ import org.ethereum.beacon.chain.pool.OffChainAggregates; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.MutableBeaconState; import org.ethereum.beacon.core.operations.Attestation; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64s; @@ -27,12 +30,14 @@ public class AttestationChurnImpl implements AttestationChurn { private final BeaconChainSpec spec; private Checkpoint justifiedCheckpoint; - private EpochNumber lowerEpoch; - private EpochNumber upperEpoch; + private EpochNumber lowerEpoch = EpochNumber.ZERO; + private EpochNumber upperEpoch = EpochNumber.ZERO; public AttestationChurnImpl(BeaconChainSpec spec, long size) { this.spec = spec; this.queue = new ChurnQueue(size); + + queue.updateEpochBoundaries(lowerEpoch, upperEpoch); } @Override @@ -43,10 +48,16 @@ public OffChainAggregates compute(BeaconTuple tuple) { updateEpochBoundaries( spec.get_previous_epoch(tuple.getState()), spec.get_current_epoch(tuple.getState())); + if (queue.isEmpty()) { + return new OffChainAggregates( + spec.signing_root(tuple.getBlock()), tuple.getState().getSlot(), Collections.emptyList()); + } + // compute coverage Map coverage = computeCoverage(tuple.getState()); // check attestations against coverage and state + MutableBeaconState state = tuple.getState().createMutableCopy(); List offChainAttestations = queue.stream() .filter( @@ -58,8 +69,17 @@ public OffChainAggregates compute(BeaconTuple tuple) { return bits.and(attestation.getAggregationBits()).isEmpty(); }) + .sorted( + Comparator.comparing(attestation -> attestation.getData().getTarget().getEpoch())) .filter( - attestation -> spec.verify_attestation_impl(tuple.getState(), attestation, false)) + attestation -> { + if (spec.verify_attestation_impl(state, attestation, false)) { + spec.process_attestation(state, attestation); + return true; + } else { + return false; + } + }) .collect(Collectors.toList()); // compute aggregates @@ -78,29 +98,35 @@ public void feedFinalizedCheckpoint(Checkpoint checkpoint) { @Override public void feedJustifiedCheckpoint(Checkpoint checkpoint) { // discard forks if justified checkpoint is updated - if (checkpoint.getEpoch().greater(justifiedCheckpoint.getEpoch())) { + if (justifiedCheckpoint == null + || checkpoint.getEpoch().greater(justifiedCheckpoint.getEpoch())) { updateJustifiedCheckpoint(checkpoint); } } + @Override + public void feedNewSlot(SlotNumber slotNumber) { + EpochNumber epoch = spec.compute_epoch_of_slot(slotNumber); + updateEpochBoundaries(epoch.equals(EpochNumber.ZERO) ? epoch : epoch.decrement(), epoch); + } + private void updateJustifiedCheckpoint(Checkpoint checkpoint) { updateEpochBoundaries(checkpoint.getEpoch(), UInt64s.max(checkpoint.getEpoch(), upperEpoch)); this.justifiedCheckpoint = checkpoint; } - private void updateEpochBoundaries(EpochNumber lowerEpoch, EpochNumber upperEpoch) { - assert lowerEpoch.lessEqual(upperEpoch); + private void updateEpochBoundaries(EpochNumber newLower, EpochNumber newUpper) { + assert newLower.lessEqual(newUpper); // return if there is nothing to update - // newLower <= lower || newUpper <= upper - if (lowerEpoch.lessEqual(this.lowerEpoch) || upperEpoch.lessEqual(this.upperEpoch)) { + if (!(newLower.greater(lowerEpoch) || newUpper.greater(upperEpoch))) { return; } - queue.updateEpochBoundaries(lowerEpoch, upperEpoch); + queue.updateEpochBoundaries(newLower, newUpper); - this.lowerEpoch = lowerEpoch; - this.upperEpoch = upperEpoch; + lowerEpoch = newLower; + upperEpoch = newUpper; } @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java index fa5e76c10..9b18501ad 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/ChurnQueue.java @@ -21,6 +21,10 @@ public ChurnQueue(long maxSize) { this.maxSize = maxSize; } + boolean isEmpty() { + return size == 0; + } + Stream stream() { return buckets.stream().map(Bucket::getAttestations).flatMap(List::stream); } @@ -45,11 +49,13 @@ void updateEpochBoundaries(EpochNumber lower, EpochNumber upper) { assert lower.lessEqual(upper); while (buckets.size() > 0 && buckets.getFirst().epoch.less(lower)) { - buckets.removeFirst(); + Bucket detached = buckets.removeFirst(); + size -= detached.attestations.size(); } while (buckets.size() > 0 && buckets.getLast().epoch.greater(upper)) { - buckets.removeLast(); + Bucket detached = buckets.removeLast(); + size -= detached.attestations.size(); } this.lowerEpoch = lower; @@ -64,7 +70,7 @@ Bucket getOrCreateBucket(EpochNumber epoch) { } Bucket newBucket = new Bucket(epoch); - buckets.add(new Bucket(epoch)); + buckets.add(newBucket); buckets.sort(Comparator.comparing(Bucket::getEpoch)); return newBucket; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java index 95cebdf20..3d4bd82b5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java @@ -6,6 +6,7 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.churn.AttestationChurn; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.OutsourcePublisher; import reactor.core.CoreSubscriber; @@ -20,6 +21,7 @@ public class ChurnProcessor extends Flux { public ChurnProcessor( AttestationChurn churn, Schedulers schedulers, + Flux newSlots, Flux chainHeads, Flux justifiedCheckpoints, Flux finalizedCheckpoints, @@ -28,12 +30,14 @@ public ChurnProcessor( Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-attestation-churn").toReactor(); chainHeads.publishOn(scheduler).subscribe(this::hookOnNext); + newSlots.publishOn(scheduler).subscribe(this.churn::feedNewSlot); justifiedCheckpoints.publishOn(scheduler).subscribe(this.churn::feedJustifiedCheckpoint); finalizedCheckpoints.publishOn(scheduler).subscribe(this.churn::feedFinalizedCheckpoint); source .publishOn(scheduler) .map(ReceivedAttestation::getMessage) - .bufferTimeout(AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL) + .bufferTimeout( + AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL, scheduler) .subscribe(this.churn::add); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java index 9edfe607d..42a9dadc9 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java @@ -39,7 +39,7 @@ public DoubleWorkProcessor( } private void hookOnNext(ReceivedAttestation attestation) { - if (!processedAttestations.add(attestation)) { + if (processedAttestations.add(attestation)) { out.publishOut(attestation); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java index ba75c82a5..0f9c1a360 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/Queue.java @@ -25,7 +25,7 @@ final class Queue { /** Max number of overall parked attestations. */ private final long maxSize; /** A lower time frame boundary of attestation queue. */ - private EpochNumber baseLine; + private EpochNumber baseLine = EpochNumber.ZERO; Queue(EpochNumber trackedEpochs, long maxSize) { assert maxSize > 0; @@ -33,7 +33,6 @@ final class Queue { this.trackedEpochs = trackedEpochs; this.maxSize = maxSize; - this.baseLine = EpochNumber.ZERO; } /** diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java index b8cd1ba5e..29a9c0aa0 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/registry/UnknownAttestationPool.java @@ -45,7 +45,7 @@ public class UnknownAttestationPool implements AttestationRegistry, StatefulProc /** A beacon chain spec. */ private final BeaconChainSpec spec; /** A lower time frame boundary of attestation queue. */ - private EpochNumber currentBaseLine; + private EpochNumber currentBaseLine = EpochNumber.ZERO; public UnknownAttestationPool( BeaconBlockStorage blockStorage, BeaconChainSpec spec, EpochNumber lookahead, long size) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java index f2fdf3b2c..e5e23d464 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AttestationVerifier.java @@ -146,11 +146,6 @@ private VerificationResult verifyGroup(AttestingTarget target, List verifyIndexed(BeaconState state, Attestation attestation) { - // skip indexed attestation verification, it's explicitly done in the next step - if (!spec.verify_attestation_impl(state, attestation, false)) { - return Optional.empty(); - } - // compute and verify indexed attestation // skip signature verification IndexedAttestation indexedAttestation = spec.get_indexed_attestation(state, attestation); diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java index b3ca11f4c..9eb0a962a 100644 --- a/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java @@ -282,6 +282,10 @@ public ExtendedSlotTransition getExtendedSlotTransition() { return extendedSlotTransition; } + public EmptySlotTransition getEmptySlotTransition() { + return emptySlotTransition; + } + public BeaconBlockVerifier getBlockVerifier() { return blockVerifier; } diff --git a/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java b/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java index 016320c69..4262d602d 100644 --- a/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java +++ b/types/src/main/java/tech/pegasys/artemis/util/collections/Bitlist.java @@ -211,7 +211,7 @@ public int size() { } public boolean isEmpty() { - return BytesValues.countZeros(this) == size; + return BytesValues.countZeros(wrapped) == wrapped.size(); } public long maxSize() { From 26da60d86e84d25312290d3a35da84ee58e7071e Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Sat, 7 Sep 2019 16:09:29 +0600 Subject: [PATCH 39/59] Use SimpleProcessor all over attestation pool --- .../chain/pool/InMemoryAttestationPool.java | 26 +++--- .../chain/pool/reactor/ChurnProcessor.java | 35 ++++---- .../pool/reactor/DoubleWorkProcessor.java | 15 ++-- .../pool/reactor/IdentificationProcessor.java | 34 ++++---- .../chain/pool/reactor/SanityProcessor.java | 30 ++++--- .../reactor/SignatureEncodingProcessor.java | 24 +++--- .../chain/pool/reactor/TimeProcessor.java | 19 +++-- .../pool/reactor/VerificationProcessor.java | 23 +++--- .../stream/AbstractDelegateProcessor.java | 79 ------------------- .../beacon/stream/OutsourcePublisher.java | 36 --------- 10 files changed, 120 insertions(+), 201 deletions(-) delete mode 100644 util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java delete mode 100644 util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 697171942..0576e225e 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -85,10 +85,8 @@ public InMemoryAttestationPool( @Override public void start() { - Scheduler parallelExecutor = - schedulers - .newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS) - .toReactor(); + org.ethereum.beacon.schedulers.Scheduler parallelExecutor = + schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); // create sources Flux sourceFx = Flux.from(source); @@ -113,8 +111,7 @@ public void start() { // check signature encoding SignatureEncodingProcessor encodingProcessor = - new SignatureEncodingProcessor( - encodingChecker, doubleWorkProcessor.publishOn(parallelExecutor)); + new SignatureEncodingProcessor(encodingChecker, parallelExecutor, doubleWorkProcessor); // identify attestation target IdentificationProcessor identificationProcessor = @@ -123,12 +120,11 @@ public void start() { // verify attestations Flux> verificationThrottle = - identificationProcessor - .getIdentified() - .publishOn(parallelExecutor) - .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL, parallelExecutor); + Flux.from(identificationProcessor.getIdentified()) + .publishOn(parallelExecutor.toReactor()) + .bufferTimeout(VERIFIER_BUFFER_SIZE, VERIFIER_INTERVAL, parallelExecutor.toReactor()); VerificationProcessor verificationProcessor = - new VerificationProcessor(verifier, verificationThrottle); + new VerificationProcessor(verifier, parallelExecutor, verificationThrottle); // feed churn ChurnProcessor churnProcessor = @@ -143,9 +139,13 @@ public void start() { Scheduler outScheduler = schedulers.events().toReactor(); // expose valid attestations - verificationProcessor.getValid().publishOn(outScheduler).subscribe(validAttestations); + Flux.from(verificationProcessor.getValid()) + .publishOn(outScheduler) + .subscribe(validAttestations); // expose not yet identified - identificationProcessor.getUnknown().publishOn(outScheduler).subscribe(unknownAttestations); + Flux.from(identificationProcessor.getUnknown()) + .publishOn(outScheduler) + .subscribe(unknownAttestations); // expose aggregates churnProcessor.publishOn(outScheduler).subscribe(offChainAggregates); // expose invalid attestations diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java index 3d4bd82b5..d3ac414b4 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java @@ -8,7 +8,8 @@ import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.OutsourcePublisher; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; import reactor.core.CoreSubscriber; import reactor.core.publisher.Flux; import reactor.core.scheduler.Scheduler; @@ -16,35 +17,41 @@ public class ChurnProcessor extends Flux { private final AttestationChurn churn; - private final OutsourcePublisher out = new OutsourcePublisher<>(); + private final SimpleProcessor out; public ChurnProcessor( AttestationChurn churn, Schedulers schedulers, - Flux newSlots, - Flux chainHeads, - Flux justifiedCheckpoints, - Flux finalizedCheckpoints, - Flux source) { + Publisher newSlots, + Publisher chainHeads, + Publisher justifiedCheckpoints, + Publisher finalizedCheckpoints, + Publisher source) { this.churn = churn; - Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-attestation-churn").toReactor(); - chainHeads.publishOn(scheduler).subscribe(this::hookOnNext); - newSlots.publishOn(scheduler).subscribe(this.churn::feedNewSlot); - justifiedCheckpoints.publishOn(scheduler).subscribe(this.churn::feedJustifiedCheckpoint); - finalizedCheckpoints.publishOn(scheduler).subscribe(this.churn::feedFinalizedCheckpoint); - source + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-churn-processor").toReactor(); + Flux.from(chainHeads).publishOn(scheduler).subscribe(this::hookOnNext); + Flux.from(newSlots).publishOn(scheduler).subscribe(this.churn::feedNewSlot); + Flux.from(justifiedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedJustifiedCheckpoint); + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedFinalizedCheckpoint); + Flux.from(source) .publishOn(scheduler) .map(ReceivedAttestation::getMessage) .bufferTimeout( AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL, scheduler) .subscribe(this.churn::add); + + out = new SimpleProcessor<>(scheduler, "ChurnProcessor.out"); } private void hookOnNext(BeaconTuple tuple) { if (churn.isInitialized()) { OffChainAggregates aggregates = churn.compute(tuple); - out.publishOut(aggregates); + out.onNext(aggregates); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java index 42a9dadc9..edcb9ba79 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java @@ -3,7 +3,8 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.OutsourcePublisher; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; import reactor.core.CoreSubscriber; import reactor.core.publisher.Flux; import reactor.core.scheduler.Scheduler; @@ -26,21 +27,23 @@ public class DoubleWorkProcessor extends Flux { private final ProcessedAttestations processedAttestations; - private final OutsourcePublisher out = new OutsourcePublisher<>(); + private final SimpleProcessor out; public DoubleWorkProcessor( ProcessedAttestations processedAttestations, Schedulers schedulers, - Flux source) { + Publisher source) { this.processedAttestations = processedAttestations; - Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-double-work").toReactor(); - source.publishOn(scheduler).subscribe(this::hookOnNext); + Scheduler scheduler = + schedulers.newSingleThreadDaemon("pool-double-work-processor").toReactor(); + out = new SimpleProcessor<>(scheduler, "DoubleWorkProcessor.out"); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); } private void hookOnNext(ReceivedAttestation attestation) { if (processedAttestations.add(attestation)) { - out.publishOut(attestation); + out.onNext(attestation); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java index 066d1f141..d6058364e 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java @@ -5,7 +5,8 @@ import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.OutsourcePublisher; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.scheduler.Scheduler; @@ -31,43 +32,46 @@ public class IdentificationProcessor { private final UnknownAttestationPool pool; - private final OutsourcePublisher identified = new OutsourcePublisher<>(); - private final OutsourcePublisher unknown = new OutsourcePublisher<>(); + private final SimpleProcessor identified; + private final SimpleProcessor unknown; public IdentificationProcessor( UnknownAttestationPool pool, Schedulers schedulers, - Flux source, - Flux newSlots, - Flux newImportedBlocks) { + Publisher source, + Publisher newSlots, + Publisher newImportedBlocks) { this.pool = pool; Scheduler scheduler = - schedulers.newSingleThreadDaemon("pool-attestation-identifier").toReactor(); - newSlots.publishOn(scheduler).subscribe(this.pool::feedNewSlot); - newImportedBlocks.publishOn(scheduler).subscribe(this::hookOnNext); - source.publishOn(scheduler).subscribe(this::hookOnNext); + schedulers.newSingleThreadDaemon("pool-identification-processor").toReactor(); + Flux.from(newSlots).publishOn(scheduler).subscribe(this.pool::feedNewSlot); + Flux.from(newImportedBlocks).publishOn(scheduler).subscribe(this::hookOnNext); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + + identified = new SimpleProcessor<>(scheduler, "IdentificationProcessor.identified"); + unknown = new SimpleProcessor<>(scheduler, "IdentificationProcessor.unknown"); } private void hookOnNext(BeaconBlock block) { - pool.feedNewImportedBlock(block).forEach(identified::publishOut); + pool.feedNewImportedBlock(block).forEach(identified::onNext); } private void hookOnNext(ReceivedAttestation attestation) { if (pool.isInitialized()) { if (pool.add(attestation)) { - unknown.publishOut(attestation); + unknown.onNext(attestation); } else { - identified.publishOut(attestation); + identified.onNext(attestation); } } } - public OutsourcePublisher getIdentified() { + public Publisher getIdentified() { return identified; } - public OutsourcePublisher getUnknown() { + public Publisher getUnknown() { return unknown; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java index 22a4e15af..9491a257d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java @@ -4,7 +4,8 @@ import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.stream.OutsourcePublisher; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; import reactor.core.scheduler.Scheduler; @@ -29,19 +30,24 @@ public class SanityProcessor { private final SanityChecker checker; - private final OutsourcePublisher valid = new OutsourcePublisher<>(); - private final OutsourcePublisher invalid = new OutsourcePublisher<>(); + private final SimpleProcessor valid; + private final SimpleProcessor invalid; public SanityProcessor( SanityChecker checker, Schedulers schedulers, - Flux source, - Flux finalizedCheckpoints) { + Publisher source, + Publisher finalizedCheckpoints) { this.checker = checker; - Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-sanity-checker").toReactor(); - finalizedCheckpoints.publishOn(scheduler).subscribe(this.checker::feedFinalizedCheckpoint); - source.publishOn(scheduler).subscribe(this::hookOnNext); + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-sanity-processor").toReactor(); + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.checker::feedFinalizedCheckpoint); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + + valid = new SimpleProcessor<>(scheduler, "SanityProcessor.valid"); + invalid = new SimpleProcessor<>(scheduler, "SanityProcessor.invalid"); } private void hookOnNext(ReceivedAttestation attestation) { @@ -50,17 +56,17 @@ private void hookOnNext(ReceivedAttestation attestation) { } if (checker.check(attestation)) { - valid.publishOut(attestation); + valid.onNext(attestation); } else { - invalid.publishOut(attestation); + invalid.onNext(attestation); } } - public Flux getValid() { + public Publisher getValid() { return valid; } - public Flux getInvalid() { + public Publisher getInvalid() { return invalid; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java index d51f5fc3f..78b04267c 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java @@ -2,7 +2,9 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; -import org.ethereum.beacon.stream.OutsourcePublisher; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; /** @@ -24,29 +26,33 @@ public class SignatureEncodingProcessor { private final SignatureEncodingChecker checker; - private final OutsourcePublisher valid = new OutsourcePublisher<>(); - private final OutsourcePublisher invalid = new OutsourcePublisher<>(); + private final SimpleProcessor valid; + private final SimpleProcessor invalid; public SignatureEncodingProcessor( - SignatureEncodingChecker checker, Flux source) { + SignatureEncodingChecker checker, + Scheduler scheduler, + Publisher source) { this.checker = checker; - source.subscribe(this::hookOnNext); + Flux.from(source).publishOn(scheduler.toReactor()).subscribe(this::hookOnNext); + valid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.valid"); + invalid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.invalid"); } private void hookOnNext(ReceivedAttestation attestation) { if (checker.check(attestation)) { - valid.publishOut(attestation); + valid.onNext(attestation); } else { - invalid.publishOut(attestation); + invalid.onNext(attestation); } } - public Flux getValid() { + public Publisher getValid() { return valid; } - public Flux getInvalid() { + public Publisher getInvalid() { return invalid; } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java index b39c82983..41f286e2b 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java @@ -6,6 +6,7 @@ import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; import reactor.core.CoreSubscriber; import reactor.core.publisher.Flux; import reactor.core.scheduler.Scheduler; @@ -35,16 +36,18 @@ public class TimeProcessor extends Flux { public TimeProcessor( TimeFrameFilter filter, Schedulers schedulers, - Flux source, - Flux finalizedCheckpoints, - Flux newSlots) { + Publisher source, + Publisher finalizedCheckpoints, + Publisher newSlots) { this.filter = filter; - Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-time-frame-filter").toReactor(); - out = new SimpleProcessor<>(scheduler, "pool-time-simple-processor"); - source.publishOn(scheduler).subscribe(this::hookOnNext); - finalizedCheckpoints.publishOn(scheduler).subscribe(this.filter::feedFinalizedCheckpoint); - newSlots.publishOn(scheduler).subscribe(this.filter::feedNewSlot); + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-time-processor").toReactor(); + out = new SimpleProcessor<>(scheduler, "TimeProcessor.out"); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.filter::feedFinalizedCheckpoint); + Flux.from(newSlots).publishOn(scheduler).subscribe(this.filter::feedNewSlot); } private void hookOnNext(ReceivedAttestation attestation) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java index 396f923fa..388fe0084 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java @@ -4,7 +4,9 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; import org.ethereum.beacon.chain.pool.verifier.VerificationResult; -import org.ethereum.beacon.stream.OutsourcePublisher; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; /** @@ -27,26 +29,29 @@ public class VerificationProcessor { private final BatchVerifier verifier; - private final OutsourcePublisher valid = new OutsourcePublisher<>(); - private final OutsourcePublisher invalid = new OutsourcePublisher<>(); + private final SimpleProcessor valid; + private final SimpleProcessor invalid; - public VerificationProcessor(BatchVerifier verifier, Flux> source) { + public VerificationProcessor( + BatchVerifier verifier, Scheduler scheduler, Publisher> source) { this.verifier = verifier; - source.subscribe(this::hookOnNext); + Flux.from(source).publishOn(scheduler.toReactor()).subscribe(this::hookOnNext); + valid = new SimpleProcessor<>(scheduler, "VerificationProcessor.valid"); + invalid = new SimpleProcessor<>(scheduler, "VerificationProcessor.invalid"); } private void hookOnNext(List batch) { VerificationResult result = verifier.verify(batch); - result.getValid().forEach(valid::publishOut); - result.getInvalid().forEach(invalid::publishOut); + result.getValid().forEach(valid::onNext); + result.getInvalid().forEach(invalid::onNext); } - public Flux getValid() { + public Publisher getValid() { return valid; } - public Flux getInvalid() { + public Publisher getInvalid() { return invalid; } } diff --git a/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java b/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java deleted file mode 100644 index a1f71b6bd..000000000 --- a/util/src/main/java/org/ethereum/beacon/stream/AbstractDelegateProcessor.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.ethereum.beacon.stream; - -import javax.annotation.Nonnull; -import org.reactivestreams.Subscription; -import reactor.core.CoreSubscriber; -import reactor.core.publisher.BaseSubscriber; -import reactor.core.publisher.DirectProcessor; -import reactor.core.publisher.FluxProcessor; - -/** - * An abstract processor built atop of reactor abstractions. - * - *

    Delegates to {@link OutsourcePublisher} which it its turn delegates to {@link - * DirectProcessor}. - * - *

    Note: DirectProcessor does not coordinate backpressure between its Subscribers and the - * upstream, but consumes its upstream in an * unbounded manner. - * - * @param a kind of input data. - * @param a kind of output data. - */ -public abstract class AbstractDelegateProcessor extends FluxProcessor { - - private final Subscriber subscriber; - private final OutsourcePublisher publisher; - - public AbstractDelegateProcessor() { - this.subscriber = new Subscriber(); - this.publisher = new OutsourcePublisher<>(); - } - - @Override - public void onSubscribe(Subscription s) { - subscriber.onSubscribe(s); - } - - @Override - public void onNext(IN in) { - subscriber.onNext(in); - } - - @Override - public void onError(Throwable t) { - subscriber.onError(t); - } - - @Override - public void onComplete() { - subscriber.onComplete(); - } - - @Override - public void subscribe(@Nonnull CoreSubscriber actual) { - publisher.subscribe(actual); - } - - private final class Subscriber extends BaseSubscriber { - @Override - protected void hookOnNext(IN value) { - AbstractDelegateProcessor.this.hookOnNext(value); - } - } - - /** - * Called when there is a new input value. - * - * @param value a value. - */ - protected abstract void hookOnNext(IN value); - - /** - * Should be called in order to publish a new value. - * - * @param value a value. - */ - protected void publishOut(OUT value) { - publisher.publishOut(value); - } -} diff --git a/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java b/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java deleted file mode 100644 index 66d78bae6..000000000 --- a/util/src/main/java/org/ethereum/beacon/stream/OutsourcePublisher.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.ethereum.beacon.stream; - -import reactor.core.CoreSubscriber; -import reactor.core.publisher.DirectProcessor; -import reactor.core.publisher.Flux; -import reactor.core.publisher.FluxSink; - -/** - * An implementation of publisher which could be manually fed with data as a source. - * - *

    Delegates to {@link DirectProcessor} and uses its sink to feed the data. - * - *

    Note: DirectProcessor does not coordinate backpressure between its Subscribers and the - * upstream, but consumes its upstream in an * unbounded manner. - * - * @param a kind of data. - */ -public class OutsourcePublisher extends Flux { - - private final DirectProcessor delegate = DirectProcessor.create(); - private final FluxSink out = delegate.sink(); - - /** - * Publishes a new value. - * - * @param value a value. - */ - public void publishOut(T value) { - out.next(value); - } - - @Override - public void subscribe(CoreSubscriber actual) { - delegate.subscribe(actual); - } -} From 49e95b4141dfdd9370b195e5b6158860635f8630 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Sat, 7 Sep 2019 16:27:11 +0600 Subject: [PATCH 40/59] Merge followups --- .../beacon/chain/DefaultBeaconChain.java | 1 + .../org/ethereum/beacon/node/ConfigUtils.java | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java index e2ba2c8a7..a7cc6c1bf 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java @@ -17,6 +17,7 @@ import org.ethereum.beacon.consensus.verifier.VerificationResult; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; diff --git a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java index 5dea1ea0e..368387866 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java @@ -1,11 +1,5 @@ package org.ethereum.beacon.node; -<<<<<<< HEAD -import java.util.List; -import java.util.Random; -import java.util.stream.Collectors; -======= ->>>>>>> develop import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.ChainStart; import org.ethereum.beacon.core.operations.Deposit; @@ -78,12 +72,8 @@ public static List createKeyPairs(ValidatorKeys keys) { genKeys.getSeed(), genKeys.getStartIndex(), genKeys.getCount()); } else if (keys instanceof InteropKeys) { InteropKeys interopKeys = (InteropKeys) keys; -<<<<<<< HEAD - return SimulationKeyPairGenerator.generateInteropKeys(interopKeys.getCount()); -======= return SimulationKeyPairGenerator.generateInteropKeys( interopKeys.getStartIndex(), interopKeys.getCount()); ->>>>>>> develop } else { throw new IllegalArgumentException("Unknown ValidatorKeys subclass: " + keys.getClass()); } @@ -135,13 +125,7 @@ public static ChainStart createChainStart( Eth1Data eth1Data = new Eth1Data( spec.hash_tree_root(depositDataList), UInt64.valueOf(deposits.size()), blockHash); -<<<<<<< HEAD - ChainStart chainStart = - new ChainStart(Time.of(eConfig.getGenesisTime().getTime() / 1000), eth1Data, deposits); - return new SimpleDepositContract(chainStart); -======= return new ChainStart(Time.of(eConfig.getGenesisTime().getTime() / 1000), eth1Data, deposits); ->>>>>>> develop } else { throw new IllegalArgumentException( "This config class is not yet supported: " + config.getClass()); From 86813ff4c03458275a9d43e2b344a21ab0ed5c15 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Sun, 8 Sep 2019 23:58:45 +0600 Subject: [PATCH 41/59] Integrate attestation pool into simulator --- .../beacon/chain/ForkChoiceProcessor.java | 7 + .../beacon/chain/LMDGhostProcessor.java | 183 ++++++++++++++++++ .../observer/ObservableStateProcessor.java | 33 +++- .../ObservableStateProcessorImpl.java | 16 -- .../observer/YetAnotherStateProcessor.java | 166 ++++++++++++++++ .../beacon/chain/pool/AttestationPool.java | 31 ++- .../chain/pool/InMemoryAttestationPool.java | 51 ++--- .../chain/pool/churn/OffChainAggregates.java | 28 --- .../chain/pool/reactor/ChurnProcessor.java | 4 +- .../pool/reactor/DoubleWorkProcessor.java | 3 +- .../pool/reactor/IdentificationProcessor.java | 6 +- .../chain/pool/reactor/SanityProcessor.java | 6 +- .../reactor/SignatureEncodingProcessor.java | 5 +- .../chain/pool/reactor/TimeProcessor.java | 3 +- .../pool/reactor/VerificationProcessor.java | 13 +- .../verifier/AggregateSignatureVerifier.java | 9 +- .../pool/verifier/VerifiableAttestation.java | 8 + .../pool/verifier/VerificationResult.java | 20 +- .../pool/InMemoryAttestationPoolTest.java | 9 +- .../beacon/start/common/Launcher.java | 60 ++++-- .../beacon/validator/api/ServiceFactory.java | 40 +--- 21 files changed, 533 insertions(+), 168 deletions(-) create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/ForkChoiceProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/LMDGhostProcessor.java create mode 100644 chain/src/main/java/org/ethereum/beacon/chain/observer/YetAnotherStateProcessor.java delete mode 100644 chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/ForkChoiceProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/ForkChoiceProcessor.java new file mode 100644 index 000000000..375a48e3d --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/ForkChoiceProcessor.java @@ -0,0 +1,7 @@ +package org.ethereum.beacon.chain; + +import org.reactivestreams.Publisher; + +public interface ForkChoiceProcessor { + Publisher getChainHeads(); +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/LMDGhostProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/LMDGhostProcessor.java new file mode 100644 index 000000000..bccbe8cb5 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/LMDGhostProcessor.java @@ -0,0 +1,183 @@ +package org.ethereum.beacon.chain; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.spec.ForkChoice.LatestMessage; +import org.ethereum.beacon.consensus.spec.ForkChoice.Store; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.ValidatorIndex; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class LMDGhostProcessor implements ForkChoiceProcessor { + + private final int SEARCH_LIMIT = Integer.MAX_VALUE; + + private final BeaconChainSpec spec; + private final BeaconChainStorage storage; + private final SimpleProcessor chainHeadStream; + + private final Map latestMessageStorage = new HashMap<>(); + private Checkpoint justifiedCheckpoint = Checkpoint.EMPTY; + private Hash32 currentHeadRoot = Hash32.ZERO; + + public LMDGhostProcessor( + BeaconChainSpec spec, + BeaconChainStorage storage, + Schedulers schedulers, + Publisher justifiedCheckpoints, + Publisher wireAttestations, + Publisher importedBlocks) { + this.spec = spec; + this.storage = storage; + + Scheduler scheduler = schedulers.newSingleThreadDaemon("lmd-ghost-processor").toReactor(); + this.chainHeadStream = new SimpleProcessor<>(scheduler, "LMDGhostProcessor.chainHeadStream"); + + Flux.from(justifiedCheckpoints).publishOn(scheduler).subscribe(this::onNewJustifiedCheckpoint); + Flux.from(wireAttestations).publishOn(scheduler).subscribe(this::onNewAttestation); + Flux.from(importedBlocks).publishOn(scheduler).subscribe(this::onNewImportedBlock); + } + + private void onNewImportedBlock(BeaconTuple tuple) { + if (!isJustifiedAncestor(tuple.getBlock())) { + return; + } + + for (Attestation attestation : tuple.getBlock().getBody().getAttestations()) { + List indices = + spec.get_attesting_indices( + tuple.getState(), attestation.getData(), attestation.getAggregationBits()); + processAttestation(indices, attestation.getData()); + } + + updateHead(); + } + + private boolean isJustifiedAncestor(BeaconBlock block) { + // genesis shortcut + if (justifiedCheckpoint.equals(Checkpoint.EMPTY) && block.getSlot().equals(SlotNumber.ZERO)) { + return true; + } + + BeaconBlock ancestor = block; + while (spec.compute_epoch_of_slot(ancestor.getSlot()) + .greaterEqual(justifiedCheckpoint.getEpoch())) { + Optional parent = storage.getBlockStorage().get(ancestor.getParentRoot()); + if (!parent.isPresent()) { + return false; + } + if (parent.get().getParentRoot().equals(justifiedCheckpoint.getRoot())) { + return true; + } + ancestor = parent.get(); + } + + return false; + } + + private void onNewAttestation(IndexedAttestation attestation) { + List indices = new ArrayList<>(attestation.getCustodyBit0Indices().listCopy()); + indices.addAll(attestation.getCustodyBit1Indices().listCopy()); + processAttestation(indices, attestation.getData()); + updateHead(); + } + + private void processAttestation(List indices, AttestationData data) { + LatestMessage message = + new LatestMessage(data.getTarget().getEpoch(), data.getBeaconBlockRoot()); + indices.forEach( + index -> { + latestMessageStorage.merge( + index, + message, + (oldMessage, newMessage) -> { + if (newMessage.getEpoch().greater(oldMessage.getEpoch())) { + return newMessage; + } else { + return oldMessage; + } + }); + }); + } + + private void updateHead() { + Hash32 newHeadRoot = getHeadRoot(); + if (!newHeadRoot.equals(currentHeadRoot)) { + BeaconTuple tuple = storage.getTupleStorage().get(newHeadRoot).get(); + currentHeadRoot = newHeadRoot; + chainHeadStream.onNext(new BeaconChainHead(tuple)); + } + } + + private Hash32 getHeadRoot() { + return spec.get_head( + new Store() { + + @Override + public Checkpoint getJustifiedCheckpoint() { + return storage.getJustifiedStorage().get().get(); + } + + @Override + public Checkpoint getFinalizedCheckpoint() { + return storage.getFinalizedStorage().get().get(); + } + + @Override + public Optional getBlock(Hash32 root) { + return storage.getBlockStorage().get(root); + } + + @Override + public Optional getState(Hash32 root) { + return storage.getStateStorage().get(root); + } + + @Override + public Optional getLatestMessage(ValidatorIndex index) { + return Optional.ofNullable(latestMessageStorage.get(index)); + } + + @Override + public List getChildren(Hash32 root) { + return storage.getBlockStorage().getChildren(root, SEARCH_LIMIT).stream() + .map(spec::signing_root) + .collect(Collectors.toList()); + } + }); + } + + private void onNewJustifiedCheckpoint(Checkpoint checkpoint) { + if (checkpoint.getEpoch().greater(justifiedCheckpoint.getEpoch())) { + justifiedCheckpoint = checkpoint; + resetLatestMessages(); + updateHead(); + } + } + + private void resetLatestMessages() { + latestMessageStorage.clear(); + } + + @Override + public Publisher getChainHeads() { + return chainHeadStream; + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java index c4b69af58..d91eb083f 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessor.java @@ -1,14 +1,41 @@ package org.ethereum.beacon.chain.observer; import org.ethereum.beacon.chain.BeaconChainHead; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.chain.pool.churn.AttestationChurnImpl; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; import org.reactivestreams.Publisher; public interface ObservableStateProcessor { void start(); - Publisher getHeadStream(); - Publisher getObservableStateStream(); - Publisher getPendingOperationsStream(); + static ObservableStateProcessor createNew( + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition, + Schedulers schedulers, + Publisher newSlots, + Publisher chainHeads, + Publisher justifiedCheckpoints, + Publisher finalizedCheckpoints, + Publisher validAttestations) { + AttestationChurn churn = new AttestationChurnImpl(spec, AttestationPool.ATTESTATION_CHURN_SIZE); + return new YetAnotherStateProcessor( + spec, + emptySlotTransition, + churn, + schedulers, + newSlots, + chainHeads, + justifiedCheckpoints, + finalizedCheckpoints, + validAttestations); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java index 686caa61f..df3cacbe4 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java @@ -14,7 +14,6 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.chain.BeaconChainHead; import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.BeaconTupleDetails; import org.ethereum.beacon.chain.LMDGhostHeadFunction; @@ -68,9 +67,7 @@ public class ObservableStateProcessorImpl implements ObservableStateProcessor { private final Map, Attestation> attestationCache = new HashMap<>(); private final Schedulers schedulers; - private final SimpleProcessor headStream; private final SimpleProcessor observableStateStream; - private final SimpleProcessor pendingOperationsStream; public ObservableStateProcessorImpl( BeaconChainStorage chainStorage, @@ -110,9 +107,7 @@ public ObservableStateProcessorImpl( this.schedulers = schedulers; this.maxEmptySlotTransitions = maxEmptySlotTransitions; - headStream = new SimpleProcessor<>(this.schedulers.events(), "ObservableStateProcessor.head"); observableStateStream = new SimpleProcessor<>(this.schedulers.events(), "ObservableStateProcessor.observableState"); - pendingOperationsStream = new SimpleProcessor<>(this.schedulers.events(), "PendingOperationsProcessor.pendingOperations"); } @Override @@ -230,7 +225,6 @@ private synchronized Map> copyAttestationCache private void newHead(BeaconTupleDetails head) { this.head = head; - headStream.onNext(new BeaconChainHead(this.head)); if (latestState == null) { latestState = head.getFinalState(); @@ -339,18 +333,8 @@ private void updateHead(BeaconState state) { newHead(tuple); } - @Override - public Publisher getHeadStream() { - return headStream; - } - @Override public Publisher getObservableStateStream() { return observableStateStream; } - - @Override - public Publisher getPendingOperationsStream() { - return pendingOperationsStream; - } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/YetAnotherStateProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/YetAnotherStateProcessor.java new file mode 100644 index 000000000..eba3fc2c4 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/YetAnotherStateProcessor.java @@ -0,0 +1,166 @@ +package org.ethereum.beacon.chain.observer; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.BeaconChainHead; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.OffChainAggregates; +import org.ethereum.beacon.chain.pool.churn.AttestationChurn; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.BeaconStateEx; +import org.ethereum.beacon.consensus.transition.BeaconStateExImpl; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.ProposerSlashing; +import org.ethereum.beacon.core.operations.Transfer; +import org.ethereum.beacon.core.operations.VoluntaryExit; +import org.ethereum.beacon.core.operations.slashing.AttesterSlashing; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.scheduler.Scheduler; + +public class YetAnotherStateProcessor implements ObservableStateProcessor { + + private final AttestationChurn churn; + private final SimpleProcessor stateStream; + private final EmptySlotTransition emptySlotTransition; + private final BeaconChainSpec spec; + private final Scheduler scheduler; + private final Publisher newSlots; + private final Publisher chainHeads; + private final Publisher justifiedCheckpoints; + private final Publisher finalizedCheckpoints; + private final Publisher validAttestations; + + private SlotNumber recentSlot = SlotNumber.ZERO; + private BeaconStateEx recentState; + private BeaconBlock recentHead; + + public YetAnotherStateProcessor( + BeaconChainSpec spec, + EmptySlotTransition emptySlotTransition, + AttestationChurn churn, + Schedulers schedulers, + Publisher newSlots, + Publisher chainHeads, + Publisher justifiedCheckpoints, + Publisher finalizedCheckpoints, + Publisher validAttestations) { + this.spec = spec; + this.emptySlotTransition = emptySlotTransition; + this.churn = churn; + this.scheduler = schedulers.newSingleThreadDaemon("yet-another-state-processor").toReactor(); + this.stateStream = new SimpleProcessor<>(scheduler, "YetAnotherStateProcessor.stateStream"); + this.newSlots = newSlots; + this.justifiedCheckpoints = justifiedCheckpoints; + this.finalizedCheckpoints = finalizedCheckpoints; + this.validAttestations = validAttestations; + this.chainHeads = chainHeads; + } + + @Override + public void start() { + Flux.from(chainHeads).publishOn(scheduler).subscribe(this::onNewHead); + Flux.from(justifiedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedJustifiedCheckpoint); + Flux.from(finalizedCheckpoints) + .publishOn(scheduler) + .subscribe(this.churn::feedFinalizedCheckpoint); + Flux.from(validAttestations) + .publishOn(scheduler) + .bufferTimeout( + AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL, scheduler) + .subscribe(this.churn::add); + Flux.from(newSlots).publishOn(scheduler).subscribe(this::onNewSlot); + } + + private void onNewHead(BeaconChainHead head) { + recentHead = head.getBlock(); + recentState = emptySlotTransition.apply(new BeaconStateExImpl(head.getState()), recentSlot); + publishObservableState(); + } + + private void onNewSlot(SlotNumber slot) { + if (slot.greater(recentSlot)) { + recentSlot = slot; + if (recentHead != null && recentState != null) { + recentState = emptySlotTransition.apply(recentState, slot); + publishObservableState(); + } + } + } + + private void publishObservableState() { + OffChainAggregates aggregates = churn.compute(BeaconTuple.of(recentHead, recentState)); + PendingOperations pendingOperations = new PendingOperationsImpl(aggregates); + stateStream.onNext(new ObservableBeaconState(recentHead, recentState, pendingOperations)); + } + + @Override + public Publisher getObservableStateStream() { + return stateStream; + } + + private final class PendingOperationsImpl implements PendingOperations { + + private final OffChainAggregates aggregates; + + private List attestations; + + public PendingOperationsImpl(OffChainAggregates aggregates) { + this.aggregates = aggregates; + } + + @Override + public List getAttestations() { + if (attestations == null) { + attestations = + aggregates.getAggregates().stream() + .map(aggregate -> aggregate.getAggregate(spec.getConstants())) + .collect(Collectors.toList()); + } + return attestations; + } + + @Override + public List peekProposerSlashings(int maxCount) { + return Collections.emptyList(); + } + + @Override + public List peekAttesterSlashings(int maxCount) { + return Collections.emptyList(); + } + + @Override + public List peekAggregateAttestations(int maxCount, SpecConstants specConstants) { + if (attestations == null) { + attestations = + aggregates.getAggregates().stream() + .limit(maxCount) + .map(aggregate -> aggregate.getAggregate(specConstants)) + .collect(Collectors.toList()); + } + return attestations.stream().limit(maxCount).collect(Collectors.toList()); + } + + @Override + public List peekExits(int maxCount) { + return Collections.emptyList(); + } + + @Override + public List peekTransfers(int maxCount) { + return Collections.emptyList(); + } + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java index b50888ba1..e9c588dcd 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/AttestationPool.java @@ -2,12 +2,10 @@ import java.time.Duration; import org.ethereum.beacon.chain.BeaconChain; -import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.chain.pool.churn.AttestationChurn; -import org.ethereum.beacon.chain.pool.churn.AttestationChurnImpl; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; @@ -16,6 +14,8 @@ import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.transition.EmptySlotTransition; import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.EpochNumber; import org.ethereum.beacon.core.types.SlotNumber; @@ -73,30 +73,31 @@ public interface AttestationPool { */ Publisher getValid(); + Publisher getValidUnboxed(); + /** - * Invalid attestations publisher. + * A publisher of valid indexed attestations. Publishes same attestations as {@link #getValid()} + * does. * * @return a publisher. */ - Publisher getInvalid(); + Publisher getValidIndexed(); /** - * Publishes attestations which block is not yet a known block. - * - *

    These attestations should be passed to a wire module in order to request a block. + * Invalid attestations publisher. * * @return a publisher. */ - Publisher getUnknownAttestations(); + Publisher getInvalid(); /** - * Publishes aggregated attestations that are not yet included on chain. + * Publishes attestations which block is not yet a known block. * - *

    It should be a source of attestations for block proposer. + *

    These attestations should be passed to a wire module in order to request a block. * * @return a publisher. */ - Publisher getAggregates(); + Publisher getUnknownAttestations(); /** Launches the pool. */ void start(); @@ -104,10 +105,8 @@ public interface AttestationPool { static AttestationPool create( Publisher source, Publisher newSlots, - Publisher justifiedCheckpoints, Publisher finalizedCheckpoints, Publisher importedBlocks, - Publisher chainHeads, Schedulers schedulers, BeaconChainSpec spec, BeaconChainStorage storage, @@ -123,22 +122,18 @@ static AttestationPool create( storage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); BatchVerifier batchVerifier = new AttestationVerifier(storage.getTupleStorage(), spec, emptySlotTransition); - AttestationChurn attestationChurn = new AttestationChurnImpl(spec, ATTESTATION_CHURN_SIZE); return new InMemoryAttestationPool( source, newSlots, - justifiedCheckpoints, finalizedCheckpoints, importedBlocks, - chainHeads, schedulers, timeFrameFilter, sanityChecker, encodingChecker, processedFilter, unknownAttestationPool, - batchVerifier, - attestationChurn); + batchVerifier); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java index 0576e225e..95848c8d4 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPool.java @@ -5,8 +5,6 @@ import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; -import org.ethereum.beacon.chain.pool.churn.AttestationChurn; -import org.ethereum.beacon.chain.pool.reactor.ChurnProcessor; import org.ethereum.beacon.chain.pool.reactor.DoubleWorkProcessor; import org.ethereum.beacon.chain.pool.reactor.IdentificationProcessor; import org.ethereum.beacon.chain.pool.reactor.SanityProcessor; @@ -17,6 +15,8 @@ import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.schedulers.Schedulers; @@ -33,10 +33,8 @@ public class InMemoryAttestationPool implements AttestationPool { private final Publisher source; private final Publisher newSlots; - private final Publisher justifiedCheckpoints; private final Publisher finalizedCheckpoints; private final Publisher importedBlocks; - private final Publisher chainHeads; private final Schedulers schedulers; private final TimeFrameFilter timeFrameFilter; @@ -45,34 +43,29 @@ public class InMemoryAttestationPool implements AttestationPool { private final ProcessedAttestations processedFilter; private final UnknownAttestationPool unknownPool; private final BatchVerifier verifier; - private final AttestationChurn attestationChurn; private final DirectProcessor invalidAttestations = DirectProcessor.create(); private final DirectProcessor validAttestations = DirectProcessor.create(); + private final DirectProcessor validIndexedAttestations = + DirectProcessor.create(); private final DirectProcessor unknownAttestations = DirectProcessor.create(); - private final DirectProcessor offChainAggregates = DirectProcessor.create(); public InMemoryAttestationPool( Publisher source, Publisher newSlots, - Publisher justifiedCheckpoints, Publisher finalizedCheckpoints, Publisher importedBlocks, - Publisher chainHeads, Schedulers schedulers, TimeFrameFilter timeFrameFilter, SanityChecker sanityChecker, SignatureEncodingChecker encodingChecker, ProcessedAttestations processedFilter, UnknownAttestationPool unknownPool, - BatchVerifier batchVerifier, - AttestationChurn attestationChurn) { + BatchVerifier batchVerifier) { this.source = source; this.newSlots = newSlots; - this.justifiedCheckpoints = justifiedCheckpoints; this.finalizedCheckpoints = finalizedCheckpoints; this.importedBlocks = importedBlocks; - this.chainHeads = chainHeads; this.schedulers = schedulers; this.timeFrameFilter = timeFrameFilter; this.sanityChecker = sanityChecker; @@ -80,7 +73,6 @@ public InMemoryAttestationPool( this.processedFilter = processedFilter; this.unknownPool = unknownPool; this.verifier = batchVerifier; - this.attestationChurn = attestationChurn; } @Override @@ -91,10 +83,8 @@ public void start() { // create sources Flux sourceFx = Flux.from(source); Flux newSlotsFx = Flux.from(newSlots); - Flux justifiedCheckpointsFx = Flux.from(justifiedCheckpoints); Flux finalizedCheckpointsFx = Flux.from(finalizedCheckpoints); Flux importedBlocksFx = Flux.from(importedBlocks); - Flux chainHeadsFx = Flux.from(chainHeads); // check time frames TimeProcessor timeProcessor = @@ -126,28 +116,18 @@ public void start() { VerificationProcessor verificationProcessor = new VerificationProcessor(verifier, parallelExecutor, verificationThrottle); - // feed churn - ChurnProcessor churnProcessor = - new ChurnProcessor( - attestationChurn, - schedulers, - newSlotsFx, - chainHeadsFx, - justifiedCheckpointsFx, - finalizedCheckpointsFx, - verificationProcessor.getValid()); - Scheduler outScheduler = schedulers.events().toReactor(); // expose valid attestations Flux.from(verificationProcessor.getValid()) .publishOn(outScheduler) .subscribe(validAttestations); + Flux.from(verificationProcessor.getValidIndexed()) + .publishOn(outScheduler) + .subscribe(validIndexedAttestations); // expose not yet identified Flux.from(identificationProcessor.getUnknown()) .publishOn(outScheduler) .subscribe(unknownAttestations); - // expose aggregates - churnProcessor.publishOn(outScheduler).subscribe(offChainAggregates); // expose invalid attestations Flux.merge( sanityProcessor.getInvalid(), @@ -162,6 +142,16 @@ public Publisher getValid() { return validAttestations; } + @Override + public Publisher getValidUnboxed() { + return validAttestations.map(ReceivedAttestation::getMessage); + } + + @Override + public Publisher getValidIndexed() { + return validIndexedAttestations; + } + @Override public Publisher getInvalid() { return invalidAttestations; @@ -171,9 +161,4 @@ public Publisher getInvalid() { public Publisher getUnknownAttestations() { return unknownAttestations; } - - @Override - public Publisher getAggregates() { - return offChainAggregates; - } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java deleted file mode 100644 index 67d98ef00..000000000 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/churn/OffChainAggregates.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.ethereum.beacon.chain.pool.churn; - -import java.util.List; -import org.ethereum.beacon.core.operations.Attestation; -import tech.pegasys.artemis.ethereum.core.Hash32; - -/** - * A DTO for aggregated attestations that are not yet included on chain. - * - *

    Beacon block proposer should be fed with this data. - */ -public class OffChainAggregates { - private final Hash32 blockRoot; - private final List aggregates; - - public OffChainAggregates(Hash32 blockRoot, List aggregates) { - this.blockRoot = blockRoot; - this.aggregates = aggregates; - } - - public Hash32 getBlockRoot() { - return blockRoot; - } - - public List getAggregates() { - return aggregates; - } -} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java index d3ac414b4..5e763ded9 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/ChurnProcessor.java @@ -30,6 +30,8 @@ public ChurnProcessor( this.churn = churn; Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-churn-processor").toReactor(); + this.out = new SimpleProcessor<>(scheduler, "ChurnProcessor.out"); + Flux.from(chainHeads).publishOn(scheduler).subscribe(this::hookOnNext); Flux.from(newSlots).publishOn(scheduler).subscribe(this.churn::feedNewSlot); Flux.from(justifiedCheckpoints) @@ -44,8 +46,6 @@ public ChurnProcessor( .bufferTimeout( AttestationPool.VERIFIER_BUFFER_SIZE, AttestationPool.VERIFIER_INTERVAL, scheduler) .subscribe(this.churn::add); - - out = new SimpleProcessor<>(scheduler, "ChurnProcessor.out"); } private void hookOnNext(BeaconTuple tuple) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java index edcb9ba79..9101b0dcb 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java @@ -37,7 +37,8 @@ public DoubleWorkProcessor( Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-double-work-processor").toReactor(); - out = new SimpleProcessor<>(scheduler, "DoubleWorkProcessor.out"); + this.out = new SimpleProcessor<>(scheduler, "DoubleWorkProcessor.out"); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java index d6058364e..9fd8e7b3a 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessor.java @@ -45,12 +45,12 @@ public IdentificationProcessor( Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-identification-processor").toReactor(); + this.identified = new SimpleProcessor<>(scheduler, "IdentificationProcessor.identified"); + this.unknown = new SimpleProcessor<>(scheduler, "IdentificationProcessor.unknown"); + Flux.from(newSlots).publishOn(scheduler).subscribe(this.pool::feedNewSlot); Flux.from(newImportedBlocks).publishOn(scheduler).subscribe(this::hookOnNext); Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); - - identified = new SimpleProcessor<>(scheduler, "IdentificationProcessor.identified"); - unknown = new SimpleProcessor<>(scheduler, "IdentificationProcessor.unknown"); } private void hookOnNext(BeaconBlock block) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java index 9491a257d..5ea29efda 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java @@ -41,13 +41,13 @@ public SanityProcessor( this.checker = checker; Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-sanity-processor").toReactor(); + this.valid = new SimpleProcessor<>(scheduler, "SanityProcessor.valid"); + this.invalid = new SimpleProcessor<>(scheduler, "SanityProcessor.invalid"); + Flux.from(finalizedCheckpoints) .publishOn(scheduler) .subscribe(this.checker::feedFinalizedCheckpoint); Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); - - valid = new SimpleProcessor<>(scheduler, "SanityProcessor.valid"); - invalid = new SimpleProcessor<>(scheduler, "SanityProcessor.invalid"); } private void hookOnNext(ReceivedAttestation attestation) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java index 78b04267c..09b54e097 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessor.java @@ -35,9 +35,10 @@ public SignatureEncodingProcessor( Publisher source) { this.checker = checker; + this.valid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.valid"); + this.invalid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.invalid"); + Flux.from(source).publishOn(scheduler.toReactor()).subscribe(this::hookOnNext); - valid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.valid"); - invalid = new SimpleProcessor<>(scheduler, "SignatureEncodingProcessor.invalid"); } private void hookOnNext(ReceivedAttestation attestation) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java index 41f286e2b..e70173b6f 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java @@ -42,7 +42,8 @@ public TimeProcessor( this.filter = filter; Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-time-processor").toReactor(); - out = new SimpleProcessor<>(scheduler, "TimeProcessor.out"); + this.out = new SimpleProcessor<>(scheduler, "TimeProcessor.out"); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); Flux.from(finalizedCheckpoints) .publishOn(scheduler) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java index 388fe0084..5401b9f2c 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/VerificationProcessor.java @@ -4,6 +4,7 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; import org.ethereum.beacon.chain.pool.verifier.VerificationResult; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; import org.ethereum.beacon.schedulers.Scheduler; import org.ethereum.beacon.stream.SimpleProcessor; import org.reactivestreams.Publisher; @@ -30,20 +31,24 @@ public class VerificationProcessor { private final BatchVerifier verifier; private final SimpleProcessor valid; + private final SimpleProcessor validIndexed; private final SimpleProcessor invalid; public VerificationProcessor( BatchVerifier verifier, Scheduler scheduler, Publisher> source) { this.verifier = verifier; + this.valid = new SimpleProcessor<>(scheduler, "VerificationProcessor.valid"); + this.validIndexed = new SimpleProcessor<>(scheduler, "VerificationProcessor.validIndexed"); + this.invalid = new SimpleProcessor<>(scheduler, "VerificationProcessor.invalid"); + Flux.from(source).publishOn(scheduler.toReactor()).subscribe(this::hookOnNext); - valid = new SimpleProcessor<>(scheduler, "VerificationProcessor.valid"); - invalid = new SimpleProcessor<>(scheduler, "VerificationProcessor.invalid"); } private void hookOnNext(List batch) { VerificationResult result = verifier.verify(batch); result.getValid().forEach(valid::onNext); + result.getValidIndexed().forEach(validIndexed::onNext); result.getInvalid().forEach(invalid::onNext); } @@ -51,6 +56,10 @@ public Publisher getValid() { return valid; } + public SimpleProcessor getValidIndexed() { + return validIndexed; + } + public Publisher getInvalid() { return invalid; } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java index 7b1b20556..d2155a973 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/verifier/AggregateSignatureVerifier.java @@ -83,6 +83,7 @@ public VerificationResult verify() { */ private VerificationResult verifyGroup(AttestationData data, List group) { final List valid = new ArrayList<>(); + final List validIndexed = new ArrayList<>(); final List invalid = new ArrayList<>(); // for aggregation sake, smaller aggregates should go first @@ -106,7 +107,10 @@ private VerificationResult verifyGroup(AttestationData data, List { + valid.add(attestation.getAttestation()); + validIndexed.add(attestation.getIndexed()); + }); } else { notAggregated = group; } @@ -115,12 +119,13 @@ private VerificationResult verifyGroup(AttestationData data, List attestations) { - return new VerificationResult(Collections.emptyList(), attestations); + return new VerificationResult(Collections.emptyList(), Collections.emptyList(), attestations); } public static final VerificationResult EMPTY = - new VerificationResult(Collections.emptyList(), Collections.emptyList()); + new VerificationResult(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); private final List valid; + private final List validIndexed; private final List invalid; - VerificationResult(List valid, List invalid) { + VerificationResult( + List valid, + List validIndexed, + List invalid) { this.valid = valid; + this.validIndexed = validIndexed; this.invalid = invalid; } @@ -27,16 +33,22 @@ public List getValid() { return valid; } + public List getValidIndexed() { + return validIndexed; + } + public List getInvalid() { return invalid; } public VerificationResult merge(VerificationResult other) { List valid = new ArrayList<>(this.valid); + List validIndexed = new ArrayList<>(this.validIndexed); List invalid = new ArrayList<>(this.invalid); valid.addAll(other.valid); + validIndexed.addAll(other.validIndexed); invalid.addAll(other.invalid); - return new VerificationResult(valid, invalid); + return new VerificationResult(valid, validIndexed, invalid); } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 048639af5..90e678c2c 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -163,10 +163,8 @@ void integrationTest() { final AttestationPool pool = AttestationPool.create( source, newSlots, - justifiedCheckpoints, finalizedCheckpoints, importedBlocks, - chainHeads, schedulers, spec, beaconChainStorage, @@ -239,23 +237,19 @@ void integrationTest2() { beaconChainStorage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); final BatchVerifier batchVerifier = new AttestationVerifier(beaconChainStorage.getTupleStorage(), spec, slotTransition); - final AttestationChurn attestationChurn = AttestationChurn.create(spec, ATTESTATION_CHURN_SIZE); final AttestationPool pool = new InMemoryAttestationPool( source, newSlots, - justifiedCheckpoints, finalizedCheckpoints, importedBlocks, - chainHeads, schedulers, timeFrameFilter, sanityChecker, encodingChecker, processedFilter, unknownAttestationPool, - batchVerifier, - attestationChurn); + batchVerifier); pool.start(); } @@ -280,7 +274,6 @@ private MutableBeaconChain createBeaconChain( return new DefaultBeaconChain( spec, - initialTransition, new EmptySlotTransition( new ExtendedSlotTransition(new PerEpochTransition(spec) { @Override diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java index 072e70f2d..6081b38a9 100644 --- a/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java @@ -1,16 +1,22 @@ package org.ethereum.beacon.start.common; +import java.util.List; import org.ethereum.beacon.bench.BenchmarkController; import org.ethereum.beacon.bench.BenchmarkController.BenchmarkRoutine; +import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.DefaultBeaconChain; +import org.ethereum.beacon.chain.ForkChoiceProcessor; +import org.ethereum.beacon.chain.LMDGhostProcessor; import org.ethereum.beacon.chain.MutableBeaconChain; import org.ethereum.beacon.chain.ProposedBlockProcessor; import org.ethereum.beacon.chain.ProposedBlockProcessorImpl; import org.ethereum.beacon.chain.SlotTicker; import org.ethereum.beacon.chain.observer.ObservableStateProcessor; -import org.ethereum.beacon.chain.observer.ObservableStateProcessorImpl; +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.BeaconChainStorageFactory; +import org.ethereum.beacon.chain.storage.util.StorageUtils; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.BeaconStateEx; import org.ethereum.beacon.consensus.ChainStart; @@ -29,7 +35,6 @@ import org.ethereum.beacon.db.InMemoryDatabase; import org.ethereum.beacon.pow.DepositContract; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.chain.storage.util.StorageUtils; import org.ethereum.beacon.util.stats.MeasurementsCollector; import org.ethereum.beacon.validator.BeaconChainProposer; import org.ethereum.beacon.validator.attester.BeaconChainAttesterImpl; @@ -41,8 +46,6 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.List; - public class Launcher { private final BeaconChainSpec spec; private final DepositContract depositContract; @@ -64,6 +67,8 @@ public class Launcher { private BeaconChainStorage beaconChainStorage; private MutableBeaconChain beaconChain; private SlotTicker slotTicker; + private AttestationPool attestationPool; + private ForkChoiceProcessor forkChoiceProcessor; private ObservableStateProcessor observableStateProcessor; private BeaconChainProposer beaconChainProposer; private BeaconChainAttesterImpl beaconChainAttester; @@ -150,14 +155,37 @@ void chainStarted(ChainStart chainStartEvent) { .publishOn(schedulers.events().toReactor()) .subscribe(allAttestations); - observableStateProcessor = new ObservableStateProcessorImpl( - beaconChainStorage, - slotTicker.getTickerStream(), - allAttestations, - beaconChain.getBlockStatesStream(), - spec, - emptySlotTransition, - schedulers); + attestationPool = + AttestationPool.create( + allAttestations.map(ReceivedAttestation::new), + slotTicker.getTickerStream(), + beaconChain.getFinalizedCheckpoints(), + Flux.from(beaconChain.getBlockStatesStream()).map(BeaconTuple::getBlock), + schedulers, + spec, + beaconChainStorage, + emptySlotTransition); + attestationPool.start(); + + forkChoiceProcessor = + new LMDGhostProcessor( + spec, + beaconChainStorage, + schedulers, + beaconChain.getJustifiedCheckpoints(), + attestationPool.getValidIndexed(), + beaconChain.getBlockStatesStream()); + + observableStateProcessor = + ObservableStateProcessor.createNew( + spec, + emptySlotTransition, + schedulers, + slotTicker.getTickerStream(), + forkChoiceProcessor.getChainHeads(), + beaconChain.getJustifiedCheckpoints(), + beaconChain.getFinalizedCheckpoints(), + attestationPool.getValidUnboxed()); observableStateProcessor.start(); if (validatorCred != null) { @@ -348,4 +376,12 @@ public MeasurementsCollector getEpochCollector() { public MeasurementsCollector getBlockCollector() { return blockCollector; } + + public AttestationPool getAttestationPool() { + return attestationPool; + } + + public ForkChoiceProcessor getForkChoiceProcessor() { + return forkChoiceProcessor; + } } diff --git a/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java b/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java index 50e60a05b..e4229c3aa 100644 --- a/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java +++ b/validator/server/src/test/java/org/ethereum/beacon/validator/api/ServiceFactory.java @@ -105,11 +105,6 @@ public static ObservableStateProcessor createObservableStateProcessor(SpecConsta @Override public void start() {} - @Override - public Publisher getHeadStream() { - return null; - } - @Override public Publisher getObservableStateStream() { return Mono.just( @@ -124,11 +119,6 @@ public Publisher getObservableStateStream() { BeaconStateEx.getEmpty(), new PendingOperationsState(Collections.emptyList()))); } - - @Override - public Publisher getPendingOperationsStream() { - return null; - } }; } @@ -138,11 +128,6 @@ public static ObservableStateProcessor createObservableStateProcessorGenesisTime @Override public void start() {} - @Override - public Publisher getHeadStream() { - return null; - } - @Override public Publisher getObservableStateStream() { MutableBeaconState state = BeaconStateEx.getEmpty().createMutableCopy(); @@ -159,11 +144,6 @@ public Publisher getObservableStateStream() { new BeaconStateExImpl(state.createImmutable()), new PendingOperationsState(Collections.emptyList()))); } - - @Override - public Publisher getPendingOperationsStream() { - return null; - } }; } @@ -173,11 +153,6 @@ public static ObservableStateProcessor createObservableStateProcessorWithValidat @Override public void start() {} - @Override - public Publisher getHeadStream() { - return null; - } - @Override public Publisher getObservableStateStream() { MutableBeaconState state = BeaconStateEx.getEmpty().createMutableCopy(); @@ -198,11 +173,6 @@ public Publisher getObservableStateStream() { new BeaconStateExImpl(state.createImmutable()), new PendingOperationsState(Collections.emptyList()))); } - - @Override - public Publisher getPendingOperationsStream() { - return null; - } }; } @@ -469,6 +439,16 @@ public BeaconTuple getRecentlyProcessed() { return null; } + @Override + public Publisher getJustifiedCheckpoints() { + return null; + } + + @Override + public Publisher getFinalizedCheckpoints() { + return null; + } + @Override public void init() {} }; From c00918e098fe6a5093f362b1c4a3b5b5769e5381 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Tue, 10 Sep 2019 14:02:48 +0300 Subject: [PATCH 42/59] pool: test valid TimeProcessor --- .../chain/pool/reactor/TimeProcessor.java | 11 ++++- .../pool/InMemoryAttestationPoolTest.java | 46 ++++++++++--------- .../chain/pool/reactor/TimeProcessorTest.java | 40 +++++++++------- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java index 41f286e2b..4ce3fa0bd 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessor.java @@ -43,11 +43,18 @@ public TimeProcessor( Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-time-processor").toReactor(); out = new SimpleProcessor<>(scheduler, "TimeProcessor.out"); - Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + + Flux.from(source) + .publishOn(scheduler) + .subscribe(this::hookOnNext); + Flux.from(finalizedCheckpoints) .publishOn(scheduler) .subscribe(this.filter::feedFinalizedCheckpoint); - Flux.from(newSlots).publishOn(scheduler).subscribe(this.filter::feedNewSlot); + + Flux.from(newSlots) + .publishOn(scheduler) + .subscribe(this.filter::feedNewSlot); } private void hookOnNext(ReceivedAttestation attestation) { diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 048639af5..25409cefa 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -1,13 +1,6 @@ package org.ethereum.beacon.chain.pool; -import static org.ethereum.beacon.chain.pool.AttestationPool.ATTESTATION_CHURN_SIZE; -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; - -import java.util.Collections; import org.ethereum.beacon.chain.BeaconTuple; -import org.ethereum.beacon.chain.DefaultBeaconChain; import org.ethereum.beacon.chain.MutableBeaconChain; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; @@ -64,6 +57,13 @@ import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.Collections; + +import static org.ethereum.beacon.chain.pool.AttestationPool.ATTESTATION_CHURN_SIZE; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; + class InMemoryAttestationPoolTest { private Schedulers schedulers = Schedulers.createControlled(); @@ -278,21 +278,23 @@ private MutableBeaconChain createBeaconChain( spec.getObjectHasher(), SerializerFactory.createSSZ(spec.getConstants())) .create(database); - return new DefaultBeaconChain( - spec, - initialTransition, - new EmptySlotTransition( - new ExtendedSlotTransition(new PerEpochTransition(spec) { - @Override - public BeaconStateEx apply(BeaconStateEx stateEx) { - return perEpochTransition.apply(stateEx); - } - }, perSlotTransition, spec)), - perBlockTransition, - blockVerifier, - stateVerifier, - chainStorage, - schedulers); +// return new DefaultBeaconChain( +// spec, +// initialTransition, +// new EmptySlotTransition( +// new ExtendedSlotTransition(new PerEpochTransition(spec) { +// @Override +// public BeaconStateEx apply(BeaconStateEx stateEx) { +// return perEpochTransition.apply(stateEx); +// } +// }, perSlotTransition, spec)), +// perBlockTransition, +// blockVerifier, +// stateVerifier, +// chainStorage, +// schedulers); + + return null; } private BeaconTuple createBlock( diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java index dc3f2e267..9d7cef586 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java @@ -16,17 +16,19 @@ import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.schedulers.ControlledSchedulers; import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.types.p2p.NodeId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.core.publisher.DirectProcessor; -import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; -import reactor.test.StepVerifier; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; class TimeProcessorTest { @@ -71,32 +73,36 @@ public SlotNumber.EpochLength getSlotsPerEpoch() { private DirectProcessor source = DirectProcessor.create(); final FluxSink sourceSink = source.sink(); - private Flux finalizedCheckpoints = Flux.empty(); - private Flux newSlots = Flux.empty(); + private DirectProcessor finalizedCheckpoints = DirectProcessor.create(); + private DirectProcessor newSlots = DirectProcessor.create(); @BeforeEach void setUp() { final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); - timeFrameFilter.feedFinalizedCheckpoint(checkpoint); - timeFrameFilter.feedNewSlot(slotNumber); - - timeProcessor = new TimeProcessor(timeFrameFilter, schedulers, source, finalizedCheckpoints, newSlots); - StepVerifier.create(timeProcessor) - .verifyComplete(); - -// timeProcessor.blockFirst() -// sourceSink.next(); - -// schedulers.addTime(1); + finalizedCheckpoints.onNext(this.checkpoint); + newSlots.onNext(slotNumber); + schedulers.addTime(Duration.ofSeconds(5)); } - @Test void testValidAttestation() { - + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + timeProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(sender); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(message); + }); } private Attestation createAttestation(BytesValue someValue) { From 11bb8009e8b3d53c63cd53a2a035470c6f3d06e7 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Tue, 10 Sep 2019 14:59:03 +0300 Subject: [PATCH 43/59] pool: test valid SanityProcessor --- .../chain/pool/reactor/SanityProcessor.java | 6 +- .../chain/pool/PoolTestConfigurator.java | 80 +++++++++++++++++++ .../pool/reactor/SanityProcessorTest.java | 52 ++++++++++++ .../chain/pool/reactor/TimeProcessorTest.java | 76 +----------------- 4 files changed, 139 insertions(+), 75 deletions(-) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java index 9491a257d..382171eee 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessor.java @@ -41,10 +41,14 @@ public SanityProcessor( this.checker = checker; Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-sanity-processor").toReactor(); + Flux.from(finalizedCheckpoints) .publishOn(scheduler) .subscribe(this.checker::feedFinalizedCheckpoint); - Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); + + Flux.from(source) + .publishOn(scheduler) + .subscribe(this::hookOnNext); valid = new SimpleProcessor<>(scheduler, "SanityProcessor.valid"); invalid = new SimpleProcessor<>(scheduler, "SanityProcessor.invalid"); diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java new file mode 100644 index 000000000..b2bc526eb --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java @@ -0,0 +1,80 @@ +package org.ethereum.beacon.chain.pool; + +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.ShardNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.schedulers.ControlledSchedulers; +import org.ethereum.beacon.schedulers.Schedulers; +import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.Bitlist; +import tech.pegasys.artemis.util.uint.UInt64; + +public class PoolTestConfigurator { + + protected final SpecConstants specConstants = + new SpecConstants() { + @Override + public SlotNumber getGenesisSlot() { + return SlotNumber.of(12345); + } + + @Override + public Time getSecondsPerSlot() { + return Time.of(1); + } + }; + + protected final BeaconChainSpec spec = BeaconChainSpec.Builder.createWithDefaultParams() + .withConstants(new SpecConstants() { + @Override + public ShardNumber getShardCount() { + return ShardNumber.of(16); + } + + @Override + public SlotNumber.EpochLength getSlotsPerEpoch() { + return new SlotNumber.EpochLength(UInt64.valueOf(4)); + } + }) + .withComputableGenesisTime(false) + .withVerifyDepositProof(false) + .withBlsVerifyProofOfPossession(false) + .withBlsVerify(false) + .withCache(true) + .build(); + + protected DirectProcessor finalizedCheckpoints = DirectProcessor.create(); + protected final ControlledSchedulers schedulers = Schedulers.createControlled(); + protected DirectProcessor source = DirectProcessor.create(); + + protected Attestation createAttestation(BytesValue someValue) { + final AttestationData attestationData = createAttestationData(); + + return new Attestation( + Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), + attestationData, + Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), + BLSSignature.wrap(Bytes96.fromHexString("cc")), + specConstants); + } + + protected AttestationData createAttestationData() { + + return new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(1), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java new file mode 100644 index 000000000..846484d53 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java @@ -0,0 +1,52 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.time.Duration; + +class SanityProcessorTest extends PoolTestConfigurator { + + private SanityProcessor sanityProcessor; + + @BeforeEach + void setUp() { + final SanityChecker sanityChecker = new SanityChecker(spec); + sanityProcessor = new SanityProcessor(sanityChecker, schedulers, source, finalizedCheckpoints); + final Checkpoint checkpoint = Checkpoint.EMPTY; + finalizedCheckpoints.onNext(checkpoint); + + schedulers.addTime(Duration.ofSeconds(5)); + } + + @Test + void testValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); +//TODO: assertThat +// StepVerifier.create(sanityProcessor.getValid()) +// .expectNext(attestation) +// .verifyComplete(); + } + + @Test + void testInvalidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); +//TODO: assertThat +// StepVerifier.create(sanityProcessor.getInvalid()) +// .expectNext(attestation) +// .verifyComplete(); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java index 9d7cef586..9c09fc6d0 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java @@ -1,79 +1,28 @@ package org.ethereum.beacon.chain.pool.reactor; +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; -import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.Attestation; -import org.ethereum.beacon.core.operations.attestation.AttestationData; -import org.ethereum.beacon.core.operations.attestation.Crosslink; -import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.types.BLSSignature; -import org.ethereum.beacon.core.types.EpochNumber; -import org.ethereum.beacon.core.types.ShardNumber; import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.core.types.Time; -import org.ethereum.beacon.crypto.Hashes; -import org.ethereum.beacon.schedulers.ControlledSchedulers; -import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.types.p2p.NodeId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.core.publisher.DirectProcessor; -import reactor.core.publisher.FluxSink; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.collections.Bitlist; -import tech.pegasys.artemis.util.uint.UInt64; import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; -class TimeProcessorTest { - - private final SpecConstants specConstants = - new SpecConstants() { - @Override - public SlotNumber getGenesisSlot() { - return SlotNumber.of(12345); - } - - @Override - public Time getSecondsPerSlot() { - return Time.of(1); - } - }; - - private final BeaconChainSpec spec = BeaconChainSpec.Builder.createWithDefaultParams() - .withConstants(new SpecConstants() { - @Override - public ShardNumber getShardCount() { - return ShardNumber.of(16); - } - - @Override - public SlotNumber.EpochLength getSlotsPerEpoch() { - return new SlotNumber.EpochLength(UInt64.valueOf(4)); - } - }) - .withComputableGenesisTime(false) - .withVerifyDepositProof(false) - .withBlsVerifyProofOfPossession(false) - .withBlsVerify(false) - .withCache(true) - .build(); +class TimeProcessorTest extends PoolTestConfigurator { private final Checkpoint checkpoint = Checkpoint.EMPTY; private final SlotNumber slotNumber = SlotNumber.of(100L); - private final ControlledSchedulers schedulers = Schedulers.createControlled(); private TimeProcessor timeProcessor; - private DirectProcessor source = DirectProcessor.create(); - final FluxSink sourceSink = source.sink(); - - private DirectProcessor finalizedCheckpoints = DirectProcessor.create(); private DirectProcessor newSlots = DirectProcessor.create(); @BeforeEach @@ -104,25 +53,4 @@ void testValidAttestation() { .isEqualTo(message); }); } - - private Attestation createAttestation(BytesValue someValue) { - final AttestationData attestationData = createAttestationData(); - - return new Attestation( - Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), - attestationData, - Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), - BLSSignature.wrap(Bytes96.fromHexString("cc")), - specConstants); - } - - private AttestationData createAttestationData() { - - return new AttestationData( - Hashes.sha256(BytesValue.fromHexString("aa")), - new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), - new Checkpoint(EpochNumber.of(1), Hashes.sha256(BytesValue.fromHexString("cc"))), - Crosslink.EMPTY); - } - } From 5e0d7bdacf243ea3c1cd91ac1eaf1874316334b9 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Tue, 10 Sep 2019 15:12:47 +0300 Subject: [PATCH 44/59] pool: correct testing valid/invalid SanityProcessor --- .../beacon/chain/pool/PoolTestConfigurator.java | 8 +++++--- .../chain/pool/reactor/SanityProcessorTest.java | 13 ++++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java index b2bc526eb..18a834a8f 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java @@ -59,8 +59,10 @@ public SlotNumber.EpochLength getSlotsPerEpoch() { protected DirectProcessor source = DirectProcessor.create(); protected Attestation createAttestation(BytesValue someValue) { - final AttestationData attestationData = createAttestationData(); + return createAttestation(someValue, createAttestationData()); + } + protected Attestation createAttestation(BytesValue someValue, AttestationData attestationData) { return new Attestation( Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), attestationData, @@ -69,12 +71,12 @@ protected Attestation createAttestation(BytesValue someValue) { specConstants); } - protected AttestationData createAttestationData() { + private AttestationData createAttestationData() { return new AttestationData( Hashes.sha256(BytesValue.fromHexString("aa")), new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), - new Checkpoint(EpochNumber.of(1), Hashes.sha256(BytesValue.fromHexString("cc"))), + new Checkpoint(EpochNumber.of(232), Hashes.sha256(BytesValue.fromHexString("cc"))), Crosslink.EMPTY); } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java index 846484d53..cf65e55e3 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SanityProcessorTest.java @@ -4,7 +4,11 @@ import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.types.p2p.NodeId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,6 +37,7 @@ void testValidAttestation() { final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); source.onNext(attestation); //TODO: assertThat +// schedulers.addTime(Duration.ofSeconds(5)); // StepVerifier.create(sanityProcessor.getValid()) // .expectNext(attestation) // .verifyComplete(); @@ -41,10 +46,16 @@ void testValidAttestation() { @Test void testInvalidAttestation() { final NodeId sender = new NodeId(new byte[100]); - final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final AttestationData invalidAttestationData = new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(2), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + final Attestation message = createAttestation(BytesValue.fromHexString("aa"), invalidAttestationData); final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); source.onNext(attestation); //TODO: assertThat +// schedulers.addTime(Duration.ofSeconds(5)); // StepVerifier.create(sanityProcessor.getInvalid()) // .expectNext(attestation) // .verifyComplete(); From 6f46160a0583ed58938c05d329d0b9c2cf806447 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Tue, 10 Sep 2019 16:44:53 +0300 Subject: [PATCH 45/59] pool: test DoubleWorkProcessor --- .../pool/reactor/DoubleWorkProcessor.java | 5 +- .../pool/reactor/DoubleWorkProcessorTest.java | 58 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessorTest.java diff --git a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java index edcb9ba79..c28055075 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessor.java @@ -35,9 +35,10 @@ public DoubleWorkProcessor( Publisher source) { this.processedAttestations = processedAttestations; - Scheduler scheduler = - schedulers.newSingleThreadDaemon("pool-double-work-processor").toReactor(); + Scheduler scheduler = schedulers.newSingleThreadDaemon("pool-double-work-processor").toReactor(); + out = new SimpleProcessor<>(scheduler, "DoubleWorkProcessor.out"); + Flux.from(source).publishOn(scheduler).subscribe(this::hookOnNext); } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessorTest.java new file mode 100644 index 000000000..910e573d1 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/DoubleWorkProcessorTest.java @@ -0,0 +1,58 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; + +class DoubleWorkProcessorTest extends PoolTestConfigurator { + + private DoubleWorkProcessor doubleWorkProcessor; + private ReceivedAttestation attestation; + + @BeforeEach + void setUp() { + final ProcessedAttestations processedAttestations = new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); + + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.of(1, 2, 3)); + attestation = new ReceivedAttestation(sender, message); + final boolean added = processedAttestations.add(attestation); + assertThat(added).isTrue(); + + doubleWorkProcessor = new DoubleWorkProcessor(processedAttestations, schedulers, source); + assertThat(doubleWorkProcessor).isNotNull(); + + doubleWorkProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(sender); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(message); + }); + } + + @Test + void testAddAttestation() { + source.onNext(attestation); + + doubleWorkProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(attestation.getSender()); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(attestation.getMessage()); + }); + } +} From 5ab558c075fd974979c751d852a61c5a617d3d4b Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Tue, 10 Sep 2019 17:29:38 +0300 Subject: [PATCH 46/59] pool: test SignatureEncodingChecker --- .../chain/pool/PoolTestConfigurator.java | 27 ++++++++- .../checker/SignatureEncodingCheckerTest.java | 59 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingCheckerTest.java diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java index 18a834a8f..972bdef57 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java @@ -4,22 +4,34 @@ import org.ethereum.beacon.core.operations.Attestation; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.operations.deposit.DepositData; import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.BLSPubkey; import org.ethereum.beacon.core.types.BLSSignature; import org.ethereum.beacon.core.types.EpochNumber; import org.ethereum.beacon.core.types.ShardNumber; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.crypto.BLS381; import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.crypto.MessageParameters; import org.ethereum.beacon.schedulers.ControlledSchedulers; import org.ethereum.beacon.schedulers.Schedulers; import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes4; +import tech.pegasys.artemis.util.bytes.Bytes48; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.Random; + +import static org.ethereum.beacon.core.spec.SignatureDomains.DEPOSIT; + public class PoolTestConfigurator { protected final SpecConstants specConstants = @@ -63,11 +75,24 @@ protected Attestation createAttestation(BytesValue someValue) { } protected Attestation createAttestation(BytesValue someValue, AttestationData attestationData) { + final Random rnd = new Random(); + BLS381.KeyPair keyPair = BLS381.KeyPair.create(BLS381.PrivateKey.create(Bytes32.random(rnd))); + Hash32 withdrawalCredentials = Hash32.random(rnd); + DepositData depositDataWithoutSignature = new DepositData( + BLSPubkey.wrap(Bytes48.leftPad(keyPair.getPublic().getEncodedBytes())), + withdrawalCredentials, + spec.getConstants().getMaxEffectiveBalance(), + BLSSignature.wrap(Bytes96.ZERO) + ); + Hash32 msgHash = spec.signing_root(depositDataWithoutSignature); + UInt64 domain = spec.compute_domain(DEPOSIT, Bytes4.ZERO); + BLS381.Signature signature = BLS381 + .sign(MessageParameters.create(msgHash, domain), keyPair); return new Attestation( Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), attestationData, Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), - BLSSignature.wrap(Bytes96.fromHexString("cc")), + BLSSignature.wrap(signature.getEncoded()), specConstants); } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingCheckerTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingCheckerTest.java new file mode 100644 index 000000000..679778e3a --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/checker/SignatureEncodingCheckerTest.java @@ -0,0 +1,59 @@ +package org.ethereum.beacon.chain.pool.checker; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.Bitlist; + +import static org.assertj.core.api.Assertions.assertThat; + +class SignatureEncodingCheckerTest extends PoolTestConfigurator { + + private SignatureEncodingChecker checker; + + @BeforeEach + void setUp() { + checker = new SignatureEncodingChecker(); + assertThat(checker).isNotNull(); + } + + @Test + void testValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.of(1, 2, 3)); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + + assertThat(checker.check(attestation)).isTrue(); + } + + @Test + void testInvalidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final AttestationData attestationData = new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(2), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + final BytesValue value = BytesValue.of(1, 2, 3); + final Attestation message = new Attestation( + Bitlist.of(value.size() * 8, value, specConstants.getMaxValidatorsPerCommittee().getValue()), + attestationData, + Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), + BLSSignature.wrap(Bytes96.fromHexString("cc")), + specConstants); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + + assertThat(checker.check(attestation)).isFalse(); + } +} From f4ba71f0901fea8331402ce5c58395e770145a1d Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Wed, 11 Sep 2019 17:17:55 +0300 Subject: [PATCH 47/59] pool: test processors by one test --- .../pool/InMemoryAttestationPoolTest.java | 37 +-------- .../chain/pool/PoolTestConfigurator.java | 4 +- .../pool/reactor/MultiProcessorTest.java | 83 +++++++++++++++++++ .../chain/pool/reactor/TimeProcessorTest.java | 2 +- 4 files changed, 87 insertions(+), 39 deletions(-) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 22e9814a8..1680b384d 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -32,17 +32,13 @@ import org.ethereum.beacon.core.BeaconBlockBody; import org.ethereum.beacon.core.BeaconState; import org.ethereum.beacon.core.operations.Attestation; -import org.ethereum.beacon.core.operations.attestation.AttestationData; -import org.ethereum.beacon.core.operations.attestation.Crosslink; import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.core.state.Checkpoint; import org.ethereum.beacon.core.state.Eth1Data; import org.ethereum.beacon.core.types.BLSSignature; -import org.ethereum.beacon.core.types.EpochNumber; import org.ethereum.beacon.core.types.ShardNumber; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.core.types.Time; -import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.db.Database; import org.ethereum.beacon.db.InMemoryDatabase; import org.ethereum.beacon.schedulers.Schedulers; @@ -52,9 +48,7 @@ import reactor.core.publisher.Flux; import reactor.test.StepVerifier; import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; import java.util.Collections; @@ -64,7 +58,7 @@ import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; -class InMemoryAttestationPoolTest { +class InMemoryAttestationPoolTest extends PoolTestConfigurator { private Schedulers schedulers = Schedulers.createControlled(); @@ -306,33 +300,4 @@ private BeaconTuple createBlock( return BeaconTuple.of( block.withStateRoot(spec.hash_tree_root(state)), new BeaconStateExImpl(state)); } - - private Attestation createAttestation(BytesValue someValue) { - final AttestationData attestationData = createAttestationData(); - final Attestation attestation = - new Attestation( - Bitlist.of(someValue.size() * 8, someValue, specConstants.getMaxValidatorsPerCommittee().getValue()), - attestationData, - Bitlist.of(8, BytesValue.fromHexString("bb"), specConstants.getMaxValidatorsPerCommittee().getValue()), - BLSSignature.wrap(Bytes96.fromHexString("cc")), - specConstants); - - return attestation; - } - - public AttestationData createAttestationData() { - final AttestationData expected = - new AttestationData( - Hashes.sha256(BytesValue.fromHexString("aa")), - new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), - new Checkpoint(EpochNumber.of(1), Hashes.sha256(BytesValue.fromHexString("cc"))), - Crosslink.EMPTY); - - return expected; - } - - @Test - void testValidAttestation() { - - } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java index 972bdef57..107a6b354 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java @@ -66,9 +66,9 @@ public SlotNumber.EpochLength getSlotsPerEpoch() { .withCache(true) .build(); - protected DirectProcessor finalizedCheckpoints = DirectProcessor.create(); + protected final DirectProcessor finalizedCheckpoints = DirectProcessor.create(); protected final ControlledSchedulers schedulers = Schedulers.createControlled(); - protected DirectProcessor source = DirectProcessor.create(); + protected final DirectProcessor source = DirectProcessor.create(); protected Attestation createAttestation(BytesValue someValue) { return createAttestation(someValue, createAttestationData()); diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java new file mode 100644 index 000000000..7953ba05a --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java @@ -0,0 +1,83 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; +import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; + +class MultiProcessorTest extends PoolTestConfigurator { + + private final Checkpoint checkpoint = Checkpoint.EMPTY; + private final SlotNumber slotNumber = SlotNumber.of(100L); + private DirectProcessor newSlots = DirectProcessor.create(); + + private TimeProcessor timeProcessor; + private SanityProcessor sanityProcessor; + private DoubleWorkProcessor doubleWorkProcessor; + + @BeforeEach + void setUp() { + final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); + timeProcessor = new TimeProcessor(timeFrameFilter, schedulers, source, finalizedCheckpoints, newSlots); + + final SanityChecker sanityChecker = new SanityChecker(spec); + sanityProcessor = new SanityProcessor(sanityChecker, schedulers, source, finalizedCheckpoints); + + final ProcessedAttestations processedAttestations = new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); + doubleWorkProcessor = new DoubleWorkProcessor(processedAttestations, schedulers, source); + + finalizedCheckpoints.onNext(this.checkpoint); + newSlots.onNext(slotNumber); + + schedulers.addTime(Duration.ofSeconds(5)); + } + + @Test + @DisplayName("Process attestation by multiple processors") + void processAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + schedulers.addTime(Duration.ofSeconds(5)); + + timeProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(sender); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(message); + }); + + //TODO: add assert to sanity checker + + doubleWorkProcessor.subscribe(s -> { + assertThat(s.getSender()) + .isNotNull() + .isEqualTo(attestation.getSender()); + + assertThat(s.getMessage()) + .isNotNull() + .isEqualTo(attestation.getMessage()); + }); + } +} diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java index 9c09fc6d0..0ddcbff66 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java @@ -21,9 +21,9 @@ class TimeProcessorTest extends PoolTestConfigurator { private final Checkpoint checkpoint = Checkpoint.EMPTY; private final SlotNumber slotNumber = SlotNumber.of(100L); + private final DirectProcessor newSlots = DirectProcessor.create(); private TimeProcessor timeProcessor; - private DirectProcessor newSlots = DirectProcessor.create(); @BeforeEach void setUp() { From 5ecceca624633f0482b7eca0d3dba5cead1f7c42 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Wed, 11 Sep 2019 17:29:37 +0300 Subject: [PATCH 48/59] pool: SignatureEncodingProcessorTest --- .../pool/reactor/MultiProcessorTest.java | 9 ++++ .../SignatureEncodingProcessorTest.java | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessorTest.java diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java index 7953ba05a..cb9751e84 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java @@ -1,8 +1,10 @@ package org.ethereum.beacon.chain.pool.reactor; +import org.ethereum.beacon.chain.pool.AttestationPool; import org.ethereum.beacon.chain.pool.PoolTestConfigurator; import org.ethereum.beacon.chain.pool.ReceivedAttestation; import org.ethereum.beacon.chain.pool.checker.SanityChecker; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.core.operations.Attestation; @@ -30,6 +32,7 @@ class MultiProcessorTest extends PoolTestConfigurator { private TimeProcessor timeProcessor; private SanityProcessor sanityProcessor; private DoubleWorkProcessor doubleWorkProcessor; + private SignatureEncodingProcessor signatureEncodingProcessor; @BeforeEach void setUp() { @@ -42,6 +45,10 @@ void setUp() { final ProcessedAttestations processedAttestations = new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); doubleWorkProcessor = new DoubleWorkProcessor(processedAttestations, schedulers, source); + final SignatureEncodingChecker checker = new SignatureEncodingChecker(); + org.ethereum.beacon.schedulers.Scheduler parallelExecutor = schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); + signatureEncodingProcessor = new SignatureEncodingProcessor(checker, parallelExecutor, source); + finalizedCheckpoints.onNext(this.checkpoint); newSlots.onNext(slotNumber); @@ -79,5 +86,7 @@ void processAttestation() { .isNotNull() .isEqualTo(attestation.getMessage()); }); + + //TODO: add assert to signatureEncodingProcessor } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessorTest.java new file mode 100644 index 000000000..0c4bd3c24 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/SignatureEncodingProcessorTest.java @@ -0,0 +1,53 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.pool.AttestationPool; +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.artemis.util.bytes.BytesValue; + +class SignatureEncodingProcessorTest extends PoolTestConfigurator { + + private SignatureEncodingProcessor signatureEncodingProcessor; + + @BeforeEach + void setUp() { + final SignatureEncodingChecker checker = new SignatureEncodingChecker(); + org.ethereum.beacon.schedulers.Scheduler parallelExecutor = schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); + signatureEncodingProcessor = new SignatureEncodingProcessor(checker, parallelExecutor, source); + } + + @Test + void testValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + //TODO: assert signatureEncodingProcessor.getValid() + } + + @Test + void testInvalidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final AttestationData invalidAttestationData = new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(2), Hashes.sha256(BytesValue.fromHexString("cc"))), + Crosslink.EMPTY); + final Attestation message = createAttestation(BytesValue.fromHexString("aa"), invalidAttestationData); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + + //TODO: assert signatureEncodingProcessor.getInvalid() + } +} From eda3012f872a459a59e593d48d623d89dd981abb Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Thu, 12 Sep 2019 18:01:37 +0300 Subject: [PATCH 49/59] test: correct test attestation to be valid --- .../chain/pool/reactor/MultiProcessorTest.java | 12 +++++++----- .../beacon/chain/pool/reactor/TimeProcessorTest.java | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java index cb9751e84..e2f9afb53 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/MultiProcessorTest.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.core.operations.Attestation; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.types.p2p.NodeId; import org.junit.jupiter.api.BeforeEach; @@ -20,7 +21,6 @@ import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; class MultiProcessorTest extends PoolTestConfigurator { @@ -36,18 +36,18 @@ class MultiProcessorTest extends PoolTestConfigurator { @BeforeEach void setUp() { - final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); + final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, EpochNumber.of(233)); timeProcessor = new TimeProcessor(timeFrameFilter, schedulers, source, finalizedCheckpoints, newSlots); final SanityChecker sanityChecker = new SanityChecker(spec); - sanityProcessor = new SanityProcessor(sanityChecker, schedulers, source, finalizedCheckpoints); + sanityProcessor = new SanityProcessor(sanityChecker, schedulers, timeProcessor, finalizedCheckpoints); final ProcessedAttestations processedAttestations = new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); - doubleWorkProcessor = new DoubleWorkProcessor(processedAttestations, schedulers, source); + doubleWorkProcessor = new DoubleWorkProcessor(processedAttestations, schedulers, sanityProcessor.getValid()); final SignatureEncodingChecker checker = new SignatureEncodingChecker(); org.ethereum.beacon.schedulers.Scheduler parallelExecutor = schedulers.newParallelDaemon("attestation-pool-%d", AttestationPool.MAX_THREADS); - signatureEncodingProcessor = new SignatureEncodingProcessor(checker, parallelExecutor, source); + signatureEncodingProcessor = new SignatureEncodingProcessor(checker, parallelExecutor, doubleWorkProcessor); finalizedCheckpoints.onNext(this.checkpoint); newSlots.onNext(slotNumber); @@ -66,6 +66,7 @@ void processAttestation() { schedulers.addTime(Duration.ofSeconds(5)); timeProcessor.subscribe(s -> { + System.out.println("TimeProcessor subscription"); assertThat(s.getSender()) .isNotNull() .isEqualTo(sender); @@ -78,6 +79,7 @@ void processAttestation() { //TODO: add assert to sanity checker doubleWorkProcessor.subscribe(s -> { + System.out.println("DoubleWorkProcessor subscription"); assertThat(s.getSender()) .isNotNull() .isEqualTo(attestation.getSender()); diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java index 0ddcbff66..4957dbdd9 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/TimeProcessorTest.java @@ -5,6 +5,7 @@ import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; import org.ethereum.beacon.core.operations.Attestation; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.types.EpochNumber; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.types.p2p.NodeId; import org.junit.jupiter.api.BeforeEach; @@ -15,7 +16,6 @@ import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; class TimeProcessorTest extends PoolTestConfigurator { @@ -28,7 +28,7 @@ class TimeProcessorTest extends PoolTestConfigurator { @BeforeEach void setUp() { - final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); + final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, EpochNumber.of(233)); timeProcessor = new TimeProcessor(timeFrameFilter, schedulers, source, finalizedCheckpoints, newSlots); finalizedCheckpoints.onNext(this.checkpoint); newSlots.onNext(slotNumber); From 85b5f68cc6f2168a207c29cf0044e831cdce0973 Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Thu, 12 Sep 2019 18:34:20 +0300 Subject: [PATCH 50/59] test: IdentificationProcessor --- .../pool/InMemoryAttestationPoolTest.java | 6 +- .../chain/pool/PoolTestConfigurator.java | 79 +++++++++++++++++++ .../reactor/IdentificationProcessorTest.java | 57 +++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessorTest.java diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java index 1680b384d..64d903068 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java @@ -5,7 +5,6 @@ import org.ethereum.beacon.chain.pool.checker.SanityChecker; import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; -import org.ethereum.beacon.chain.pool.churn.AttestationChurn; import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; @@ -53,7 +52,6 @@ import java.util.Collections; -import static org.ethereum.beacon.chain.pool.AttestationPool.ATTESTATION_CHURN_SIZE; import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; @@ -248,7 +246,7 @@ void integrationTest2() { pool.start(); } - private MutableBeaconChain createBeaconChain( + protected MutableBeaconChain createBeaconChain( BeaconChainSpec spec, StateTransition perSlotTransition, Schedulers schedulers) { Time start = Time.castFrom(UInt64.valueOf(schedulers.getCurrentTime() / 1000)); ChainStart chainStart = new ChainStart(start, Eth1Data.EMPTY, Collections.emptyList()); @@ -284,7 +282,7 @@ private MutableBeaconChain createBeaconChain( return null; } - private BeaconTuple createBlock( + protected BeaconTuple createBlock( BeaconTuple parent, BeaconChainSpec spec, long currentTime, StateTransition perSlotTransition) { diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java index 107a6b354..d7f887b9f 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java @@ -1,12 +1,36 @@ package org.ethereum.beacon.chain.pool; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.DefaultBeaconChain; +import org.ethereum.beacon.chain.MutableBeaconChain; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.BeaconStateEx; +import org.ethereum.beacon.consensus.BlockTransition; +import org.ethereum.beacon.consensus.ChainStart; +import org.ethereum.beacon.consensus.StateTransition; +import org.ethereum.beacon.consensus.transition.BeaconStateExImpl; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.consensus.transition.ExtendedSlotTransition; +import org.ethereum.beacon.consensus.transition.InitialStateTransition; +import org.ethereum.beacon.consensus.transition.PerEpochTransition; +import org.ethereum.beacon.consensus.transition.PerSlotTransition; +import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; +import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; +import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; +import org.ethereum.beacon.consensus.verifier.VerificationResult; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconBlockBody; +import org.ethereum.beacon.core.BeaconState; import org.ethereum.beacon.core.operations.Attestation; import org.ethereum.beacon.core.operations.attestation.AttestationData; import org.ethereum.beacon.core.operations.attestation.Crosslink; import org.ethereum.beacon.core.operations.deposit.DepositData; import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.core.state.Checkpoint; +import org.ethereum.beacon.core.state.Eth1Data; import org.ethereum.beacon.core.types.BLSPubkey; import org.ethereum.beacon.core.types.BLSSignature; import org.ethereum.beacon.core.types.EpochNumber; @@ -16,6 +40,7 @@ import org.ethereum.beacon.crypto.BLS381; import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.crypto.MessageParameters; +import org.ethereum.beacon.db.Database; import org.ethereum.beacon.schedulers.ControlledSchedulers; import org.ethereum.beacon.schedulers.Schedulers; import reactor.core.publisher.DirectProcessor; @@ -28,6 +53,7 @@ import tech.pegasys.artemis.util.collections.Bitlist; import tech.pegasys.artemis.util.uint.UInt64; +import java.util.Collections; import java.util.Random; import static org.ethereum.beacon.core.spec.SignatureDomains.DEPOSIT; @@ -66,6 +92,7 @@ public SlotNumber.EpochLength getSlotsPerEpoch() { .withCache(true) .build(); + protected StateTransition perSlotTransition = new PerSlotTransition(spec); protected final DirectProcessor finalizedCheckpoints = DirectProcessor.create(); protected final ControlledSchedulers schedulers = Schedulers.createControlled(); protected final DirectProcessor source = DirectProcessor.create(); @@ -104,4 +131,56 @@ private AttestationData createAttestationData() { new Checkpoint(EpochNumber.of(232), Hashes.sha256(BytesValue.fromHexString("cc"))), Crosslink.EMPTY); } + + protected BeaconTuple createBlock( + BeaconTuple parent, + BeaconChainSpec spec, long currentTime, + StateTransition perSlotTransition) { + BeaconBlock block = + new BeaconBlock( + spec.get_current_slot(parent.getState(), currentTime), + spec.signing_root(parent.getBlock()), + Hash32.ZERO, + BeaconBlockBody.getEmpty(spec.getConstants()), + BLSSignature.ZERO); + BeaconState state = perSlotTransition.apply(new BeaconStateExImpl(parent.getState())); + + return BeaconTuple.of( + block.withStateRoot(spec.hash_tree_root(state)), new BeaconStateExImpl(state)); + } + + protected MutableBeaconChain createBeaconChain(BeaconChainSpec spec, StateTransition perSlotTransition, Schedulers schedulers) { + + final Time start = Time.castFrom(UInt64.valueOf(schedulers.getCurrentTime() / 1000)); + final ChainStart chainStart = new ChainStart(start, Eth1Data.EMPTY, Collections.emptyList()); + + final BlockTransition initialTransition = new InitialStateTransition(chainStart, spec); + final BlockTransition perBlockTransition = StateTransitionTestUtil.createPerBlockTransition(); + final StateTransition perEpochTransition = StateTransitionTestUtil.createStateWithNoTransition(); + + final BeaconBlockVerifier blockVerifier = (block, state) -> VerificationResult.PASSED; + final BeaconStateVerifier stateVerifier = (block, state) -> VerificationResult.PASSED; + + final Database database = Database.inMemoryDB(); + final SerializerFactory ssz = SerializerFactory.createSSZ(spec.getConstants()); + final BeaconChainStorage chainStorage = new SSZBeaconChainStorageFactory(spec.getObjectHasher(), ssz) + .create(database); + + final EmptySlotTransition preBlockTransition = new EmptySlotTransition( + new ExtendedSlotTransition(new PerEpochTransition(spec) { + @Override + public BeaconStateEx apply(BeaconStateEx stateEx) { + return perEpochTransition.apply(stateEx); + } + }, perSlotTransition, spec)); + + return new DefaultBeaconChain( + spec, + preBlockTransition, + perBlockTransition, + blockVerifier, + stateVerifier, + chainStorage, + schedulers); + } } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessorTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessorTest.java new file mode 100644 index 000000000..b09437752 --- /dev/null +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/reactor/IdentificationProcessorTest.java @@ -0,0 +1,57 @@ +package org.ethereum.beacon.chain.pool.reactor; + +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.MutableBeaconChain; +import org.ethereum.beacon.chain.pool.PoolTestConfigurator; +import org.ethereum.beacon.chain.pool.ReceivedAttestation; +import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; +import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.db.InMemoryDatabase; +import org.ethereum.beacon.types.p2p.NodeId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.DirectProcessor; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; +import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; + +class IdentificationProcessorTest extends PoolTestConfigurator { + + private IdentificationProcessor identificationProcessor; + private final DirectProcessor newSlots = DirectProcessor.create(); + private final DirectProcessor importedBlocks = DirectProcessor.create(); + + @BeforeEach + void setUp() { + final InMemoryDatabase db = new InMemoryDatabase(); + final BeaconChainStorage beaconChainStorage = + new SSZBeaconChainStorageFactory( + spec.getObjectHasher(), SerializerFactory.createSSZ(specConstants)) + .create(db); + final UnknownAttestationPool unknownAttestationPool = new UnknownAttestationPool(beaconChainStorage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); + identificationProcessor = new IdentificationProcessor(unknownAttestationPool, schedulers, source, newSlots, importedBlocks); + newSlots.onNext(SlotNumber.of(500)); + + final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); + beaconChain.init(); + final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); + final BeaconTuple aTuple = createBlock(recentlyProcessed, spec, + schedulers.getCurrentTime(), perSlotTransition); + final BeaconBlock aBlock = aTuple.getBlock(); + importedBlocks.onNext(aBlock); + } + + @Test + void testPublishValidAttestation() { + final NodeId sender = new NodeId(new byte[100]); + final Attestation message = createAttestation(BytesValue.fromHexString("aa")); + final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); + source.onNext(attestation); + } +} From a1c2751dd986592119c0a5a5edc042e7eac0973b Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Tue, 17 Sep 2019 18:28:00 +0300 Subject: [PATCH 51/59] test: initialize tuple storage --- .../beacon/chain/pool/PoolTestConfigurator.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java index d7f887b9f..42bbb4426 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/pool/PoolTestConfigurator.java @@ -6,6 +6,7 @@ import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; import org.ethereum.beacon.chain.storage.impl.SerializerFactory; +import org.ethereum.beacon.chain.storage.util.StorageUtils; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.consensus.BeaconStateEx; import org.ethereum.beacon.consensus.BlockTransition; @@ -127,8 +128,8 @@ private AttestationData createAttestationData() { return new AttestationData( Hashes.sha256(BytesValue.fromHexString("aa")), - new Checkpoint(EpochNumber.of(231), Hashes.sha256(BytesValue.fromHexString("bb"))), - new Checkpoint(EpochNumber.of(232), Hashes.sha256(BytesValue.fromHexString("cc"))), + new Checkpoint(EpochNumber.of(125), Hashes.sha256(BytesValue.fromHexString("bb"))), + new Checkpoint(EpochNumber.of(126), Hashes.sha256(BytesValue.fromHexString("cc"))), Crosslink.EMPTY); } @@ -154,7 +155,7 @@ protected MutableBeaconChain createBeaconChain(BeaconChainSpec spec, StateTransi final Time start = Time.castFrom(UInt64.valueOf(schedulers.getCurrentTime() / 1000)); final ChainStart chainStart = new ChainStart(start, Eth1Data.EMPTY, Collections.emptyList()); - final BlockTransition initialTransition = new InitialStateTransition(chainStart, spec); + final InitialStateTransition initialTransition = new InitialStateTransition(chainStart, spec); final BlockTransition perBlockTransition = StateTransitionTestUtil.createPerBlockTransition(); final StateTransition perEpochTransition = StateTransitionTestUtil.createStateWithNoTransition(); @@ -174,6 +175,9 @@ public BeaconStateEx apply(BeaconStateEx stateEx) { } }, perSlotTransition, spec)); + final BeaconStateEx initialState = initialTransition.apply(spec.get_empty_block()); + StorageUtils.initializeStorage(chainStorage, spec, initialState); + return new DefaultBeaconChain( spec, preBlockTransition, From 01c0c7c64cc5e9e998e51f28e4f13983d7d14eea Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Wed, 18 Sep 2019 11:58:45 +0300 Subject: [PATCH 52/59] test: remove inmemory attestation pool test --- .../pool/InMemoryAttestationPoolTest.java | 301 ------------------ 1 file changed, 301 deletions(-) delete mode 100644 chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java diff --git a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java b/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java deleted file mode 100644 index 64d903068..000000000 --- a/chain/src/test/java/org/ethereum/beacon/chain/pool/InMemoryAttestationPoolTest.java +++ /dev/null @@ -1,301 +0,0 @@ -package org.ethereum.beacon.chain.pool; - -import org.ethereum.beacon.chain.BeaconTuple; -import org.ethereum.beacon.chain.MutableBeaconChain; -import org.ethereum.beacon.chain.pool.checker.SanityChecker; -import org.ethereum.beacon.chain.pool.checker.SignatureEncodingChecker; -import org.ethereum.beacon.chain.pool.checker.TimeFrameFilter; -import org.ethereum.beacon.chain.pool.registry.ProcessedAttestations; -import org.ethereum.beacon.chain.pool.registry.UnknownAttestationPool; -import org.ethereum.beacon.chain.pool.verifier.AttestationVerifier; -import org.ethereum.beacon.chain.pool.verifier.BatchVerifier; -import org.ethereum.beacon.chain.storage.BeaconChainStorage; -import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; -import org.ethereum.beacon.chain.storage.impl.SerializerFactory; -import org.ethereum.beacon.consensus.BeaconChainSpec; -import org.ethereum.beacon.consensus.BeaconStateEx; -import org.ethereum.beacon.consensus.BlockTransition; -import org.ethereum.beacon.consensus.ChainStart; -import org.ethereum.beacon.consensus.StateTransition; -import org.ethereum.beacon.consensus.transition.BeaconStateExImpl; -import org.ethereum.beacon.consensus.transition.EmptySlotTransition; -import org.ethereum.beacon.consensus.transition.ExtendedSlotTransition; -import org.ethereum.beacon.consensus.transition.InitialStateTransition; -import org.ethereum.beacon.consensus.transition.PerEpochTransition; -import org.ethereum.beacon.consensus.transition.PerSlotTransition; -import org.ethereum.beacon.consensus.util.StateTransitionTestUtil; -import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; -import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; -import org.ethereum.beacon.consensus.verifier.VerificationResult; -import org.ethereum.beacon.core.BeaconBlock; -import org.ethereum.beacon.core.BeaconBlockBody; -import org.ethereum.beacon.core.BeaconState; -import org.ethereum.beacon.core.operations.Attestation; -import org.ethereum.beacon.core.spec.SpecConstants; -import org.ethereum.beacon.core.state.Checkpoint; -import org.ethereum.beacon.core.state.Eth1Data; -import org.ethereum.beacon.core.types.BLSSignature; -import org.ethereum.beacon.core.types.ShardNumber; -import org.ethereum.beacon.core.types.SlotNumber; -import org.ethereum.beacon.core.types.Time; -import org.ethereum.beacon.db.Database; -import org.ethereum.beacon.db.InMemoryDatabase; -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.types.p2p.NodeId; -import org.junit.jupiter.api.Test; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; -import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.uint.UInt64; - -import java.util.Collections; - -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_ATTESTATION_LOOKAHEAD; -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_PROCESSED_ATTESTATIONS; -import static org.ethereum.beacon.chain.pool.AttestationPool.MAX_UNKNOWN_ATTESTATIONS; - -class InMemoryAttestationPoolTest extends PoolTestConfigurator { - - private Schedulers schedulers = Schedulers.createControlled(); - - private SpecConstants specConstants = - new SpecConstants() { - @Override - public SlotNumber getGenesisSlot() { - return SlotNumber.of(12345); - } - - @Override - public Time getSecondsPerSlot() { - return Time.of(1); - } - }; - private BeaconChainSpec spec = BeaconChainSpec.Builder.createWithDefaultParams() - .withConstants(new SpecConstants() { - @Override - public ShardNumber getShardCount() { - return ShardNumber.of(16); - } - - @Override - public SlotNumber.EpochLength getSlotsPerEpoch() { - return new SlotNumber.EpochLength(UInt64.valueOf(4)); - } - }) - .withComputableGenesisTime(false) - .withVerifyDepositProof(false) - .withBlsVerifyProofOfPossession(false) - .withBlsVerify(false) - .withCache(true) - .build(); - - private InMemoryDatabase db = new InMemoryDatabase(); - private BeaconChainStorage beaconChainStorage = - new SSZBeaconChainStorageFactory( - spec.getObjectHasher(), SerializerFactory.createSSZ(specConstants)) - .create(db); - - private PerEpochTransition perEpochTransition = new PerEpochTransition(spec); - private StateTransition perSlotTransition = new PerSlotTransition(spec); - private EmptySlotTransition slotTransition = - new EmptySlotTransition( - new ExtendedSlotTransition(perEpochTransition, perSlotTransition, spec)); - - @Test - void integrationTest() { - final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); - beaconChain.init(); - final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); - final BeaconTuple aTuple = createBlock(recentlyProcessed, spec, - schedulers.getCurrentTime(), perSlotTransition); - final BeaconBlock aBlock = aTuple.getBlock(); - - final NodeId sender = new NodeId(new byte[100]); - final Attestation message = createAttestation(BytesValue.fromHexString("aa")); - final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); - final Publisher source = Flux.just(attestation); - StepVerifier.create(source) - .expectNext(attestation) - .expectComplete() - .verify(); - - final SlotNumber slotNumber = SlotNumber.of(100L); - final Publisher newSlots = Flux.just(slotNumber); - StepVerifier.create(newSlots) - .expectNext(slotNumber) - .expectComplete() - .verify(); - - final Checkpoint checkpoint = Checkpoint.EMPTY; - final Publisher finalizedCheckpoints = Flux.just(checkpoint); - StepVerifier.create(finalizedCheckpoints) - .expectNext(checkpoint) - .expectComplete() - .verify(); - final Publisher justifiedCheckpoints = Flux.just(checkpoint); - StepVerifier.create(justifiedCheckpoints) - .expectNext(checkpoint) - .expectComplete() - .verify(); - - final Publisher importedBlocks = Flux.just(aBlock); - StepVerifier.create(importedBlocks) - .expectNext(aBlock) - .expectComplete() - .verify(); - - final Publisher chainHeads = Flux.just(recentlyProcessed); - StepVerifier.create(chainHeads) - .expectNext(recentlyProcessed) - .expectComplete() - .verify(); - - final AttestationPool pool = AttestationPool.create( - source, - newSlots, - finalizedCheckpoints, - importedBlocks, - schedulers, - spec, - beaconChainStorage, - slotTransition - ); - - pool.start(); - } - - @Test - void integrationTest2() { - final MutableBeaconChain beaconChain = createBeaconChain(spec, perSlotTransition, schedulers); - beaconChain.init(); - final BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); - final BeaconTuple aTuple = createBlock(recentlyProcessed, spec, - schedulers.getCurrentTime(), perSlotTransition); - final BeaconBlock aBlock = aTuple.getBlock(); - - final NodeId sender = new NodeId(new byte[100]); - final Attestation message = createAttestation(BytesValue.fromHexString("aa")); - final ReceivedAttestation attestation = new ReceivedAttestation(sender, message); - final Publisher source = Flux.just(attestation); - StepVerifier.create(source) - .expectNext(attestation) - .expectComplete() - .verify(); - - final SlotNumber slotNumber = SlotNumber.of(100L); - final Publisher newSlots = Flux.just(slotNumber); - StepVerifier.create(newSlots) - .expectNext(slotNumber) - .expectComplete() - .verify(); - - final Checkpoint checkpoint = Checkpoint.EMPTY; - final Publisher finalizedCheckpoints = Flux.just(checkpoint); - StepVerifier.create(finalizedCheckpoints) - .expectNext(checkpoint) - .expectComplete() - .verify(); - - final Publisher justifiedCheckpoints = Flux.just(checkpoint); - StepVerifier.create(justifiedCheckpoints) - .expectNext(checkpoint) - .expectComplete() - .verify(); - - final Publisher importedBlocks = Flux.just(aBlock); - StepVerifier.create(importedBlocks) - .expectNext(aBlock) - .expectComplete() - .verify(); - - final Publisher chainHeads = Flux.just(aTuple); - StepVerifier.create(chainHeads) - .expectNext(aTuple) - .expectComplete() - .verify(); - - final TimeFrameFilter timeFrameFilter = new TimeFrameFilter(spec, MAX_ATTESTATION_LOOKAHEAD); - timeFrameFilter.feedFinalizedCheckpoint(checkpoint); - timeFrameFilter.feedNewSlot(slotNumber); - - final SanityChecker sanityChecker = new SanityChecker(spec); - final SignatureEncodingChecker encodingChecker = new SignatureEncodingChecker(); - final ProcessedAttestations processedFilter = - new ProcessedAttestations(spec::hash_tree_root, MAX_PROCESSED_ATTESTATIONS); - final UnknownAttestationPool unknownAttestationPool = - new UnknownAttestationPool( - beaconChainStorage.getBlockStorage(), spec, MAX_ATTESTATION_LOOKAHEAD, MAX_UNKNOWN_ATTESTATIONS); - final BatchVerifier batchVerifier = - new AttestationVerifier(beaconChainStorage.getTupleStorage(), spec, slotTransition); - - final AttestationPool pool = new InMemoryAttestationPool( - source, - newSlots, - finalizedCheckpoints, - importedBlocks, - schedulers, - timeFrameFilter, - sanityChecker, - encodingChecker, - processedFilter, - unknownAttestationPool, - batchVerifier); - - pool.start(); - } - - protected MutableBeaconChain createBeaconChain( - BeaconChainSpec spec, StateTransition perSlotTransition, Schedulers schedulers) { - Time start = Time.castFrom(UInt64.valueOf(schedulers.getCurrentTime() / 1000)); - ChainStart chainStart = new ChainStart(start, Eth1Data.EMPTY, Collections.emptyList()); - BlockTransition initialTransition = - new InitialStateTransition(chainStart, spec); - BlockTransition perBlockTransition = - StateTransitionTestUtil.createPerBlockTransition(); - StateTransition perEpochTransition = - StateTransitionTestUtil.createStateWithNoTransition(); - - BeaconBlockVerifier blockVerifier = (block, state) -> VerificationResult.PASSED; - BeaconStateVerifier stateVerifier = (block, state) -> VerificationResult.PASSED; - Database database = Database.inMemoryDB(); - BeaconChainStorage chainStorage = new SSZBeaconChainStorageFactory( - spec.getObjectHasher(), SerializerFactory.createSSZ(spec.getConstants())) - .create(database); - -// return new DefaultBeaconChain( -// spec, -// new EmptySlotTransition( -// new ExtendedSlotTransition(new PerEpochTransition(spec) { -// @Override -// public BeaconStateEx apply(BeaconStateEx stateEx) { -// return perEpochTransition.apply(stateEx); -// } -// }, perSlotTransition, spec)), -// perBlockTransition, -// blockVerifier, -// stateVerifier, -// chainStorage, -// schedulers); - - return null; - } - - protected BeaconTuple createBlock( - BeaconTuple parent, - BeaconChainSpec spec, long currentTime, - StateTransition perSlotTransition) { - BeaconBlock block = - new BeaconBlock( - spec.get_current_slot(parent.getState(), currentTime), - spec.signing_root(parent.getBlock()), - Hash32.ZERO, - BeaconBlockBody.getEmpty(spec.getConstants()), - BLSSignature.ZERO); - BeaconState state = perSlotTransition.apply(new BeaconStateExImpl(parent.getState())); - - return BeaconTuple.of( - block.withStateRoot(spec.hash_tree_root(state)), new BeaconStateExImpl(state)); - } -} From 33f0629e4b875de38ec9c1674922b73ad3be3eb6 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 19 Sep 2019 00:11:14 +0300 Subject: [PATCH 53/59] Modify update finality to comply with the fork choice spec. --- .../beacon/chain/DefaultBeaconChain.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java index a7cc6c1bf..3763c1c9d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java @@ -126,7 +126,7 @@ public synchronized ImportResult insert(BeaconBlock block) { BeaconTuple newTuple = BeaconTuple.of(block, postBlockState); tupleStorage.put(newTuple); - updateFinality(parentState, postBlockState); + updateFinality(postBlockState); chainStorage.commit(); @@ -153,21 +153,32 @@ public BeaconTuple getRecentlyProcessed() { return recentlyProcessed; } - private void updateFinality(BeaconState previous, BeaconState current) { - if (!previous.getFinalizedCheckpoint().equals(current.getFinalizedCheckpoint())) { + private void updateFinality(BeaconState current) { + boolean finalizedStorageUpdated = false; + boolean justifiedStorageUpdated = false; + if (current + .getFinalizedCheckpoint() + .getEpoch() + .greater(fetchFinalizedCheckpoint().getEpoch())) { chainStorage.getFinalizedStorage().set(current.getFinalizedCheckpoint()); + finalizedStorageUpdated = true; + } + if (current + .getCurrentJustifiedCheckpoint() + .getEpoch() + .greater(fetchJustifiedCheckpoint().getEpoch())) { + chainStorage.getJustifiedStorage().set(current.getCurrentJustifiedCheckpoint()); + justifiedStorageUpdated = true; + } + // publish updates after both storages have been updated + // the order can be important if a finalizedCheckpointStream subscriber will look + // into justified storage + // in general, it may be important to publish after commit has succeeded + if (finalizedStorageUpdated) { finalizedCheckpointStream.onNext(current.getFinalizedCheckpoint()); } - if (!previous.getCurrentJustifiedCheckpoint().equals(current.getCurrentJustifiedCheckpoint())) { - // store new justified checkpoint if its epoch greater than previous one - if (current - .getCurrentJustifiedCheckpoint() - .getEpoch() - .greater(fetchJustifiedCheckpoint().getEpoch())) { - chainStorage.getJustifiedStorage().set(current.getCurrentJustifiedCheckpoint()); - } - - justifiedCheckpointStream.onNext(current.getFinalizedCheckpoint()); + if (justifiedStorageUpdated) { + justifiedCheckpointStream.onNext(current.getCurrentJustifiedCheckpoint()); } } From 94cd7be226eaaa340e3cee7c12526ff14011b495 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 19 Sep 2019 00:12:17 +0300 Subject: [PATCH 54/59] Modify get_head to comply with the fork_choice spec: add tie break. --- .../java/org/ethereum/beacon/consensus/spec/ForkChoice.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java index 4afb89087..bc3853bb8 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java @@ -10,6 +10,7 @@ import org.ethereum.beacon.core.types.Gwei; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.core.types.ValidatorIndex; +import org.javatuples.Pair; import tech.pegasys.artemis.ethereum.core.Hash32; /** @@ -109,7 +110,7 @@ default Hash32 get_head(Store store) { } head = children.stream() - .max(Comparator.comparing(root -> get_latest_attesting_balance(store, root))) + .max(Comparator.comparing(root -> Pair.with(get_latest_attesting_balance(store, root), root))) .get(); } } From 07d2003d0a5f440643470647b68be5064cda6f7a Mon Sep 17 00:00:00 2001 From: Eugene Ustimenko Date: Thu, 19 Sep 2019 14:04:20 +0300 Subject: [PATCH 55/59] attestation-pool: update eth2.0-spec-tests --- test/src/test/resources/eth2.0-spec-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/resources/eth2.0-spec-tests b/test/src/test/resources/eth2.0-spec-tests index aaa1673f5..ae6dd9011 160000 --- a/test/src/test/resources/eth2.0-spec-tests +++ b/test/src/test/resources/eth2.0-spec-tests @@ -1 +1 @@ -Subproject commit aaa1673f508103e11304833e0456e4149f880065 +Subproject commit ae6dd9011df05fab8c7e651c09cf9c940973bf81 From 4c6541d09318febd8c238d3d7c6d5b887bb75804 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 19 Sep 2019 00:11:14 +0300 Subject: [PATCH 56/59] Modify update finality to comply with the fork choice spec. --- .../beacon/chain/DefaultBeaconChain.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java index a7cc6c1bf..3763c1c9d 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java @@ -126,7 +126,7 @@ public synchronized ImportResult insert(BeaconBlock block) { BeaconTuple newTuple = BeaconTuple.of(block, postBlockState); tupleStorage.put(newTuple); - updateFinality(parentState, postBlockState); + updateFinality(postBlockState); chainStorage.commit(); @@ -153,21 +153,32 @@ public BeaconTuple getRecentlyProcessed() { return recentlyProcessed; } - private void updateFinality(BeaconState previous, BeaconState current) { - if (!previous.getFinalizedCheckpoint().equals(current.getFinalizedCheckpoint())) { + private void updateFinality(BeaconState current) { + boolean finalizedStorageUpdated = false; + boolean justifiedStorageUpdated = false; + if (current + .getFinalizedCheckpoint() + .getEpoch() + .greater(fetchFinalizedCheckpoint().getEpoch())) { chainStorage.getFinalizedStorage().set(current.getFinalizedCheckpoint()); + finalizedStorageUpdated = true; + } + if (current + .getCurrentJustifiedCheckpoint() + .getEpoch() + .greater(fetchJustifiedCheckpoint().getEpoch())) { + chainStorage.getJustifiedStorage().set(current.getCurrentJustifiedCheckpoint()); + justifiedStorageUpdated = true; + } + // publish updates after both storages have been updated + // the order can be important if a finalizedCheckpointStream subscriber will look + // into justified storage + // in general, it may be important to publish after commit has succeeded + if (finalizedStorageUpdated) { finalizedCheckpointStream.onNext(current.getFinalizedCheckpoint()); } - if (!previous.getCurrentJustifiedCheckpoint().equals(current.getCurrentJustifiedCheckpoint())) { - // store new justified checkpoint if its epoch greater than previous one - if (current - .getCurrentJustifiedCheckpoint() - .getEpoch() - .greater(fetchJustifiedCheckpoint().getEpoch())) { - chainStorage.getJustifiedStorage().set(current.getCurrentJustifiedCheckpoint()); - } - - justifiedCheckpointStream.onNext(current.getFinalizedCheckpoint()); + if (justifiedStorageUpdated) { + justifiedCheckpointStream.onNext(current.getCurrentJustifiedCheckpoint()); } } From 67541c039deaf212ab2684b6f8b8f6f4f47709e4 Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Thu, 19 Sep 2019 00:12:17 +0300 Subject: [PATCH 57/59] Modify get_head to comply with the fork_choice spec: add tie break. --- .../java/org/ethereum/beacon/consensus/spec/ForkChoice.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java index 4afb89087..bc3853bb8 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java @@ -10,6 +10,7 @@ import org.ethereum.beacon.core.types.Gwei; import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.core.types.ValidatorIndex; +import org.javatuples.Pair; import tech.pegasys.artemis.ethereum.core.Hash32; /** @@ -109,7 +110,7 @@ default Hash32 get_head(Store store) { } head = children.stream() - .max(Comparator.comparing(root -> get_latest_attesting_balance(store, root))) + .max(Comparator.comparing(root -> Pair.with(get_latest_attesting_balance(store, root), root))) .get(); } } From 1bc6ce986e488f8238de69ccc1e23338b0b3356d Mon Sep 17 00:00:00 2001 From: Alex Vlasov/ericsson49 Date: Fri, 20 Sep 2019 15:40:16 +0300 Subject: [PATCH 58/59] Check that BytesValue are compared the same way in Java and in python3 (i.e. lexicographically, from left to right, unsigned bytes). --- .../artemis/util/bytes/BytesValueTest.java | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java b/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java index 762682236..27abc3fc0 100644 --- a/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java +++ b/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java @@ -16,7 +16,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.vertx.core.buffer.Buffer; -import net.consensys.cava.bytes.MutableBytes; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -857,6 +856,49 @@ public void testBytesValuesComparatorReturnsMatchUnsignedValueByteValue() { assertThat(small.compareTo(otherSmall)).isEqualTo(0); } + @Test + public void testBytesValueComparator_lexicographical_order() { + checkOrder(h("0x00"), h("0x01")); + checkOrder(h("0x01"), h("0x7f")); + checkOrder(h("0x7f"), h("0x80")); + checkOrder(h("0x80"), h("0xff")); + + checkOrder(h("0x0000"), h("0x0001")); + checkOrder(h("0x0001"), h("0x007f")); + checkOrder(h("0x007f"), h("0x0080")); + checkOrder(h("0x0080"), h("0x00ff")); + + checkOrder(h("0x0000"), h("0x0100")); + checkOrder(h("0x0100"), h("0x7f00")); + checkOrder(h("0x7f00"), h("0x8000")); + checkOrder(h("0x8000"), h("0xff00")); + + checkOrder(h("0x00"), h("0x0100")); + checkOrder(h("0x01"), h("0x7f00")); + checkOrder(h("0x7f"), h("0x8000")); + checkOrder(h("0x80"), h("0xff00")); + + checkOrder(h("0x00"), h("0x01ff")); + checkOrder(h("0x01"), h("0x7fff")); + checkOrder(h("0x7f"), h("0x80ff")); + checkOrder(h("0x80"), h("0xffff")); + + checkOrder(h("0x0001"), h("0x0100")); + checkOrder(h("0x007f"), h("0x7f00")); + checkOrder(h("0x0080"), h("0x8000")); + checkOrder(h("0x00ff"), h("0xff00")); + + checkOrder(h("0x000001"), h("0x010000")); + checkOrder(h("0x00007f"), h("0x7f0000")); + checkOrder(h("0x000080"), h("0x800000")); + checkOrder(h("0x0000ff"), h("0xff0000")); + } + + static void checkOrder(BytesValue lesser, BytesValue greater) { + assertThat(lesser).isLessThan(greater); + assertThat(greater).isGreaterThan(lesser); + } + @Test public void testGetSetBit() { MutableBytesValue bytes = MutableBytesValue.create(4); From cf24e15b46b1ed91de45177137e4ac86a0ede8a2 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 23 Sep 2019 14:32:13 +0600 Subject: [PATCH 59/59] Replace string comparison with bytes' in get_winning_crosslink_and_attesting_indices --- .../org/ethereum/beacon/consensus/spec/EpochProcessing.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java index ac816272b..e007069c1 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java @@ -131,7 +131,7 @@ default Pair> get_winning_crosslink_and_attestin Gwei b2 = get_attesting_balance(state, attestations.stream().filter(a -> a.getData().getCrosslink().equals(c2)).collect(toList())); if (b1.equals(b2)) { - return c1.getDataRoot().toString().compareTo(c2.getDataRoot().toString()); + return c1.getDataRoot().compareTo(c2.getDataRoot()); } else { return b1.compareTo(b2); }