approval-voting improvement: include all tranche0 assignments in one …

…certificate (#1178)

**_PR migrated from https://github.com/paritytech/polkadot/pull/6782_** 

This PR will upgrade the network protocol to version 3 -> VStaging which
will later be renamed to V3. This version introduces a new kind of
assignment certificate that will be used for tranche0 assignments.
Instead of issuing/importing one tranche0 assignment per candidate,
there will be just one certificate per relay chain block per validator.
However, we will not be sending out the new assignment certificates,
yet. So everything should work exactly as before. Once the majority of
the validators have been upgraded to the new protocol version we will
enable the new certificates (starting at a specific relay chain block)
with a new client update.

There are still a few things that need to be done:

- [x] Use bitfield instead of Vec<CandidateIndex>:
paritytech/polkadot#6802
  - [x] Fix existing approval-distribution and approval-voting tests
  - [x] Fix bitfield-distribution and statement-distribution tests
  - [x] Fix network bridge tests
  - [x] Implement todos in the code
  - [x] Add tests to cover new code
  - [x] Update metrics
  - [x] Remove the approval distribution aggression levels: TBD PR
  - [x] Parachains DB migration 
  - [x] Test network protocol upgrade on Versi
  - [x] Versi Load test
  - [x] Add Zombienet test
  - [x] Documentation updates
- [x] Fix for sending DistributeAssignment for each candidate claimed by
a v2 assignment (warning: Importing locally an already known assignment)
 - [x]  Fix AcceptedDuplicate
 - [x] Fix DB migration so that we can still keep old data.
 - [x] Final Versi burn in

---------

Signed-off-by: Andrei Sandu <andrei-mihail@parity.io>
Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>
Co-authored-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>

.gitlab/pipeline/zombienet/polkadot.yml

@@ -105,6 +105,14 @@ zombienet-polkadot-functional-0005-parachains-disputes-past-session:
       --local-dir="${LOCAL_DIR}/functional"
       --test="0005-parachains-disputes-past-session.zndsl"
 
+zombienet-polkadot-functional-0006-parachains-max-tranche0:
+  extends:
+    - .zombienet-polkadot-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}/functional"
+      --test="0006-parachains-max-tranche0.zndsl"
+
 zombienet-polkadot-smoke-0001-parachains-smoke-test:
   extends:
     - .zombienet-polkadot-common

polkadot/Cargo.toml

@@ -64,6 +64,8 @@ jemalloc-allocator = [
 	"polkadot-node-core-pvf/jemalloc-allocator",
 	"polkadot-overseer/jemalloc-allocator",
 ]
+network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ]
+
 
 # Enables timeout-based tests supposed to be run only in CI environment as they may be flaky
 # when run locally depending on system load

polkadot/cli/Cargo.toml

@@ -74,3 +74,5 @@ runtime-metrics = [
 	"polkadot-node-metrics/runtime-metrics",
 	"service/runtime-metrics",
 ]
+
+network-protocol-staging = [ "service/network-protocol-staging" ]

polkadot/node/core/approval-voting/Cargo.toml

@@ -17,6 +17,7 @@ schnorrkel = "0.9.1"
 kvdb = "0.13.0"
 derive_more = "0.99.17"
 thiserror = "1.0.48"
+itertools = "0.10.5"
 
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }
@@ -30,6 +31,9 @@ sp-consensus = { path = "../../../../substrate/primitives/consensus/common", def
 sp-consensus-slots = { path = "../../../../substrate/primitives/consensus/slots", default-features = false }
 sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false, features = ["full_crypto"] }
 sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false }
+rand_core = "0.5.1"
+rand_chacha = { version = "0.3.1" }
+rand = "0.8.5"
 
 [dev-dependencies]
 async-trait = "0.1.57"
@@ -43,3 +47,5 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 assert_matches = "1.4.0"
 kvdb-memorydb = "0.13.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
+log = "0.4.17"
+env_logger = "0.9.0"

polkadot/node/core/approval-voting/src/approval_checking.rs

@@ -17,7 +17,7 @@
 //! Utilities for checking whether a candidate has been approved under a given block.
 
 use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice};
-use polkadot_node_primitives::approval::DelayTranche;
+use polkadot_node_primitives::approval::v1::DelayTranche;
 use polkadot_primitives::ValidatorIndex;
 
 use crate::{
@@ -472,9 +472,9 @@ mod tests {
 		}
 		.into();
 
-		let approval_entry = approval_db::v1::ApprovalEntry {
+		let approval_entry = approval_db::v2::ApprovalEntry {
 			tranches: Vec::new(),
-			assignments: BitVec::default(),
+			assigned_validators: BitVec::default(),
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -509,22 +509,22 @@ mod tests {
 			candidate.mark_approval(ValidatorIndex(i));
 		}
 
-		let approval_entry = approval_db::v1::ApprovalEntry {
+		let approval_entry = approval_db::v2::ApprovalEntry {
 			tranches: vec![
-				approval_db::v1::TrancheEntry {
+				approval_db::v2::TrancheEntry {
 					tranche: 0,
 					assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(),
 				},
-				approval_db::v1::TrancheEntry {
+				approval_db::v2::TrancheEntry {
 					tranche: 1,
 					assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(),
 				},
-				approval_db::v1::TrancheEntry {
+				approval_db::v2::TrancheEntry {
 					tranche: 2,
 					assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(),
 				},
 			],
-			assignments: bitvec![u8, BitOrderLsb0; 1; 10],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -581,22 +581,22 @@ mod tests {
 			candidate.mark_approval(ValidatorIndex(i));
 		}
 
-		let approval_entry = approval_db::v1::ApprovalEntry {
+		let approval_entry = approval_db::v2::ApprovalEntry {
 			tranches: vec![
-				approval_db::v1::TrancheEntry {
+				approval_db::v2::TrancheEntry {
 					tranche: 0,
 					assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(),
 				},
-				approval_db::v1::TrancheEntry {
+				approval_db::v2::TrancheEntry {
 					tranche: 1,
 					assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(),
 				},
-				approval_db::v1::TrancheEntry {
+				approval_db::v2::TrancheEntry {
 					tranche: 2,
 					assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(),
 				},
 			],
-			assignments: bitvec![u8, BitOrderLsb0; 1; 10],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -647,9 +647,9 @@ mod tests {
 		let no_show_duration = 10;
 		let needed_approvals = 4;
 
-		let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
+		let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
 			tranches: Vec::new(),
-			assignments: bitvec![u8, BitOrderLsb0; 0; 5],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -691,9 +691,9 @@ mod tests {
 		let no_show_duration = 10;
 		let needed_approvals = 4;
 
-		let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
+		let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
 			tranches: Vec::new(),
-			assignments: bitvec![u8, BitOrderLsb0; 0; 10],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -731,9 +731,9 @@ mod tests {
 		let no_show_duration = 10;
 		let needed_approvals = 4;
 
-		let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
+		let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
 			tranches: Vec::new(),
-			assignments: bitvec![u8, BitOrderLsb0; 0; 10],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -776,9 +776,9 @@ mod tests {
 		let needed_approvals = 4;
 		let n_validators = 8;
 
-		let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
+		let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
 			tranches: Vec::new(),
-			assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -843,9 +843,9 @@ mod tests {
 		let needed_approvals = 4;
 		let n_validators = 8;
 
-		let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
+		let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
 			tranches: Vec::new(),
-			assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -934,9 +934,9 @@ mod tests {
 		let needed_approvals = 4;
 		let n_validators = 8;
 
-		let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
+		let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
 			tranches: Vec::new(),
-			assignments: bitvec![u8, BitOrderLsb0; 0; n_validators],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -1041,15 +1041,15 @@ mod tests {
 			candidate.mark_approval(ValidatorIndex(i));
 		}
 
-		let approval_entry = approval_db::v1::ApprovalEntry {
+		let approval_entry = approval_db::v2::ApprovalEntry {
 			tranches: vec![
 				// Assignments with invalid validator indexes.
-				approval_db::v1::TrancheEntry {
+				approval_db::v2::TrancheEntry {
 					tranche: 1,
 					assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(),
 				},
 			],
-			assignments: bitvec![u8, BitOrderLsb0; 1; 3],
+			assigned_validators: bitvec![u8, BitOrderLsb0; 1; 3],
 			our_assignment: None,
 			our_approval_sig: None,
 			backing_group: GroupIndex(0),
@@ -1094,12 +1094,12 @@ mod tests {
 		];
 
 		for test_tranche in test_tranches {
-			let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry {
+			let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry {
 				tranches: Vec::new(),
 				backing_group: GroupIndex(0),
 				our_assignment: None,
 				our_approval_sig: None,
-				assignments: bitvec![u8, BitOrderLsb0; 0; 3],
+				assigned_validators: bitvec![u8, BitOrderLsb0; 0; 3],
 				approved: false,
 			}
 			.into();

polkadot/node/core/approval-voting/src/approval_db/mod.rs

@@ -31,3 +31,4 @@
 //! time being we share the same DB with the rest of Substrate.
 
 pub mod v1;
+pub mod v2;