Merge branch 'main' into optimisation/parallel-test-runs

.env.example

@@ -1,42 +1,56 @@
+##### ORCHESTRATOR #####
+
 HOST=
 PORT=
-DATABASE_URL=
-MADARA_RPC_URL=
-DA_LAYER=
-SETTLEMENT_LAYER=
 
-# Ethereum
-ETHEREUM_PRIVATE_KEY=
-ETHEREUM_RPC_URL=
-MEMORY_PAGES_CONTRACT_ADDRESS=
-STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS=
+##### AWS CONFIG #####
 
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=
+AWS_ENDPOINT_URL=
 
-# Starknet
-STARKNET_PUBLIC_KEY=
-STARNET_PRIVATE_KEY=
-STARKNET_RPC_URL=
-STARKNET_CAIRO_CORE_CONTRACT_ADDRESS=
+##### STORAGE #####
 
-# MongoDB connection string
-MONGODB_CONNECTION_STRING=
+DATA_STORAGE=
+AWS_S3_BUCKET_NAME=
 
 # AWS
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_REGION=
 
-# SQS
+##### QUEUE #####
+
+QUEUE_PROVIDER=
 SQS_JOB_PROCESSING_QUEUE_URL=
 SQS_JOB_VERIFICATION_QUEUE_URL=
+SQS_JOB_HANDLE_FAILURE_QUEUE_URL=
+SQS_WORKER_TRIGGER_QUEUE_URL=
 
-# S3
-AWS_S3_BUCKET_NAME=
-AWS_S3_BUCKET_REGION=
+##### DATABASE #####
+
+DATABASE=
+MONGODB_CONNECTION_STRING=
+
+##### PROVER #####
 
-# Sharp Services
+PROVER_SERVICE=
 SHARP_CUSTOMER_ID=
+SHARP_URL=
+# [IMP!!!] These are test certificates (they don't work)
 SHARP_USER_CRT=
 SHARP_USER_KEY=
 SHARP_SERVER_CRT=
-SHARP_PROOF_LAYOUT=
+SHARP_PROOF_LAYOUT=
+
+##### ON CHAIN CONFIG #####
+
+DA_LAYER=
+SETTLEMENT_LAYER=
+ETHEREUM_RPC_URL=
+MADARA_RPC_URL=
+MEMORY_PAGES_CONTRACT_ADDRESS=
+PRIVATE_KEY=
+ETHEREUM_PRIVATE_KEY=
+STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS=

.env.test

@@ -1,4 +1,9 @@
-##### AWS config #####
+##### ORCHESTRATOR #####
+
+HOST=127.0.0.1
+PORT=3000
+
+##### AWS CONFIG #####
 
 AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID"
 AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY"
@@ -8,32 +13,45 @@ AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566"
 AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"
 AWS_S3_BUCKET_REGION="us-east-1"
 
-#### QUEUE ####
+##### STORAGE #####
+
+DATA_STORAGE="s3"
+AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"
+
+##### QUEUE #####
+
 # Add without trailing slash, to be handled wherever used.
 QUEUE_BASE_URL = "http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000"
+QUEUE_PROVIDER="sqs"
+SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue"
+SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue"
+SQS_JOB_HANDLE_FAILURE_QUEUE_URL=
+SQS_WORKER_TRIGGER_QUEUE_URL=
 
-##### On chain config #####
+##### DATABASE #####
 
-MADARA_RPC_URL="http://localhost:3000"
-ETHEREUM_RPC_URL="http://localhost:3001"
-MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead"
-PRIVATE_KEY="0xdead"
-ETHEREUM_PRIVATE_KEY="0x000000000000000000000000000000000000000000000000000000000000beef"
-STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead"
+DATABASE="mongodb"
+MONGODB_CONNECTION_STRING="mongodb://localhost:27017"
 
-##### Config URLs #####
+##### PROVER #####
 
-DA_LAYER="ethereum"
 PROVER_SERVICE="sharp"
-SETTLEMENT_LAYER="ethereum"
-DATA_STORAGE="s3"
-MONGODB_CONNECTION_STRING="mongodb://localhost:27017"
-
-# Sharp Services
 SHARP_CUSTOMER_ID="sharp_consumer_id"
 SHARP_URL="http://127.0.0.1:5000"
 # [IMP!!!] These are test certificates (they don't work)
 SHARP_USER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR4ekNDQXErZ0F3SUJBZ0lVTjBSK0xpb1MzL2ZadUZsK291RjZNNFk2RnRZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2N6RUxNQWtHQTFVRUJoTUNTVTR4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpERU5NQXNHQTFVRUF3d0VVMVJTU3pFZE1Cc0dDU3FHClNJYjNEUUVKQVJZT1lXSmpRR3RoY201dmRDNTRlWG93SGhjTk1qUXdPREV6TVRNd05UTTBXaGNOTWpVd09ERXoKTVRNd05UTTBXakJ6TVFzd0NRWURWUVFHRXdKSlRqRVRNQkVHQTFVRUNBd0tVMjl0WlMxVGRHRjBaVEVoTUI4RwpBMVVFQ2d3WVNXNTBaWEp1WlhRZ1YybGtaMmwwY3lCUWRIa2dUSFJrTVEwd0N3WURWUVFEREFSVFZGSkxNUjB3Ckd3WUpLb1pJaHZjTkFRa0JGZzVoWW1OQWEyRnlibTkwTG5oNWVqQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQUQKZ2dFUEFEQ0NBUW9DZ2dFQkFOSEtaUGRqWSs4QWo4ZFV2V0xReEl5NTNrK1BHY001T2FlYnpTV3FER0xGSlBOdgpkVzJvWjFCSnNEb2hobWZFSCt5ZEFoQXEvbzc4NDljblg2VDJTOVhta25wdnNud2dRckU5Z3lqSmV3MUxBRzNHCm10U0lOMWJJSm9peWJ3QUR5NGxPd0xrVzUzdFdueHBSazVVVmZUU1hLYVRRTnlHd2o3Q2xMSGthcnlZYVk3OVkKOXlHMFJ2RkFkb1IzczBveWthNkFLV0d1WjhOdWd4NTY2bysyWllRenJteWVNU1NGYkhNdW1aUkxYb0hpazhBSgpLZXJ0bnNBRC9LMVJRYm80Y21ubHFoTVRhQktiTEFVVjVteFVvMlpveFBJVU9tREE5N3IyMmRTYkRkRlVjeC9kCjhQcDB6VXNycXdQckJlcW5SMXdLOE80MUlHajUzRnUzVmxDeS94MENBd0VBQWFOVE1GRXdIUVlEVlIwT0JCWUUKRkc0T0lvKzcvckJyZlR4S2FFMGx2L1dwRDJ3UE1COEdBMVVkSXdRWU1CYUFGRzRPSW8rNy9yQnJmVHhLYUUwbAp2L1dwRDJ3UE1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFEMURDZkR3CnpoSXRGMWd5YVdhWURZRHErZjJSUHBFRWVaWk1BSDdJV0ZTajRrTzhmVHN1RnN6bFoyNXNlR3ZHYW4xQ3F4alQKYnJ3MXliVlJQeGZMUWgxRlZMMGhFeDZWYXhGditxMmtqUmlCQmZURFBxWGxYcmpaaUYrZTNPS3lKSVhnNkpIUAppbVpBV0dyRFBHNkorQi90bHRaQ3VLZVhLK1FUcnRSOVVCL29hOWVaQWc5RXNkOVJsZDRNeVo5b0NtdUNPU1hmCnk1THFkVlgrNENpTnJXQ3BwM1B2M2MyL28rZ0RMQjUzZ252R056RjR6Q1FIZ0RtN0RNZnpmZlY1TUMwV1MvWXkKVnpyUG11Sys0Y0tSK3dMOFZITVNEeC9ybTFhYnh0dEN2VW92MUw5dVZ1QUNGc29yNmdsR0N1RDNNQ0dIa0pNNgpxaS8rM1haeHhxeGw1Rzg9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
 SHARP_USER_KEY="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFJ5bVQzWTJQdkFJL0gKVkwxaTBNU011ZDVQanhuRE9UbW5tODBscWd4aXhTVHpiM1Z0cUdkUVNiQTZJWVpueEIvc25RSVFLdjZPL09QWApKMStrOWt2VjVwSjZiN0o4SUVLeFBZTW95WHNOU3dCdHhwclVpRGRXeUNhSXNtOEFBOHVKVHNDNUZ1ZDdWcDhhClVaT1ZGWDAwbHltazBEY2hzSSt3cFN4NUdxOG1HbU8vV1BjaHRFYnhRSGFFZDdOS01wR3VnQ2xocm1mRGJvTWUKZXVxUHRtV0VNNjVzbmpFa2hXeHpMcG1VUzE2QjRwUEFDU25xN1o3QUEveXRVVUc2T0hKcDVhb1RFMmdTbXl3RgpGZVpzVktObWFNVHlGRHBnd1BlNjl0blVtdzNSVkhNZjNmRDZkTTFMSzZzRDZ3WHFwMGRjQ3ZEdU5TQm8rZHhiCnQxWlFzdjhkQWdNQkFBRUNnZ0VBQU9mcDFiT2xLOVFKeXVlUHhjeDIvTkNVcUMxTEJDL01FdkEyUzVKWGFWbkcKbGhLR0pFb1U0Q0RoVk83dUlLYVZLTFZvMjk4RHFHUnBLM1d0RVE1TE40bytXYTcveTA5c1drMlVzbWxrVWFOZwpSaGtVZEJSK2dLNXVsQ3FKRml2dUJoTEQvRWlnQ1VWUGZKS2JtNG96TnpYcjVSMU5ENlV1aWFtODdtenlFcTBLCmZsVXlhR0RZNGdIdFNBOVBENVBFYlUveFpKeitKaHk5T2l3aVRXV0MrSHoyb2c3UWRDRDE2RlhGcit2VHpQN0MKb2tFb0VDZFNPRWlMalVENjBhS2ZxRmFCVm5MTkVudC9QSytmY1RBM05mNGtSMnFDNk9ZWjVFb09zYm1ka29ZTgpyU3NJZW9XblMxOEhvekZud2w3Z05wTUtjNmRzQzRBTldOVDFsTkhCb1FLQmdRRHlaUDFJSlppZUh6NlExaUVTCm5zd2tnblZCQUQ0SlVLR1ZDMHA3dk4yclNDZXh4c05ZZXFPTEEyZGZCUGpOVjd3blFKcUgxT05XellOMUJVSUUKeThLTCtFZVl6Q3RZa21LL21wSGJIMzNjd2tJODBuMHJROU1BalZMTlJ2YVVEOWp1NFBsRzFqaEFZUVVyTkViZQpKRlVpSk83aDVQa1llZG50SitqSHFpQnRoUUtCZ1FEZGtPbndmL0szYk4xenR0bXZQd0VicjhkVWJjRVh5NDFOCkl5VWwrZW1WSlgzYktKM0duNDZnQ2RsTTdkYmpwS3JVZ3oxL2JsZTgvMkVFckJvSEFRNkMrU2pEaGhvL01CbnIKekZheTBoK3YxbjBnZnNNVzRoOEF4cEFwc25OYnh6K2g1Wm5uSnRTd0srUjB3U0VJVVEzRjAxL2hMWWhLQ2l5OApwbW5HQi9hU3VRS0JnRzdxd1cvVExGd214ZlYyMXBsenFzeUdHZXVObGRXalhOMGIxcEI2b3lDdW11TmhwYUFHCk5uSDFNOGNxT2tPVWd4ZWZHMWRPbGx6eEc5ZGZlWTlDUWhyVW1NYVZucndmK0NuZkxDRU43d1VtcXpLenl1MFMKVXlwc2dOaElRYXNNK1dLTjllTnhRVHBNYXhZVERONjMxM0VSWDNKazJZdFdydDh6cFBSQXFDZ1ZBb0dCQU54egpUa0NMbmJ6aFphbTNlZm9DenlCMEVma3dSdHBkSGxkc3E0NlFqTmRuK1VSd3NpTXBLR2lWeEE3bDZsU1B4NlV3CmU2VHA3Z1JQZUlHRWwxVDJ1VENacGZSODNtcVdlb1FCeVJXZE9nZmplcFkxYWZpL3ZhY3c2Y21ERTRKeXloNVUKYTMveFE5ZVJwSHFDbWxKREMxZ1V5eVlwL3B2a2FjUytNeW5sVEhHSkFvR0FQekdTSzdXOHBUYldSVEFoaTVrSQpwZk5kWk1tcnRodUxNT3F6TGhyRjZublpldk9OdTBoYXVhZktlVElFd2w0clhYZHFKQlJBaWZKMFFsLzZKWFFkCmd1VzFrZWk1Ui8rUFZ5eUhab042c3NXSTNWYklwUUloUmt6UENnTDZhbHEwSzFpT1dlV1lIOHdORGRRdlB1T2UKRkZPOEovSzNxV0NtWjU0ODBBbTNhT0U9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K"
 SHARP_SERVER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQWxPZ0F3SUJBZ0lVRUR0Rjd0YVNTUnVQQTJ6Uk1aNWNzY2JCRm5jd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1JURUxNQWtHQTFVRUJoTUNTVTR4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpEQWVGdzB5TkRBNE1UTXhNekEzTVROYUZ3MHlOVEE0Ck1UTXhNekEzTVROYU1FVXhDekFKQmdOVkJBWVRBa2xPTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXcKSHdZRFZRUUtEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQgpBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFRHcEEwNEZ1QlNFaE5PNVYvMGxTaDkvSEgxeVRZT2dRVFdoOG43eDlRCnZGMHpvZFZueVFIdjE5elU5eVdia2xvOEkvOXFBVm9lRzdXTnpUVFg2Q295ZlNjb1YvazN0Q2UwVnVWMlFJTVQKdW82SzJSU05CVHB1TlNqNTlzUiszVTQ2OFRBQnY0YVpsYjU4TU5CRXM3MVRieVpLRHBGRVRkMkg3T0ZKajg4QQpNRi9MaXJkeDZPOFdZL0tDeisxd1ZXL1JRdytYYjRJSWx4bXJFOC9UZ3FNSEo4dFUxYkZiOWJNcTEvOTN5YWtJClU1V2J2NVhXKzFwZFVyTUFNcTFFaC9vZThMN2pFaFdvZXZrNzgyU0kwUk0xeG5MaEtrUUVBYXd6Zkg2ODZiR2YKUHQ3RkFIQ1pGaWJ4KzZzSkg0R1M3S25iK0x5bk9ud3phMWZPUXZEZmcvRm5BZ01CQUFHalV6QlJNQjBHQTFVZApEZ1FXQkJUYlFUdmlUTW1xNXlNK2ZJRVI4VjdTZk1pK3B6QWZCZ05WSFNNRUdEQVdnQlRiUVR2aVRNbXE1eU0rCmZJRVI4VjdTZk1pK3B6QVBCZ05WSFJNQkFmOEVCVEFEQVFIL01BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRREYKTllyRnpBa2RIVkhjUkd5SUNsTi9IVGswaldOcTVSdTB1RUpDQ21Dbm9ZY1pRSTlDQlcwTkl3dGpZUkpTKzR1UwordWh4VWpSYTA5YXdOWDhvYmU0dDZjK25HRnhZMGZqamk0cGZnbU1kMWNJeGdsM3E3Nlp0ZkllRGR6alRLRXN1CjRFUTVadnEwMnJvTEZ0ZjEvL3dRVG0xNkNKdFpGWnhNZ1phYnNxc2JRc3M2dWdMUGtTTmdBWjI1L2VhcWhnQ20KTjFUV2FxL0xJMVBLSkxPK085NFlMa2FsNVpyOTJCOXk4Q0VKVUVuSTA1R1N1MmJUOFM2a0ZBMEpadEszTW9SbwpqRWZWV1lQVHR5TFR4amNvRndCcDlHaXZYSDdSdHBxMDlmSmFhU1pNekxmNGlyNHpBdXprbExBNWZvampPNXlKCllnYlVaQUU2aS81N1NFWjR3VmxTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
 SHARP_PROOF_LAYOUT="small"
+
+##### ON CHAIN CONFIG #####
+
+DA_LAYER="ethereum"
+SETTLEMENT_LAYER="ethereum"
+SETTLEMENT_RPC_URL="http://localhost:3001"
+MADARA_RPC_URL="http://localhost:3000"
+L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
+MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead"
+PRIVATE_KEY="0xdead"
+ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead"

.github/workflows/coverage.yml

@@ -1,8 +1,10 @@
 name: Rust Test & Coverage
 
 on:
-  workflow_dispatch:
   workflow_call:
+    secrets:
+      ETHEREUM_BLAST_RPC_URL:
+        required: true
 
 jobs:
   coverage:
@@ -40,6 +42,20 @@ jobs:
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
 
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Check Anvil Installation
+        run: |
+          if command -v anvil &> /dev/null
+          then
+              echo "Anvil is installed. Version information:"
+              anvil --version
+          else
+              echo "Anvil is not installed or not in PATH"
+              exit 1
+          fi
+
       - name: Install cargo-llvm-cov & nextest
         uses: taiki-e/install-action@cargo-llvm-cov
 
@@ -50,12 +66,10 @@ jobs:
         run: |
           wget -P ./crates/prover-services/sharp-service/tests/artifacts https://madara-orchestrator-sharp-pie.s3.amazonaws.com/238996-SN.zip
 
-      - name: Clean workspace
-        run: |
-          cargo clean --doc
-
-      - name: Run tests llvm-cov
-        run: cargo llvm-cov nextest --release --lcov --output-path lcov.info --test-threads=1
+      - name: Run llvm-cov tests
+        env:
+          ETHEREUM_BLAST_RPC_URL: ${{ secrets.ETHEREUM_BLAST_RPC_URL }}
+        run: RUST_LOG=debug RUST_BACKTRACE=1 cargo llvm-cov nextest --release --lcov --output-path lcov.info --test-threads=1
 
       - name: Coveralls
         uses: coverallsapp/github-action@v2

.github/workflows/pull-request.yml

@@ -25,3 +25,4 @@ jobs:
     name: Run coverage
     uses: ./.github/workflows/coverage.yml
     needs: rust_build
+    secrets: inherit

CHANGELOG.md

@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Added
 
+- Tests for Settlement client.
+- Worker queues to listen for trigger events.
 - Tests for prover client.
 - Added testcontainer for parallel testing.
 - added new creator for SqsQueue.
@@ -30,13 +32,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Changed
 
+- refactor AWS config usage and clean .env files
 - GitHub's coverage CI yml file for localstack and db testing.
 - Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder.
 - `.env` file requires two more variables which are queue urls for processing
   and verification.
 
 ## Removed
 
+- revert CI changes from settlement client PR.
 - `init_config` from all the tests.
 - `fetch_from_test` argument