Fixes #7004: making azure-key-valt refresh test stable.

integration-test-groups/azure/azure-key-vault/README.adoc

@@ -80,3 +80,9 @@ Following properties are generated by the script and are required for the test e
     export AZURE_STORAGE_ACCOUNT_KEY=<storage account key required for context refresh configuration>
 ----
 
+=== Limitations
+
+Do not execute the tests in parallel, without changing the Event Hubs resource!
+
+Even thought each test is using a unique secret name regexp, the refresh trigger task receives all events, which are send to the event hub.
+Therefore an event send by the first test might be consumed by the second test, which would fail the first test.

integration-test-groups/azure/azure-key-vault/src/main/java/org/apache/camel/quarkus/component/azure/key/vault/it/AzureKeyVaultResource.java

@@ -36,10 +36,13 @@
 import org.apache.camel.ResolveEndpointFailedException;
 import org.apache.camel.component.azure.key.vault.KeyVaultConstants;
 import org.apache.camel.impl.event.CamelContextReloadedEvent;
+import org.jboss.logging.Logger;
 
 @Path("/azure-key-vault")
 @ApplicationScoped
 public class AzureKeyVaultResource {
+    private static final Logger LOG = Logger.getLogger(AzureKeyVaultResource.class);
+
     @Inject
     ProducerTemplate producerTemplate;
 
@@ -49,6 +52,7 @@ public class AzureKeyVaultResource {
     static final AtomicBoolean contextReloaded = new AtomicBoolean(false);
 
     void onReload(@Observes CamelContextReloadedEvent event) {
+        LOG.info("AzureKeyVaultResource onReload");
         contextReloaded.set(true);
     }
 

integration-test-groups/azure/azure-key-vault/src/main/resources/application.properties

@@ -15,6 +15,8 @@
 ## limitations under the License.
 ## ---------------------------------------------------------------------------
 camel.vault.azure.vaultName = ${AZURE_VAULT_NAME:cq-vault-testing}
+camel.main.context-reload-enabled = true
+
 #following properties are added by the test profile if needed
 #camel.vault.azure.tenantId = ${AZURE_TENANT_ID:placeholderTenantId}
 #camel.vault.azure.clientId = ${AZURE_CLIENT_ID:placeholderClientId}

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AbstractAzureKeyVaultContextReloadTest.java

@@ -18,15 +18,13 @@
 
 import java.util.LinkedList;
 import java.util.List;
-import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
 import com.azure.messaging.eventhubs.EventData;
 import com.azure.messaging.eventhubs.EventHubClientBuilder;
-import com.azure.messaging.eventhubs.EventHubConsumerAsyncClient;
 import com.azure.messaging.eventhubs.EventHubProducerClient;
-import com.azure.messaging.eventhubs.models.EventPosition;
 import io.restassured.RestAssured;
+import org.eclipse.microprofile.config.ConfigProvider;
 import org.hamcrest.CoreMatchers;
 import org.jboss.logging.Logger;
 import org.junit.jupiter.api.Test;
@@ -38,26 +36,20 @@
 abstract class AbstractAzureKeyVaultContextReloadTest {
 
     private static final Logger LOG = Logger.getLogger(AbstractAzureKeyVaultContextReloadTest.class);
-    private static final String SECRET_NAME_FOR_REFRESH_PREFIX = "cq-secret-context-refresh-";
     private static final String AZURE_VAULT_EVENT_HUBS_CONNECTION_STRING = "AZURE_VAULT_EVENT_HUBS_CONNECTION_STRING";
 
-    private final boolean useIdentity;
-
-    public AbstractAzureKeyVaultContextReloadTest(boolean useIdentity) {
-        this.useIdentity = useIdentity;
-    }
-
     private String generateRefreshEvent(String secretName) {
         return "[{\n" +
-                "  \"subject\": \"" + SECRET_NAME_FOR_REFRESH_PREFIX + (useIdentity ? "Identity-" : "") + ".*\",\n" +
+                "  \"subject\": \"" + secretName + "\",\n" +
                 "  \"eventType\": \"Microsoft.KeyVault.SecretNewVersionCreated\"\n" +
                 "}]";
     }
 
     @Test
     void contextReload() {
-        String secretName = SECRET_NAME_FOR_REFRESH_PREFIX + (useIdentity ? "Identity-" : "") + UUID.randomUUID();
+        String secretName = ConfigProvider.getConfig().getValue("camel.vault.azure.secrets", String.class).replace(".*", "");
         String secretValue = "Hello Camel Quarkus Azure Key Vault From Refresh";
+        boolean reloadDetected = false;
         try {
             // Create secret
             RestAssured.given()
@@ -66,12 +58,21 @@ void contextReload() {
                     .then()
                     .statusCode(200)
                     .body(is(secretName));
+            LOG.infof("Secret created: %s", secretName);
 
             // Retrieve secret
             RestAssured.given()
                     .get("/azure-key-vault/secret/true/{secretName}", secretName)
                     .then()
                     .statusCode(200);
+            LOG.info("Secret verified before refresh.");
+
+            LOG.info("Wait some time for listener to be initialized");
+            try {
+                Thread.sleep(5000);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
 
             //force reload by sending a msg
             try (EventHubProducerClient client = new EventHubClientBuilder()
@@ -81,44 +82,52 @@ void contextReload() {
                 EventData eventData = new EventData(generateRefreshEvent(secretName).getBytes());
                 List<EventData> finalEventData = new LinkedList<>();
                 finalEventData.add(eventData);
+                LOG.info("Sending refresh event.");
                 client.send(finalEventData);
             } catch (Exception e) {
                 LOG.info("Failed to send a refresh message", e);
             }
 
             //await context reload
-            Awaitility.await().pollInterval(10, TimeUnit.SECONDS).atMost(1, TimeUnit.MINUTES).untilAsserted(
+            Awaitility.await().pollInterval(10, TimeUnit.SECONDS).atMost(2, TimeUnit.MINUTES).untilAsserted(
                     () -> {
                         RestAssured.get("/azure-key-vault/context/reload")
                                 .then()
                                 .statusCode(200)
                                 .body(CoreMatchers.is("true"));
                     });
+            reloadDetected = true;
         } finally {
-
-            //move cursor of events to ignore old ones (old events are deleted after 1 hour)
-            try {
-                String connectionString = System.getenv(AZURE_VAULT_EVENT_HUBS_CONNECTION_STRING);
-                String consumerGroup = EventHubClientBuilder.DEFAULT_CONSUMER_GROUP_NAME;
-
-                try (EventHubConsumerAsyncClient consumer = new EventHubClientBuilder()
-                        .connectionString(connectionString)
-                        .consumerGroup(consumerGroup)
-                        .buildAsyncConsumerClient()) {
-
-                    // Move consumer to the latest position, skipping old messages
-                    consumer.receiveFromPartition("0", EventPosition.latest())
-                            .subscribe(event -> {
-                                System.out.println("Processing new event: " + event.toString());
-                            }, error -> {
-                                System.err.println("Error receiving events: " + error);
-                            });
-                }
-            } catch (Exception e) {
-                LOG.info("Failed to clear event hub.", e);
-            }
-
-            AzureKeyVaultUtil.deleteSecretImmediately(secretName);
+            // meant to be commented.
+            // during development, it may be handy to mark eventhub as completely read. (in case the test is not reading all the messages by itself)
+            // Please uncomment the rest of the code to make cursor reset to the latest position after test execution
+
+            //            following code moves the cursor of the hub to the latest position, thus marking all events as read
+            //            partition 0 is hardcoded (the resource script creates only one partition)
+            //            by default this functionality should not be executed, as the eventbus might be shared for more tests/purposes
+            //            even if event stays, it is removed by retention policy in some time (i.e. 1 hpr)
+            //            try {
+            //                String connectionString = System.getenv(AZURE_VAULT_EVENT_HUBS_CONNECTION_STRING);
+            //                String consumerGroup = EventHubClientBuilder.DEFAULT_CONSUMER_GROUP_NAME;
+            //
+            //                try (EventHubConsumerAsyncClient consumer = new EventHubClientBuilder()
+            //                        .connectionString(connectionString)
+            //                        .consumerGroup(consumerGroup)
+            //                        .buildAsyncConsumerClient()) {
+            //
+            //                    // Move consumer to the latest position, skipping old messages
+            //                    consumer.receiveFromPartition("0", EventPosition.latest())
+            //                            .subscribe(event -> {
+            //                                System.out.println("Processing new event: " + event.toString());
+            //                            }, error -> {
+            //                                System.err.println("Error receiving events: " + error);
+            //                            });
+            //                }
+            //            } catch (Exception e) {
+            //                LOG.info("Failed to clear event hub.", e);
+            //            }
+
+            AzureKeyVaultUtil.deleteSecretImmediately(secretName, true);
         }
     }
 }

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AbstractAzureKeyVaultTest.java

@@ -76,6 +76,7 @@ void secretCreateRetrieveDeletePurge() {
                     .then()
                     .statusCode(200)
                     .body(is(secret));
+
         } finally {
             AzureKeyVaultUtil.deleteSecretImmediately(secretName, useIdentity);
         }

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AzureKeyVaultContextReloadTest.java

@@ -36,7 +36,4 @@
 @TestProfile(AzureKeyVaultContextReloadTestProfile.class)
 @QuarkusTest
 class AzureKeyVaultContextReloadTest extends AbstractAzureKeyVaultContextReloadTest {
-    public AzureKeyVaultContextReloadTest() {
-        super(false);
-    }
 }

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AzureKeyVaultContextReloadTestProfile.java

@@ -18,6 +18,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 
 import io.quarkus.test.junit.QuarkusTestProfile;
 
@@ -32,12 +33,11 @@ public Map<String, String> getConfigOverrides() {
         props.put("camel.vault.azure.clientSecret", System.getenv("AZURE_CLIENT_SECRET"));
         props.put("camel.vault.azure.refreshEnabled", "true");
         props.put("camel.vault.azure.refreshPeriod", "1000");
-        props.put("camel.vault.azure.secrets", "cq-secret-context-refresh.*");
+        props.put("camel.vault.azure.secrets", String.format("cq-secret-context-refresh-%s.*", UUID.randomUUID()));
         props.put("camel.vault.azure.eventhubConnectionString", System.getenv("AZURE_VAULT_EVENT_HUBS_CONNECTION_STRING"));
         props.put("camel.vault.azure.blobAccountName", System.getenv("AZURE_STORAGE_ACCOUNT_NAME"));
         props.put("camel.vault.azure.blobContainerName", System.getenv("AZURE_VAULT_EVENT_HUBS_BLOB_CONTAINER_NAME"));
         props.put("camel.vault.azure.blobAccessKey", System.getenv("AZURE_STORAGE_ACCOUNT_KEY"));
-        props.put("camel.main.context-reload-enabled", "true");
 
         return props;
     }

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AzureKeyVaultContextReloadWithIdentityTest.java

@@ -36,7 +36,4 @@
 @TestProfile(AzureKeyVaultContextReloadWithIdentityTestProfile.class)
 @QuarkusTest
 class AzureKeyVaultContextReloadWithIdentityTest extends AbstractAzureKeyVaultContextReloadTest {
-    public AzureKeyVaultContextReloadWithIdentityTest() {
-        super(true);
-    }
 }

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AzureKeyVaultContextReloadWithIdentityTestProfile.java

@@ -18,6 +18,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 
 import io.quarkus.test.junit.QuarkusTestProfile;
 
@@ -29,12 +30,11 @@ public Map<String, String> getConfigOverrides() {
         Map<String, String> props = new HashMap<>();
         props.put("camel.vault.azure.refreshEnabled", "true");
         props.put("camel.vault.azure.refreshPeriod", "1000");
-        props.put("camel.vault.azure.secrets", "cq-secret-context-refresh.*");
+        props.put("camel.vault.azure.secrets", String.format("cq-secret-context-refresh-identity-%s.*", UUID.randomUUID()));
         props.put("camel.vault.azure.eventhubConnectionString", System.getenv("AZURE_VAULT_EVENT_HUBS_CONNECTION_STRING"));
         props.put("camel.vault.azure.blobAccountName", System.getenv("AZURE_STORAGE_ACCOUNT_NAME"));
         props.put("camel.vault.azure.blobContainerName", System.getenv("AZURE_VAULT_EVENT_HUBS_BLOB_CONTAINER_NAME"));
         props.put("camel.vault.azure.blobAccessKey", System.getenv("AZURE_STORAGE_ACCOUNT_KEY"));
-        props.put("camel.main.context-reload-enabled", "true");
         props.put("camel.vault.azure.azureIdentityEnabled", "true");
 
         return props;

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AzureKeyVaultTest.java

@@ -66,7 +66,7 @@ void wrongClientTest() {
             tryToDeleteSecret = false;
         } finally {
             if (tryToDeleteSecret) {
-                AzureKeyVaultUtil.deleteSecretImmediately(secretName);
+                AzureKeyVaultUtil.deleteSecretImmediately(secretName, true);
             }
         }
     }

integration-test-groups/azure/azure-key-vault/src/test/java/org/apache/camel/quarkus/component/azure/key/vault/it/AzureKeyVaultUtil.java

@@ -16,33 +16,101 @@
  */
 package org.apache.camel.quarkus.component.azure.key.vault.it;
 
+import com.azure.core.credential.TokenCredential;
+import com.azure.core.exception.HttpResponseException;
+import com.azure.core.exception.ResourceNotFoundException;
+import com.azure.identity.ClientSecretCredentialBuilder;
+import com.azure.security.keyvault.secrets.SecretClient;
+import com.azure.security.keyvault.secrets.SecretClientBuilder;
+import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
 import io.restassured.RestAssured;
+import org.jboss.logging.Logger;
 
 public class AzureKeyVaultUtil {
+    private static final Logger LOG = Logger.getLogger(AzureKeyVaultUtil.class);
 
-    static void deleteSecretImmediately(String secretName) {
-        //we need to se identity by default, as the non-identity routes may not start
-        AzureKeyVaultUtil.deleteSecretImmediately(secretName, true);
-    }
+    private static final int MAX_RETRIES = 5;
+    private static final int RETRY_DELAY_MS = 5000; // 5 seconds
 
     static void deleteSecretImmediately(String secretName, boolean useIdentity) {
-        // Delete secret
-        RestAssured.given()
-                .delete("/azure-key-vault/secret/" + useIdentity + "/{secretName}", secretName)
-                .then()
-                .statusCode(200);
-
-        // Purge secret
-        RestAssured.given()
-                .delete("/azure-key-vault/secret/" + useIdentity + "/{secretName}/purge", secretName)
-                .then()
-                .statusCode(200);
-
-        // Confirm deletion
-        RestAssured.given()
-                .queryParam("identity", useIdentity)
-                .get("/azure-key-vault/secret/" + useIdentity + "/{secretName}", secretName)
-                .then()
-                .statusCode(500);
+
+        boolean deleted = false;
+
+        try {
+            // Delete secret
+            RestAssured.given()
+                    .delete("/azure-key-vault/secret/" + useIdentity + "/{secretName}", secretName)
+                    .then()
+                    .statusCode(200);
+
+            // Purge secret
+            RestAssured.given()
+                    .delete("/azure-key-vault/secret/" + useIdentity + "/{secretName}/purge", secretName)
+                    .then()
+                    .statusCode(200);
+
+            // Confirm deletion
+            RestAssured.given()
+                    .queryParam("identity", useIdentity)
+                    .get("/azure-key-vault/secret/" + useIdentity + "/{secretName}", secretName)
+                    .then()
+                    .statusCode(500);
+            deleted = true;
+        } finally {
+            if (!deleted) {
+                // in case the deletion via component fails, delete directly via client
+                deleteSecretImmediatelyViaClient(secretName);
+            }
+        }
+
+    }
+
+    private static void deleteSecretImmediatelyViaClient(String secretName) {
+
+        //create client
+        String keyVaultUri = "https://" + System.getenv("AZURE_VAULT_NAME") + ".vault.azure.net";
+        TokenCredential credential = ((ClientSecretCredentialBuilder) ((ClientSecretCredentialBuilder) (new ClientSecretCredentialBuilder())
+                .tenantId(System.getenv("AZURE_TENANT_ID"))).clientId(System.getenv("AZURE_CLIENT_ID")))
+                .clientSecret(System.getenv("AZURE_CLIENT_SECRET")).build();
+
+        SecretClient client = (new SecretClientBuilder()).vaultUrl(keyVaultUri).credential(credential).buildClient();
+
+        try {
+            KeyVaultSecret secret = client.getSecret(secretName);
+
+            if (secret != null) {
+                client.beginDeleteSecret(secretName);
+            }
+
+        } catch (ResourceNotFoundException e) {
+            //already deleted
+        } finally {
+            //purge secret in all cases to be sure it is purged
+            try {
+                client.purgeDeletedSecret(secretName);
+            } catch (HttpResponseException e) {
+                if (e.getResponse().getStatusCode() == 409) { // Conflict: Object is being deleted
+                    int attempt = 0;
+                    while (attempt++ < MAX_RETRIES) {
+                        LOG.infof("Attempt %d to delete secret '%s'.", attempt, secretName);
+                        try {
+                            Thread.sleep(RETRY_DELAY_MS);
+                        } catch (InterruptedException ex) {
+                            LOG.errorf("Purging of secret `%s` failed", secretName, ex);
+                        }
+                        try {
+                            client.purgeDeletedSecret(secretName);
+                            break;
+                        } catch (HttpResponseException ex) {
+                            LOG.errorf("Purging of secret `%s` failed", secretName, ex);
+                        }
+
+                    }
+                    if (attempt >= MAX_RETRIES) {
+                        LOG.errorf("Purging of secret `%s` failed after %d attempts.", secretName, attempt);
+                    }
+                }
+            }
+        }
     }
 }