Test/component updates 02082022

.ado/pipelines/README.md

@@ -91,9 +91,9 @@ All pipelines are using Azure DevOps service connections to connect to Microsoft
 As part of the pipelines, basic Smoke Tests against the APIs are executed:
 
 - GET call to the Health Service `/health/stamp` API. Expected result: HTTP 200
-- GET call the CatalogService `/api/1.0/catalogitem` API to retrieve a list of existing items. Expected result: HTTP 200
-- POST call to the CatalogService `/api/1.0/catalogitem/{itemId}/comments` API to create a new comment for an existing item. Expected result: HTTP 202
-- GET call to the CatalogService `/api/1.0/catalogitem/{itemId}/comments/{commentId}` API to retrieve the previously created comment. Expected result: HTTP 200
+- GET call the CatalogService `/catalogservice/api/1.0/catalogitem` API to retrieve a list of existing items. Expected result: HTTP 200
+- POST call to the CatalogService `/catalogservice/api/1.0/catalogitem/{itemId}/comments` API to create a new comment for an existing item. Expected result: HTTP 202
+- GET call to the CatalogService `/catalogservice/api/1.0/catalogitem/{itemId}/comments/{commentId}` API to retrieve the previously created comment. Expected result: HTTP 200
 
 The calls are first executed against the individual stamps to test the availability of each regional deployment and afterwards against the global Front Door endpoint, which distributes the requests to the different stamps.
 

.ado/pipelines/azure-deploy-grafana.yaml

@@ -0,0 +1,136 @@
+name: '[${{ parameters.environment }}] $(BuildDefinitionName).$(DayOfYear)$(Rev:.r)'
+
+trigger: none
+
+pr: none
+
+parameters:
+- name: environment
+  displayName: 'Environment (Subscriptions)'
+  type: string
+  default: 'e2e'
+  values:
+  - e2e
+  - int
+  - prod
+
+pool:
+  name: 'e2e-private-agents' # self-hosted agentpool
+
+variables:
+- name:  'environment' # set the environment name as variable
+  value: '${{ parameters.environment }}'
+- name:  'workingDirectory'       # Terraform working directory
+  value: 'src/infra/monitoring/grafana'
+- name:  'azureServiceConnection' # environment specific service connection
+  value: 'alwayson-${{ parameters.environment }}-serviceconnection'
+
+- template: config/configuration.yaml # contains version numbers and global configuration settings
+- template: config/variables-values-${{ parameters.environment }}.yaml # contains environment-specific values (non-secret)
+
+stages:
+
+- stage: deployglobalinfra
+  displayName: 'Deploy Global Infra'
+  jobs:
+    - job: deployterraform
+      displayName: 'Deploy Resources'
+      steps:
+      - checkout: self # checkout github repository
+
+      - template: templates/steps-terraform-init.yaml # Initialize Terraform
+        parameters:
+          terraformStorageAccountName:        '$(terraformStorageAccount)'
+          terraformStorageResourceGroupName:  '$(terraformResourceGroup)'
+          terraformStateFilename:             'terraform-grafana-$(prefix).state'
+          terraformWorkingDirectory:          '$(workingDirectory)/terraform/globalresources'
+
+      - template: templates/steps-terraform-apply.yaml
+        parameters:
+          jobName:                    'GlobalInfra'
+          terraformWorkingDirectory:  '$(workingDirectory)/terraform/globalresources'
+          customPrefix:               '$(prefix)'
+          customSuffix:               ''
+          customAttributes:           '-var=queued_by="$(Build.QueuedBy)"
+                                      -var=branch="$(Build.SourceBranch)"
+                                      -var=contact_email="$(contactEmail)"
+                                      -var=environment="$(environment)"
+                                      -var=''stamps=$(stampLocationsGrafana)'''
+
+- stage: buildcontainer
+  displayName: 'Build Docker Container'
+  dependsOn: deployglobalinfra
+  jobs:
+  - template: templates/jobs-container-build.yaml
+    parameters:
+      jobName:                        'alwayson_grafana' # unique pipeline job name
+      containerImageName:             'alwayson-grafana'   # container image name for Grafana
+      containerImageDockerFile:       './Dockerfile'  # dockerfile used to build Grafana
+      workingDirectory:               '$(workingDirectory)'
+
+
+- stage: deploystampinfra
+  displayName: 'Deploy Stamp Infra'
+  jobs:
+    - job: deployterraform
+      displayName: 'Deploy Resources'
+      steps:
+      - checkout: self # checkout github repository
+      - download: current # Download build artefacts
+
+      - template: templates/steps-terraform-init.yaml # Initialize Terraform
+        parameters:
+          terraformStorageAccountName:        '$(terraformStorageAccount)'
+          terraformStorageResourceGroupName:  '$(terraformResourceGroup)'
+          terraformStateFilename:             'terraform-grafana-stamp-$(prefix).state'
+          terraformWorkingDirectory:          '$(workingDirectory)/terraform/stamps'
+
+      # Get the deployment output from the global resources. We need this for the FDID.
+      - template: templates/steps-parse-terraform-output.yaml
+        parameters:
+          workingDirectory:           '$(pipeline.workspace)/terraformOutputGlobalInfra'
+
+      # Get the full image name/tags from the artefact created during build
+      - template: templates/steps-parse-containerimagename.yaml
+        parameters:
+          workingDirectory:           '$(pipeline.workspace)'
+          imageName:                  'alwayson_grafana'
+
+      - task: PowerShell@2
+        displayName: 'Prepare stamp input variables'
+        inputs:
+          targetType: inline
+          script: |
+
+            $locations = '$(stampLocationsGrafana)' | ConvertFrom-Json
+
+            $stamps = @{
+              primary = @{
+                location           = $locations[0]
+                vnet_address_space = "10.1.0.0/16"
+              };
+              secondary = @{
+                location           = $locations[1]
+                vnet_address_space = "10.2.0.0/16"
+              };
+            }
+
+            $stamps = $stamps | ConvertTo-Json -Compress
+            echo "##vso[task.setvariable variable=grafanaStamps]$stamps"
+
+      # Deploy the stamp resources for both regions
+      - template: templates/steps-terraform-apply.yaml
+        parameters:
+          jobName:                    'StampInfra'
+          terraformWorkingDirectory:  '$(workingDirectory)/terraform/stamps'
+          customPrefix:               '$(prefix)'
+          customSuffix:               ''
+          customAttributes:           '-var=queued_by="$(Build.QueuedBy)"
+                                      -var=branch="$(Build.SourceBranch)"
+                                      -var=contact_email="$(contactEmail)"
+                                      -var=environment="$(environment)"
+                                      -var=wapp_container_image="$(ImageName-alwayson_grafana)"
+                                      -var=acr_resource_id="$(acr_resource_id)"
+                                      -var=''stamps=$(grafanaStamps)''
+                                      -var=frontdoor_fqdn="$(frontdoor_fqdn)"
+                                      -var=frontdoor_header_id="$(frontdoor_header_id)"'

.ado/pipelines/azure-release-e2e.yaml

@@ -18,7 +18,7 @@ pool:
 
 parameters:
 - name: destroyEnvironment
-  displayName: "Destroy Environment at the end"
+  displayName: "Destroy environment at the end"
   type: boolean
   default: true
 

.ado/pipelines/config/configuration.yaml

@@ -5,17 +5,17 @@ variables:
   value: 'foundational-connected'
 
 - name: 'terraformVersion'    # Terraform Version
-  value: '1.1.7'
+  value: '1.2.4'
 - name: 'kubernetesVersion'   # kubernetes version used for aks clusters
-  value: '1.22.6'
+  value: '1.23.5'
 - name: 'helmVersion'         # helm package manager version
-  value: 'v3.8.1'
+  value: 'v3.9.0'
 - name: 'ingressNginxVersion' # nginx ingress controller helm chart version
-  value: '4.1.0'
+  value: '4.1.4'
 - name: 'certManagerVersion'  # cert-manager helm chart version
-  value: 'v1.8.0'
+  value: 'v1.8.2'
 - name: 'dotnetSdkVersion'    # dotnet sdk version
-  value: '6.0.200'
+  value: '6.0.301'
 - name: 'uiAppNodeVersion'    # nodejs version for the UI app
   value: '16.x'
 
@@ -42,4 +42,4 @@ variables:
 - name: 'smokeTestRetryWaitSeconds' # How long to wait between each retry of the smoke tests
   value: '60'
 - name: 'skipComponentGovernanceDetection' # This is a Microsoft-specific setting. Having this in place ensures we don't create work items with CELA every time we use an open-source component
-  value: 'true'
+  value: 'true'

.ado/pipelines/config/variables-values-int.yaml

@@ -11,6 +11,9 @@ variables:
 - name: 'stampLocations'
   value: '["westeurope","northeurope"]' # Check which regions are valid. There is a list in /src/infra/README.md
 
+- name: 'stampLocationsGrafana'
+  value: '["eastus", "brazilsouth"]' # Check which regions are valid. There is a list in /src/infra/README.md
+
 # Other settings
 - name:  'contactEmail'
   value: 'noreply@microsoft.com'  # <===== CHANGE THIS VALUE TO ONE OF YOUR REAL EMAIL ADDRESSES (OR A GROUP)!

.ado/pipelines/config/variables-values-prod.yaml

@@ -11,6 +11,9 @@ variables:
 - name: 'stampLocations'
   value: '["northeurope", "canadacentral"]' # Check which regions are valid. There is a list in /src/infra/README.md
 
+- name: 'stampLocationsGrafana'
+  value: '["eastus", "brazilsouth"]' # Check which regions are valid. There is a list in /src/infra/README.md
+
 # Other settings
 - name:  'contactEmail'
   value: 'noreply@microsoft.com'  # <===== CHANGE THIS VALUE TO ONE OF YOUR REAL EMAIL ADDRESSES (OR A GROUP)!

.ado/pipelines/scripts/README.md

@@ -0,0 +1,3 @@
+# Pipeline scripts
+
+This folder contains scripts for which are referenced by various pipeline tasks.

.ado/scripts/SmokeTest.ps1 → .ado/pipelines/scripts/Run-SmokeTests.ps1

@@ -7,7 +7,7 @@ param(
 # -----------
 # Load helper functions.
 # -----------
-. $env:SYSTEM_DEFAULTWORKINGDIRECTORY/.ado/scripts/Invoke-WebRequestWithRetry.ps1
+. $env:SYSTEM_DEFAULTWORKINGDIRECTORY/.ado/pipelines/scripts/Invoke-WebRequestWithRetry.ps1
 
 # -----------
 # Execute smoke tests.
@@ -33,7 +33,7 @@ Write-Output "*******************"
 # request body needs to be a valid object expected by the API - keep up to date when the contract changes
 $post_comment_body = @{
   "authorName" = "Smoke Test Author"
-  "text" = "Just a smoke test"
+  "text"       = "Just a smoke test"
 } | ConvertTo-JSON
 
 
@@ -48,8 +48,8 @@ if ($mode -eq "stamp") {
 
   # setting header with X-Azure-FDID for HTTP-based smoke tests (required to access the individual stamps directly, bypassing Front Door)
   $header = @{
-    "X-Azure-FDID"="$frontdoorHeaderId"
-    "X-TEST-DATA"="true" # Header to indicate that posted comments and rating are just for test and can be deleted again by the app
+    "X-Azure-FDID" = "$frontdoorHeaderId"
+    "X-TEST-DATA"  = "true" # Header to indicate that posted comments and rating are just for test and can be deleted again by the app
   }
 
   # loop through stamps from pipeline artifact json
@@ -70,7 +70,7 @@ if ($mode -eq "stamp") {
 }
 else {
   $header = @{
-    "X-TEST-DATA"="true"
+    "X-TEST-DATA" = "true"
   }
 
   $props = @{
@@ -85,7 +85,7 @@ else {
 Write-Output "*** Testing $($targets.Count) targets"
 
 # loop through targets - either multiple stamps or one front door (global)
-foreach($target in $targets) {
+foreach ($target in $targets) {
 
   # shorthand for easier manipulation in strings
   $targetFqdn = $target.ApiEndpointFqdn
@@ -95,14 +95,14 @@ foreach($target in $targets) {
 
   # test health endpoints for stamps only
   if ($mode -eq "stamp") {
-    $stampHealthUrl = "https://$targetFqdn/health/stamp"
+    $stampHealthUrl = "https://$targetFqdn/healthservice/health/stamp"
     Write-Output "*** Call - Stamp Health ($mode)"
 
     # custom retry loop to handle the situation when the SSL certificate is not valid yet and Invoke-WebRequest throws an exception
     Invoke-WebRequestWithRetry -Uri $stampHealthUrl -Method 'GET' -Headers $header -MaximumRetryCount $smokeTestRetryCount -RetryWaitSeconds $smokeTestRetryWaitSeconds
   }
 
-  $listCatalogUrl = "https://$targetFqdn/api/1.0/catalogitem"
+  $listCatalogUrl = "https://$targetFqdn/catalogservice/api/1.0/catalogitem"
   Write-Output "*** Call - List Catalog ($mode)"
   $responseListCatalog = Invoke-WebRequestWithRetry -Uri $listCatalogUrl -Method 'get' -Headers $header -MaximumRetryCount $smokeTestRetryCount -RetryWaitSeconds $smokeTestRetryWaitSeconds
   $responseListCatalog
@@ -115,11 +115,11 @@ foreach($target in $targets) {
 
   $randomItem = Get-Random $allItems
 
-  $itemUrl = "https://$targetFqdn/api/1.0/catalogitem/$($randomItem.id)"
+  $itemUrl = "https://$targetFqdn/catalogservice/api/1.0/catalogitem/$($randomItem.id)"
   Write-Output "*** Call - Get get item ($($randomItem.id)) ($mode)"
   Invoke-WebRequestWithRetry -Uri $itemUrl -Method 'GET' -Headers $header -MaximumRetryCount $smokeTestRetryCount -RetryWaitSeconds $smokeTestRetryWaitSeconds
 
-  $postCommentUrl = "https://$targetFqdn/api/1.0/catalogitem/$($randomItem.id)/comments"
+  $postCommentUrl = "https://$targetFqdn/catalogservice/api/1.0/catalogitem/$($randomItem.id)/comments"
   Write-Output "*** Call - Post new comment to item $($randomItem.id) ($mode)"
 
   $responsePostComment = Invoke-WebRequestWithRetry -Uri $postCommentUrl -Method 'POST' -Headers $header -Body $post_comment_body -MaximumRetryCount $smokeTestRetryCount -RetryWaitSeconds $smokeTestRetryWaitSeconds -ExpectedResponseCode 202
@@ -129,12 +129,10 @@ foreach($target in $targets) {
   Start-Sleep 10
 
   # The 202-response to POST new comment contains in the 'Location' header the URL under which the new comment will be accessible
-  $getCommentUrl = $responsePostComment.Headers['Location'][0]
+  $getCommentPath = $responsePostComment.Headers['Location'][0]
 
-  if ($mode -eq "stamp") {
-    # The Location header contains the global FQDN of the Front Door entry point. For the the individual cluster, we need to change the URL
-    $getCommentUrl = $getCommentUrl -replace $frontdoorFqdn,$targetFqdn
-  }
+  # The location header does not contain the host part of the URL so we need to prepend it
+  $getCommentUrl = "https://$($targetFqdn)$($getCommentPath)"
 
   Write-Output "*** Call - Get newly created comment ($mode)"
   Invoke-WebRequestWithRetry -Uri $getCommentUrl -Method 'GET' -Headers $header -MaximumRetryCount $smokeTestRetryCount -RetryWaitSeconds $smokeTestRetryWaitSeconds