Test opendev watcher master content in CI job

In order to test opendev watcher master content in CI job, this pr
adds following thigs:
* opendev-master-watcher-operator-pipeline project template which
  will run meta content provider from master and then Deploy EDPM
  job watcher-operator-validation-master from mastter content.

* It changes the existing job watcher-operator-validation (still
  deploying the antelope content) with update_containers: false to avoid
  using meta content provider containers.

* It also updates the existing watcher-operator-validation and watcher-operator-kuttl
  job to use openstack-meta-content-provider-master as a job
  dependencies.

Depends-On: https://review.opendev.org/c/openstack/watcher/+/938893

Signed-off-by: Chandan Kumar (raukadah) <raukadah@gmail.com>

.zuul.yaml

@@ -2,20 +2,20 @@
 - project:
     name: openstack-k8s-operators/watcher-operator
     default-branch: main
+    templates:
+      - opendev-master-watcher-operator-pipeline
     github-check:
       jobs:
         - noop
         - watcher-operator-doc-preview
-        - openstack-meta-content-provider:
-            vars:
-              cifmw_operator_build_meta_build: false
         - watcher-operator-validation
         - watcher-operator-kuttl
 
 - job:
     name: watcher-operator-base
+    override-checkout: main
     parent: podified-multinode-edpm-deployment-crc-2comp
-    dependencies: ["openstack-meta-content-provider"]
+    dependencies: ["openstack-meta-content-provider-master"]
     description: |
       A multinode EDPM Zuul job which has one ansible controller, one
       extracted crc and two computes. It will be used for testing watcher-operator.
@@ -40,9 +40,19 @@
       A zuul job to validate the watcher operator and its service deployment.
     vars:
       run_tempest: false
+      # Donot use openstack services from meta content provider master
+      # job.
+      cifmw_update_containers: false
+    extra-vars:
+      # Override zuul meta content provider provided content_provider_dlrn_md5_hash
+      # var. As returned dlrn md5 hash comes from master release but job is using
+      # antelope content.
+      content_provider_dlrn_md5_hash: ''
 
 - job:
     name: watcher-operator-kuttl
+    override-checkout: main
+    dependencies: ["openstack-meta-content-provider-master"]
     parent: cifmw-multinode-kuttl-operator-target
     description: |
       A zuul job to run watcher operator kuttl tests.
@@ -61,6 +71,11 @@
           source: "{{ watcher_hook }}"
           extra_vars:
             watcher_catalog_image: "{{ content_provider_registry_ip }}:5001/openstack-k8s-operators/watcher-operator-index:{{ zuul.patchset }}"
+    extra-vars:
+      # Override zuul meta content provider provided content_provider_dlrn_md5_hash
+      # var. As returned dlrn md5 hash comes from master release but job is using
+      # antelope content.
+      content_provider_dlrn_md5_hash: ''
 
 - job:
     name: watcher-operator-doc-preview
@@ -71,3 +86,52 @@
     vars:
       asciidoc_available: true
       doc_available: false
+
+##########################################################
+#                                                        #
+#               Master Zuul Jobs                         #
+#                                                        #
+##########################################################
+
+
+- job:
+    name: openstack-meta-content-provider-master
+    override-checkout: main
+    description: |
+      A zuul job building content from OpenDev master release.
+    parent: openstack-meta-content-provider
+    vars:
+      cifmw_operator_build_meta_build: false
+      cifmw_bop_openstack_release: master
+      cifmw_bop_dlrn_baseurl: "https://trunk.rdoproject.org/centos9-master"
+      cifmw_repo_setup_branch: master
+
+- job:
+    name: watcher-operator-validation-master
+    parent: watcher-operator-validation
+    override-checkout: main
+    description: |
+      A Zuul job consuming content from openstack-meta-content-provider-master
+      and deploying EDPM with master content.
+    dependencies: ["openstack-meta-content-provider-master"]
+    vars:
+      cifmw_repo_setup_branch: master
+      # To consume containers from meta content provider
+      cifmw_update_containers_openstack: true
+
+##########################################################
+#                                                        #
+#               Project Template                         #
+#                                                        #
+##########################################################
+
+- project-template:
+    name: opendev-master-watcher-operator-pipeline
+    description: |
+      Project template to run meta content provider and
+      EDPM job with master opendev and github operator
+      content.
+    github-check:
+      jobs:
+        - openstack-meta-content-provider-master
+        - watcher-operator-validation-master

ci/playbooks/deploy_watcher_service.yaml

@@ -8,13 +8,27 @@
     KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
     PATH: "{{ cifmw_path }}"
   tasks:
+    - name: Fetch dlrn md5_hash from master DLRN repo
+      ansible.builtin.uri:
+        url: "https://{{ watcher_dlrn_server | default('trunk.rdoproject.org') }}/{{ watcher_dlrn_namespace | default('centos9-master') }}/{{ watcher_dlrn_tag | default('current-podified') }}/delorean.repo.md5"
+        return_content: true
+      register: latest_dlrn_tag
+      until: latest_dlrn_tag.status == 200
+      retries: 6
+      delay: 5
+
     - name: Install Watcher Operator
+      vars:
+        _tag: "{{ latest_dlrn_tag.content }}"
       cifmw.general.ci_script:
         output_dir: "{{ cifmw_basedir }}/artifacts"
         chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/watcher-operator"
         script: make watcher
         extra_args:
           CATALOG_IMAGE: "{{ watcher_catalog_image | default('quay.io/openstack-k8s-operators/watcher-operator-index:latest') }}"
+          WATCHER_API_IMAGE_URL_DEFAULT: "{{ content_provider_os_registry_url }}/openstack-watcher-api:{{ _tag }}"
+          WATCHER_DECISION_ENGINE_IMAGE_URL_DEFAULT: "{{ content_provider_os_registry_url }}/openstack-watcher-decision-engine:{{ _tag }}"
+          WATCHER_APPLIER_IMAGE_URL_DEFAULT: "{{ content_provider_os_registry_url }}/openstack-watcher-applier:{{ _tag }}"
 
     - name: Deploy Watcher service
       when: deploy_watcher_service | default('true') | bool