Merge pull request #783 from IBM/dev

Dev to main

roles/afm_cos_configure/tasks/afm_configure.yml

@@ -22,30 +22,57 @@
         when:
            - scale_existing_bucket.stdout_lines is defined
 
+      - name: create | Find existing Bucket(s)
+        shell: mmafmcoskeys all get --report | grep -v HEADER | cut -d '=' -f 1 | uniq
+        register: scale_existing_bucket_list
+        changed_when: false
+        failed_when: false
+
+      - debug:
+           msg: "{{scale_existing_bucket_list.stdout_lines}}"
+        changed_when: false
+        failed_when: false
+        when:
+           - scale_existing_bucket_list.stdout_lines is defined
+
       - name: create | Initialize
         set_fact:
           extra_option_flag: ""
+          all_existing_bucket: []
+
+      - set_fact:
+            all_existing_bucket: "{{ scale_existing_bucket.stdout_lines + scale_existing_bucket_list.stdout_lines }}"
+
+
+      - debug:
+           msg: "{{all_existing_bucket}}"
 
       - name: create | Set existing bucket if it is defined
         set_fact:
           extra_option_flag: "{{ extra_option_flag }} --bucket {{ item.bucket }}"
         when:
-          - item.bucket in scale_existing_bucket.stdout_lines
+          - item.bucket in all_existing_bucket
           - item.bucket is defined
 
       - name: create | Set new bucket if it is defined
         set_fact:
           extra_option_flag: "{{ extra_option_flag }} --new-bucket {{ item.bucket }}"
         when:
-          - item.bucket not in scale_existing_bucket.stdout_lines
+          - item.bucket not in all_existing_bucket
           - item.bucket is defined
 
       - name: create | Set mode if it is defined
         set_fact:
           extra_option_flag: "{{ extra_option_flag }} --mode {{ item.mode | default('sw') }}"
+
+      - name: create | Set extra parameter config
+        set_fact:
+          extra_option_flag: "{{ extra_option_flag }} --gcs"
+        when: "scale_afm_cos_config_params is defined and
+                (scale_afm_cos_config_params.is_gcs is defined and scale_afm_cos_config_params.is_gcs|bool)"
 
       - name: configure | Create a AFM cos relationship with filesets
-        command: "{{ scale_command_path }}mmafmcosconfig {{ item.filesystem }} {{ item.fileset }}  --endpoint {{ item.endpoint }}  {{ extra_option_flag }}"
+        command: "{{ scale_command_path }}mmafmcosconfig {{ item.filesystem }} {{ item.fileset }}  --endpoint {{ item.endpoint }}  {{ extra_option_flag }} --directory-object"
         register: scale_afm_cos_define
         failed_when: scale_afm_cos_define.rc != 0
         when:

roles/afm_cos_configure/tasks/afm_gateway.yml

@@ -7,25 +7,48 @@
   add_host:
     name: "{{ item }}"
     groups: scale_gateway_nodes
-  when: hostvars[item].scale_cluster_gateway | bool
+  when:
+    - hostvars[item].scale_cluster_gateway is defined 
+    - hostvars[item].scale_cluster_gateway | bool
   with_items: "{{ ansible_play_hosts }}"
   changed_when: false
 
 - name: configure | Setting server licenses on AFM gateway nodes
   vars:
      scale_gateway_node: "{{ groups['scale_gateway_nodes'] | list }}"
   command: "{{ scale_command_path }}mmchlicense server --accept -N {{ scale_gateway_node | join(',') }}"
-  when: groups['scale_gateway_nodes'] | list | length > 0
+  when:
+    - groups['scale_gateway_nodes'] is defined 
+    - groups['scale_gateway_nodes'] | list | length > 0
   run_once: true
 
 - name: configure | Enabling AFM gateway
   vars:
      scale_gateway_node: "{{ groups['scale_gateway_nodes'] | list }}"
   command: "{{ scale_command_path }}mmchnode -N {{ scale_gateway_node | join(',') }} --gateway"
-  when: groups['scale_gateway_nodes'] | list | length > 0
+  when:
+    - groups['scale_gateway_nodes'] is defined 
+    - groups['scale_gateway_nodes'] | list | length > 0
   register: result_gateway
   run_once: true
 
 - debug:
     msg: "{{ result_gateway.cmd }}"
   when: result_gateway.cmd is defined
+
+- name: storage | Mount filesystem(s) on AFM gateway nodes
+  vars:
+     scale_gateway_node: "{{ groups['scale_gateway_nodes'] | list }}"
+  command: "{{ scale_command_path }}mmmount all -N {{ scale_gateway_node | join(',') }}"
+  changed_when: false
+  run_once: true
+  failed_when: false
+  ignore_errors: yes
+  register: scale_mount_fs_cloud
+  when:
+    - groups['scale_gateway_nodes'] is defined 
+    - groups['scale_gateway_nodes'] | list | length > 0
+
+- debug:
+    msg: "{{ scale_mount_fs_cloud }}"
+  when: scale_mount_fs_cloud is defined

roles/ces_common/tasks/configure.yml

@@ -134,6 +134,29 @@
         ( scale_service_status.rc == 0 )
   run_once: true
 
+- name: configure | Check if S3 is running
+  shell:
+   cmd: "{{ scale_command_path }}mmces service list|grep S3"
+  register: scale_service_status
+  when: inventory_hostname in scale_protocol_node_list
+  ignore_errors: true
+  failed_when: false
+  run_once: true
+
+- name: configure | Add S3 service to the list
+  set_fact:
+   scale_service_list: "{{ scale_service_list + [scale_service_status.stderr|regex_search('S3')] }}"
+  when: (inventory_hostname in scale_protocol_node_list) and
+        ( scale_service_status.rc > 0 )
+  run_once: true
+
+- name: configure | Add S3 service to the list
+  set_fact:
+   scale_service_list: "{{ scale_service_list + ['S3'] }}"
+  when: (inventory_hostname in scale_protocol_node_list) and
+        ( scale_service_status.rc == 0 )
+  run_once: true
+
 - import_role:
    name: ibm.spectrum_scale.nfs_install
   when: scale_ces_disabled_nodes|length > 0 and 'NFS' in scale_service_list
@@ -146,6 +169,10 @@
    name: ibm.spectrum_scale.obj_install
   when: scale_ces_disabled_nodes|length > 0 and 'OBJ' in scale_service_list
 
+- import_role:
+   name: ibm.spectrum_scale.s3_install
+  when: scale_ces_disabled_nodes|length > 0 and 'S3' in scale_service_list
+
 - name: configure | Prepare ces nodes string
   set_fact:
    scale_ces_nodes: "{{ scale_ces_nodes + ',' + item|string }}"

roles/ces_common/tasks/main.yml

@@ -5,13 +5,15 @@
           (scale_protocols.nfs is defined and scale_protocols.nfs|bool) or
           (scale_protocols.smb is defined and scale_protocols.smb|bool) or
           (scale_protocols.object is defined and scale_protocols.object|bool) or
-          (scale_protocols.hdfs is defined and scale_protocols.hdfs|bool)"
+          (scale_protocols.hdfs is defined and scale_protocols.hdfs|bool) or
+          (scale_protocols.s3 is defined and scale_protocols.s3|bool)"
   tags: always
 
 - include_tasks: configure.yml
   when: "scale_protocols is defined and
           (scale_protocols.nfs is defined and scale_protocols.nfs|bool) or
           (scale_protocols.smb is defined and scale_protocols.smb|bool) or
           (scale_protocols.object is defined and scale_protocols.object|bool) or
-          (scale_protocols.hdfs is defined and scale_protocols.hdfs|bool)"
+          (scale_protocols.hdfs is defined and scale_protocols.hdfs|bool) or
+          (scale_protocols.s3 is defined and scale_protocols.s3|bool)"
   tags: configure

roles/core_upgrade/tasks/install_repository.yml

@@ -7,12 +7,17 @@
 - name: Initialize
   set_fact:
     is_scale_java_pkg_installed: false
+    base_repo_url: "gpfs_rpms/"
+
+- set_fact:
+    base_repo_url: ""
+  when: scale_baseurl is defined
 
 - name: upgrade | Configure GPFS YUM repository
   yum_repository:
     name: spectrum-scale-gpfs
     description: IBM Spectrum Scale (GPFS)
-    baseurl: "{{ scale_install_repository_url }}gpfs_rpms/"
+    baseurl: "{{ scale_install_repository_url }}{{ base_repo_url }}"
     gpgcheck: "{{ scale_install_gpgcheck }}"
     repo_gpgcheck: no
     sslverify: no
@@ -22,6 +27,7 @@
     - ansible_pkg_mgr == 'yum' or ansible_pkg_mgr == 'dnf'
     - scale_install_repository_url is defined
     - scale_install_repository_url != 'existing'
+
 #
 # Configure apt repository
 #

roles/gui_upgrade/tasks/install_repository.yml

@@ -1,14 +1,21 @@
 ---
 # YUM repository installation method
 
+- name: initialize
+  set_fact:
+    gui_url: "gpfs_rpms/"
+
+- set_fact:
+     gui_url: ""
+  when: scale_baseurl is defined
 #
 # Configure YUM repository
 #
 - name: upgrade | Configure GUI YUM repository
   yum_repository:
     name: spectrum-scale-gui
     description: IBM Spectrum Scale (GUI)
-    baseurl: "{{ scale_install_repository_url }}gpfs_rpms/"
+    baseurl: "{{ scale_install_repository_url }}{{ gui_url }}"
     gpgcheck: "{{ scale_install_gpgcheck }}"
     repo_gpgcheck: no
     sslverify: no

roles/hdfs_prepare/tasks/java_home.yml

@@ -66,4 +66,51 @@
     msg: "JAVA_HOME not set properly"
   when: 
     - ansible_fqdn in scale_hdfs_nodes_list or inventory_hostname in scale_hdfs_nodes_list
-    - jvm_list.rc != 0
+    - jvm_list.rc != 0
+
+- name: check | Fetch hdfs extracted tar
+  set_fact:
+     hdfs_dependency_jars_dir: "hadoop-3.1.4"
+
+- name: Check and fetch gpfs.hdfs-protocol version
+  shell: "rpm -q gpfs.hdfs-protocol --qf %{VERSION}-%{RELEASE}"
+  register: gpfs_hdfs_protocol_version
+  when:
+    - ansible_fqdn in scale_hdfs_nodes_list or inventory_hostname in scale_hdfs_nodes_list
+    - transparency_322_enabled|bool
+  ignore_errors: true
+
+- debug:
+    msg: "gpfs_hdfs_protocol_version: {{ gpfs_hdfs_protocol_version}}"
+
+- name: Check gpfs.hdfs-protocol version for standalone installation
+  fail:
+    msg: >
+      "Standalone installation of gpfs.hdfs-protocol version is not supported. It can only be upgraded"
+      " from gpfs.hdfs-protocol version 3.2.2-5. For additional information, refer to the documentation available at the following link:"
+      " https://www.ibm.com/docs/en/storage-scale-bda?topic=hdfs-setup-transparency-cluster."
+  when:
+    - ansible_fqdn in scale_hdfs_nodes_list or inventory_hostname in scale_hdfs_nodes_list
+    - transparency_322_enabled|bool
+    - gpfs_hdfs_protocol_version.rc == 0
+    - gpfs_hdfs_protocol_version.stdout_lines[0] < '3.2.2-5'
+
+- debug:
+    msg: "hdfs_dependency_jars_dir: {{ hdfs_dependency_jars_dir }}"
+
+- name: check | verify dependency jars
+  command: "ls /opt/hadoop/jars/{{ hdfs_dependency_jars_dir }}"
+  register: dep_jars
+  when:
+    - ansible_fqdn in scale_hdfs_nodes_list or inventory_hostname in scale_hdfs_nodes_list
+    - transparency_322_enabled|bool == False
+
+- fail:
+    msg: >
+      "Dependency jars not exist in /opt/hadoop/jars directory, which are essential prerequisites, For further details, "
+      "please consult the documentation via the following link: https://www.ibm.com/docs/en/storage-scale-bda?topic=hdfs-setup"
+  when:
+    - ansible_fqdn in scale_hdfs_nodes_list or inventory_hostname in scale_hdfs_nodes_list
+    - transparency_322_enabled|bool == False
+    - dep_jars.rc != 0
+

roles/hdfs_upgrade/tasks/prepare_env.yml

@@ -20,7 +20,7 @@
 
 - name:
   set_fact:
-    transparency_33_enabled: "{{ transparency_version.stdout|bool }}"
+    transparency_33_enabled: "{{ transparency_version_33.stdout|bool }}"
   when: 
     - transparency_version_33.stdout is defined
     - transparency_version_33.stdout|bool
@@ -29,7 +29,7 @@
 
 - name:
   set_fact:
-    transparency_322_enabled: "{{ transparency_version.stdout|bool }}"
+    transparency_322_enabled: "{{ transparency_version_322.stdout|bool }}"
   when: 
     - transparency_version_322.stdout is defined
     - transparency_version_322.stdout|bool

roles/perfmon_upgrade/tasks/install_repository.yml

@@ -79,6 +79,10 @@
     - ansible_distribution in scale_ubuntu_distribution
     - scale_version >= "5.1.4.0"
 
+- set_fact:
+    scale_zimon_url: ""
+  when: scale_baseurl is defined
+
 - name: upgrade | Configure ZIMon YUM repository
   yum_repository:
     name: spectrum-scale-zimon

roles/remotemount_configure/defaults/main.yml

@@ -16,6 +16,8 @@ scale_remotemount_endpoint: "{{ scale_remotemount_scalemgmt_endpoint }}/remotemo
 
 scale_remotemount_validate_certs_uri: 'no'
 
+scale_cluster_authorization_cleanup: true
+
 # Temporary Storage for Public Key, Only used when debuging
 scale_remotemount_client_access_key: /tmp/client_cluster.pub
 
@@ -87,4 +89,4 @@ scale_remotemount_storage_contactnodes_filter: '?fields=roles.gatewayNode%2Cnetw
 #        "managerNode": false,
 #        "otherNodeRoles": "perfmonNode",
 #        "quorumNode": true,
-#        "snmpNode": false
+#        "snmpNode": false

roles/remotemount_configure/tasks/cleanup_remote_mount.yml

@@ -88,7 +88,9 @@
 # We want to delete the one where the owningCluster name matches what we are trying to do a remote mount on
 - name: Cleanup | Client Cluster (access) | Delete the Remote Mount/clusters connection on a loop.
   include_tasks: delete_remote_cluster.yml
-  when: item.owningCluster == owning_cluster_name
+  when:
+     - item.owningCluster == owning_cluster_name
+     - scale_cluster_authorization_cleanup is defined and scale_cluster_authorization_cleanup | bool
   loop: "{{ remote_clusters_result.json.owningClusters }}"
   run_once: True
 
@@ -139,12 +141,15 @@
       retries: "{{ scale_remotemount_restapi_retries_count }}"
       delay: "{{ scale_remotemount_restapi_retries_delay }}"
   #when: not remote_clusters_results.failed
-  when: remote_clusters_results.json.status.code == 200
+  when: 
+    - remote_clusters_results.json.status.code == 200
+    - scale_cluster_authorization_cleanup is defined and scale_cluster_authorization_cleanup | bool
   run_once: True
 
 - name: "Cleanup | Storage Cluster (owner) | Output from delete the Client Cluster, ('{{ access_cluster_name }}')"
   run_once: True
   debug:
     msg: "The is no Client/Accessing cluster named: ({{ access_cluster_name }}) - Message from RestAPI: {{ remote_clusters_results.json.status.message }}"
   when:
-    - remote_clusters_results.json.status.code == 400
+    - remote_clusters_results.json.status.code == 400
+    - scale_cluster_authorization_cleanup is defined and scale_cluster_authorization_cleanup | bool

roles/remotemount_configure/tasks/remotecluster.yml

@@ -288,7 +288,7 @@
           owning_nodes_name: "{{ owning_nodes_name + [item.adminNodeName] }}"
       with_items: "{{ owning_cluster_nodes.json.nodes }}"
       run_once: True
-
+  
 #
 # This Section is when using daemonNodeName
 #
@@ -427,19 +427,21 @@
   ignore_errors: true
   run_once: True
 
-
 - name: "Mount Filesystem | Storage Cluster (owner) | Check if filesystems is accessible on Client Cluster ('{{ access_cluster_name }}') - Debug"
   run_once: True
   debug:
-    msg: "{{ remote_clusters_results.json.remoteClusters[0] | json_query('owningClusterFilesystems[*].filesystem') | join(', ')  }}"
-  when: scale_remotemount_debug | bool
+    msg: "{{ remote_clusters_results.json.remoteClusters[0].owningClusterFilesystems| map(attribute='filesystem') | list | join(', ') }}"
+  when:
+   - scale_remotemount_debug | bool
+   - remote_clusters_results.json.remoteClusters[0].owningClusterFilesystems is defined
 
 # Set current filesystem access and run only the access task on filesystem, this is a list
 # this could also be changed to a import_task with loop.
 
 - set_fact:
-    current_scale_remotemount_storage_filesystem_name: "{{ remote_clusters_results.json.remoteClusters[0] | json_query('owningClusterFilesystems[*].filesystem') | join(', ')  }}"
+    current_scale_remotemount_storage_filesystem_name: "{{ remote_clusters_results.json.remoteClusters[0].owningClusterFilesystems| map(attribute='filesystem') | list | join(', ') }}"
   run_once: True
+  when: remote_clusters_results.json.remoteClusters[0].owningClusterFilesystems is defined
 
 - name: Step 6 - Configure and Mount filesystems
   debug:

roles/s3_configure/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# Default variables for the IBM Spectrum Scale (S3) cluster role -
+# either edit this file or define your own variables to override the defaults