Merge pull request #634 from splunk/develop

Release/8.2.1

docs/ADVANCED.md

@@ -231,6 +231,18 @@ When deploying distributed Splunk Enterprise environments, apps should be instal
 
 To install an app from elsewhere, provide a path to a compressed `splunkApp.spl` file (either through a filesystem or URL) as seen above. For proper installation, apps should be compressed using `tar` in a GNU/Linux environment, as apps compressed on OSX or other BSD-variant operating systems have been known to cause issues.
 
+For roles that manage clusters, such as Cluster Master and SH Deployer, the apps specified in `apps_location` will be pushed to the cluster and disabled on the local CM or Deployer instance.  For apps that need to be installed locally on these instances, use the `apps_location_local` variable.  Apps specified here will not be pushed to the cluster but installed locally in the same way that apps are installed on a standalone instance.  Both `apps_location` and `apps_location_local` can be used concurrently to allow for different sets of apps to be installed locally vs. pushed to the cluster for CM and Deployer instance.  The syntax is the same as `apps_location`:
+```
+splunk:
+  ...
+  apps_location:
+    - /tmp/cluster_app.tgz
+  apps_location_local:
+    - /tmp/local_app.tgz
+    - http://webserver.com/path/to/local_auth_app.spl
+  ...
+```
+
 ---
 
 ## SmartStore

docs/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Navigation
 
+* [8.2.1](#821)
 * [8.2.0](#820)
 * [8.1.4](#814)
 * [8.1.3](#813)
@@ -50,6 +51,17 @@
 
 ---
 
+## 8.2.1
+
+#### What's New?
+* Support for installing apps locally on a Cluster Manager or Deployer instance using `apps_location_local`. Both `apps_location` and `apps_location_local` can be used concurrently and follow the same syntax. See our [documentation](https://splunk.github.io/splunk-ansible/ADVANCED.html#apps) for details.
+
+#### Changes
+* Enabled `force_basic_auth` to reduce requests to splunkd
+* Bugfixes
+
+---
+
 ## 8.2.0
 
 #### What's New?

inventory/environ.py

@@ -146,6 +146,7 @@ def getDefaultVars():
     getSplunkBuild(defaultVars)
     getSplunkbaseToken(defaultVars)
     getSplunkApps(defaultVars)
+    getSplunkAppsLocal(defaultVars)
     getLaunchConf(defaultVars)
     getDFS(defaultVars)
     getUFSplunkVariables(defaultVars)
@@ -377,6 +378,28 @@ def getSplunkApps(vars_scope):
                 appList.append(app)
     vars_scope["splunk"]["apps_location"] = appList
 
+def getSplunkAppsLocal(vars_scope):
+    """
+    Determine the set of Splunk apps to install locally only as union of defaults.yml and environment variables
+    """
+    # Check if theres a local apps list for CM/Deployer roles
+    appListLocal = []
+    if not "apps_location_local" in vars_scope["splunk"]:
+        vars_scope["splunk"]["apps_location_local"] = []
+    # From default.yml
+    elif type(vars_scope["splunk"]["apps_location_local"]) == str:
+        appListLocal = vars_scope["splunk"]["apps_location_local"].split(",")
+    elif type(vars_scope["splunk"]["apps_location_local"]) == list:
+        appListLocal = vars_scope["splunk"]["apps_location_local"]
+    # From environment variables
+    apps = os.environ.get("SPLUNK_APPS_URL_LOCAL")
+    if apps:
+        apps = apps.split(",")
+        for app in apps:
+            if app not in appListLocal:
+                appListLocal.append(app)
+    vars_scope["splunk"]["apps_location_local"] = appListLocal
+
 def getSecrets(vars_scope):
     """
     Parse sensitive passphrases

roles/splunk_cluster_master/tasks/generate_ess_bundle.yml

@@ -17,6 +17,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200
     timeout: 10

roles/splunk_cluster_master/tasks/initialize_cluster_master.yml

@@ -5,6 +5,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: False
     body:
       name: "indexer_discovery"

roles/splunk_cluster_master/tasks/main.yml

@@ -3,6 +3,8 @@
   when:
     - "'apps_location' in splunk and splunk.apps_location"
     - splunk.deployment_server is not defined or not splunk.deployment_server
+  vars:
+    app_list: "{{ splunk.apps_location }}"
 
 - include_tasks: ../../../roles/splunk_common/tasks/set_as_deployment_client.yml
   when: splunk.deployment_server is defined and splunk.deployment_server
@@ -22,6 +24,13 @@
 - include_tasks: ../../../roles/splunk_common/tasks/disable_installed_apps.yml
   when: "'apps_location' in splunk and splunk.apps_location"
 
+# Now install local apps in apps_location_local exists
+- include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
+  when:
+    - "'apps_location_local' in splunk and splunk.apps_location_local"
+  vars:
+    app_list: "{{ splunk.apps_location_local }}"
+
 - include_tasks: setup_multisite.yml
   when:
     - splunk.site is defined

roles/splunk_common/tasks/check_for_required_restarts.yml

@@ -5,6 +5,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200,404
     timeout: 10

roles/splunk_common/tasks/disable_popups.yml

@@ -5,6 +5,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200
     timeout: 10
@@ -19,6 +20,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     body: "{{ item.value }}"
     validate_certs: false
     status_code: 200,201,409

roles/splunk_common/tasks/enable_dfs.yml

@@ -5,6 +5,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       disabled: "false"
@@ -24,6 +25,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       dfc_num_slots: "{{ splunk.dfs.dfc_num_slots }}"
@@ -42,6 +44,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       phased_execution: "true"
@@ -59,6 +62,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     use_proxy: no
   register: check_dfs_job_extractor_result
@@ -71,6 +75,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       name: "search_optimization::dfs_job_extractor"
@@ -90,6 +95,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       enabled: "true"

roles/splunk_common/tasks/enable_forwarder_monitoring.yml

@@ -9,6 +9,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200,201
     timeout: 10
@@ -22,6 +23,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     body:
       search: "{{ dmc_forwarder_build_assets.json['entry'][0]['content']['search'] }}"
       request.ui_dispatch_app: splunk_monitoring_console
@@ -41,6 +43,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200,201
     use_proxy: no

roles/splunk_common/tasks/install_apps.yml

@@ -5,6 +5,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       name: "{{ app_url }}"
@@ -75,6 +76,7 @@
       method: POST
       user: "{{ splunk.admin_user }}"
       password: "{{ splunk.password }}"
+      force_basic_auth: yes
       validate_certs: false
       body:
         name: "{{ app_filepath }}"

roles/splunk_common/tasks/licenses/enable_forwarder_license.yml

@@ -5,6 +5,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200,404
     timeout: 10
@@ -19,6 +20,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       is_active: 1

roles/splunk_common/tasks/licenses/enable_free_license.yml

@@ -5,6 +5,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200,404
     timeout: 10
@@ -19,6 +20,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       is_active: 1

roles/splunk_common/tasks/premium_apps/configure_ess.yml

@@ -5,6 +5,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200
     timeout: 10

roles/splunk_common/tasks/premium_apps/configure_itsi.yml

@@ -5,6 +5,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body: "name=itsi_admin&imported_roles=itoa_user&imported_roles=itoa_analyst&imported_roles=itoa_admin"
     headers:
@@ -27,6 +28,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body: "roles=itsi_admin&roles=admin"
     headers:

roles/splunk_common/tasks/provision_apps.yml

@@ -2,7 +2,7 @@
 - include_tasks: install_apps.yml
   vars:
     app_url: "{{ item }}"
-  loop: "{{ splunk.apps_location }}"
+  loop: "{{ app_list }}"
 
 - name: Flush restart handlers
   meta: flush_handlers

roles/splunk_common/tasks/set_as_hec_receiver.yml

@@ -7,6 +7,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       disabled: "{% if ('hec' in splunk and 'enable' in splunk.hec and splunk.hec.enable | bool) or ('hec_disabled' in splunk and not splunk.hec_disabled | bool) %}0{% else %}1{% endif %}"
@@ -26,6 +27,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200,404
     timeout: 60
@@ -41,6 +43,7 @@
     method: DELETE
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200,404
     timeout: 60
@@ -58,6 +61,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       name: "splunk_hec_token"

roles/splunk_common/tasks/start_splunk.yml

@@ -26,6 +26,9 @@
     register: start_splunk
     changed_when: start_splunk.rc == 0 and 'already running' not in start_splunk.stdout
     when: not splunk.enable_service
+    until: start_splunk.rc == 0
+    retries: 5
+    delay: 10
     become: yes
     become_user: "{{ splunk.user }}"
 

roles/splunk_deployer/tasks/main.yml

@@ -25,6 +25,7 @@
     method: POST
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     body:
       deployer_push_mode: "{{ splunk.shc.deployer_push_mode }}"
@@ -57,6 +58,8 @@
   when:
     - "'apps_location' in splunk and splunk.apps_location"
     - splunk.deployment_server is not defined or not splunk.deployment_server
+  vars:
+    app_list: "{{ splunk.apps_location }}"
 
 - include_tasks: ../../../roles/splunk_common/tasks/set_as_deployment_client.yml
   when: splunk.deployment_server is defined and splunk.deployment_server
@@ -67,4 +70,10 @@
 - include_tasks: ../../../roles/splunk_common/tasks/disable_installed_apps.yml
   when: "'apps_location' in splunk and splunk.apps_location"
 
+# Now install local apps in apps_location_local exists
+- include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
+  when: "'apps_location_local' in splunk and splunk.apps_location_local"
+  vars:
+    app_list: "{{ splunk.apps_location_local }}"
+
 - include_tasks: ../../../roles/splunk_common/tasks/check_for_required_restarts.yml

roles/splunk_heavy_forwarder/tasks/main.yml

@@ -14,6 +14,8 @@
 
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
   when: "'apps_location' in splunk and splunk.apps_location"
+  vars:
+    app_list: "{{ splunk.apps_location }}"
 
 - name: Execute Splunk commands
   command: "{{ splunk.exec }} {{ item }} -auth {{ splunk.admin_user }}:{{ splunk.password }}"

roles/splunk_license_master/tasks/main.yml

@@ -3,6 +3,8 @@
 
 - include_tasks: ../../splunk_common/tasks/provision_apps.yml
   when: "'apps_location' in splunk and splunk.apps_location"
+  vars:
+    app_list: "{{ splunk.apps_location }}"
 
 - name: Enable license master local indexing
   ini_file:

roles/splunk_monitor/tasks/adding_peers.yml

@@ -5,6 +5,7 @@
     method: GET
     user: "{{ splunk.admin_user }}"
     password: "{{ splunk.password }}"
+    force_basic_auth: yes
     validate_certs: false
     status_code: 200
     return_content: yes