change network to host and optimize tasks

molecule/default/converge-vars/beats.yml

@@ -55,7 +55,7 @@ beats_config:
 beats_installation_name: molecule
 
 # Index management
-beats_index_template_download_dir: /tmp/
+beats_index_template_download_dir: ~/elk/beats/
 beats_extract_index_template: true
 beats_setup_index_management: "{{ beats_check_output }}"
 beats_setup_pipelines: "{{ beats_check_output }}"

tasks/beats/core/keystore.yml

@@ -1,17 +1,11 @@
 ---
 
-- name: Check if a keystore exists
-  command: "docker exec {{ beats_flavor }} test -f /usr/share/{{ beats_flavor }}/data/{{ beats_flavor }}.keystore"
+- name: "{{ beats_flavor }}: Check if a keystore exists on Linux"
+  stat:
+    path: "{{ beats_path_data }}/{{ beats_flavor }}.keystore"
   register: result
-  ignore_errors: true
-  changed_when: false
 
 
 - name: Create keystore
-  command: "docker exec {{ beats_flavor }} {{ beats_flavor }} 
-      --path.data /usr/share/{{ beats_flavor }}/data 
-      --path.config /usr/share/{{ beats_flavor }} 
-      --path.home {{ beats_path_home }} 
-      --path.logs /usr/share/{{ beats_flavor }}/logs keystore create"
-  when: result.rc != 0
-
+  command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} keystore create"
+  when: not result.stat.exists | bool

tasks/beats/core/version.yml

@@ -1,10 +1,6 @@
 ---
 - name: "{{ beats_flavor }}: Get version"
-  command: "docker exec {{ beats_flavor }} {{ beats_flavor }} 
-      --path.data /usr/share/{{ beats_flavor }}/data 
-      --path.config /usr/share/{{ beats_flavor }} 
-      --path.home {{ beats_path_home }} 
-      --path.logs /usr/share/{{ beats_flavor }}/logs version"
+  command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} version"
   register: _beats_version_output
   changed_when: false
 

tasks/beats/flavor-specific/filebeat.yml

@@ -63,7 +63,7 @@
           docker rm -f -v {{ beats_flavor }} || :
 
           # Run the Beat container
-          docker run --network=elastic --user root --rm --name={{ beats_flavor }}  \
+          docker run --network host --user root --rm --name={{ beats_flavor }}  \
             -v  "/var/lib/docker/containers:/var/lib/docker/containers:ro" \
             -v  "/var/run/docker.sock:/var/run/docker.sock:ro" \
             -v  {{ beats_certificates_dir }}:{{ beats_certificates_dir }} \
@@ -122,11 +122,7 @@
         loop_var: _beats__module
 
     - name: "{{ beats_flavor }}: Enable filebeat modules on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} \
-          --path.data /usr/share/{{ beats_flavor }}/data \
-          --path.config /usr/share/{{ beats_flavor }} \
-          --path.home {{ beats_path_home }} \
-          --path.logs /usr/share/{{ beats_flavor }}/logs modules enable {{ modules_to_enable }}"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} modules enable {{ modules_to_enable }}"
       when: modules_to_enable | length > 0
       vars:
         modules_to_enable: >-
@@ -182,11 +178,7 @@
         loop_var: _beats__module
 
     - name: "{{ beats_flavor }}: Disable filebeat modules on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} \
-          --path.data /usr/share/{{ beats_flavor }}/data \
-          --path.config /usr/share/{{ beats_flavor }} \
-          --path.home {{ beats_path_home }} \
-          --path.logs /usr/share/{{ beats_flavor }}/logs modules disable {{ modules_to_disable }}"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} modules disable {{ modules_to_disable }}"
       when: modules_to_disable | length > 0
       vars:
         modules_to_disable: >-

tasks/beats/flavor-specific/heartbeat.yml

@@ -17,7 +17,7 @@
           docker rm -f -v {{ beats_flavor }} || :
 
           # Run the Beat container
-          docker run --network=elastic --user root --cap-add NET_RAW --rm --name={{ beats_flavor }}  \
+          docker run --network host --user root --cap-add NET_RAW --rm --name={{ beats_flavor }}  \
             -v  {{ beats_certificates_dir }}:{{ beats_certificates_dir }} \
             -v  {{ beats_path_config }}/scripts:{{ beats_path_config }}/scripts \
             -v {{ beats_path_config }}/{{ beats_flavor }}.yml:{{ beats_path_config }}/{{ beats_flavor }}.yml \

tasks/beats/flavor-specific/metricbeat.yml

@@ -17,7 +17,7 @@
             docker rm -f -v {{ beats_flavor }} || :
 
             # Run the MetricBeat container
-            docker run --network=elastic --user root  --rm --name={{ beats_flavor }}  \
+            docker run --network host --user root  --rm --name={{ beats_flavor }}  \
               -v "/var/run/docker.sock:/var/run/docker.sock:ro" \
               -v "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro" \
               -v "/proc:/hostfs/proc:ro" \
@@ -95,11 +95,7 @@
 - name: "{{ beats_flavor }}: Enable metricbeat modules"
   block:
     - name: "{{ beats_flavor }}: Enable metricbeat modules on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} \
-          --path.data /usr/share/{{ beats_flavor }}/data \
-          --path.config /usr/share/{{ beats_flavor }} \
-          --path.home {{ beats_path_home }} \
-          --path.logs /usr/share/{{ beats_flavor }}/logs modules enable {{ beats_metricbeat_modules | join(' ') }}"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} modules enable {{ beats_metricbeat_modules | join(' ') }}"
       when: ansible_os_family != 'Windows'
 
     - name: "{{ beats_flavor }}: Enable metricbeat modules on Windows"

tasks/beats/outputs/index.yml

@@ -4,11 +4,7 @@
   block:
     - name: "{{ beats_flavor }}: Extract the beats index template on Linux"
       shell: >-
-        docker exec {{ beats_flavor }} {{ beats_flavor }} \
-          --path.data /usr/share/{{ beats_flavor }}/data \
-          --path.config /usr/share/{{ beats_flavor }} \
-          --path.home {{ beats_path_home }} \
-          --path.logs /usr/share/{{ beats_flavor }}/logs export template > \
+        docker exec {{ beats_flavor }} {{ beats_linux_exec }} export template > \
           /tmp/{{ beats_flavor }}.{{ beats_full_version }}.template.json
       args:
         executable: /bin/bash

tasks/beats/outputs/keystore.yml

@@ -3,11 +3,7 @@
 - name: "{{ beats_flavor }}: output keystore for Linux"
   block:
     - name: "{{ beats_flavor }}: Remove existing keystore records on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} 
-      --path.data /usr/share/{{ beats_flavor }}/data 
-      --path.config /usr/share/{{ beats_flavor }} 
-      --path.home {{ beats_path_home }} 
-      --path.logs /usr/share/{{ beats_flavor }}/logs keystore remove {{ _beats__record }}"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} keystore remove {{ _beats__record }}"
       register: result
       failed_when:
         - result is failed
@@ -24,23 +20,15 @@
         - KIBANA_CERT_KEY_PASSPHRASE
 
     - name: "{{ beats_flavor }}: List keystore settings on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} \
-      --path.data /usr/share/{{ beats_flavor }}/data \
-      --path.config /usr/share/{{ beats_flavor }} \
-      --path.home {{ beats_path_home }} \
-      --path.logs /usr/share/{{ beats_flavor }}/logs keystore list"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} keystore list"
       register: _beats_keystore_settings
       changed_when: false
 
     - name: "{{ beats_flavor }}: Add passwords and passphrases to keystore on Linux"
       shell: |
         set -o pipefail;
         echo '{{ lookup("vars", _beats__record.1) }}' |
-          docker exec {{ beats_flavor }} {{ beats_flavor }} \
-          --path.data /usr/share/{{ beats_flavor }}/data \
-          --path.config /usr/share/{{ beats_flavor }} \
-          --path.home {{ beats_path_home }} \
-          --path.logs /usr/share/{{ beats_flavor }}/logs keystore add \
+          docker exec {{ beats_flavor }} {{ beats_linux_exec }} keystore add \
           {{ _beats__record.0 }} --stdin
       args:
         executable: /bin/bash

tasks/beats/setup.yml

@@ -3,11 +3,7 @@
 - name: "{{ beats_flavor }}: Setup Kibana dashboards manually"
   block:
     - name: "{{ beats_flavor }}: Setup kibana dashboards manually on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} 
-          --path.data /usr/share/{{ beats_flavor }}/data 
-          --path.config /usr/share/{{ beats_flavor }} 
-          --path.home {{ beats_path_home }} 
-          --path.logs /usr/share/{{ beats_flavor }}/logs setup --dashboards"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} setup --dashboards"
       when: ansible_os_family != 'Windows'
 
   when:
@@ -18,23 +14,15 @@
 - name: "{{ beats_flavor }}: Load index templates, ilm policy manually"
   block:
     - name: "{{ beats_flavor }}: Load index templates, ilm policy manually on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} 
-          --path.data /usr/share/{{ beats_flavor }}/data 
-          --path.config /usr/share/{{ beats_flavor }} 
-          --path.home {{ beats_path_home }} 
-          --path.logs /usr/share/{{ beats_flavor }}/logs setup --index-management"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} setup --index-management"
       when: ansible_os_family != 'Windows'
 
   when: beats_setup_index_management | bool
 
 - name: "{{ beats_flavor }}: Setup ingest pipelines"
   block:
     - name: "{{ beats_flavor }}: Setup ingest pipelines on Linux"
-      command: "docker exec {{ beats_flavor }} {{ beats_flavor }} 
-          --path.data /usr/share/{{ beats_flavor }}/data 
-          --path.config /usr/share/{{ beats_flavor }} 
-          --path.home {{ beats_path_home }} 
-          --path.logs /usr/share/{{ beats_flavor }}/logs setup --pipelines"
+      command: "docker exec {{ beats_flavor }} {{ beats_linux_exec }} setup --pipelines"
       when: ansible_os_family != 'Windows'
 
   when:

tasks/beats/variables.yml

@@ -1,17 +1,5 @@
 ---
 
-# Packages {{{
-- name: "Include OS specific packages"
-  include_vars: "{{ _beats__file }}"
-  loop: "{{ query('first_found', files) }}"
-  loop_control:
-    loop_var: _beats__file
-  vars:
-    files:
-      - "{{ role_path }}/vars/packages/{{ ansible_distribution_release | d('') }}.yml"
-      - "{{ role_path }}/vars/packages/{{ ansible_os_family }}.yml"
-      - "{{ role_path }}/vars/main.yml"  # fallback
-# }}}
 # Paths {{{
 - name: "{{ beats_flavor }}: Load default paths"
   include_vars: "{{ role_path }}/vars/paths/{{ ansible_os_family }}.yml"

tasks/elastic/indexes.yml

@@ -2,7 +2,7 @@
 
 - name: Import index templates
   uri:
-    url: "{{ elastic_node_address }}/_template/{{ _elasticsearch__template.name }}"
+    url: "{{ elastic_node_address }}/_index_template/{{ _elasticsearch__template.name }}"
     method: PUT
     body: "{{ lookup('file', _elasticsearch__template.file) }}"
     body_format: json
@@ -22,7 +22,7 @@
   block:
     - name: Read existing template
       uri:
-        url: "{{ elastic_node_address }}/_template/{{ _elasticsearch__template.name }}"
+        url: "{{ elastic_node_address }}/_index_template/{{ _elasticsearch__template.name }}"
         method: GET
         headers:
           Content-Type: application/json
@@ -37,9 +37,9 @@
       loop_control:
         loop_var: _elasticsearch__template
 
-    - name: Import overriden index template
+    - name: Import overridden index template
       uri:
-        url: "{{ elastic_node_address }}/_template/{{ _elasticsearch__template.name }}"
+        url: "{{ elastic_node_address }}/_index_template/{{ _elasticsearch__template.name }}"
         method: PUT
         body: "{{ _existing_templates.results[i].json[_elasticsearch__template.name]
                   | combine(_elasticsearch__template.overrides, recursive=true) }}"

templates/filebeat

@@ -6,7 +6,7 @@ docker pull docker.elastic.co/beats/{{ beats_flavor }}:{{ beats_major_version }}
 docker rm -f -v {{ beats_flavor }} || :
 
 # Run the Beat container
-docker run --network=elastic --user root --rm --name={{ beats_flavor }}  \
+docker run --network host --user root --rm --name={{ beats_flavor }}  \
   -v  "/var/lib/docker/containers:/var/lib/docker/containers:ro" \
   -v  "/var/run/docker.sock:/var/run/docker.sock:ro" \
   -v  {{ beats_certificates_dir }}:{{ beats_certificates_dir }} \

templates/heartbeat

@@ -6,7 +6,7 @@ docker pull docker.elastic.co/beats/{{ beats_flavor }}:{{ beats_major_version }}
 docker rm -f -v {{ beats_flavor }} || :
 
 # Run the Beat container
-docker run --network=elastic --user root --cap-add NET_RAW --rm --name={{ beats_flavor }}  \
+docker run --network host --user root --cap-add NET_RAW --rm --name={{ beats_flavor }}  \
   -v  {{ beats_certificates_dir }}:{{ beats_certificates_dir }} \
   -v  {{ beats_path_config }}/scripts:{{ beats_path_config }}/scripts \
   -v {{ beats_path_config }}/{{ beats_flavor }}.yml:{{ beats_path_config }}/{{ beats_flavor }}.yml \

templates/journalbeat

@@ -6,7 +6,7 @@ docker pull docker.elastic.co/beats/{{ beats_flavor }}:{{ beats_major_version }}
 docker rm -f -v {{ beats_flavor }} || :
 
 # Run the MetricBeat container
-docker run --network=elastic --user root  --rm --name={{ beats_flavor }}  \
+docker run --network host --user root  --rm --name={{ beats_flavor }}  \
   -v "/var/log/journal:/var/log/journal" \
   -v "/etc/machine-id:/etc/machine-id" \
   -v "/run/systemd:/run/systemd" \

templates/metricbeat

@@ -6,7 +6,7 @@ docker pull docker.elastic.co/beats/{{ beats_flavor }}:{{ beats_major_version }}
 docker rm -f -v {{ beats_flavor }} || :
 
 # Run the MetricBeat container
-docker run --network=elastic --user root  --rm --name={{ beats_flavor }}  \
+docker run --network host --user root  --rm --name={{ beats_flavor }}  \
   -v "/var/run/docker.sock:/var/run/docker.sock:ro" \
   -v "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro" \
   -v "/proc:/hostfs/proc:ro" \

templates/packetbeat

@@ -6,7 +6,7 @@ docker pull docker.elastic.co/beats/{{ beats_flavor }}:{{ beats_major_version }}
 docker rm -f -v {{ beats_flavor }} || :
 
 # Run the Beat container
-docker run --network=elastic --user root --cap-add NET_RAW --cap-add NET_ADMIN --rm --name={{ beats_flavor }}  \
+docker run --network host --user root --cap-add NET_RAW --cap-add NET_ADMIN --rm --name={{ beats_flavor }}  \
   -v  {{ beats_certificates_dir }}:{{ beats_certificates_dir }} \
   -v  {{ beats_path_config }}/scripts:{{ beats_path_config }}/scripts \
   -v {{ beats_path_config }}/{{ beats_flavor }}.yml:{{ beats_path_config }}/{{ beats_flavor }}.yml \