Merge pull request #427 from splunk/develop

Release/8.1.0

Makefile

@@ -7,8 +7,8 @@ SPLUNK_ANSIBLE_BRANCH ?= develop
 SPLUNK_COMPOSE ?= cluster_absolute_unit.yaml
 # Set Splunk version/build parameters here to define downstream URLs and file names
 SPLUNK_PRODUCT := splunk
-SPLUNK_VERSION := 8.0.6
-SPLUNK_BUILD := 152fb4b2bb96
+SPLUNK_VERSION := 8.1.0
+SPLUNK_BUILD := f57c09e87251
 ifeq ($(shell arch), s390x)
 	SPLUNK_ARCH = s390x
 else
@@ -58,7 +58,7 @@ ansible:
 	@cat splunk-ansible/version.txt
 
 ##### Base images #####
-base: base-debian-9 base-debian-10 base-centos-7 base-redhat-8 base-windows-2016
+base: base-debian-9 base-debian-10 base-centos-7 base-centos-8 base-redhat-8 base-windows-2016
 
 base-debian-10:
 	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-debian-10:${IMAGE_VERSION} ./base/debian-10
@@ -69,14 +69,17 @@ base-debian-9:
 base-centos-7:
 	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-centos-7:${IMAGE_VERSION} ./base/centos-7
 
+base-centos-8:
+	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-centos-8:${IMAGE_VERSION} ./base/centos-8
+
 base-redhat-8:
 	docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} --label version=${SPLUNK_VERSION} -t base-redhat-8:${IMAGE_VERSION} ./base/redhat-8
 
 base-windows-2016:
 	docker build ${DOCKER_BUILD_FLAGS} -t base-windows-2016:${IMAGE_VERSION} ./base/windows-2016
 
 ##### Minimal images #####
-minimal: minimal-debian-9 minimal-debian-10 minimal-centos-7 minimal-redhat-8
+minimal: minimal-debian-9 minimal-debian-10 minimal-centos-7 minimal-centos-8 minimal-redhat-8
 
 minimal-debian-9: base-debian-9
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -99,6 +102,13 @@ minimal-centos-7: base-centos-7
 		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
 		--target minimal -t minimal-centos-7:${IMAGE_VERSION} .	
 
+minimal-centos-8: base-centos-8
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f splunk/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
+		--target minimal -t minimal-centos-8:${IMAGE_VERSION} .
+
 minimal-redhat-8: base-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f splunk/common-files/Dockerfile \
@@ -107,7 +117,7 @@ minimal-redhat-8: base-redhat-8
 		--target minimal -t minimal-redhat-8:${IMAGE_VERSION} .
 
 ##### Bare images #####
-bare: bare-debian-9 bare-debian-10 bare-centos-7 bare-redhat-8
+bare: bare-debian-9 bare-debian-10 bare-centos-7 bare-centos-8 bare-redhat-8
 
 bare-debian-9: base-debian-9
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -128,7 +138,14 @@ bare-centos-7: base-centos-7
 		-f splunk/common-files/Dockerfile \
 		--build-arg SPLUNK_BASE_IMAGE=base-centos-7 \
 		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
-		--target bare -t bare-centos-7:${IMAGE_VERSION} .	
+		--target bare -t bare-centos-7:${IMAGE_VERSION} .
+
+bare-centos-8: base-centos-7
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f splunk/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
+		--target bare -t bare-centos-8:${IMAGE_VERSION} .	
 
 bare-redhat-8: base-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -138,7 +155,7 @@ bare-redhat-8: base-redhat-8
 		--target bare -t bare-redhat-8:${IMAGE_VERSION} .
 
 ##### Splunk images #####
-splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-redhat-8
+splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-centos-8 splunk-redhat-8
 
 splunk-debian-9: base-debian-9 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -161,6 +178,13 @@ splunk-centos-7: base-centos-7 ansible
 		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
 		-t splunk-centos-7:${IMAGE_VERSION} .
 
+splunk-centos-8: base-centos-8 ansible 
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f splunk/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \
+		-t splunk-centos-8:${IMAGE_VERSION} .
+
 splunk-redhat-8: base-redhat-8 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f splunk/common-files/Dockerfile \
@@ -176,7 +200,7 @@ splunk-windows-2016: base-windows-2016 ansible
 		-t splunk-windows-2016:${IMAGE_VERSION} .
 
 ##### UF images #####
-uf: ansible uf-debian-9 uf-debian-10 uf-centos-7 uf-redhat-8
+uf: ansible uf-debian-9 uf-debian-10 uf-centos-7 uf-centos-8 uf-redhat-8
 
 ufbare-debian-9: base-debian-9 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -213,6 +237,13 @@ uf-centos-7: base-centos-7 ansible
 		--build-arg SPLUNK_BUILD_URL=${UF_LINUX_BUILD_URL} \
 		-t uf-centos-7:${IMAGE_VERSION} .
 
+uf-centos-8: base-centos-8 ansible
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f uf/common-files/Dockerfile \
+		--build-arg SPLUNK_BASE_IMAGE=base-centos-8 \
+		--build-arg SPLUNK_BUILD_URL=${UF_LINUX_BUILD_URL} \
+		-t uf-centos-8:${IMAGE_VERSION} .
+
 uf-redhat-8: base-redhat-8 ansible
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f uf/common-files/Dockerfile \
@@ -229,7 +260,7 @@ uf-windows-2016: base-windows-2016 ansible
 
 
 ##### Python 3 support #####
-splunk-py23: splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-redhat-8
+splunk-py23: splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-centos-8 splunk-py23-redhat-8
 
 splunk-py23-debian-9: splunk-debian-9
 	docker build ${DOCKER_BUILD_FLAGS} \
@@ -249,6 +280,12 @@ splunk-py23-centos-7: splunk-centos-7
 		--build-arg SPLUNK_PRODUCT=splunk \
 		-t splunk-py23-centos-7:${IMAGE_VERSION} .
 
+splunk-py23-centos-8: splunk-centos-8
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f py23-image/centos-8/Dockerfile \
+		--build-arg SPLUNK_PRODUCT=splunk \
+		-t splunk-py23-centos-8:${IMAGE_VERSION} .
+
 splunk-py23-redhat-8: splunk-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f py23-image/redhat-8/Dockerfile \
@@ -275,6 +312,12 @@ uf-py23-centos-7: uf-centos-7
 		--build-arg SPLUNK_PRODUCT=uf \
 		-t uf-py23-centos-7:${IMAGE_VERSION} .
 
+uf-py23-centos-8: uf-centos-8
+	docker build ${DOCKER_BUILD_FLAGS} \
+		-f py23-image/centos-8/Dockerfile \
+		--build-arg SPLUNK_PRODUCT=uf \
+		-t uf-py23-centos-8:${IMAGE_VERSION} .
+
 uf-py23-redhat-8: uf-redhat-8
 	docker build ${DOCKER_BUILD_FLAGS} \
 		-f py23-image/redhat-8/Dockerfile \

base/centos-7/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2018 Splunk
+# Copyright 2018-2020 Splunk
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

base/centos-7/install.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Splunk
+# Copyright 2018-2020 Splunk
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

base/centos-8/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2018-2020 Splunk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM centos:8
+LABEL maintainer="support@splunk.com"
+
+ARG SCLOUD_URL
+ENV SCLOUD_URL ${SCLOUD_URL}
+
+COPY install.sh /install.sh
+RUN /install.sh && rm -rf /install.sh

base/centos-8/install.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright 2018-2020 Splunk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+yum -y install glibc-locale-source glibc-langpack-en
+
+localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+export LANG=en_US.utf8
+
+yum -y update && yum -y install wget sudo epel-release make
+yum -y install ansible python3-requests python3-jmespath
+
+# Install busybox
+wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch`
+chmod +x /bin/busybox
+
+# Install scloud
+wget -O /usr/bin/scloud.tar.gz ${SCLOUD_URL}
+tar -xf /usr/bin/scloud.tar.gz -C /usr/bin/
+rm /usr/bin/scloud.tar.gz
+
+cd /bin
+ln -s busybox killall
+ln -s busybox netstat
+ln -s busybox nslookup
+ln -s busybox readline
+ln -s busybox route
+ln -s busybox syslogd
+ln -s busybox traceroute
+chmod u+s /bin/ping
+groupadd sudo
+
+echo "
+## Allows people in group sudo to run all commands
+%sudo  ALL=(ALL)       ALL" >> /etc/sudoers
+
+# symlink for python3
+ln -s /bin/python3 /bin/python
+
+# Clean
+yum clean all
+rm -rf /install.sh /anaconda-post.log /var/log/anaconda/*

docs/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Navigation
 
+* [8.1.0](#810)
 * [8.0.6](#806)
 * [8.0.5.1](#8051)
 * [8.0.5](#805)
@@ -37,6 +38,26 @@
 
 ---
 
+## 8.1.0
+
+#### What's New?
+* Releasing new images to support Splunk Enterprise release.
+
+#### docker-splunk changes:
+* Bumping Splunk version. For details, see [Fixed issues for 8.1.0](https://docs.splunk.com/Documentation/Splunk/8.1.0/ReleaseNotes/Fixedissues)
+* Set bash as the default shell
+* CI bugfixes
+
+#### splunk-ansible changes:
+* Added environment variables to configure HTTPS on Splunkd. See [Supported environment variables](https://github.com/splunk/splunk-ansible/blob/develop/docs/ADVANCED.md#supported-environment-variables) for details.
+    * `SPLUNKD_SSL_` prefixed environment variables
+    * `splunk.ssl` section in `default.yml`
+* Enabled multisite for the `splunk_monitor` role
+* Enabled local indexing on the license master
+* Bugfixes and cleanup
+
+---
+
 ## 8.0.6
 
 #### What's New?
@@ -45,7 +66,7 @@
 #### docker-splunk changes:
 * Bumping Splunk version. For details, see [Fixed issues for 8.0.6](https://docs.splunk.com/Documentation/Splunk/8.0.6/ReleaseNotes/Fixedissues)
 * Test rewrite for parallelization
-* Decouples `etc` backup directory from `SPLUNK_HOME`
+* Decoupled `etc` backup directory from `SPLUNK_HOME`
 * Added tests and documentation for new features
 
 #### splunk-ansible changes:
@@ -97,7 +118,7 @@
 #### splunk-ansible changes:
 * Support for setting the [deployer push mode](https://docs.splunk.com/Documentation/Splunk/latest/DistSearch/PropagateSHCconfigurationchanges#Choose_a_deployer_push_mode) to control how apps are bundled and distributed to cluster members:
     * `shc.deployer_push_mode` in `default.yml`
-* Added the config variable `auxiliary_cluster_masters` to support enabling a search head to search across multiple indexer clusters. See [Multi-Cluster Search](advanced/MULTICLUSTERSEARCH.md) for details on configuration.
+* Added the config variable `auxiliary_cluster_masters` to support enabling a search head to search across multiple indexer clusters. See [Multi-Cluster Search](https://github.com/splunk/splunk-ansible/blob/develop/docs/advanced/MULTICLUSTERSEARCH.md) for details on configuration.
 * Documentation on executing `splunk-ansible` remotely, through a controller node such as Ansible Tower/AWX
 * Set custom Splunkd connection timeout using either:
     * `splunk.connection_timeout` in `default.yml`

py23-image/centos-8/Dockerfile

@@ -0,0 +1,30 @@
+ARG SPLUNK_PRODUCT=splunk
+FROM ${SPLUNK_PRODUCT}-centos-8:latest
+USER root
+
+RUN yum -y update
+RUN yum -y install gcc openssl-devel bzip2-devel libffi-devel python3-pip python2 python2-pip
+
+# manual installation of python 3.7 as default distro version is 3.6
+RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \
+    && tar xzf Python-3.7.4.tgz \
+    && cd Python-3.7.4 \
+    && ./configure --enable-optimizations --prefix=/usr \
+    && make install \
+    && cd .. \
+    && rm Python-3.7.4.tgz \
+    && rm -r Python-3.7.4 \
+    && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+    && python3.7 get-pip.py \
+    && rm -f get-pip.py \
+    # pip version is not automatically "fixed", unlike debian-based
+    && ln -sf /usr/bin/pip2 /usr/bin/pip \
+    && ln -sf /usr/bin/pip3.7 /usr/bin/pip3
+    # add python alias
+    # && ln -s /bin/python3 /bin/python
+
+RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \
+    && yum autoremove -y \
+    && yum clean all
+RUN pip3 --no-cache-dir install ansible requests \
+    && pip --no-cache-dir install pyyaml requests

splunk/common-files/Dockerfile

@@ -55,7 +55,7 @@ COPY [ "splunk/common-files/updateetc.sh", "/sbin/" ]
 
 # Setup users and groups
 RUN groupadd -r -g ${GID} ${SPLUNK_GROUP} \
-    && useradd -r -m -u ${UID} -g ${GID} ${SPLUNK_USER} \
+    && useradd -r -m -u ${UID} -g ${GID} -s /bin/bash ${SPLUNK_USER} \
     && chmod 755 /sbin/updateetc.sh
 
 COPY --from=package --chown=splunk:splunk /minimal /opt
@@ -98,7 +98,7 @@ COPY splunk-ansible ${SPLUNK_ANSIBLE_HOME}
 RUN sed -i -e 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL\nansible ALL=(splunk)NOPASSWD:ALL/g' /etc/sudoers \
     && echo 'Create the ansible user/group' \
     && groupadd -r ${ANSIBLE_GROUP} \
-    && useradd -r -m -g ${ANSIBLE_GROUP} ${ANSIBLE_USER} \
+    && useradd -r -m -g ${ANSIBLE_GROUP} -s /bin/bash ${ANSIBLE_USER} \
     && usermod -aG sudo ${ANSIBLE_USER} \
     && usermod -aG ${ANSIBLE_GROUP} ${SPLUNK_USER} \
     && echo 'Container Artifact Directory is a place for all artifacts and logs that are generated by the provisioning process. The directory is owned by the user "ansible".' \

splunk/common-files/createdefaults.py

@@ -1,5 +1,5 @@
 #! /usr/bin/python
-# Copyright 2018 Splunk
+# Copyright 2018-2020 Splunk
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import six
 import sys
 import uuid
 import random
@@ -32,10 +33,14 @@
 def random_generator(size=24):
     # Use System Random for
     rng = random.SystemRandom()
-    bytes = [chr(rng.randrange(256)) for i in range(size)]
-    s = ''.join(bytes)
+    b = [chr(rng.randrange(256)) for i in range(size)]
+    s = ''.join(b)
+    if six.PY2:
+        s = base64.b64encode(s)
+    else:
+        s = base64.b64encode(s.encode()).decode()
+    return s
 
-    return base64.b64encode(s)
 
 # if there are no environment vars set, lets make some safe defaults
 if not splunk_hec_token: