diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 22e2a9b6f..7799c672b 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -91,7 +91,9 @@ pipeline { dir("${BASE_DIR}") { script { parallel([ - 'stack-command': generateTestCommandStage(command: 'test-stack-command', artifacts: ['build/elastic-stack-dump/stack/logs/*.log', 'build/elastic-stack-dump/stack/logs/fleet-server-internal/*']), + 'stack-command-default': generateTestCommandStage(command: 'test-stack-command-default', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']), + 'stack-command-7x': generateTestCommandStage(command: 'test-stack-command-7x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']), + 'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*']), 'check-packages': generateTestCommandStage(command: 'test-check-packages', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check/logs/*.log', 'build/elastic-stack-dump/check/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log']), 'profiles-command': generateTestCommandStage(command: 'test-profiles-command'), diff --git a/Makefile b/Makefile index f86268ab0..37ed82181 100644 --- a/Makefile +++ b/Makefile @@ -38,9 +38,17 @@ test-go-ci: $(MAKE) test-go | go run github.com/tebeka/go2xunit > "$(PWD)/build/test-results/TEST-unit.xml" go run github.com/boumenot/gocover-cobertura < $(CODE_COVERAGE_REPORT_NAME_UNIT).out > $(CODE_COVERAGE_REPORT_NAME_UNIT).xml -test-stack-command: +test-stack-command-default: ./scripts/test-stack-command.sh +test-stack-command-7x: + ./scripts/test-stack-command.sh 7.16.0-SNAPSHOT + +test-stack-command-8x: + ./scripts/test-stack-command.sh 8.0.0-SNAPSHOT + +test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-8x + test-check-packages: ./scripts/test-check-packages.sh diff --git a/internal/install/install.go b/internal/install/install.go index fe64337b2..a6d29406d 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -104,7 +104,7 @@ func migrateIfNeeded(elasticPackagePath *locations.LocationManager) error { oldFiles := []string{ filepath.Join(elasticPackagePath.StackDir(), string(profile.SnapshotFile)), filepath.Join(elasticPackagePath.StackDir(), string(profile.PackageRegistryDockerfileFile)), - filepath.Join(elasticPackagePath.StackDir(), string(profile.KibanaConfigFile)), + filepath.Join(elasticPackagePath.StackDir(), string(profile.KibanaConfigDefaultFile)), filepath.Join(elasticPackagePath.StackDir(), string(profile.PackageRegistryConfigFile)), } diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index e742f9c1f..8a02ab27e 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -43,8 +43,8 @@ services: retries: 600 interval: 1s volumes: - - ./kibana.config.yml:/usr/share/kibana/config/kibana.yml - - ../../../stack/healthcheck.sh:/usr/share/kibana/healthcheck.sh + - "./kibana.config.${STACK_VERSION_VARIANT}.yml:/usr/share/kibana/config/kibana.yml" + - "../../../stack/healthcheck.sh:/usr/share/kibana/healthcheck.sh" ports: - "127.0.0.1:5601:5601" diff --git a/internal/profile/_static/kibana_config_8x.yml b/internal/profile/_static/kibana_config_8x.yml new file mode 100644 index 000000000..5d5a56188 --- /dev/null +++ b/internal/profile/_static/kibana_config_8x.yml @@ -0,0 +1,15 @@ +server.name: kibana +server.host: "0.0.0.0" + +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +elasticsearch.username: elastic +elasticsearch.password: changeme + +monitoring.ui.container.elasticsearch.enabled: true + +xpack.fleet.registryUrl: "http://package-registry:8080" +xpack.fleet.agents.enabled: true +xpack.fleet.agents.elasticsearch.hosts: ["http://elasticsearch:9200"] +xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"] + +xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" diff --git a/internal/profile/_static/kibana_config.yml b/internal/profile/_static/kibana_config_default.yml similarity index 100% rename from internal/profile/_static/kibana_config.yml rename to internal/profile/_static/kibana_config_default.yml diff --git a/internal/profile/files.go b/internal/profile/files.go index 0bd5d1c1b..9a4dc445d 100644 --- a/internal/profile/files.go +++ b/internal/profile/files.go @@ -23,11 +23,14 @@ type simpleFile struct { const profileStackPath = "stack" -// configfilesDiffer checks to see if a local configItem differs from the one it knows. -func (cfg simpleFile) configfilesDiffer() (bool, error) { +// configFilesDiffer checks to see if a local configItem differs from the one it knows. +func (cfg simpleFile) configFilesDiffer() (bool, error) { changes, err := os.ReadFile(cfg.path) + if err != nil && errors.Is(err, os.ErrNotExist) { + return false, nil + } if err != nil { - return false, errors.Wrapf(err, "error reading %s", KibanaConfigFile) + return false, errors.Wrapf(err, "error reading %s", cfg.path) } if string(changes) != cfg.body { return true, nil diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 5eb16751e..3b8a4c793 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -36,7 +36,8 @@ type configFile string // managedProfileFiles is the list of all files managed in a profile // If you create a new file that's managed by a profile, it needs to go in this list var managedProfileFiles = map[configFile]NewConfig{ - KibanaConfigFile: newKibanaConfig, + KibanaConfigDefaultFile: newKibanaConfigDefault, + KibanaConfig8xFile: newKibanaConfig8x, PackageRegistryDockerfileFile: newPackageRegistryDockerfile, PackageRegistryConfigFile: newPackageRegistryConfig, SnapshotFile: newSnapshotFile, @@ -240,7 +241,7 @@ func (profile Profile) localFilesChanged() (bool, error) { if cfgName == PackageProfileMetaFile { continue } - changes, err := cfgFile.configfilesDiffer() + changes, err := cfgFile.configFilesDiffer() if err != nil { return false, errors.Wrap(err, "error checking config file") } diff --git a/internal/profile/static.go b/internal/profile/static.go index 7bc218a8a..3a7020c7e 100644 --- a/internal/profile/static.go +++ b/internal/profile/static.go @@ -25,18 +25,31 @@ func newSnapshotFile(_ string, profilePath string) (*simpleFile, error) { }, nil } -// KibanaConfigFile is the main kibana config file -const KibanaConfigFile configFile = "kibana.config.yml" +// KibanaConfigDefaultFile is the default kibana config file +const KibanaConfigDefaultFile configFile = "kibana.config.default.yml" -//go:embed _static/kibana_config.yml -var kibanaConfigYml string +//go:embed _static/kibana_config_default.yml +var kibanaConfigDefaultYml string -// newKibanaConfig returns a Managed Config -func newKibanaConfig(_ string, profilePath string) (*simpleFile, error) { +func newKibanaConfigDefault(_ string, profilePath string) (*simpleFile, error) { return &simpleFile{ - name: string(KibanaConfigFile), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfigFile)), - body: kibanaConfigYml, + name: string(KibanaConfigDefaultFile), + path: filepath.Join(profilePath, profileStackPath, string(KibanaConfigDefaultFile)), + body: kibanaConfigDefaultYml, + }, nil +} + +// KibanaConfig8xFile is the Kibana config file for 8.x stack family +const KibanaConfig8xFile configFile = "kibana.config.8x.yml" + +//go:embed _static/kibana_config_8x.yml +var kibanaConfig8xYml string + +func newKibanaConfig8x(_ string, profilePath string) (*simpleFile, error) { + return &simpleFile{ + name: string(KibanaConfig8xFile), + path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig8xFile)), + body: kibanaConfig8xYml, }, nil } diff --git a/internal/stack/compose.go b/internal/stack/compose.go index 64ec44d34..2d2651044 100644 --- a/internal/stack/compose.go +++ b/internal/stack/compose.go @@ -14,6 +14,28 @@ import ( "github.com/elastic/elastic-package/internal/profile" ) +type envBuilder struct { + vars []string +} + +func newEnvBuilder() *envBuilder { + return new(envBuilder) +} + +func (eb *envBuilder) withEnvs(envs []string) *envBuilder { + eb.vars = append(eb.vars, envs...) + return eb +} + +func (eb *envBuilder) withEnv(env string) *envBuilder { + eb.vars = append(eb.vars, env) + return eb +} + +func (eb *envBuilder) build() []string { + return eb.vars +} + func dockerComposeBuild(options Options) error { c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) if err != nil { @@ -26,7 +48,11 @@ func dockerComposeBuild(options Options) error { } opts := compose.CommandOptions{ - Env: append(appConfig.StackImageRefs(options.StackVersion).AsEnv(), options.Profile.ComposeEnvVars()...), + Env: newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnv(stackVariantAsEnv(options.StackVersion)). + withEnvs(options.Profile.ComposeEnvVars()). + build(), Services: withIsReadyServices(withDependentServices(options.Services)), } @@ -48,7 +74,11 @@ func dockerComposePull(options Options) error { } opts := compose.CommandOptions{ - Env: append(appConfig.StackImageRefs(options.StackVersion).AsEnv(), options.Profile.ComposeEnvVars()...), + Env: newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnv(stackVariantAsEnv(options.StackVersion)). + withEnvs(options.Profile.ComposeEnvVars()). + build(), Services: withIsReadyServices(withDependentServices(options.Services)), } @@ -75,7 +105,11 @@ func dockerComposeUp(options Options) error { } opts := compose.CommandOptions{ - Env: append(appConfig.StackImageRefs(options.StackVersion).AsEnv(), options.Profile.ComposeEnvVars()...), + Env: newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnv(stackVariantAsEnv(options.StackVersion)). + withEnvs(options.Profile.ComposeEnvVars()). + build(), ExtraArgs: args, Services: withIsReadyServices(withDependentServices(options.Services)), } @@ -98,7 +132,11 @@ func dockerComposeDown(options Options) error { } downOptions := compose.CommandOptions{ - Env: append(appConfig.StackImageRefs(options.StackVersion).AsEnv(), options.Profile.ComposeEnvVars()...), + Env: newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnv(stackVariantAsEnv(options.StackVersion)). + withEnvs(options.Profile.ComposeEnvVars()). + build(), // Remove associated volumes. ExtraArgs: []string{"--volumes", "--remove-orphans"}, } diff --git a/internal/stack/shellinit.go b/internal/stack/shellinit.go index 926e896e2..029976ae2 100644 --- a/internal/stack/shellinit.go +++ b/internal/stack/shellinit.go @@ -39,7 +39,8 @@ type kibanaConfiguration struct { // ShellInit method exposes environment variables that can be used for testing purposes. func ShellInit(elasticStackProfile *profile.Profile) (string, error) { // Read Elasticsearch username and password from Kibana configuration file. - body, err := os.ReadFile(elasticStackProfile.FetchPath(profile.KibanaConfigFile)) + // FIXME read credentials from correct Kibana config file, not default + body, err := os.ReadFile(elasticStackProfile.FetchPath(profile.KibanaConfigDefaultFile)) if err != nil { return "", errors.Wrap(err, "error reading Kibana config file") } diff --git a/internal/stack/variants.go b/internal/stack/variants.go new file mode 100644 index 000000000..a069b3053 --- /dev/null +++ b/internal/stack/variants.go @@ -0,0 +1,25 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "fmt" + "strings" +) + +// stackVariantAsEnv function returns a stack variant based on the given stack version. +// We identified two variants: +// * default, covers all of 7.x branches +// * 8x, supports different configuration options in Kibana +func stackVariantAsEnv(version string) string { + return fmt.Sprintf("STACK_VERSION_VARIANT=%s", selectStackVersion(version)) +} + +func selectStackVersion(version string) string { + if strings.HasPrefix(version, "8.") { + return "8x" + } + return "default" +} diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index 32e2b88b8..c09f2bf95 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -2,11 +2,13 @@ set -euxo pipefail +VERSION=${1:-default} + cleanup() { r=$? # Dump stack logs - elastic-package stack dump -v --output build/elastic-stack-dump/stack + elastic-package stack dump -v --output build/elastic-stack-dump/stack/${VERSION} # Take down the stack elastic-package stack down -v @@ -24,11 +26,16 @@ cleanup() { trap cleanup EXIT +ARG_VERSION="" +if [ "${VERSION}" != "default" ]; then + ARG_VERSION="--version ${VERSION}" +fi + # Update the stack -elastic-package stack update -v +elastic-package stack update -v ${ARG_VERSION} # Boot up the stack -elastic-package stack up -d -v +elastic-package stack up -d -v ${ARG_VERSION} # Verify it's accessible eval "$(elastic-package stack shellinit)"