Merge branch 'main' into sample-data-docs

Author: ruflin

.buildkite/pipeline.trigger.integration.tests.sh

@@ -3,6 +3,7 @@
 # exit immediately on failure, or if an undefined variable is used
 set -eu
 
+
 # begin the pipeline.yml file
 echo "steps:"
 echo "  - group: \":terminal: Integration test suite\""
@@ -110,14 +111,21 @@ echo "          - build/elastic-stack-dump/install-zip-shellinit/logs/*.log"
 
 echo "      - label: \":go: Running integration test: test-profiles-command\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command"
+echo "        env:"
+echo "          DOCKER_COMPOSE_VERSION: \"false\""
+echo "          DOCKER_VERSION: \"false\""
 echo "        agents:"
-echo "          provider: \"gcp\""
+echo "          image: \"${LINUX_AGENT_IMAGE}\""
+echo "          cpu: \"8\""
+echo "          memory: \"4G\""
 
 echo "      - label: \":go: Running integration test: test-check-update-version\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-update-version"
 echo "        env:"
 echo "          DEFAULT_VERSION_TAG: v0.80.0"
+echo "          DOCKER_COMPOSE_VERSION: \"false\""
+echo "          DOCKER_VERSION: \"false\""
 echo "        agents:"
 echo "          image: \"${LINUX_AGENT_IMAGE}\""
 echo "          cpu: \"8\""
-echo "          memory: \"4G\""
+echo "          memory: \"4G\""

.buildkite/pipeline.yml

@@ -1,7 +1,8 @@
 env:
   SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
+  DOCKER_COMPOSE_VERSION: "v2.24.1"
+  DOCKER_VERSION: "24.0.7"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"

.buildkite/scripts/install_deps.sh

@@ -34,6 +34,44 @@ add_bin_path(){
     export PATH="${WORKSPACE}/bin:${PATH}"
 }
 
+with_docker() {
+    if [[ "${DOCKER_VERSION:-"false"}" == "false" ]]; then
+        echo "Skip docker installation"
+        return
+    fi
+    local ubuntu_version
+    local ubuntu_codename
+    local architecture
+    ubuntu_version="$(lsb_release -rs)" # 20.04
+    ubuntu_codename="$(lsb_release -sc)" # focal
+    architecture=$(dpkg --print-architecture)
+    local debian_version="5:${DOCKER_VERSION}-1~ubuntu.${ubuntu_version}~${ubuntu_codename}"
+
+    sudo sudo mkdir -p /etc/apt/keyrings
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+    echo "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce=${debian_version}"
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce-cli=${debian_version}"
+    sudo systemctl start docker
+}
+
+with_docker_compose_plugin() {
+    if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then
+        echo "Skip docker compose installation (plugin)"
+        return
+    fi
+    create_bin_folder
+    check_platform_architecture
+
+    local DOCKER_CONFIG="$HOME/.docker/cli-plugins"
+    mkdir -p "$DOCKER_CONFIG"
+
+    retry 5 curl -SL -o ${DOCKER_CONFIG}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}"
+    chmod +x ${DOCKER_CONFIG}/docker-compose
+    docker compose version
+}
+
 with_kubernetes() {
     create_bin_folder
     check_platform_architecture

.buildkite/scripts/integration_tests.sh

@@ -109,6 +109,12 @@ add_bin_path
 echo "--- install go"
 with_go
 
+echo "--- install docker"
+with_docker
+
+echo "--- install docker-compose plugin"
+with_docker_compose_plugin
+
 if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then
     echo "--- install kubectl & kind"
     with_kubernetes

Makefile

@@ -66,7 +66,7 @@ test-stack-command-86:
 	./scripts/test-stack-command.sh 8.6.2
 
 test-stack-command-8x:
-	./scripts/test-stack-command.sh 8.11.0-SNAPSHOT
+	./scripts/test-stack-command.sh 8.12-SNAPSHOT
 
 test-stack-command-with-apm-server:
 	APM_SERVER_ENABLED=true ./scripts/test-stack-command.sh

README.md

@@ -625,6 +625,48 @@ The following settings are available per profile:
 * `stack.serverless.region` can be used to select the region to use when starting
   serverless projects.
 
+## Useful environment variables
+
+There are available some environment variables that could be used to change some of the
+`elastic-package` settings:
+
+- Related to `docker-compose` / `docker compose` commands:
+    - `ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT`: If set to `true`, it disables the progress output from `docker compose`/`docker-compose` commands.
+        - For versions v2 `< 2.19.0`, it sets `--ansi never` flag.
+        - For versions v2 `>= 2.19.0`, it sets `--progress plain` flag and `--quiet-pull` for `up` sub-command`.
+
+
+- Related to global `elastic-package` settings:
+    - `ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED`: if set to `true`, `elastic-package` is not going to check
+      for newer versions.
+    - `ELASTIC_PACKAGE_PROFILE`: Name of the profile to be using.
+    - `ELASTIC_PACKAGE_DATA_HOME`: Custom path to be used for `elastic-package` data directory. By default this is `~/.elastic-package`.
+
+- Related to the build process:
+    - `ELASTIC_PACKAGE_REPOSITORY_LICENSE`: Path to the default repository license.
+    - `ELASTIC_PACKAGE_LINKS_FILE_PATH`: Path to the links table file (e.g. `links_table.yml`) with the link definitions to be used in the build process of a package.
+
+- Related to signing packages:
+    - `ELASTIC_PACKAGE_SIGNER_PRIVATE_KEYFILE`: Path to the private key file to sign packages.
+    - `ELASTIC_PACKAGE_SIGNER_PASSPHRASE`: Passphrase to use the private key file.
+
+- Related to tests:
+    - `ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS`: If set to `true`, the results from pipeline tests are not compared to avoid errors from GeoIP.
+
+- To configure the Elastic stack to be used by `elastic-package`:
+    - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)
+    - `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`: User name to connect to elasticsearch (e.g. elastic)
+    - `ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD`: Password of that user.
+    - `ELASTIC_PACKAGE_ELASTICSEARCH_KIBANA_HOST`: Kibana URL (e.g. https://127.0.0.1:5601)
+    - `ELASTIC_PACKAGE_ELASTICSEARCH_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services.
+
+- To configure an external metricstore while running benchmarks (more info at [system benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_benchmarking.md#setting-up-an-external-metricstore) or [rally benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/rally_benchmarking.md#setting-up-an-external-metricstore)):
+    - `ELASTIC_PACKAGE_ESMETRICSTORE_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)
+    - `ELASTIC_PACKAGE_ESMETRICSTORE_USERNAME`: Username to connect to elasticsearch (e.g. elastic)
+    - `ELASTIC_PACKAGE_ESMETRICSTORE_PASSWORD`: Password for the user.
+    - `ELASTIC_PACKAGE_ESMETRICSTORE_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services.
+
+
 ## Release process
 
 This project uses [GoReleaser](https://goreleaser.com/) to release a new version of the application (semver). Release publishing

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/elastic/go-licenser v0.4.1
 	github.com/elastic/go-resource v0.1.1
 	github.com/elastic/go-ucfg v0.8.6
-	github.com/elastic/package-spec/v3 v3.0.3
+	github.com/elastic/package-spec/v3 v3.0.4
 	github.com/fatih/color v1.16.0
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.11.0
@@ -68,11 +68,11 @@ require (
 	github.com/dnephin/pflag v1.0.7 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/elastic/gojsonschema v1.2.1 // indirect
-	github.com/elastic/kbncontent v0.1.1 // indirect
+	github.com/elastic/kbncontent v0.1.3 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
-	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.8.1 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect

go.sum

@@ -100,10 +100,10 @@ github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0
 github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
 github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc=
 github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg=
-github.com/elastic/kbncontent v0.1.1 h1:X6ZXKPTW3MwFZPLMZcPAbpUIsKQhFImhBGKOd83+UFo=
-github.com/elastic/kbncontent v0.1.1/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI=
-github.com/elastic/package-spec/v3 v3.0.3 h1:dkYujRWPwnE77kYfMKQ1W2YkDSHwY0ej9HG8F5wT2zY=
-github.com/elastic/package-spec/v3 v3.0.3/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90=
+github.com/elastic/kbncontent v0.1.3 h1:qT0RkshRhiAgH51dDD983tJloeLLMCYE/qlRRtn3R8k=
+github.com/elastic/kbncontent v0.1.3/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI=
+github.com/elastic/package-spec/v3 v3.0.4 h1:IZDRpVoMPztSgONUGbTR6re0VxpkitMDR/HwcLQpt7g=
+github.com/elastic/package-spec/v3 v3.0.4/go.mod h1:zMVyksPTDFsWxxdbw3ahXBs6O0G2tULA7HXjFl8w8zA=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
@@ -114,8 +114,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
-github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/evanphx/json-patch/v5 v5.8.1 h1:iPEdwg0XayoS+E7Mth9JxwUtOgyVxnDTXHtKhZPlZxA=
+github.com/evanphx/json-patch/v5 v5.8.1/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=

internal/compose/compose.go

@@ -33,9 +33,12 @@ const (
 )
 
 var (
-	DisableANSIComposeEnv             = environment.WithElasticPackagePrefix("COMPOSE_DISABLE_ANSI")
-	DisablePullProgressInformationEnv = environment.WithElasticPackagePrefix("COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION")
-	EnableComposeStandaloneEnv        = environment.WithElasticPackagePrefix("COMPOSE_ENABLE_STANDALONE")
+	EnableComposeStandaloneEnv     = environment.WithElasticPackagePrefix("COMPOSE_ENABLE_STANDALONE")
+	DisableVerboseOutputComposeEnv = environment.WithElasticPackagePrefix("COMPOSE_DISABLE_VERBOSE_OUTPUT")
+)
+
+const (
+	defaultComposeProgressOutput = "plain"
 )
 
 // Project represents a Docker Compose project.
@@ -47,6 +50,8 @@ type Project struct {
 	dockerComposeStandalone        bool
 	disableANSI                    bool
 	disablePullProgressInformation bool
+	progressOutput                 string
+	composeVersion                 *semver.Version
 }
 
 // Config represents a Docker Compose configuration file.
@@ -204,13 +209,16 @@ func NewProject(name string, paths ...string) (*Project, error) {
 	}
 	logger.Debug(versionMessage)
 
-	v, ok = os.LookupEnv(DisableANSIComposeEnv)
+	v, ok = os.LookupEnv(DisableVerboseOutputComposeEnv)
 	if !c.dockerComposeV1 && ok && strings.ToLower(v) != "false" {
-		c.disableANSI = true
-	}
-
-	v, ok = os.LookupEnv(DisablePullProgressInformationEnv)
-	if ok && strings.ToLower(v) != "false" {
+		if c.composeVersion.LessThan(semver.MustParse("2.19.0")) {
+			c.disableANSI = true
+		} else {
+			// --ansi never looks is ignored by "docker compose" and latest versions of "docker-compose"
+			// adding --progress plain is a similar result as --ansi never
+			// if set to "--progress quiet", there is no output at all from docker compose commands
+			c.progressOutput = defaultComposeProgressOutput
+		}
 		c.disablePullProgressInformation = true
 	}
 
@@ -442,6 +450,10 @@ func (p *Project) baseArgs() []string {
 		args = append(args, "--ansi", "never")
 	}
 
+	if p.progressOutput != "" {
+		args = append(args, "--progress", p.progressOutput)
+	}
+
 	args = append(args, "-p", p.name)
 	return args
 }
@@ -504,6 +516,7 @@ func (p *Project) dockerComposeVersion() (*semver.Version, error) {
 	if err != nil {
 		return nil, fmt.Errorf("docker compose version is not a valid semver (value: %s): %w", dcVersion, err)
 	}
+	p.composeVersion = ver
 	return ver, nil
 }
 

internal/install/stack_version.go

@@ -6,5 +6,5 @@ package install
 
 const (
 	// DefaultStackVersion is the default version of the stack
-	DefaultStackVersion = "8.10.1"
+	DefaultStackVersion = "8.11.4"
 )