Merge remote-tracking branch 'origin/main' into profile-resources-refactor

Author: jsoriano

.buildkite/hooks/pre-command

@@ -26,10 +26,17 @@ GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-ac
 AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
 GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
+SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials
+PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader
+PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials
 
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+fi
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then
     export ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH})
@@ -56,4 +63,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" &&
     export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
     export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
     export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
+
+    # signing job
+    export SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${SIGNING_PACKAGES_GCS_CREDENTIALS_PATH})
+
+    # publishing job
+    export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
 fi

.buildkite/pipeline.package-storage-publish.yml

@@ -1,7 +1,7 @@
-steps:
-  - label: "Example Test"
-    command: echo "Hello!"
+env:
+  SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
 
+steps:
   - label: ":go: Build package"
     key: build-package
     command:
@@ -11,10 +11,15 @@ steps:
       image: "golang:1.19.5"
       cpu: "8"
       memory: "4G"
+    artifact_paths:
+      - build/packages/*.zip
 
-  - label: "Test"
+  - label: "Sign and Publish package"
     key: sign-publish
     command: ".buildkite/scripts/signAndPublishPackage.sh"
+    depends_on:
+      - build-package
+    timeout_in_minutes: 30
     agents:
       provider: "gcp"
       image: family/core-ubuntu-2004

.buildkite/pipeline.trigger.integration.tests.sh

@@ -56,6 +56,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "      - label: \":go: Running integration test: ${package_name}\""
     echo "        key: \"integration-parallel-${package_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+    echo "        env:"
+    echo "          UPLOAD_SAFE_LOGS: 1"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"

.buildkite/pipeline.yml

@@ -1,6 +1,7 @@
 env:
   SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   DOCKER_COMPOSE_VERSION: "1.25.5" # "v2.15.1"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
   KIND_VERSION: 'v0.17.0'
   K8S_VERSION: 'v1.26.0'
 
@@ -56,11 +57,8 @@ steps:
 
   - label: ":github: Release"
     key: "release"
-    # build.tag != null && build.branch == "main"
     if: |
       build.tag =~ /^v[0-9]+[.][0-9]+[.][0-9]+$$/
     command: ".buildkite/scripts/release.sh"
     agents:
-      image: "golang:1.19.5"
-      cpu: "8"
-      memory: "4G"
+      provider: "gcp"

.buildkite/scripts/install_deps.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/tooling.sh
+
+add_bin_path(){
+    mkdir -p ${WORKSPACE}/bin
+    export PATH="${WORKSPACE}/bin:${PATH}"
+}
+
+with_kubernetes() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+    chmod +x ${WORKSPACE}/bin/kind
+    kind version
+    which kind
+
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x ${WORKSPACE}/bin/kubectl
+    kubectl version --client
+    which kubectl
+}
+
+with_go() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
+    chmod +x ${WORKSPACE}/bin/gvm
+    eval "$(gvm $(cat .go-version))"
+    go version
+    which go
+    export PATH="$(go env GOPATH)/bin:${PATH}"
+}
+
+with_docker_compose() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
+    chmod +x ${WORKSPACE}/bin/docker-compose
+    docker-compose version
+}

.buildkite/scripts/integration_tests.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
-
 set -euo pipefail
 
-PARALLEL_TARGET="test-check-packages-parallel"
-KIND_TARGET="test-check-packages-with-kind"
+WORKSPACE="$(pwd)"
+TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
+
+cleanup() {
+    echo "Deleting temporal files..."
+    cd ${WORKSPACE}
+    rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
+    echo "Done."
+}
+trap cleanup EXIT
 
 usage() {
     echo "$0 [-t <target>] [-h]"
@@ -13,38 +20,18 @@ usage() {
     echo -e "\t-h: Show this message"
 }
 
-with_kubernetes() {
-    # FIXME add retry logic
-    mkdir -p ${WORKSPACE}/bin
-    curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
-    chmod +x ${WORKSPACE}/bin/kind
-    kind version
-    which kind
-
-    mkdir -p ${WORKSPACE}/bin
-    curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-    chmod +x ${WORKSPACE}/bin/kubectl
-    kubectl version --client
-    which kubectl
-}
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
 
-with_go() {
-    # FIXME add retry logic
-    mkdir -p ${WORKSPACE}/bin
-    curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
-    chmod +x ${WORKSPACE}/bin/gvm
-    eval "$(gvm $(cat .go-version))"
-    go version
-    which go
-}
+PARALLEL_TARGET="test-check-packages-parallel"
+KIND_TARGET="test-check-packages-with-kind"
+TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 
-with_docker_compose() {
-    # FIXME add retry logic
-    mkdir -p ${WORKSPACE}/bin
-    curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
-    chmod +x ${WORKSPACE}/bin/docker-compose
-    docker-compose version
-}
+JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
+
+REPO_NAME=$(repoName "${BUILDKITE_REPO}")
+REPO_BUILD_TAG="${REPO_NAME}/${BUILDKITE_BUILD_NUMBER}"
 
 TARGET=""
 PACKAGE=""
@@ -79,14 +66,39 @@ if [[ "${TARGET}" == "" ]]; then
     exit 1
 fi
 
-echo "Current path: $(pwd)"
-WORKSPACE="$(pwd)"
-export PATH="${WORKSPACE}/bin:${PATH}"
-echo "Path: $PATH"
+google_cloud_auth_safe_logs() {
+    local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE})
+    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
+
+    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+
+    google_cloud_auth "${secretFileLocation}"
+
+    echo "${gsUtilLocation}"
+}
+
+upload_safe_logs() {
+    local bucket="$1"
+    local source="$2"
+    local target="$3"
+
+    if ! ls ${source} 2>&1 > /dev/null ; then
+        echo "upload_safe_logs: artifacts files not found, nothing will be archived"
+        return
+    fi
+
+    local gsUtilLocation=$(google_cloud_auth_safe_logs)
+
+    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+
+    rm -rf "${gsUtilLocation}"
+    unset GOOGLE_APPLICATIONS_CREDENTIALS
+}
+
+add_bin_path
 
 echo "--- install go"
 with_go
-export PATH="$(go env GOPATH)/bin:${PATH}"
 
 echo "--- install docker-compose"
 with_docker_compose
@@ -100,6 +112,19 @@ echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
     make install
     make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
+
+    if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then
+        upload_safe_logs \
+            "${JOB_GCS_BUCKET_INTERNAL}" \
+            "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*" \
+            "insecure-logs/${PACKAGE}/"
+
+        upload_safe_logs \
+            "${JOB_GCS_BUCKET_INTERNAL}" \
+            "build/container-logs/*.log" \
+            "insecure-logs/${PACKAGE}/container-logs/"
+    fi
+    make check-git-clean
     exit 0
 fi
 

.buildkite/scripts/release.sh

@@ -2,6 +2,20 @@
 
 set -euo pipefail
 
+cleanup() {
+    rm -rf ${WORKSPACE}
+}
+trap cleanup exit
+
+WORKSPACE="/tmp/bin-buildkite/"
+
+VERSION=""
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
+add_bin_path
+with_go
+
 echo "--- fetching tags"
 # Ensure that tags are present so goreleaser can build the changelog from the last release.
 git rev-parse --is-shallow-repository
@@ -10,4 +24,3 @@ git fetch origin --tags
 echo "--- running goreleaser"
 # Run latest version of goreleaser
 curl -sL https://git.io/goreleaser | bash
-