From 4f5bc2b266d01405d73efbe66c9f4bc4e5db4ed2 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Sun, 11 Feb 2024 17:09:48 +0100 Subject: [PATCH 01/15] Generate pipeline from latest 'cloudfoundry-community/pipeline-templates' project --- ci/pipeline.yml | 539 ++++++++++++++++++++ ci/repipe | 151 ++++++ ci/scripts/bump-dependent-release | 89 ++++ ci/scripts/export-release | 82 +++ ci/scripts/shipit | 127 +++++ ci/scripts/testflight | 194 +++++++ ci/scripts/update-blob | 85 +++ ci/scripts/update-manifest-compiled-release | 65 +++ ci/scripts/update-manifest-release | 62 +++ ci/scripts/use-compiled-releases | 49 ++ ci/scripts/wait-until-proxy-bosh-cf-running | 60 +++ ci/settings.yml | 46 ++ 12 files changed, 1549 insertions(+) create mode 100644 ci/pipeline.yml create mode 100755 ci/repipe create mode 100755 ci/scripts/bump-dependent-release create mode 100755 ci/scripts/export-release create mode 100755 ci/scripts/shipit create mode 100755 ci/scripts/testflight create mode 100755 ci/scripts/update-blob create mode 100755 ci/scripts/update-manifest-compiled-release create mode 100755 ci/scripts/update-manifest-release create mode 100755 ci/scripts/use-compiled-releases create mode 100755 ci/scripts/wait-until-proxy-bosh-cf-running create mode 100644 ci/settings.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 00000000..6d17e23f --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,539 @@ +--- +# +# ci/pipeline.yml +# +# Pipeline structure file for a BOSH Release pipeline +# +# DO NOT MAKE CHANGES TO THIS FILE. Instead, modify +# ci/settings.yml and override what needs overridden. +# This uses spruce, so you have some options there. +# +# author: James Hunt +# created: 2016-03-30 + +meta: + name: (( param "Please name your pipeline" )) + release: (( grab meta.name )) + target: (( param "Please identify the name of the target Concourse CI" )) + url: (( param "Please specify the full URL of the target Concourse CI" )) + pipeline: (( concat meta.name "-boshrelease" )) + manifest: + directory: manifests + path: (( concat meta.manifest.directory "/" meta.name ".yml" )) + vars: "--- {}" + vars-pr: (( grab meta.manifest.vars )) + operator_file_paths: "" # comma (or space) separated list relative to repo root + + git: + email: (( param "Please provide the git email for automated commits" )) + name: (( param "Please provide the git name for automated commits" )) + + image: + name: starkandwayne/concourse + tag: latest + registry: + username: (( param "Please set your Docker registry username for your pipeline image" )) + password: (( param "Please set your Docker registry password for your pipeline image" )) + + aws: + bucket: (( grab meta.pipeline )) + region_name: us-east-1 + access_key: (( param "Please set your AWS Access Key ID for your pipeline S3 Bucket" )) + secret_key: (( param "Please set your AWS Secret Key ID for your pipeline S3 Bucket" )) + + github: + uri: (( concat "git@github.com:" meta.github.owner "/" meta.github.repo )) + owner: (( param "Please specify the name of the user / organization that owns the Github repository" )) + repo: (( param "Please specify the name of the Github repository" )) + branch: master + pr_base_branch: (( grab meta.github.branch )) + release_branch: (( grab meta.github.branch )) + private_key: (( param "Please generate an SSH Deployment Key for this repo and specify it here" )) + access_token: (( param "Please generate a Personal Access Token to be used for creating github releases (do you have a ci-bot?)" )) + + bosh: + stemcell: + major: 1 + cpi: aws-xen-hvm + os: ubuntu-jammy + target: ((bosh-lite-environment)) + cacert: ((bosh-lite-ca-cert)) + username: ((bosh-lite-client)) + password: ((bosh-lite-client-secret)) + deployment: (( concat meta.name "-testflight" )) + deployment-pr: (( concat meta.name "-testflight-pr" )) + + slack: + webhook: (( param "Please specify your Slack Incoming Webhook Integration URL" )) + channel: (( param "Please specify the channel (#name) or user (@user) to send messages to" )) + username: concourse + icon: https://cl.ly/2F421Y300u07/concourse-logo-blue-transparent.png + success_moji: ":airplane_departure:" + fail_moji: ":airplane_arriving:" + upset_moji: ":sadpanda:" + pipeline_url: (( concat meta.url "/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}" )) + fail_link: (( concat "<" meta.slack.pipeline_url "/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME}| Concourse Failure! " meta.slack.upset_moji ">" )) + fail_text: '(( concat meta.slack.fail_link " " meta.pipeline ": `${BUILD_JOB_NAME}` job failed" ))' + +groups: + - name: (( grab meta.pipeline )) + jobs: + - testflight + - testflight-pr + - rc + - shipit + - name: versioning + jobs: + - major + - minor + - patch + - name: compiled-releases + jobs: + - compile-release + - use-compiled-releases + +jobs: + - name: testflight + public: true + serial: true + plan: + - in_parallel: + - { get: git, trigger: true } + - { get: image } + - task: testflight + image: image + config: + platform: linux + inputs: [ name: git ] + caches: [ path: /root/.bosh/cache ] + run: + path: ./git/ci/scripts/testflight + args: [] + params: + REPO_ROOT: git + BOSH_ENVIRONMENT: (( grab meta.bosh.target )) + BOSH_CA_CERT: (( grab meta.bosh.cacert )) + BOSH_CLIENT: (( grab meta.bosh.username )) + BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) + BOSH_DEPLOYMENT: (( grab meta.bosh.deployment )) + TEST_ERRANDS: (( grab meta.test-errands || meta.test-errand || ~ )) + AWS_ACCESS_KEY: (( grab meta.aws.access_key )) + AWS_SECRET_KEY: (( grab meta.aws.secret_key )) + MANIFEST_PATH: (( grab meta.manifest.path )) + MANIFEST_VARS: (( grab meta.manifest.vars )) + MANIFEST_OP_PATHS: (( grab meta.manifest.operator_file_paths )) + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + + - name: testflight-pr + public: true + serial: true + plan: + - in_parallel: + - { get: git-pull-requests, trigger: true, version: every } + - { get: image } + - put: git-pull-requests + params: + path: git-pull-requests + status: PENDING + - task: testflight + image: image + config: + platform: linux + inputs: [ name: git-pull-requests ] + caches: [ path: /root/.bosh/cache ] + run: + path: ./git-pull-requests/ci/scripts/testflight + args: [] + params: + REPO_ROOT: git-pull-requests + BOSH_ENVIRONMENT: (( grab meta.bosh.target )) + BOSH_CA_CERT: (( grab meta.bosh.cacert )) + BOSH_CLIENT: (( grab meta.bosh.username )) + BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) + BOSH_DEPLOYMENT: (( grab meta.bosh.deployment-pr )) + TEST_ERRANDS: (( grab meta.test-errands || meta.test-errand || ~ )) + AWS_ACCESS_KEY: (( grab meta.aws.access_key )) + AWS_SECRET_KEY: (( grab meta.aws.secret_key )) + MANIFEST_PATH: (( grab meta.manifest.path )) + MANIFEST_VARS: (( grab meta.manifest.vars-pr )) + MANIFEST_OP_PATHS: (( grab meta.manifest.operator_file_paths )) + on_success: + put: git-pull-requests + params: + path: git-pull-requests + status: SUCCESS + on_failure: + put: git-pull-requests + params: + path: git-pull-requests + status: FAILURE + - task: pr-success-message + image: image + config: + platform: linux + inputs: [ name: git-pull-requests ] + outputs: [ name: message ] + run: + path: bash + args: + - -c + - | + set -ueo pipefail + pr_author=$( < git-pull-requests/.git/resource/author) + pr_author_email=$(< git-pull-requests/.git/resource/author_email) + pr_title=$( < git-pull-requests/.git/resource/title) + pr_url=$( < git-pull-requests/.git/resource/url) + echo "Pull request “_<${pr_url}|${pr_title}>_”" \ + "by *${pr_author:-${pr_author_email}}*" \ + "has passed testflight :tada:" \ + "Merge when ready ${SUCCESS_MOJI}:"$'\n'"${pr_url}" \ + > message/body + params: + SUCCESS_MOJI: (( grab meta.slack.success_moji )) + on_success: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text_file: message/body + + - name: rc + public: true + serial: true + plan: + - in_parallel: + - { get: git, trigger: true, passed: [ testflight ] } + - { get: version, params: { pre: rc } } + - { get: image } + - task: release-notes + image: image + config: + platform: linux + inputs: [ name: git ] + run: + path: sh + args: + - -ce + - | + cd git + if [ -f ci/release_notes.md ]; then + echo "###### RELEASE NOTES ###############" + echo + cat ci/release_notes.md + echo + echo "########################################" + echo + else + echo "NO RELEASE NOTES HAVE BEEN WRITTEN" + echo "You *might* want to do that before" + echo "hitting (+) on that shipit job..." + echo + fi + - put: version + params: { file: version/number } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + + - name: minor + public: true + serial: true + plan: + - { get: version, trigger: false, params: { bump: minor } } + - { put: version, params: { file: version/number } } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + + - name: patch + public: true + serial: true + plan: + - { get: version, trigger: false, params: { bump: patch } } + - { put: version, params: { file: version/number } } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + + - name: major + public: true + serial: true + plan: + - { get: version, trigger: false, params: { bump: major } } + - { put: version, params: { file: version/number } } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + + - name: shipit + public: true + serial: true + plan: + - in_parallel: + - { get: version, passed: [ rc ], params: { bump: final } } + - { get: git, passed: [ rc ] } + - { get: image } + - task: release + image: image + config: + platform: linux + inputs: + - name: version + - name: git + outputs: + - name: gh + - name: pushme + - name: notifications + run: + path: ./git/ci/scripts/shipit + args: [] + params: + REPO_ROOT: git + VERSION_FROM: version/number + RELEASE_ROOT: gh + GH_RELEASE_NAME: (( grab meta.release )) + REPO_OUT: pushme + NOTIFICATION_OUT: notifications + RELEASE_BRANCH: (( grab meta.github.release_branch )) + GITHUB_OWNER: (( grab meta.github.owner )) + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + AWS_ACCESS_KEY: (( grab meta.aws.access_key )) + AWS_SECRET_KEY: (( grab meta.aws.secret_key )) + MANIFESTS_DIR: (( grab meta.manifest.directory )) + + - put: git + params: + rebase: true + repository: pushme + - put: s3-tarball + params: + file: (( concat "gh/artifacts/" meta.name "-*.tgz" )) + - put: github + params: + name: gh/name + tag: gh/tag + body: gh/notes.md + globs: [ gh/artifacts/* ] + - put: version + params: + bump: patch + - put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text_file: notifications/message + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + + - name: compile-release + public: true + serial: true + plan: + - in_parallel: + - get: git + - get: github + trigger: true + - get: (( concat meta.bosh.stemcell.os "-stemcell-" meta.bosh.stemcell.major )) + params: { tarball: false } + - get: image + - task: export-release + image: image + config: + platform: linux + inputs: + - name: git + - name: (( concat meta.bosh.stemcell.os "-stemcell-" meta.bosh.stemcell.major )) + path: stemcell + - name: github + path: release + outputs: + - name: compiled-release + run: + path: git/ci/scripts/export-release + params: + BOSH_ENVIRONMENT: (( grab meta.bosh.target )) + BOSH_CA_CERT: (( grab meta.bosh.cacert )) + BOSH_CLIENT: (( grab meta.bosh.username )) + BOSH_CLIENT_SECRET: (( grab meta.bosh.password )) + STEMCELL_CPI: (( grab meta.bosh.stemcell.cpi )) + STEMCELL_OS: (( grab meta.bosh.stemcell.os )) + - put: compiled-release + params: + file: (( concat "compiled-release/compiled-releases/" meta.name "/*.tgz" )) + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + + - name: use-compiled-releases + public: true + serial: true + plan: + - in_parallel: + - get: git + - get: github + passed: [compile-release] + trigger: true + - get: (( concat meta.bosh.stemcell.os "-stemcell-" meta.bosh.stemcell.major )) + passed: [ compile-release ] + params: { tarball: false } + - get: compiled-release + passed: [ compile-release ] + - get: image + - task: use-compiled-releases + image: image + config: + platform: linux + inputs: + - name: git + - name: (( concat meta.bosh.stemcell.os "-stemcell-" meta.bosh.stemcell.major )) + path: stemcell + - name: github + path: release + - name: compiled-release + outputs: + - name: pushme + run: + path: git/ci/scripts/use-compiled-releases + params: + REPO_ROOT: git + REPO_OUT: pushme + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + BRANCH: (( grab meta.github.branch )) + STEMCELL_OS: (( grab meta.bosh.stemcell.os )) + MANIFESTS_DIR: (( grab meta.manifest.directory )) + - put: git + params: + rebase: true + repository: pushme + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + +resource_types: + - name: slack-notification + type: registry-image + source: + repository: cfcommunity/slack-notification-resource + + - name: pull-request + type: registry-image + source: + repository: cfcommunity/github-pr-resource + +resources: + - name: image + type: registry-image + icon: docker + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + username: (( grab meta.image.registry.username )) + password: (( grab meta.image.registry.password )) + + - name: git + type: git + icon: github + source: + uri: (( grab meta.github.uri )) + branch: (( grab meta.github.branch )) + private_key: (( grab meta.github.private_key )) + + - name: git-pull-requests + type: pull-request + icon: github + source: + access_token: (( grab meta.github.access_token )) + repository: (( concat meta.github.owner "/" meta.github.repo )) + base_branch: (( grab meta.github.pr_base_branch )) + + - name: version + type: semver + icon: aws + source: + driver: s3 + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + key: version + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + initial_version: (( grab meta.initial_version || "0.0.1" )) + + - name: notify + type: slack-notification + icon: slack + source: + url: (( grab meta.slack.webhook )) + + - name: github + type: github-release + icon: github + source: + user: (( grab meta.github.owner )) + repository: (( grab meta.github.repo )) + access_token: (( grab meta.github.access_token )) + + - name: s3-tarball + type: s3 + icon: aws + source: + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + regexp: (( concat meta.name "-(.*).tgz" )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + + - name: (( concat meta.bosh.stemcell.os "-stemcell-" meta.bosh.stemcell.major )) + type: bosh-io-stemcell + icon: ubuntu + source: + name: (( concat "bosh-" meta.bosh.stemcell.cpi "-" meta.bosh.stemcell.os "-go_agent" )) + version_family: (( concat meta.bosh.stemcell.major ".latest" )) + + - name: compiled-release + type: s3 + icon: aws + source: + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + regexp: (( concat "compiled-releases/" meta.name "/.*-(\\d+).tgz" )) diff --git a/ci/repipe b/ci/repipe new file mode 100755 index 00000000..1be1a0b2 --- /dev/null +++ b/ci/repipe @@ -0,0 +1,151 @@ +#!/bin/bash +# +# ci/repipe +# +# Script for merging together pipeline configuration files +# (via Spruce!) and configuring Concourse. +# +# author: James Hunt +# Dennis Bell +# created: 2016-03-04 + +need_command() { + local cmd=${1:?need_command() - no command name given} + + if [[ ! -x "$(command -v $cmd)" ]]; then + echo >&2 "${cmd} is not installed." + if [[ "${cmd}" == "spruce" ]]; then + echo >&2 "Please download it from https://github.com/geofffranks/spruce/releases" + fi + exit 2 + fi +} + +NO_FLY= +SAVE_MANIFEST= +VALIDATE_PIPELINE= +NON_INTERACTIVE= + +cleanup() { + rm -f save-manifest.yml + if [[ -n ${SAVE_MANIFEST} && -e .deploy.yml ]]; then + mv .deploy.yml save-manifest.yml + fi + rm -f .deploy.yml +} + +usage() { + echo Command line arguments: + echo "no-fly Do not execute any fly commands" + echo "save-manifest Save manifest to file save-manifest" + echo "validate Validate pipeline instead of set pipeline" + echo "validate-strict Validate pipeline with strict mode" + echo "non-interactive Run set-pipeline in non-interactive mode" + echo "open Open pipeline dashboard to browser (if possible)" +} + +open_pipeline() { + url=$(show_pipeline_url) + cleanup + if [[ -x /usr/bin/open ]]; then + exec /usr/bin/open "$url" + else + echo "Sorry, but I was not able to automagically open" + echo "your Concourse Pipeline in the browser." + echo + echo "Here's a link you can click on, though:" + echo + echo " $url" + echo + exit 0; + fi +} + +show_pipeline_url() { + spruce merge --skip-eval pipeline.yml ${settings_file} > .deploy.yml + concourse_url=$(spruce json .deploy.yml | jq -r ".meta.url") + team=$(spruce json .deploy.yml | jq -r ".meta.team // \"main\"") + pipeline=$(spruce merge --skip-eval \ + --cherry-pick meta.pipeline \ + --cherry-pick meta.name \ + .deploy.yml | spruce merge - | spruce json | jq -r ".meta.pipeline") + + echo "$concourse_url/teams/$team/pipelines/$pipeline" + exit 0 +} + +for arg do + case "${arg}" in + no-fly|no_fly) NO_FLY="yes" ;; + save-manifest|save_manifest) SAVE_MANIFEST="yes" ;; + validate) VALIDATE_PIPELINE="normal" ;; + validate-strict|validate_strict) VALIDATE_PIPELINE="strict" ;; + non-interactive|non_interactive) NON_INTERACTIVE="--non-interactive" ;; + url) SHOW_PIPELINE_URL="yes" ;; + open) OPEN_PIPELINE="yes" ;; + help|-h|--help) usage; exit 0 ;; + *) echo Invalid argument + usage + exit 1 + esac +done + +cd $(dirname $BASH_SOURCE[0]) +echo >&2 "Working in $(pwd)" +need_command spruce + +# Allow for target-specific settings +settings_file="$(ls -1 settings.yml ${CONCOURSE_TARGET:+"settings-${CONCOURSE_TARGET}.yml"} 2>/dev/null | head -n1)" +if [[ -z "$settings_file" ]] +then + echo >&2 "Missing local settings in ci/settings.yml${CONCOURSE_TARGET:+" or ci/settings-${CONCOURSE_TARGET}.yml"}!" + exit 1 +fi + +echo >&2 "Using settings found in ${settings_file}" + +set -e +trap "cleanup" QUIT TERM EXIT INT + +[[ -n ${SHOW_PIPELINE_URL} ]] && { show_pipeline_url; exit 0; } +[[ -n ${OPEN_PIPELINE} ]] && { open_pipeline; exit 0; } + +spruce merge pipeline.yml ${settings_file} > .deploy.yml +PIPELINE=$(spruce json .deploy.yml | jq -r '.meta.pipeline // ""') +if [[ -z ${PIPELINE} ]]; then + echo >&2 "Missing pipeline name in ci/settings.yml!" + exit 1 +fi + +TARGET_FROM_SETTINGS=$(spruce json .deploy.yml | jq -r '.meta.target // ""') +if [[ -z ${CONCOURSE_TARGET} ]]; then + TARGET=${TARGET_FROM_SETTINGS} +elif [[ "$CONCOURSE_TARGET" != "$TARGET_FROM_SETTINGS" ]] +then + echo >&2 "Target in {$settings_file} differs from target in \$CONCOURSE_TARGET" + echo >&2 " \$CONCOURSE_TARGET: $CONCOURSE_TARGET" + echo >&2 " Target in file: $TARGET_FROM_SETTINGS" + exit 1 +else + TARGET=${CONCOURSE_TARGET} +fi + +if [[ -z ${TARGET} ]]; then + echo >&2 "Missing Concourse Target in ci/settings.yml!" + exit 1 +fi + +fly_cmd="${FLY_CMD:-fly}" + +[[ -n ${NO_FLY} ]] && { echo no fly execution requested ; exit 0; } + +case "${VALIDATE_PIPELINE}" in + normal) fly_opts="validate-pipeline" ;; + strict) fly_opts="validate-pipeline --strict" ;; + *) fly_opts="set-pipeline ${NON_INTERACTIVE} --pipeline ${PIPELINE}" ;; +esac + +set +x +$fly_cmd --target ${TARGET} ${fly_opts} --config .deploy.yml +[[ -n ${VALIDATE_PIPELINE} ]] && exit 0 +$fly_cmd --target ${TARGET} unpause-pipeline --pipeline ${PIPELINE} diff --git a/ci/scripts/bump-dependent-release b/ci/scripts/bump-dependent-release new file mode 100755 index 00000000..e73c5256 --- /dev/null +++ b/ci/scripts/bump-dependent-release @@ -0,0 +1,89 @@ +#!/bin/bash + +# The goal of this script is to retain the pretty nature of manifests, whilst updating +# the "releases:" section. +# +# If we use `spruce merge` to do the job, the entire manifest will be reordered - +# keys will be sorted alphabetically. Comments will be lost. +# +# Instead, we assume that "releases:" is the last section of the manifest. +set -e + +: ${REPO_ROOT:?required} +: ${REPO_OUT:?required} +: ${RELEASE:?required} +: ${NAME:?required} +: ${GIT_EMAIL:?required} +: ${GIT_NAME:?required} +: ${BRANCH:?required} + +if [[ ! -f ${RELEASE}/version ]]; then + echo "Director ${RELEASE} must have file /version" + exit 1 +fi + +git clone ${REPO_ROOT} ${REPO_OUT} + +version=$(cat ${RELEASE}/version) + +# So, this is a nested bash/spruce/jq combo. +# What is happening here is that the "releases:" section of each deployment manifest +# is being updated with the new version/sha1 for the release. +# +# We use "spruce json manifest.yml | jq '.releases'" to extract the existing releases array +# and the '.releases | map(if .name == $name)' will modify a specific element of the array +# +# This gives us a modified "releases: [{...}, {...}]" segment of the final deployment manifest. +# We now need to merge this back into the original manifest. +# +# But, I don't want to just use `spruce merge` for this as it will reorder the manifest and +# make it ugly. If I didn't care about the manifest's aesthetics then this whole script +# would be simpler. I want the original layout of the manifest to be retained; and so +# we will just chomp out the original "releases:" section at the end of the file and +# paste in the updated releases section. +function bump_version { + manifest_path=$1 + releases_updated=$(spruce merge < $manifest_path <>ci/release_notes.md +* Bumped ${NAME} to v${version} +EOF + + +if [[ -z $(git config --global user.email) ]]; then + git config --global user.email "${GIT_EMAIL}" +fi +if [[ -z $(git config --global user.name) ]]; then + git config --global user.name "${GIT_NAME}" +fi + +git merge --no-edit ${BRANCH} +git add -A +git status +git commit -m "bump ${NAME} v${version}" diff --git a/ci/scripts/export-release b/ci/scripts/export-release new file mode 100755 index 00000000..71e5dfb9 --- /dev/null +++ b/ci/scripts/export-release @@ -0,0 +1,82 @@ +#!/bin/bash + +set -eu + +: ${BOSH_ENVIRONMENT:?required} +: ${BOSH_CA_CERT:?required} +: ${BOSH_CLIENT:?required} +: ${BOSH_CLIENT_SECRET:?required} + +# +# stemcell metadata/upload +# + +STEMCELL_CPI=${STEMCELL_CPI:-aws-xen-hvm} +STEMCELL_OS=${STEMCELL_OS:-"ubuntu-jammy"} +STEMCELL_VERSION=$(cat stemcell/version) + +stemcells_count=$( + bosh stemcells --json \ + | jq --raw-output \ + --arg "cpi" "${STEMCELL_CPI}" \ + --arg "os" "${STEMCELL_OS}" \ + --arg "version" "${STEMCELL_VERSION}" \ + '[ .Tables[0].Rows[] + | select(.os == $os + and (.name | contains($cpi)) + and .version == $version) + ] | length' +) +if [[ ${stemcells_count} -gt 0 ]]; then + bosh -n upload-stemcell "https://bosh.io/d/stemcells/bosh-${STEMCELL_CPI}-${STEMCELL_OS}-go_agent?v=${STEMCELL_VERSION}" +fi + +# +# release metadata/upload +# + +cd release +tar -xzf *.tgz $( tar -tzf *.tgz | grep 'release.MF' ) +release_name=$(bosh int release.MF --path /name) +release_version=$(bosh int release.MF --path /version) + +bosh -n upload-release *.tgz +cd ../ + +# +# compilation deployment +# + +cat > manifest.yml < +# created: 2016-03-30 + +set -eu + +header() { + echo + echo "###############################################" + echo + echo $* + echo +} + +: ${REPO_ROOT:?required} +: ${RELEASE_ROOT:?required} +: ${GH_RELEASE_NAME:?required} +: ${REPO_OUT:?required} +RELEASE_BRANCH=${RELEASE_BRANCH:-"master"} +: ${GITHUB_OWNER:?required} +: ${VERSION_FROM:?required} +: ${GIT_EMAIL:?required} +: ${GIT_NAME:?required} + +if [[ ! -f ${VERSION_FROM} ]]; then + echo >&2 "Version file (${VERSION_FROM}) not found. Did you misconfigure Concourse?" + exit 2 +fi +version=$(cat ${VERSION_FROM}) +if [[ -z ${version} ]]; then + echo >&2 "Version file (${VERSION_FROM}) was empty. Did you misconfigure Concourse?" + exit 2 +fi + +if [[ ! -f ${REPO_ROOT}/ci/release_notes.md ]]; then + echo >&2 "ci/release_notes.md not found. Did you forget to write them?" + exit 1 +fi + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +############################################################### + +git clone ${REPO_ROOT} ${REPO_OUT} + +pushd ${REPO_OUT} + +if [[ -f ".gitattributes" ]] && grep -qF "lfs" ".gitattributes"; then + git lfs install +fi + +release_name=$(bosh int config/final.yml --path /final_name 2> /dev/null) \ + || release_name=$(bosh int config/final.yml --path /name) + +blobstore_provider=$(bosh int config/final.yml --path /blobstore/provider) +if [[ ${blobstore_provider} == "s3" && -n ${AWS_ACCESS_KEY} ]]; then + cat > config/private.yml < ${RELEASE_ROOT}/tag +echo "${GH_RELEASE_NAME} v${version}" > ${RELEASE_ROOT}/name +mv ${REPO_OUT}/releases/*/*-${version}.tgz ${RELEASE_ROOT}/artifacts +mv ${REPO_OUT}/ci/release_notes.md ${RELEASE_ROOT}/notes.md +cat >> ${RELEASE_ROOT}/notes.md < ${RELEASE_ROOT}/notification < New ${release_name} v${version} released! +EOF + + +header "Update git repo with final release..." +if [[ -z $(git config --global user.email) ]]; then + git config --global user.email "${GIT_EMAIL}" +fi +if [[ -z $(git config --global user.name) ]]; then + git config --global user.name "${GIT_NAME}" +fi + +pushd ${REPO_OUT} +${script_dir}/update-manifest-release ${release_name} ${version} ${release_url} ${release_sha1} + +git merge --no-edit "${RELEASE_BRANCH}" +git add -A +git status +git commit -m "release v${version}" +popd + +cat > ${NOTIFICATION_OUT:-notifications}/message <. +EOS diff --git a/ci/scripts/testflight b/ci/scripts/testflight new file mode 100755 index 00000000..5c40a1b0 --- /dev/null +++ b/ci/scripts/testflight @@ -0,0 +1,194 @@ +#!/bin/bash +# +# ci/scripts/testflight +# +# Script for testing a BOSH release using bosh +# +# author: James Hunt + +set -eu + +: ${BOSH_ENVIRONMENT:?required} +: ${BOSH_CA_CERT:?required} +: ${BOSH_CLIENT:?required} +: ${BOSH_CLIENT_SECRET:?required} +: ${BOSH_DEPLOYMENT:?required} +: ${MANIFEST_PATH:?required} +EXPECT_GIT_CHANGES=${EXPECT_GIT_CHANGES:-""} +DEBUG=${DEBUG:-} +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +header() { + echo + echo "###############################################" + echo + echo $* + echo +} + +delete_deployment() { + header "Cleaning up deployment..." + bosh -n delete-deployment --force +} + +deployment_cleanup_trap() { + echo; echo; echo + sleep 10 + delete_deployment +} + + +cd ${REPO_ROOT:?required} + +if [[ -f ".gitattributes" ]] && grep -qF "lfs" ".gitattributes"; then + git lfs install +fi +if [[ "${EXPECT_GIT_CHANGES:-X}" != "X" \ + && -z "$(git status --porcelain)" ]]; then + echo "Skipping testflight, as git changes are expected, but there are actually none" + exit 0 +fi + +header "Pulling in any git submodules..." +git submodule update --init --recursive --force + +header "Confirming testflight inputs" + +echo "Confirming deployment manifest ${MANIFEST_PATH} exists" +if [[ ! -f ${MANIFEST_PATH} ]]; then + echo "Deployment manifest ${MANIFEST_PATH} does not exist" + exit 1 +fi + +echo "Confirming that \$MANIFEST_VARS is valid" +mkdir -p tmp + +# $MANIFEST_VARS can either be "key: value" or "key=value" +# and will be converted to "key: value" YAML +echo "${MANIFEST_VARS:-"{} ---"}" | sed -e "s/ *= */: /g" > tmp/vars.yml +if [[ ! -z $DEBUG ]]; then + echo "Variables passed to deployment manifest:" + cat tmp/vars.yml +fi +# convert YAML to JSON to check YAML validity +spruce json tmp/vars.yml > /dev/null + +echo "Confirming each operator file exists: ${MANIFEST_OP_PATHS:-ok, none specified.}" +op_patch_file_errors= +for op_patch_file in ${MANIFEST_OP_PATHS//,/ } ; do + if [[ ! -f $op_patch_file ]]; then + op_patch_file_errors=1 + echo "Operator file missing: ${op_patch_file}" + fi +done +if [[ ! -z ${op_patch_file_errors} ]]; then + exit 1 +fi + +${script_dir}/wait-until-proxy-bosh-cf-running +if [[ "${PROXY_IP:-X}" != "X" ]] ; then export BOSH_ALL_PROXY=socks5://localhost:9999 ; fi + +delete_deployment + +header "Creating candidate BOSH release..." +bosh -n reset-release # in case dev_releases/ is in repo accidentally + +blobstore_provider=$(bosh int config/final.yml --path /blobstore/provider) +if [[ ${blobstore_provider} == "s3" && -n ${AWS_ACCESS_KEY} ]]; then + cat > config/private.yml < /dev/null) \ + || release_name=$(bosh int config/final.yml --path /name) + +header "Choosing cloud-config options" +vm_type=$(bosh int <(bosh cloud-config) --path /vm_types/0/name) +disk_type=$(bosh int <(bosh cloud-config) --path /disk_types/0/name) +network=$(bosh int <(bosh cloud-config) --path /networks/0/name) + +release_final_version=$(spruce json dev_releases/*/index.yml | jq -r ".builds[].version" | sed -e "s%+.*%%") +release_dev_version="${release_final_version}.latest" + +cat > tmp/deployment.yml <> tmp/deployment.yml <> tmp/deployment.yml < tmp/manifest.yml + +trap "deployment_cleanup_trap" EXIT + +bosh -n deploy tmp/manifest.yml + +TEST_ERRAND=${TEST_ERRAND:-} # backwards compatibility +TEST_ERRANDS=${TEST_ERRANDS:-$TEST_ERRAND} +if [[ -n ${TEST_ERRANDS} ]]; then + for errand in ${TEST_ERRANDS}; do + header "Running '${errand}' errand" + bosh -n run-errand ${errand} + done +else + echo "No test errands specified, skipping." +fi + +echo +echo "SUCCESS" +exit 0 diff --git a/ci/scripts/update-blob b/ci/scripts/update-blob new file mode 100755 index 00000000..04a5d22a --- /dev/null +++ b/ci/scripts/update-blob @@ -0,0 +1,85 @@ +#!/bin/bash + +set -e + +: ${REPO_ROOT:?required} +: ${REPO_OUT:?required} +: ${BLOB_DIR:?required} +: ${BLOB_NAME:?required} +: ${BLOB_BINARY:?required} +: ${BLOB_CLEANUP:?required} +: ${BLOB_DESTINATION:?required} +: ${GIT_EMAIL:?required} +: ${GIT_NAME:?required} +: ${BRANCH:?required} + +# if git-release, then look for tag +# git describe --tags `git rev-list --tags --max-count=1` +# then convert v2.23.0 to VERSION=2.23.0 +[[ -f ${BLOB_DIR}/version ]] || { + # Perhaps move this into helper script that creates a github-release style folder + pushd ${BLOB_DIR} + tag=$(git describe --tags `git rev-list --tags --max-count=1`) + popd + resource_dir=${BLOB_NAME}-${tag//v/} + git clone ${BLOB_DIR} ${resource_dir} + rm -rf ${resource_dir}/.git + echo ${tag} > ${BLOB_DIR}/tag + echo ${tag//v/} > ${BLOB_DIR}/version + tar czf ${BLOB_DIR}/${BLOB_NAME}-${tag//v/}.tar.gz ${resource_dir} +} +version=$(cat ${BLOB_DIR}/version) + +pushd ${REPO_ROOT} + +if [[ -f ".gitattributes" ]] && grep -qF "lfs" ".gitattributes"; then + git lfs install +fi + +blobstore_provider=$(bosh int config/final.yml --path /blobstore/provider) +if [[ ${blobstore_provider} == "s3" && -n ${AWS_ACCESS_KEY} ]]; then + cat > config/private.yml <>ci/release_notes.md +* Bumped ${BLOB_URL} to v${version} +EOF + popd + + # GIT! + if [[ -z $(git config --global user.email) ]]; then + git config --global user.email "${GIT_EMAIL}" + fi + if [[ -z $(git config --global user.name) ]]; then + git config --global user.name "${GIT_NAME}" + fi + + (cd ${REPO_ROOT} + git merge --no-edit ${BRANCH} + git add -A + git status + git commit -m "Bumped ${BLOB_NAME} to v${version}") +fi + +# so that future steps in the pipeline can push our changes +cp -a ${REPO_ROOT} ${REPO_OUT} diff --git a/ci/scripts/update-manifest-compiled-release b/ci/scripts/update-manifest-compiled-release new file mode 100755 index 00000000..6a78f89d --- /dev/null +++ b/ci/scripts/update-manifest-compiled-release @@ -0,0 +1,65 @@ +#!/bin/bash + +# The goal of this script is to retain the pretty nature of manifests, whilst updating +# the "releases:" section with a stemcell-restricted compiled release +# +# If we use `spruce merge` to do the job, the entire manifest will be reordered - +# keys will be sorted alphabetically. Comments will be lost. +# +# Instead, we assume that "releases:" is the last section of the manifest. +set -e + +NAME=$1 +VERSION=$2 +RELEASE_TGZ_URL=$3 +RELEASE_TGZ_SHA1=$4 +STEMCELL_OS=$5 +STEMCELL_VERSION=$6 + +if [[ "${STEMCELL_VERSION:-X}" == "X" ]]; then + echo "USAGE: update-manifest-release NAME VERSION URL SHA1 STEMCELL_OS STEMCELL_VERSION" + exit 2 +fi +: ${MANIFESTS_DIR:?required} + +# So, this is a nested bash/spruce/jq combo. +# What is happening here is that the "releases:" section of each deployment manifest +# is being updated with the new version/sha1 for the release. +# +# We use "spruce json manifest.yml | jq '.releases'" to extract the existing releases array +# and the '.releases | map(if .name == $name)' will modify a specific element of the array +# +# This gives us a modified "releases: [{...}, {...}]" segment of the final deployment manifest. +# We now need to merge this back into the original manifest. +# +# But, I don't want to just use `spruce merge` for this as it will reorder the manifest and +# make it ugly. If I didn't care about the manifest's aesthetics then this whole script +# would be simpler. I want the original layout of the manifest to be retained; and so +# we will just chomp out the original "releases:" section at the end of the file and +# paste in the updated releases section. +function bump_version { + manifest_path=$1 + releases_updated=$(spruce merge < $manifest_path < $manifest_path < proxy/ssh/private_key + chmod 600 proxy/ssh/private_key + + header "Checking jumpbox available..." + set +e + until ssh ${PROXY_USERNAME}@${PROXY_IP} -i proxy/ssh/private_key \ + -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=60 \ + "whoami" + do + echo "Waiting until jumpbox/proxy available..." + done + set -e + + header "Starting socks5 proxy..." + ssh ${PROXY_USERNAME}@${PROXY_IP} -i proxy/ssh/private_key -N -D 9999 & + sleep 10 + + echo BOSH_ALL_PROXY=socks5://localhost:9999 + export BOSH_ALL_PROXY=socks5://localhost:9999 +fi + +header "Checking bosh available..." +until bosh env; do + echo "Waiting until bosh available..." + sleep 60 +done + +if [[ -f git/tmp/vars.yml ]]; then + cf_api_url=$(bosh int git/tmp/vars.yml --path /cf-api-url) + if [[ "${cf_api_url:-X}" != "X" ]]; then + header "Checking Cloud Foundry available..." + until cf api $cf_api_url --skip-ssl-validation; do + echo "Waiting until Cloud Foundry available..." + sleep 60 + done + fi +fi + +echo +echo diff --git a/ci/settings.yml b/ci/settings.yml new file mode 100644 index 00000000..059d05b1 --- /dev/null +++ b/ci/settings.yml @@ -0,0 +1,46 @@ +--- +meta: + name: bosh-virtualbox-cpi + target: gk + url: https://ci.gstack.io + + test-errands: ~ + + initial_version: 1.0.0 + + manifest: + directory: manifests + + git: + email: ((git-commit-email)) + name: ((git-commit-name)) + + image: + registry: + username: ((docker_registry_username)) + password: ((docker-registry-password)) + + aws: + bucket: (( grab meta.pipeline )) + region_name: us-east-1 + access_key: ((aws-access-key)) + secret_key: ((aws-secret-key)) + + github: + owner: cloudfoundry-community + repo: (( grab meta.pipeline )) + branch: master + private_key: ((github-private-key)) + access_token: ((github-access-token)) + + bosh: + stemcell: + cpi: warden-boshlite + + slack: + webhook: ((slack-webhook)) + username: ((slack-username)) + icon: ((slack-icon-url)) + channel: (( concat "#" meta.name "-boshrelease" )) + blob_success: '(( concat ": New version of was detected, and updated in master. <" meta.url "/teams//pipelines/| Cut a new release?>" ))' + blob_failure: '(( concat ": :airplane_arriving: <" meta.url "/teams//pipelines//jobs//builds/| Failed to update the blob for >" ))' From 8fb0f64920c0fa79655b860d876c30cfaa568c85 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Sun, 11 Feb 2024 17:09:30 +0100 Subject: [PATCH 02/15] Customize pipeline --- .../workflows/trigger-git-resource-check.yml | 14 +++ .../workflows/trigger-pr-resource-check.yml | 14 +++ ci/settings.yml | 90 +++++++++++++++---- manifests/bosh-virtualbox-cpi.yml | 41 +++++++++ 4 files changed, 144 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/trigger-git-resource-check.yml create mode 100644 .github/workflows/trigger-pr-resource-check.yml create mode 100644 manifests/bosh-virtualbox-cpi.yml diff --git a/.github/workflows/trigger-git-resource-check.yml b/.github/workflows/trigger-git-resource-check.yml new file mode 100644 index 00000000..a51fcd34 --- /dev/null +++ b/.github/workflows/trigger-git-resource-check.yml @@ -0,0 +1,14 @@ +name: Trigger Concourse git resource +on: [ push, workflow_dispatch ] +jobs: + trigger-resource-check: + runs-on: ubuntu-latest + steps: + - name: Trigger resource check + uses: gstackio/trigger-concourse-resource-check-action@v1 + with: + concourse-url: https://ci.gstack.io + concourse-team: gk-plat-devs + concourse-pipeline: bosh-virtualbox-cpi-release + concourse-resource: git + concourse-webhook-token: ${{ secrets.CONCOURSE_WEBHOOK_TOKEN }} diff --git a/.github/workflows/trigger-pr-resource-check.yml b/.github/workflows/trigger-pr-resource-check.yml new file mode 100644 index 00000000..10b1b36c --- /dev/null +++ b/.github/workflows/trigger-pr-resource-check.yml @@ -0,0 +1,14 @@ +name: Trigger Concourse pull-request resource +on: [ push, pull_request, workflow_dispatch ] +jobs: + trigger-resource-check: + runs-on: ubuntu-latest + steps: + - name: Trigger resource check + uses: gstackio/trigger-concourse-resource-check-action@v1 + with: + concourse-url: https://ci.gstack.io + concourse-team: gk-plat-devs + concourse-pipeline: bosh-virtualbox-cpi-release + concourse-resource: git-pull-requests + concourse-webhook-token: ${{ secrets.CONCOURSE_WEBHOOK_TOKEN }} diff --git a/ci/settings.yml b/ci/settings.yml index 059d05b1..615ddfce 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -1,12 +1,15 @@ --- meta: - name: bosh-virtualbox-cpi - target: gk - url: https://ci.gstack.io + name: bosh-virtualbox-cpi + release: VirtualBox CPI Bosh Release + target: (( grab meta.team )) + url: https://ci.gstack.io + pipeline: (( concat meta.name "-release" )) + team: gk-plat-devs test-errands: ~ - initial_version: 1.0.0 + initial_version: "0.4.3" manifest: directory: manifests @@ -16,22 +19,29 @@ meta: name: ((git-commit-name)) image: + name: (( concat meta.private-registry.host "/gstack/gk-ops" )) registry: - username: ((docker_registry_username)) - password: ((docker-registry-password)) + username: (( grab meta.private-registry.username )) + password: (( grab meta.private-registry.password )) + + private-registry: + host: harbor.ci.gstack.io + username: ((private-registry-username)) + password: ((private-registry-password)) aws: bucket: (( grab meta.pipeline )) - region_name: us-east-1 + region_name: eu-west-3 access_key: ((aws-access-key)) secret_key: ((aws-secret-key)) github: - owner: cloudfoundry-community - repo: (( grab meta.pipeline )) - branch: master - private_key: ((github-private-key)) - access_token: ((github-access-token)) + owner: gstackio + repo: (( grab meta.pipeline )) + branch: ci-pipeline + pr_base_branch: master + private_key: ((github-private-key)) + access_token: ((github-access-token)) bosh: stemcell: @@ -41,6 +51,56 @@ meta: webhook: ((slack-webhook)) username: ((slack-username)) icon: ((slack-icon-url)) - channel: (( concat "#" meta.name "-boshrelease" )) - blob_success: '(( concat ": New version of was detected, and updated in master. <" meta.url "/teams//pipelines/| Cut a new release?>" ))' - blob_failure: '(( concat ": :airplane_arriving: <" meta.url "/teams//pipelines//jobs//builds/| Failed to update the blob for >" ))' + channel: "#oss-pipelines-notifications" + # blob_success: '(( concat ": New version of was detected, and updated in master. <" meta.url "/teams//pipelines/| Cut a new release?>" ))' + # blob_failure: '(( concat ": :airplane_arriving: <" meta.url "/teams//pipelines//jobs//builds/| Failed to update the blob for >" ))' + +groups: + + # NOTICE: we deactivate all S3 access, as our release a git-LSF blobstore + - (( delete name "compiled-releases" )) + +jobs: + + - (( merge on name )) + + - name: shipit + plan: + # NOTICE: we deactivate all S3 access, as our release a git-LSF blobstore + - (( delete 3 )) # 'put: s3-tarball' step + + # NOTICE: we deactivate all S3 access, as our release a git-LSF blobstore + - (( delete name "compile-release" )) + - (( delete name "use-compiled-releases" )) + +resources: + + - (( merge on name )) + + - name: git + check_every: 24h + webhook_token: ((concourse-webhook-token)) + + - name: git-pull-requests + check_every: 24h + webhook_token: ((concourse-webhook-token)) + + - name: version + icon: github + source: + bucket: (( prune )) + region_name: (( prune )) + key: (( prune )) + access_key_id: (( prune )) + secret_access_key: (( prune )) + driver: git + uri: git@github.com:gstackio/gk-pipelines-compass.git + branch: master + file: (( concat "versions/" meta.pipeline )) + private_key: ((github-private-key)) + git_user: "((git-commit-name)) <((git-commit-email))>" + + # NOTICE: we deactivate all S3 access, as our release a git-LSF blobstore + - (( delete name "s3-tarball" )) + - (( delete name "compiled-release" )) + - (( delete name "(( concat meta.bosh.stemcell.os "-stemcell-" meta.bosh.stemcell.major ))" )) diff --git a/manifests/bosh-virtualbox-cpi.yml b/manifests/bosh-virtualbox-cpi.yml new file mode 100644 index 00000000..8d9747f0 --- /dev/null +++ b/manifests/bosh-virtualbox-cpi.yml @@ -0,0 +1,41 @@ +--- +name: bosh-virtualbox-cpi + +instance_groups: + - name: vbox-cpi + instances: 1 + azs: [ z1 ] + jobs: + - name: virtualbox_cpi + release: bosh-virtualbox-cpi + properties: + agent: + mbus: nats://bosh-director.example.org:4222 + host: vbox-host.example.org + private_key: ((vbox_ssh.private_key)) + username: example-vbox-username + stemcell: default + vm_type: default + networks: [ name: default ] + +variables: + - name: vbox_ssh + type: ssh + +update: + serial: true + canaries: 1 + canary_watch_time: 1000-5000 + max_in_flight: 1 + update_watch_time: 1000-5000 + +stemcells: + - alias: default + os: ubuntu-jammy + version: latest + +releases: +- name: bosh-virtualbox-cpi + version: 0.4.2 + url: https://bosh.io/d/github.com/cloudfoundry/bosh-virtualbox-cpi-release?v=0.4.2 + sha1: 5f12883286390406947305aac08541f7f52f1a05 From cba3b4ac551fdfcc9b5dadc9a9f0c93aee7c5285 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Sun, 11 Feb 2024 17:58:44 +0100 Subject: [PATCH 03/15] Fix build/test scripts after project has adopted go modules --- src/bosh-virtualbox-cpi/bin/build | 2 +- src/bosh-virtualbox-cpi/bin/test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bosh-virtualbox-cpi/bin/build b/src/bosh-virtualbox-cpi/bin/build index 3303c779..27c4e3dc 100755 --- a/src/bosh-virtualbox-cpi/bin/build +++ b/src/bosh-virtualbox-cpi/bin/build @@ -4,4 +4,4 @@ set -e bin=$(dirname $0) -go build -o $bin/cpi github.com/cppforlife/bosh-virtualbox-cpi/main +go build -o $bin/cpi bosh-virtualbox-cpi/main diff --git a/src/bosh-virtualbox-cpi/bin/test b/src/bosh-virtualbox-cpi/bin/test index 181761ce..7132cf39 100755 --- a/src/bosh-virtualbox-cpi/bin/test +++ b/src/bosh-virtualbox-cpi/bin/test @@ -5,7 +5,7 @@ result=0 bin=$(dirname $0) echo -e "\n Formatting packages..." -go fmt github.com/cppforlife/bosh-virtualbox-cpi/... +go fmt bosh-virtualbox-cpi/... let "result+=$?" echo -e "\n Running build script to confirm everything compiles..." From 94fe0bc7a2b23822df6749a7cc04b2a3862a3b76 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Sun, 11 Feb 2024 17:59:18 +0100 Subject: [PATCH 04/15] Ginkgo tests are supposed to run, though there are none yet --- src/bosh-virtualbox-cpi/bin/test | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bosh-virtualbox-cpi/bin/test b/src/bosh-virtualbox-cpi/bin/test index 7132cf39..1208e2f1 100755 --- a/src/bosh-virtualbox-cpi/bin/test +++ b/src/bosh-virtualbox-cpi/bin/test @@ -13,6 +13,10 @@ $bin/build $bin/build-linux-amd64 let "result+=$?" +echo -e "\n Running ginkgo tests..." +ACK_GINKGO_RC="true" ginkgo -r . +let "result+=$?" + if [ $result -eq 0 ]; then echo -e "\nSUITE SUCCESS" else From a71e4b40b8e07ef8a8ce2ecb4aaea53418b11580 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 04:51:54 +0100 Subject: [PATCH 05/15] Add smoke tests --- ci/settings.yml | 2 +- jobs/vbox-cpi-smoke-tests/monit | 0 jobs/vbox-cpi-smoke-tests/spec | 9 +++++++++ jobs/vbox-cpi-smoke-tests/templates/bin/run | 10 ++++++++++ manifests/bosh-virtualbox-cpi.yml | 5 +++++ 5 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 jobs/vbox-cpi-smoke-tests/monit create mode 100644 jobs/vbox-cpi-smoke-tests/spec create mode 100644 jobs/vbox-cpi-smoke-tests/templates/bin/run diff --git a/ci/settings.yml b/ci/settings.yml index 615ddfce..81048c6e 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -7,7 +7,7 @@ meta: pipeline: (( concat meta.name "-release" )) team: gk-plat-devs - test-errands: ~ + test-errands: vbox-cpi-smoke-tests initial_version: "0.4.3" diff --git a/jobs/vbox-cpi-smoke-tests/monit b/jobs/vbox-cpi-smoke-tests/monit new file mode 100644 index 00000000..e69de29b diff --git a/jobs/vbox-cpi-smoke-tests/spec b/jobs/vbox-cpi-smoke-tests/spec new file mode 100644 index 00000000..f5f4af8c --- /dev/null +++ b/jobs/vbox-cpi-smoke-tests/spec @@ -0,0 +1,9 @@ +--- +name: vbox-cpi-smoke-tests + +templates: + bin/run: bin/run + +packages: [] + +properties: {} diff --git a/jobs/vbox-cpi-smoke-tests/templates/bin/run b/jobs/vbox-cpi-smoke-tests/templates/bin/run new file mode 100644 index 00000000..6d7c4542 --- /dev/null +++ b/jobs/vbox-cpi-smoke-tests/templates/bin/run @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -eo pipefail + +( + set -x + payload='{ "method": "info", "arguments": [], "context": {} }' + /var/vcap/jobs/virtualbox_cpi/bin/cpi <<< "${payload}" 2> /dev/null \ + | python3 -m "json.tool" +) diff --git a/manifests/bosh-virtualbox-cpi.yml b/manifests/bosh-virtualbox-cpi.yml index 8d9747f0..af8c1966 100644 --- a/manifests/bosh-virtualbox-cpi.yml +++ b/manifests/bosh-virtualbox-cpi.yml @@ -6,6 +6,7 @@ instance_groups: instances: 1 azs: [ z1 ] jobs: + - name: virtualbox_cpi release: bosh-virtualbox-cpi properties: @@ -14,6 +15,10 @@ instance_groups: host: vbox-host.example.org private_key: ((vbox_ssh.private_key)) username: example-vbox-username + + - name: vbox-cpi-smoke-tests + release: bosh-virtualbox-cpi + stemcell: default vm_type: default networks: [ name: default ] From 39b1a8e44560511cb311447e9c9cc04b211279ff Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 00:37:52 +0100 Subject: [PATCH 06/15] Add jobs to update golang package and go language and dependencies --- ci/settings.yml | 385 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 385 insertions(+) diff --git a/ci/settings.yml b/ci/settings.yml index 81048c6e..c241d13e 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -57,6 +57,13 @@ meta: groups: + - (( merge on name )) + + - name: bump + jobs: + - bump-golang-blob + - bump-deps + # NOTICE: we deactivate all S3 access, as our release a git-LSF blobstore - (( delete name "compiled-releases" )) @@ -64,6 +71,339 @@ jobs: - (( merge on name )) + - name: testflight + serial_groups: [ testflight ] + + - name: bump-golang-blob + # See also: golang-release/ci/tasks/shared/bump-golang-package.yml + # Used in: https://github.com/cloudfoundry/bosh-google-cpi-release/blob/1525ee4/ci/pipeline.yml#L364-L387 + public: true + serial: true + serial_groups: [ testflight ] + plan: + - in_parallel: + - { get: weekly, trigger: true } + - { get: git } + - { get: golang-release } + - { get: gk-automation } + - { get: image } + + - task: vendor-bosh-packages + image: image + config: + platform: linux + inputs: + - { name: git, path: repo } + - { name: golang-release } + caches: [ path: /root/.bosh/cache ] + outputs: + - name: repo-bumped + - name: package-info + run: + path: bash + args: + - -c + - | + set -ueo pipefail + build_dir="${PWD}" + find "repo" -mindepth 1 -maxdepth 1 -print0 \ + | xargs -0 -I{} cp -a {} "repo-bumped" + cd "repo-bumped" || exit 115 + blobstore_provider=$(bosh int config/final.yml --path /blobstore/provider) + if [[ ${blobstore_provider} == "s3" && -n ${AWS_ACCESS_KEY} ]]; then + cat > config/private.yml < /dev/null + if [[ -f ".gitattributes" ]] && grep -qF "lfs" ".gitattributes"; then + git lfs install + fi + git_changes=$(git status --porcelain) + popd > /dev/null + + go_prev_ver=$(< golang-info/previous-version) + go_curr_ver=$(< golang-info/version) + + version_details="${go_curr_ver}" + if [[ ${go_curr_ver} != ${go_prev_ver} ]]; then + version_details="from ${go_prev_ver} to ${go_curr_ver}" + fi + echo "INFO: Generated commit message:" + echo "Update Golang package ${version_details}" \ + | tee messages/commit-message + + echo "INFO: Generated slack notification:" + if [[ -z "${git_changes}" ]]; then + : > messages/notif-body + echo "(none, implying no notification, as there are no changes)" + else + echo "${PIPELINE_NAME}: successfully updated" \ + "Golang package ${version_details}," \ + "with passing testflight! :tada:" \ + "<${PIPELINE_URL}|Ship it when ready!> ${SUCCESS_MOJI}" \ + | tee messages/notif-body + fi + params: + SUCCESS_MOJI: (( grab meta.slack.success_moji )) + PIPELINE_URL: (( grab meta.slack.pipeline_url )) + PIPELINE_NAME: (( grab meta.pipeline )) + + - task: git-commit + image: image + file: gk-automation/tasks/git/commit.yml + input_mapping: + repo: repo-bumped + commit-info: messages + params: + GIT_COMMIT_NAME: (( grab meta.git.name )) + GIT_COMMIT_EMAIL: (( grab meta.git.email )) + GIT_DIFF_OPTS: --color-words + + - put: git + params: + repository: repo-committed + rebase: true + + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + on_success: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text_file: messages/notif-body + + - name: bump-deps + # See also: golang-release/ci/tasks/shared/bump-deps.yml + # Used in: https://github.com/cloudfoundry/bosh-google-cpi-release/blob/1525ee4/ci/pipeline.yml#L389-L411 + public: true + serial: true + plan: + - in_parallel: + - { get: weekly, trigger: true } + - { get: git } + - { get: gk-automation } + - { get: golang-runner, resource: golang-alpine } # format: rootfs + - { get: golang-alpine, params: { format: oci } } + - { get: alpine-latest, params: { format: oci } } + - { get: image } + + - task: bump-golang-deps + image: golang-runner + config: + platform: linux + inputs: [ { name: git, path: repo } ] + caches: [ path: /go ] + outputs: + - name: repo-bumped + - name: golang-info + run: + path: sh + args: + - -ec + - | + build_dir="${PWD}" + find "repo" -mindepth 1 -maxdepth 1 -print0 \ + | xargs -0 -I{} cp -a {} "repo-bumped" + cd "repo-bumped/${SOURCE_PATH}" || exit 115 + + awk '/^go /{print $2}' go.mod > "${build_dir}/golang-info/previous-version" + ( + set -x + go_version=$( + go version \ + | awk '{sub(/go/,"",$3); sub(/\.[[:digit:]]+$/,"",$3); print $3}' + ) + # sed -i -Ee "s/^go [[:digit:].]+\$/go ${go_version}/" go.mod + go mod edit -go="${go_version}" + go get -t -u ./... + go mod tidy + if [ -f "vendor/modules.txt" ]; then + go mod vendor + fi + + echo "${go_version}" > "${build_dir}/golang-info/version" + ) + params: + SOURCE_PATH: src/bosh-virtualbox-cpi + + - task: test-golang-code + image: golang-runner + config: + platform: linux + inputs: [ name: repo-bumped ] + caches: [ path: /go ] + run: + dir: repo-bumped/src/bosh-virtualbox-cpi + path: sh + args: + - -exc + - | + apk add --no-cache --no-progress git git-lfs + if [[ -f ".gitattributes" ]] && grep -qF "lfs" ".gitattributes"; then + git lfs install + fi + git_changes=$(git status --porcelain) + if [[ -z "${git_changes}" ]]; then + echo "INFO: skipping compile/test, as there are no changes." + exit + fi + + apk add --no-cache --no-progress bash + go install github.com/onsi/ginkgo/v2/ginkgo@latest + bin/test + + - task: generate-messages + image: image + config: + platform: linux + inputs: + - name: golang-info + - name: repo-bumped + outputs: [ name: messages ] + run: + path: bash + args: + - -c + - | + set -ueo pipefail + + pushd "repo-bumped" > /dev/null + if [[ -f ".gitattributes" ]] && grep -qF "lfs" ".gitattributes"; then + git lfs install + fi + git_changes=$(git status --porcelain) + popd > /dev/null + + go_prev_ver=$(< golang-info/previous-version) + go_curr_ver=$(< golang-info/version) + + version_details="${go_curr_ver}" + if [[ ${go_curr_ver} != ${go_prev_ver} ]]; then + version_details="from ${go_prev_ver} to ${go_curr_ver} and its" + fi + echo "INFO: Generated commit message:" + echo "Update Golang ${version_details} dependencies" \ + | tee messages/commit-message + + echo "INFO: Generated slack notification:" + if [[ -z "${git_changes}" ]]; then + : > messages/notif-body + echo "(none, implying no notification, as there are no changes)" + else + echo "${PIPELINE_NAME}: successfully bumped" \ + "Golang ${version_details} dependencies," \ + "with passing tests! :tada:" \ + "<${PIPELINE_URL}|Ship it when ready!> ${SUCCESS_MOJI}" \ + | tee messages/notif-body + fi + params: + SUCCESS_MOJI: (( grab meta.slack.success_moji )) + PIPELINE_URL: (( grab meta.slack.pipeline_url )) + PIPELINE_NAME: (( grab meta.pipeline )) + + - task: git-commit + image: image + file: gk-automation/tasks/git/commit.yml + input_mapping: + repo: repo-bumped + commit-info: messages + params: + GIT_COMMIT_NAME: (( grab meta.git.name )) + GIT_COMMIT_EMAIL: (( grab meta.git.email )) + GIT_DIFF_OPTS: --color-words + + - put: git + params: + repository: repo-committed + rebase: true + + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: (( grab meta.slack.fail_text )) + on_success: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text_file: messages/notif-body + - name: shipit plan: # NOTICE: we deactivate all S3 access, as our release a git-LSF blobstore @@ -104,3 +444,48 @@ resources: - (( delete name "s3-tarball" )) - (( delete name "compiled-release" )) - (( delete name "(( concat meta.bosh.stemcell.os "-stemcell-" meta.bosh.stemcell.major ))" )) + + - (( merge on name )) + + - name: golang-alpine + type: registry-image + icon: docker + check_every: 24h + source: + repository: golang + variant: alpine + + - name: alpine-latest + type: registry-image + icon: docker + check_every: 24h + source: + repository: alpine + semver_constraint: "< 1970" # Avoid YYYYMMDD tags like '20231219' + + - name: gk-automation + type: git + icon: github + check_every: 24h + source: + uri: git@github.com:gstackio/gk-automation.git + branch: master + private_key: (( grab meta.github.private_key )) + + - name: weekly + type: time + icon: alarm + source: + location: Europe/Paris + start: "3:00" + stop: "4:30" + days: [ Saturday ] + + - name: golang-release + type: git + icon: github + check_every: 24h + source: + uri: https://github.com/cloudfoundry/bosh-package-golang-release.git + branch: main + private_key: (( grab meta.github.private_key )) From ad2081b0995bc531260af442f19a2c5c41dd1b17 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 01:50:18 +0100 Subject: [PATCH 07/15] Migrate to ginkgo v2 --- src/bosh-virtualbox-cpi/bin/test | 2 +- src/bosh-virtualbox-cpi/cpi/cpi_suite_test.go | 2 +- src/bosh-virtualbox-cpi/disk/disk_suite_test.go | 2 +- src/bosh-virtualbox-cpi/driver/driver_suite_test.go | 2 +- src/bosh-virtualbox-cpi/driver/local_runner_test.go | 2 +- src/bosh-virtualbox-cpi/driver/ssh_runner_test.go | 2 +- src/bosh-virtualbox-cpi/main/main_suite_test.go | 2 +- src/bosh-virtualbox-cpi/stemcell/stemcell_suite_test.go | 2 +- src/bosh-virtualbox-cpi/vm/vm_suite_test.go | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bosh-virtualbox-cpi/bin/test b/src/bosh-virtualbox-cpi/bin/test index 1208e2f1..b9a883b4 100755 --- a/src/bosh-virtualbox-cpi/bin/test +++ b/src/bosh-virtualbox-cpi/bin/test @@ -14,7 +14,7 @@ $bin/build-linux-amd64 let "result+=$?" echo -e "\n Running ginkgo tests..." -ACK_GINKGO_RC="true" ginkgo -r . +ginkgo -r . let "result+=$?" if [ $result -eq 0 ]; then diff --git a/src/bosh-virtualbox-cpi/cpi/cpi_suite_test.go b/src/bosh-virtualbox-cpi/cpi/cpi_suite_test.go index d344ee0b..d9fd6912 100644 --- a/src/bosh-virtualbox-cpi/cpi/cpi_suite_test.go +++ b/src/bosh-virtualbox-cpi/cpi/cpi_suite_test.go @@ -3,7 +3,7 @@ package cpi_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/src/bosh-virtualbox-cpi/disk/disk_suite_test.go b/src/bosh-virtualbox-cpi/disk/disk_suite_test.go index ede118b6..76bd860e 100644 --- a/src/bosh-virtualbox-cpi/disk/disk_suite_test.go +++ b/src/bosh-virtualbox-cpi/disk/disk_suite_test.go @@ -3,7 +3,7 @@ package disk_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/src/bosh-virtualbox-cpi/driver/driver_suite_test.go b/src/bosh-virtualbox-cpi/driver/driver_suite_test.go index 35dff671..14b86cc9 100644 --- a/src/bosh-virtualbox-cpi/driver/driver_suite_test.go +++ b/src/bosh-virtualbox-cpi/driver/driver_suite_test.go @@ -3,7 +3,7 @@ package driver_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/src/bosh-virtualbox-cpi/driver/local_runner_test.go b/src/bosh-virtualbox-cpi/driver/local_runner_test.go index a9c559cc..a62f3267 100644 --- a/src/bosh-virtualbox-cpi/driver/local_runner_test.go +++ b/src/bosh-virtualbox-cpi/driver/local_runner_test.go @@ -3,7 +3,7 @@ package driver_test import ( boshlog "github.com/cloudfoundry/bosh-utils/logger" boshsys "github.com/cloudfoundry/bosh-utils/system" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "bosh-virtualbox-cpi/driver" diff --git a/src/bosh-virtualbox-cpi/driver/ssh_runner_test.go b/src/bosh-virtualbox-cpi/driver/ssh_runner_test.go index b5b49ffb..16c06e83 100644 --- a/src/bosh-virtualbox-cpi/driver/ssh_runner_test.go +++ b/src/bosh-virtualbox-cpi/driver/ssh_runner_test.go @@ -4,7 +4,7 @@ import ( "os" boshlog "github.com/cloudfoundry/bosh-utils/logger" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "bosh-virtualbox-cpi/driver" diff --git a/src/bosh-virtualbox-cpi/main/main_suite_test.go b/src/bosh-virtualbox-cpi/main/main_suite_test.go index cbddca29..75499947 100644 --- a/src/bosh-virtualbox-cpi/main/main_suite_test.go +++ b/src/bosh-virtualbox-cpi/main/main_suite_test.go @@ -3,7 +3,7 @@ package main_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/src/bosh-virtualbox-cpi/stemcell/stemcell_suite_test.go b/src/bosh-virtualbox-cpi/stemcell/stemcell_suite_test.go index 66c5ce93..5aee79cd 100644 --- a/src/bosh-virtualbox-cpi/stemcell/stemcell_suite_test.go +++ b/src/bosh-virtualbox-cpi/stemcell/stemcell_suite_test.go @@ -3,7 +3,7 @@ package stemcell_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/src/bosh-virtualbox-cpi/vm/vm_suite_test.go b/src/bosh-virtualbox-cpi/vm/vm_suite_test.go index 15cc6536..c14ee8c0 100644 --- a/src/bosh-virtualbox-cpi/vm/vm_suite_test.go +++ b/src/bosh-virtualbox-cpi/vm/vm_suite_test.go @@ -3,7 +3,7 @@ package vm_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) From 4073176e2984fd37200eb674ac4bbeef79e7fdf3 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 02:03:59 +0100 Subject: [PATCH 08/15] Format code when running bin/test --- src/bosh-virtualbox-cpi/cpi/factory.go | 2 +- .../cpi/factory_options.go | 2 +- src/bosh-virtualbox-cpi/cpi/stemcells.go | 2 +- src/bosh-virtualbox-cpi/disk/disk.go | 2 +- src/bosh-virtualbox-cpi/disk/factory.go | 2 +- src/bosh-virtualbox-cpi/stemcell/stemcell.go | 2 +- src/bosh-virtualbox-cpi/vm/directories.go | 20 +++++++++---------- src/bosh-virtualbox-cpi/vm/factory.go | 2 +- src/bosh-virtualbox-cpi/vm/iso9660_writer.go | 15 +++++++------- .../vm/portdevices/port_devices.go | 2 +- src/bosh-virtualbox-cpi/vm/vm.go | 2 +- src/bosh-virtualbox-cpi/vm/vm_agent.go | 2 +- src/bosh-virtualbox-cpi/vm/vm_disks.go | 2 +- 13 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/bosh-virtualbox-cpi/cpi/factory.go b/src/bosh-virtualbox-cpi/cpi/factory.go index 0addb5b5..bb930f27 100644 --- a/src/bosh-virtualbox-cpi/cpi/factory.go +++ b/src/bosh-virtualbox-cpi/cpi/factory.go @@ -1,11 +1,11 @@ package cpi import ( + apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" boshcmd "github.com/cloudfoundry/bosh-utils/fileutil" boshlog "github.com/cloudfoundry/bosh-utils/logger" boshsys "github.com/cloudfoundry/bosh-utils/system" boshuuid "github.com/cloudfoundry/bosh-utils/uuid" - apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bdisk "bosh-virtualbox-cpi/disk" "bosh-virtualbox-cpi/driver" diff --git a/src/bosh-virtualbox-cpi/cpi/factory_options.go b/src/bosh-virtualbox-cpi/cpi/factory_options.go index 564283a7..906e38bf 100644 --- a/src/bosh-virtualbox-cpi/cpi/factory_options.go +++ b/src/bosh-virtualbox-cpi/cpi/factory_options.go @@ -3,8 +3,8 @@ package cpi import ( "path/filepath" - bosherr "github.com/cloudfoundry/bosh-utils/errors" apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" + bosherr "github.com/cloudfoundry/bosh-utils/errors" bpds "bosh-virtualbox-cpi/vm/portdevices" ) diff --git a/src/bosh-virtualbox-cpi/cpi/stemcells.go b/src/bosh-virtualbox-cpi/cpi/stemcells.go index b341d0cc..5bee52b0 100644 --- a/src/bosh-virtualbox-cpi/cpi/stemcells.go +++ b/src/bosh-virtualbox-cpi/cpi/stemcells.go @@ -1,8 +1,8 @@ package cpi import ( - bosherr "github.com/cloudfoundry/bosh-utils/errors" apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" + bosherr "github.com/cloudfoundry/bosh-utils/errors" bstem "bosh-virtualbox-cpi/stemcell" ) diff --git a/src/bosh-virtualbox-cpi/disk/disk.go b/src/bosh-virtualbox-cpi/disk/disk.go index bcb89655..2b7c0cb2 100644 --- a/src/bosh-virtualbox-cpi/disk/disk.go +++ b/src/bosh-virtualbox-cpi/disk/disk.go @@ -3,9 +3,9 @@ package disk import ( "path/filepath" + apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bosherr "github.com/cloudfoundry/bosh-utils/errors" boshlog "github.com/cloudfoundry/bosh-utils/logger" - apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" "bosh-virtualbox-cpi/driver" ) diff --git a/src/bosh-virtualbox-cpi/disk/factory.go b/src/bosh-virtualbox-cpi/disk/factory.go index 2447ed12..0b417e3d 100644 --- a/src/bosh-virtualbox-cpi/disk/factory.go +++ b/src/bosh-virtualbox-cpi/disk/factory.go @@ -4,10 +4,10 @@ import ( "path/filepath" "strconv" + apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bosherr "github.com/cloudfoundry/bosh-utils/errors" boshlog "github.com/cloudfoundry/bosh-utils/logger" boshuuid "github.com/cloudfoundry/bosh-utils/uuid" - apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" "bosh-virtualbox-cpi/driver" ) diff --git a/src/bosh-virtualbox-cpi/stemcell/stemcell.go b/src/bosh-virtualbox-cpi/stemcell/stemcell.go index 12a77506..6e85ac26 100644 --- a/src/bosh-virtualbox-cpi/stemcell/stemcell.go +++ b/src/bosh-virtualbox-cpi/stemcell/stemcell.go @@ -1,9 +1,9 @@ package stemcell import ( + apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bosherr "github.com/cloudfoundry/bosh-utils/errors" boshlog "github.com/cloudfoundry/bosh-utils/logger" - apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" "bosh-virtualbox-cpi/driver" ) diff --git a/src/bosh-virtualbox-cpi/vm/directories.go b/src/bosh-virtualbox-cpi/vm/directories.go index cb8a3685..dab22d5c 100644 --- a/src/bosh-virtualbox-cpi/vm/directories.go +++ b/src/bosh-virtualbox-cpi/vm/directories.go @@ -3,8 +3,8 @@ package vm // more or less vendored from github.com/johto/iso9660wrap/blob/master/directories.go import ( - "time" "fmt" + "time" ) func WriteDirectoryRecord(w *SectorWriter, identifier string, firstSectorNum uint32) (uint32, error) { @@ -18,14 +18,14 @@ func WriteDirectoryRecord(w *SectorWriter, identifier string, firstSectorNum uin w.WriteBothEndianDWord(firstSectorNum) w.WriteBothEndianDWord(SectorSize) // directory length writeDirectoryRecordtimestamp(w, time.Now()) - w.WriteByte(byte(3)) // bitfield; directory - w.WriteByte(byte(0)) // file unit size for an interleaved file - w.WriteByte(byte(0)) // interleave gap size for an interleaved file + w.WriteByte(byte(3)) // bitfield; directory + w.WriteByte(byte(0)) // file unit size for an interleaved file + w.WriteByte(byte(0)) // interleave gap size for an interleaved file w.WriteBothEndianWord(1) // volume sequence number w.WriteByte(byte(len(identifier))) w.WriteString(identifier) // optional padding to even length - if recordLength % 2 == 1 { + if recordLength%2 == 1 { recordLength++ w.WriteByte(0) } @@ -39,18 +39,18 @@ func WriteFileRecordHeader(w *SectorWriter, identifier string, firstSectorNum ui recordLength := 33 + len(identifier) w.WriteByte(byte(recordLength)) - w.WriteByte(0) // number of sectors in extended attribute record + w.WriteByte(0) // number of sectors in extended attribute record w.WriteBothEndianDWord(firstSectorNum) // first sector w.WriteBothEndianDWord(fileSize) writeDirectoryRecordtimestamp(w, time.Now()) - w.WriteByte(byte(0)) // bitfield; normal file - w.WriteByte(byte(0)) // file unit size for an interleaved file - w.WriteByte(byte(0)) // interleave gap size for an interleaved file + w.WriteByte(byte(0)) // bitfield; normal file + w.WriteByte(byte(0)) // file unit size for an interleaved file + w.WriteByte(byte(0)) // interleave gap size for an interleaved file w.WriteBothEndianWord(1) // volume sequence number w.WriteByte(byte(len(identifier))) w.WriteString(identifier) // optional padding to even length - if recordLength % 2 == 1 { + if recordLength%2 == 1 { recordLength++ w.WriteByte(0) } diff --git a/src/bosh-virtualbox-cpi/vm/factory.go b/src/bosh-virtualbox-cpi/vm/factory.go index 94242051..45508ff1 100644 --- a/src/bosh-virtualbox-cpi/vm/factory.go +++ b/src/bosh-virtualbox-cpi/vm/factory.go @@ -3,10 +3,10 @@ package vm import ( "path/filepath" + apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bosherr "github.com/cloudfoundry/bosh-utils/errors" boshlog "github.com/cloudfoundry/bosh-utils/logger" boshuuid "github.com/cloudfoundry/bosh-utils/uuid" - apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bdisk "bosh-virtualbox-cpi/disk" "bosh-virtualbox-cpi/driver" diff --git a/src/bosh-virtualbox-cpi/vm/iso9660_writer.go b/src/bosh-virtualbox-cpi/vm/iso9660_writer.go index d3ea9986..8c9c464b 100644 --- a/src/bosh-virtualbox-cpi/vm/iso9660_writer.go +++ b/src/bosh-virtualbox-cpi/vm/iso9660_writer.go @@ -4,11 +4,11 @@ package vm import ( "encoding/binary" + "fmt" "io" "math" "strings" "time" - "fmt" ) const SectorSize uint32 = 2048 @@ -28,9 +28,9 @@ func (w *SectorWriter) Write(p []byte) (uint32, error) { } w.p += l _, err := w.w.Write(p) - if err != nil { - return 0, err - } + if err != nil { + return 0, err + } return l, nil } @@ -45,7 +45,7 @@ func (w *SectorWriter) WriteUnspecifiedDateTime() (uint32, error) { func (w *SectorWriter) WriteDateTime(t time.Time) (uint32, error) { f := t.UTC().Format("20060102150405") - f += "00" // 1/100 + f += "00" // 1/100 f += "\x00" // UTC offset if len(f) != 17 { return 0, fmt.Errorf("date and time field %q is of unexpected length %d", f, len(f)) @@ -65,7 +65,7 @@ func (w *SectorWriter) WritePaddedString(str string, length uint32) (uint32, err if l > 32 { return 0, fmt.Errorf("padded string %q exceeds length %d", str, length) } else if l < 32 { - w.WriteString(strings.Repeat(" ", int(32 - l))) + w.WriteString(strings.Repeat(" ", int(32-l))) } return 32, nil } @@ -122,9 +122,8 @@ func (w *SectorWriter) Reset() { w.p = 0 } - type ISO9660Writer struct { - sw *SectorWriter + sw *SectorWriter sectorNum uint32 } diff --git a/src/bosh-virtualbox-cpi/vm/portdevices/port_devices.go b/src/bosh-virtualbox-cpi/vm/portdevices/port_devices.go index 950e0149..2a913906 100644 --- a/src/bosh-virtualbox-cpi/vm/portdevices/port_devices.go +++ b/src/bosh-virtualbox-cpi/vm/portdevices/port_devices.go @@ -5,9 +5,9 @@ import ( "regexp" "strings" + apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bosherr "github.com/cloudfoundry/bosh-utils/errors" boshlog "github.com/cloudfoundry/bosh-utils/logger" - apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" "bosh-virtualbox-cpi/driver" ) diff --git a/src/bosh-virtualbox-cpi/vm/vm.go b/src/bosh-virtualbox-cpi/vm/vm.go index 5c5c8e6c..8e04baa6 100644 --- a/src/bosh-virtualbox-cpi/vm/vm.go +++ b/src/bosh-virtualbox-cpi/vm/vm.go @@ -5,9 +5,9 @@ import ( "fmt" "strconv" + apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" bosherr "github.com/cloudfoundry/bosh-utils/errors" boshlog "github.com/cloudfoundry/bosh-utils/logger" - apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" "bosh-virtualbox-cpi/driver" bpds "bosh-virtualbox-cpi/vm/portdevices" diff --git a/src/bosh-virtualbox-cpi/vm/vm_agent.go b/src/bosh-virtualbox-cpi/vm/vm_agent.go index d2ba3b24..2fd4bb4b 100644 --- a/src/bosh-virtualbox-cpi/vm/vm_agent.go +++ b/src/bosh-virtualbox-cpi/vm/vm_agent.go @@ -1,8 +1,8 @@ package vm import ( - bosherr "github.com/cloudfoundry/bosh-utils/errors" apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" + bosherr "github.com/cloudfoundry/bosh-utils/errors" ) func (vm VMImpl) ConfigureAgent(agentEnv apiv1.AgentEnv) error { diff --git a/src/bosh-virtualbox-cpi/vm/vm_disks.go b/src/bosh-virtualbox-cpi/vm/vm_disks.go index 32a6f085..e7cc3115 100644 --- a/src/bosh-virtualbox-cpi/vm/vm_disks.go +++ b/src/bosh-virtualbox-cpi/vm/vm_disks.go @@ -4,8 +4,8 @@ import ( "encoding/json" "strings" - bosherr "github.com/cloudfoundry/bosh-utils/errors" apiv1 "github.com/cloudfoundry/bosh-cpi-go/apiv1" + bosherr "github.com/cloudfoundry/bosh-utils/errors" bdisk "bosh-virtualbox-cpi/disk" ) From d087eb1456219ea7722dde4aefcc2917fbcb9153 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 04:58:27 +0100 Subject: [PATCH 09/15] Add 'version' files to vendored packages, as required by the Concourse job that bumps these packages --- packages/golang-1-darwin/version | 1 + packages/golang-1-linux/version | 1 + 2 files changed, 2 insertions(+) create mode 100644 packages/golang-1-darwin/version create mode 100644 packages/golang-1-linux/version diff --git a/packages/golang-1-darwin/version b/packages/golang-1-darwin/version new file mode 100644 index 00000000..0e1f39b8 --- /dev/null +++ b/packages/golang-1-darwin/version @@ -0,0 +1 @@ +1.17.2 \ No newline at end of file diff --git a/packages/golang-1-linux/version b/packages/golang-1-linux/version new file mode 100644 index 00000000..0e1f39b8 --- /dev/null +++ b/packages/golang-1-linux/version @@ -0,0 +1 @@ +1.17.2 \ No newline at end of file From 276cb243639a1dab40f3de5c51e9f61b5a11dae8 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 05:59:23 +0100 Subject: [PATCH 10/15] Adapt CPI packaging script to updated 'compile.env' convention as of Golang release v0.117.0 --- packages/virtualbox_cpi/packaging | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/virtualbox_cpi/packaging b/packages/virtualbox_cpi/packaging index 72be9b7c..70dda299 100644 --- a/packages/virtualbox_cpi/packaging +++ b/packages/virtualbox_cpi/packaging @@ -1,19 +1,19 @@ set -e -x +current_os=$(uname -s | tr '[:upper:]' '[:lower:]') if [ -z "$BOSH_PACKAGES_DIR" ]; then - pkg_dir=$(readlink -nf /var/vcap/packages/golang-1-darwin) + pkg_dir=$(readlink -nf /var/vcap/packages/golang-1-${current_os}) else - pkg_dir=$BOSH_PACKAGES_DIR/golang-1-darwin + pkg_dir=$BOSH_PACKAGES_DIR/golang-1-${current_os} fi -source ${pkg_dir}/bosh/compile.env.generic +source ${pkg_dir}/bosh/compile.env # Since HOME is not set we must set GOPATH and GOCACHE mkdir -p /tmp/go/.cache export GOPATH=/tmp/go export GOCACHE=${GOPATH}/.cache - cd ${BOSH_COMPILE_TARGET}/bosh-virtualbox-cpi mkdir -p ${BOSH_INSTALL_TARGET}/bin From 2931e587d5e86c9e5878a7d7fcd0ccd2000a6c1b Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 15:01:58 +0100 Subject: [PATCH 11/15] Switch pipeline back to 'master' after having merged the 'ci-pipeline' branch --- ci/settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/settings.yml b/ci/settings.yml index c241d13e..da86e916 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -38,7 +38,7 @@ meta: github: owner: gstackio repo: (( grab meta.pipeline )) - branch: ci-pipeline + branch: master pr_base_branch: master private_key: ((github-private-key)) access_token: ((github-access-token)) From 4cee77e1278cb921f1714103bccfaf8bf28a2177 Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 17:40:27 +0100 Subject: [PATCH 12/15] Add release notes, listing recent achievements --- ci/release_notes.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ci/release_notes.md diff --git a/ci/release_notes.md b/ci/release_notes.md new file mode 100644 index 00000000..57595293 --- /dev/null +++ b/ci/release_notes.md @@ -0,0 +1,11 @@ +### Improvements + +- Generated pipeline from Cloud Foundry community's [pipeline-templates](https://github.com/cloudfoundry-community/pipeline-templates), with custom jobs for bumping golang bosh package, go language version, and go dependencies versions. +- Added a smoke test job. +- Bumped all Golang dependencies. +- Bumped the Golang package. +- Rebuilt CPI binary using latest Go v1.22.0. + +### Fixes + +- Fixes cloudfoundry/bosh-virtualbox-cpi-release#33. From d4d636dc5236a7c92173a54352f7bb00e1675f6c Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 18:41:51 +0100 Subject: [PATCH 13/15] Fix shipit to support Git-LFS blobs --- ci/pipeline.yml | 1 + ci/scripts/shipit | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 6d17e23f..dbaf0692 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -303,6 +303,7 @@ jobs: inputs: - name: version - name: git + caches: [ path: /root/.bosh/cache ] outputs: - name: gh - name: pushme diff --git a/ci/scripts/shipit b/ci/scripts/shipit index a9245d3e..fbff8f98 100755 --- a/ci/scripts/shipit +++ b/ci/scripts/shipit @@ -47,7 +47,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ############################################################### -git clone ${REPO_ROOT} ${REPO_OUT} +find "${REPO_ROOT}" -mindepth 1 -maxdepth 1 -print0 \ + | xargs -0 -I{} cp -a {} "${REPO_OUT}" pushd ${REPO_OUT} From c8bb96078fce7659f90caec0c24fbbedc4e7a62d Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 19:12:26 +0100 Subject: [PATCH 14/15] Move the ops-files so that the 'shipit' job can properly apply its mass update on all '*.yml' deployment manifests --- manifests/{ => operators}/gui.yml | 0 manifests/{ => operators}/local-release.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename manifests/{ => operators}/gui.yml (100%) rename manifests/{ => operators}/local-release.yml (100%) diff --git a/manifests/gui.yml b/manifests/operators/gui.yml similarity index 100% rename from manifests/gui.yml rename to manifests/operators/gui.yml diff --git a/manifests/local-release.yml b/manifests/operators/local-release.yml similarity index 100% rename from manifests/local-release.yml rename to manifests/operators/local-release.yml From 7213b6a06d395525ba86bcddfb7fec52c513707e Mon Sep 17 00:00:00 2001 From: Benjamin Gandon Date: Tue, 13 Feb 2024 19:33:48 +0100 Subject: [PATCH 15/15] Fix issue in 'shipit' when GitHub repo name doesn't follow the '-boshrelease' suffix convention --- ci/pipeline.yml | 1 + ci/scripts/shipit | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index dbaf0692..41477ec8 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -320,6 +320,7 @@ jobs: NOTIFICATION_OUT: notifications RELEASE_BRANCH: (( grab meta.github.release_branch )) GITHUB_OWNER: (( grab meta.github.owner )) + GITHUB_REPO: (( grab meta.github.repo )) GIT_EMAIL: (( grab meta.git.email )) GIT_NAME: (( grab meta.git.name )) AWS_ACCESS_KEY: (( grab meta.aws.access_key )) diff --git a/ci/scripts/shipit b/ci/scripts/shipit index fbff8f98..f33fa356 100755 --- a/ci/scripts/shipit +++ b/ci/scripts/shipit @@ -24,6 +24,7 @@ header() { : ${REPO_OUT:?required} RELEASE_BRANCH=${RELEASE_BRANCH:-"master"} : ${GITHUB_OWNER:?required} +: ${GITHUB_REPO:?required} : ${VERSION_FROM:?required} : ${GIT_EMAIL:?required} : ${GIT_NAME:?required} @@ -81,7 +82,7 @@ popd release_tgz=${REPO_OUT}/releases/${release_name}/${release_name}-${version}.tgz export release_sha1=$(sha1sum $release_tgz | head -n1 | awk '{print $1}') -export release_url="https://github.com/${GITHUB_OWNER}/${release_name}-boshrelease/releases/download/v${version}/${release_name}-${version}.tgz" +export release_url="https://github.com/${GITHUB_OWNER}/${GITHUB_REPO}/releases/download/v${version}/${release_name}-${version}.tgz" echo "RELEASE_SHA1=${release_sha1}" mkdir -p ${RELEASE_ROOT}/artifacts @@ -124,5 +125,5 @@ git commit -m "release v${version}" popd cat > ${NOTIFICATION_OUT:-notifications}/message <. +New ${release_name} v${version} released. . EOS