diff --git a/ci/buildhost-image.sh b/ci/buildhost-image.sh new file mode 100755 index 000000000..1ffdc6744 --- /dev/null +++ b/ci/buildhost-image.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -ex + +platform=$1 +c=$platform +base=$platform-upgraded +built=$platform-buildhost +if buildah images --format '{{.Name}}:{{.Tag}}' | grep $built; then + echo "container image $built already exists, remove if you want to rebuild" + continue +fi + +buildah rm $c || true +buildah --name $c from $base +buildah copy $c . /buildscripts/ci + +# This section is debian specific for now. TODO: add alternatives when we add more platforms. +buildah run $c apt install -y procps wget sudo +buildah run $c apt remove -y cfengine-nova || true + +buildah run $c rm -rf /var/cfengine || true +# touch flag file for policy to know it is in a container and avoid some aspects of configuration +buildah run $c touch /etc/cfengine-in-container.flag +buildah run $c /buildscripts/ci/setup-cfengine-build-host.sh | tee setup-cfengine-build-host.log +# the above, if errored out, is not causing an error, need to fix that. +grep -i error setup-cfengine-build-host.log && exit 1 +buildah tag $c $c-$(date +%F) +buildah commit $c $c-buildhost +buildah rm $c diff --git a/ci/cfengine-build-host-setup.cf b/ci/cfengine-build-host-setup.cf index 3f01dcbc5..dcadf7eb9 100644 --- a/ci/cfengine-build-host-setup.cf +++ b/ci/cfengine-build-host-setup.cf @@ -30,8 +30,6 @@ bundle agent cfengine_build_host_setup "fakeroot"; "flex"; "gdb"; - "libncurses5" comment => "added for debian-10"; - "libncurses5-dev" comment => "added for debian-10"; "libexpat1-dev"; "libmodule-load-conditional-perl"; "libpam0g-dev"; @@ -43,6 +41,15 @@ bundle agent cfengine_build_host_setup "rsync" comment => "added for debian-10"; "systemd-coredump" comment => "added step to jenkins testing-pr job to query for coredumps on failures"; + debian_10|ubuntu_20:: + "libncurses5"; + "libncurses5-dev"; + + (debian|ubuntu).!(debian_10|ubuntu_20):: + "libncurses6"; + "libncurses-dev"; + + mingw_build_host:: "wine:i386"; "mingw-w64"; @@ -148,6 +155,8 @@ bundle agent cfengine_build_host_setup "systemssl_build_host" expression => fileexists("/etc/cfengine-systemssl-build-host.flag"); "bootstrap_pr_host" expression => fileexists("/etc/cfengine-bootstrap-pr-host.flag"); "containers_host" expression => fileexists("/etc/cfengine-containers-host.flag"); + "not_in_container" expression => not(fileexists("/etc/cfengine-in-container.flag")), + comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not."; linux:: "have_coredumpctl" expression => returnszero("command -v coredumpctl", "useshell"); "missing_opt_jdk21" expression => not(fileexists("/opt/jdk-21.0.1")); @@ -173,7 +182,7 @@ bundle agent cfengine_build_host_setup comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option"; commands: - have_coredumpctl:: + have_coredumpctl.not_in_container:: "sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e'" -> { "ENT-12669" } comment => "Ensure that core_pattern is proper for systemd-coredump if coredumpctl is present.", contain => in_shell; @@ -200,7 +209,7 @@ bundle agent cfengine_build_host_setup ubuntu_16:: "have_i386_architecture" expression => strcmp(execresult("${paths.dpkg} --print-foreign-architectures", "noshell"), "i386"); ubuntu:: - "have_localhost_localdomain_hostname" expression => strcmp(execresult("${paths.hostname} -f", "useshell"), "localhost.localdomain"); + "localhost_localdomain_hostname_missing" expression => not(strcmp(execresult("${paths.hostname} -f", "useshell"), "localhost.localdomain")); opensuse|suse|sles:: "have_$(suse_users_and_groups)_group" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/group >/dev/null", "useshell"); "have_$(suse_users_and_groups)_user" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/passwd >/dev/null", "useshell"); @@ -271,7 +280,7 @@ jenkins_builds ALL=NOPASSWD: /usr/bin/podman mingw_build_host.!have_i386_architecture:: "${paths.dpkg} --add-architecture i386"; - ubuntu.!have_localhost_localdomain_hostname:: + ubuntu.not_in_container.localhost_localdomain_hostname_missing:: "/usr/bin/hostnamectl set-hostname localhost.localdomain" comment => "hack for aws ubuntu hosts having unique ip-n-n-n-n hostnames, we need localhost.localdomain"; !have_daemon_group.(suse|sles|opensuse):: diff --git a/ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 b/ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 new file mode 100644 index 000000000..3a99eea98 --- /dev/null +++ b/ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 @@ -0,0 +1 @@ +1217b72455f09776442c5abb40b66f49629175b298c0b7baf0d5ba5ffe94e5ed cfengine-masterfiles-3.26.0-1.pkg.tar.gz diff --git a/ci/clean-buildhost-images.sh b/ci/clean-buildhost-images.sh new file mode 100755 index 000000000..731a9aeba --- /dev/null +++ b/ci/clean-buildhost-images.sh @@ -0,0 +1 @@ +buildah images | grep buildhost | awk '{print $3}' | xargs buildah rmi diff --git a/ci/images.sh b/ci/images.sh new file mode 100644 index 000000000..fecc590df --- /dev/null +++ b/ci/images.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ex + +# todo: centos7, opensuse/leap:15 (no :12), registry.access.redhat.com/ubi9 (-minimal, -init, -micro (standard)) +# run this on x86 and arm hardware to cover "all the bases" :) +for platform in $(cat platform-container-image.list); do + ./upgrade-image.sh $platform + ./buildhost-image.sh $platform +done diff --git a/ci/linux-install-jdk21.sh b/ci/linux-install-jdk21.sh index 49305d92e..43c3ccabd 100755 --- a/ci/linux-install-jdk21.sh +++ b/ci/linux-install-jdk21.sh @@ -1,29 +1,38 @@ #!/usr/bin/env bash set -e -# install jdk "manually" -# depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*` -cd /opt -baseurl=https://download.oracle.com/java/21/latest/ -version=21.0.7 -if uname -m | grep aarch64; then - tarball=jdk-21_linux-aarch64_bin.tar.gz - sha=47372cfa9244dc74ec783a1b287381502419b564fbd0b18abc8f2d6b19ac865e -else - tarball=jdk-21_linux-x64_bin.tar.gz - sha=267b10b14b4e5fada19aca3be3b961ce4f81f1bd3ffcd070e90a5586106125eb -fi -wget --quiet "$baseurl$tarball" -echo "$sha" "$tarball" | sha256sum --check - -sudo tar xf "$tarball" -sudo tee /etc/profile.d/jdk.sh << EOF + +install_jdk() { + # install jdk "manually" + # depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*` + cd /opt + baseurl=https://download.oracle.com/java/21/latest/ + version=21.0.7 + if uname -m | grep aarch64; then + tarball=jdk-21_linux-aarch64_bin.tar.gz + sha=47372cfa9244dc74ec783a1b287381502419b564fbd0b18abc8f2d6b19ac865e + else + tarball=jdk-21_linux-x64_bin.tar.gz + sha=267b10b14b4e5fada19aca3be3b961ce4f81f1bd3ffcd070e90a5586106125eb + fi + wget --quiet "$baseurl$tarball" + echo "$sha" "$tarball" | sha256sum --check - + tar xf "$tarball" + tee /etc/profile.d/jdk.sh << EOF export JAVA_HOME="/opt/jdk-$version" export PATH=\$PATH:\$JAVA_HOME/bin EOF -sudo chown -R root:jenkins "/opt/jdk-$version" -sudo chmod -R g+rx "/opt/jdk-$version" -if command -v update-alternatives; then - sudo update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999 + chown -R root:jenkins "/opt/jdk-$version" + chmod -R g+rx "/opt/jdk-$version" + if command -v update-alternatives; then + update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999 + else + ln -s "/opt/jdk-$version/bin/java" /usr/bin/java + fi + cd - +} + +if [ "$(whoami)" = "root" ]; then + install_jdk else - sudo ln -s "/opt/jdk-$version/bin/java" /usr/bin/java + sudo bash -c install_jdk fi -cd - diff --git a/ci/platform-container-image.list b/ci/platform-container-image.list new file mode 100644 index 000000000..8aa2bc9f7 --- /dev/null +++ b/ci/platform-container-image.list @@ -0,0 +1,5 @@ +debian:11 +debian:12 +ubuntu:20.04 +ubuntu:22.04 +ubuntu:24.04 diff --git a/ci/setup-cfengine-build-host.sh b/ci/setup-cfengine-build-host.sh index 2c027f3d3..7cf18a455 100755 --- a/ci/setup-cfengine-build-host.sh +++ b/ci/setup-cfengine-build-host.sh @@ -2,7 +2,7 @@ shopt -s expand_aliases # TODO get latest LTS dynamically -CFE_VERSION=3.24.2 +CFE_VERSION=3.26.0 # install needed packages and software for a build host set -ex @@ -12,18 +12,25 @@ if [ "$(id -u)" != "0" ]; then fi ls -la /home/ +if ! id -u jenkins; then + useradd jenkins -p jenkins +fi +mkdir -p /home/jenkins chown -R jenkins /home/jenkins +echo "checking for CFEngine install..." if [ -d /var/cfengine ]; then echo "Error: CFEngine already installed on this host. Will not proceed trying to setup build host with CFEngine temporary install." exit 1 fi - function cleanup() { set -ex if command -v apt 2>/dev/null; then + # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false + rm /bin/systemctl + ln -s /bin/echo /bin/systemctl apt remove -y cfengine-nova || true elif command -v yum 2>/dev/null; then yum erase -y cfengine-nova || true @@ -34,7 +41,8 @@ function cleanup() exit 1 fi echo "Ensuring CFEngine fully uninstalled/cleaned up" - rm -rf /var/cfengine /opt/cfengine /var/log/CFE* /var/log/postgresql.log || true +# keep these logs around for debugging failed setup runs +# rm -rf /var/cfengine /opt/cfengine /var/log/CFE* /var/log/postgresql.log || true if command -v pkill; then pkill -9 cf-agent || true pkill -9 cf-serverd || true diff --git a/ci/upgraded-image.sh b/ci/upgraded-image.sh new file mode 100755 index 000000000..a819e6f60 --- /dev/null +++ b/ci/upgraded-image.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -ex + +platform=$1 +upgraded=$platform-upgraded +if buildah images --format '{{.Name}}:{{.Tag}}' | grep $upgraded; then + echo "container image $upgraded already exists. buildah rm $upgraded if you want to regenerate" + continue +fi +if ! buildah ps | grep $platform; then + buildah --name $platform from $platform +fi +buildah copy $platform .. /buildscripts/ci +buildah run $platform apt update -y +buildah run $platform apt upgrade -y +buildah tag $platform $platform-$(date +%F) +buildah commit $platform $upgraded +buildah rm $platform