ENT-13030: fixup ubuntu 24 buildhost policy and added containers scripts to test the policy

ci/buildhost-image.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -ex
+
+platform=$1
+c=$platform
+base=$platform-upgraded
+built=$platform-buildhost
+if buildah images --format '{{.Name}}:{{.Tag}}' | grep $built; then
+  echo "container image $built already exists, remove if you want to rebuild"
+  continue
+fi
+
+buildah rm $c || true
+buildah --name $c from $base
+buildah copy $c . /buildscripts/ci
+
+# This section is debian specific for now. TODO: add alternatives when we add more platforms.
+buildah run $c apt install -y procps wget sudo
+buildah run $c apt remove -y cfengine-nova || true
+
+buildah run $c rm -rf /var/cfengine || true
+# touch flag file for policy to know it is in a container and avoid some aspects of configuration
+buildah run $c touch /etc/cfengine-in-container.flag
+buildah run $c /buildscripts/ci/setup-cfengine-build-host.sh | tee setup-cfengine-build-host.log
+# the above, if errored out, is not causing an error, need to fix that.
+grep -i error setup-cfengine-build-host.log && exit 1
+buildah tag $c $c-$(date +%F)
+buildah commit $c $c-buildhost
+buildah rm $c

ci/cfengine-build-host-setup.cf

@@ -30,8 +30,6 @@ bundle agent cfengine_build_host_setup
       "fakeroot";
       "flex";
       "gdb";
-      "libncurses5" comment => "added for debian-10";
-      "libncurses5-dev" comment => "added for debian-10";
       "libexpat1-dev";
       "libmodule-load-conditional-perl";
       "libpam0g-dev";
@@ -43,6 +41,15 @@ bundle agent cfengine_build_host_setup
       "rsync" comment => "added for debian-10";
       "systemd-coredump" comment => "added step to jenkins testing-pr job to query for coredumps on failures";
 
+    debian_10|ubuntu_20::
+      "libncurses5";
+      "libncurses5-dev";
+
+    (debian|ubuntu).!(debian_10|ubuntu_20)::
+      "libncurses6";
+      "libncurses-dev";
+
+
     mingw_build_host::
       "wine:i386";
       "mingw-w64";
@@ -148,6 +155,8 @@ bundle agent cfengine_build_host_setup
       "systemssl_build_host" expression => fileexists("/etc/cfengine-systemssl-build-host.flag");
       "bootstrap_pr_host" expression => fileexists("/etc/cfengine-bootstrap-pr-host.flag");
       "containers_host" expression => fileexists("/etc/cfengine-containers-host.flag");
+      "not_in_container" expression => not(fileexists("/etc/cfengine-in-container.flag")),
+        comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not.";
     linux::
       "have_coredumpctl" expression => returnszero("command -v coredumpctl", "useshell");
       "missing_opt_jdk21" expression => not(fileexists("/opt/jdk-21.0.1"));
@@ -173,7 +182,7 @@ bundle agent cfengine_build_host_setup
         comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option";
 
   commands:
-    have_coredumpctl::
+    have_coredumpctl.not_in_container::
       "sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e'" -> { "ENT-12669" }
         comment => "Ensure that core_pattern is proper for systemd-coredump if coredumpctl is present.",
         contain => in_shell;
@@ -200,7 +209,7 @@ bundle agent cfengine_build_host_setup
     ubuntu_16::
       "have_i386_architecture" expression => strcmp(execresult("${paths.dpkg} --print-foreign-architectures", "noshell"), "i386");
     ubuntu::
-      "have_localhost_localdomain_hostname" expression => strcmp(execresult("${paths.hostname} -f", "useshell"), "localhost.localdomain");
+      "localhost_localdomain_hostname_missing" expression => not(strcmp(execresult("${paths.hostname} -f", "useshell"), "localhost.localdomain"));
     opensuse|suse|sles::
       "have_$(suse_users_and_groups)_group" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/group >/dev/null", "useshell");
       "have_$(suse_users_and_groups)_user" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/passwd >/dev/null", "useshell");
@@ -271,7 +280,7 @@ jenkins_builds ALL=NOPASSWD: /usr/bin/podman
     mingw_build_host.!have_i386_architecture::
       "${paths.dpkg} --add-architecture i386";
 
-    ubuntu.!have_localhost_localdomain_hostname::
+    ubuntu.not_in_container.localhost_localdomain_hostname_missing::
       "/usr/bin/hostnamectl set-hostname localhost.localdomain"
         comment => "hack for aws ubuntu hosts having unique ip-n-n-n-n hostnames, we need localhost.localdomain";
     !have_daemon_group.(suse|sles|opensuse)::

ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256

@@ -0,0 +1 @@
+1217b72455f09776442c5abb40b66f49629175b298c0b7baf0d5ba5ffe94e5ed  cfengine-masterfiles-3.26.0-1.pkg.tar.gz

ci/clean-buildhost-images.sh

@@ -0,0 +1 @@
+buildah images | grep buildhost | awk '{print $3}' | xargs buildah rmi

ci/images.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ex
+
+# todo: centos7, opensuse/leap:15 (no :12), registry.access.redhat.com/ubi9 (-minimal, -init, -micro (standard))
+# run this on x86 and arm hardware to cover "all the bases" :)
+for platform in $(cat platform-container-image.list); do
+  ./upgrade-image.sh $platform
+  ./buildhost-image.sh $platform
+done

ci/linux-install-jdk21.sh

@@ -1,29 +1,38 @@
 #!/usr/bin/env bash
 set -e
-# install jdk "manually"
-# depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*`
-cd /opt
-baseurl=https://download.oracle.com/java/21/latest/
-version=21.0.7
-if uname -m | grep aarch64; then
-  tarball=jdk-21_linux-aarch64_bin.tar.gz
-  sha=47372cfa9244dc74ec783a1b287381502419b564fbd0b18abc8f2d6b19ac865e
-else
-  tarball=jdk-21_linux-x64_bin.tar.gz
-  sha=267b10b14b4e5fada19aca3be3b961ce4f81f1bd3ffcd070e90a5586106125eb
-fi
-wget --quiet "$baseurl$tarball"
-echo "$sha" "$tarball" | sha256sum --check -
-sudo tar xf "$tarball"
-sudo tee /etc/profile.d/jdk.sh << EOF
+
+install_jdk() {
+  # install jdk "manually"
+  # depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*`
+  cd /opt
+  baseurl=https://download.oracle.com/java/21/latest/
+  version=21.0.7
+  if uname -m | grep aarch64; then
+    tarball=jdk-21_linux-aarch64_bin.tar.gz
+    sha=47372cfa9244dc74ec783a1b287381502419b564fbd0b18abc8f2d6b19ac865e
+  else
+    tarball=jdk-21_linux-x64_bin.tar.gz
+    sha=267b10b14b4e5fada19aca3be3b961ce4f81f1bd3ffcd070e90a5586106125eb
+  fi
+  wget --quiet "$baseurl$tarball"
+  echo "$sha" "$tarball" | sha256sum --check -
+  tar xf "$tarball"
+  tee /etc/profile.d/jdk.sh << EOF
 export JAVA_HOME="/opt/jdk-$version"
 export PATH=\$PATH:\$JAVA_HOME/bin
 EOF
-sudo chown -R root:jenkins "/opt/jdk-$version"
-sudo chmod -R g+rx "/opt/jdk-$version"
-if command -v update-alternatives; then
-  sudo update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999
+  chown -R root:jenkins "/opt/jdk-$version"
+  chmod -R g+rx "/opt/jdk-$version"
+  if command -v update-alternatives; then
+    update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999
+  else
+    ln -s "/opt/jdk-$version/bin/java" /usr/bin/java
+  fi
+  cd -
+}
+
+if [ "$(whoami)" = "root" ]; then
+  install_jdk
 else
-  sudo ln -s "/opt/jdk-$version/bin/java" /usr/bin/java
+  sudo bash -c install_jdk
 fi
-cd -

ci/platform-container-image.list

@@ -0,0 +1,5 @@
+debian:11
+debian:12
+ubuntu:20.04
+ubuntu:22.04
+ubuntu:24.04

ci/setup-cfengine-build-host.sh

@@ -2,7 +2,7 @@
 shopt -s expand_aliases
 
 # TODO get latest LTS dynamically
-CFE_VERSION=3.24.2
+CFE_VERSION=3.26.0
 
 # install needed packages and software for a build host
 set -ex
@@ -12,18 +12,25 @@ if [ "$(id -u)" != "0" ]; then
 fi
 
 ls -la /home/
+if ! id -u jenkins; then
+  useradd jenkins -p jenkins
+fi
+mkdir -p /home/jenkins
 chown -R jenkins /home/jenkins
 
+echo "checking for CFEngine install..."
 if [ -d /var/cfengine ]; then
   echo "Error: CFEngine already installed on this host. Will not proceed trying to setup build host with CFEngine temporary install."
   exit 1
 fi
 
-
 function cleanup()
 {
   set -ex
   if command -v apt 2>/dev/null; then
+    # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false
+    rm /bin/systemctl
+    ln -s /bin/echo /bin/systemctl
     apt remove -y cfengine-nova || true
   elif command -v yum 2>/dev/null; then
     yum erase -y cfengine-nova || true
@@ -34,7 +41,8 @@ function cleanup()
     exit 1
   fi
   echo "Ensuring CFEngine fully uninstalled/cleaned up"
-  rm -rf /var/cfengine /opt/cfengine /var/log/CFE* /var/log/postgresql.log || true
+# keep these logs around for debugging failed setup runs
+#  rm -rf /var/cfengine /opt/cfengine /var/log/CFE* /var/log/postgresql.log || true
   if command -v pkill; then
     pkill -9 cf-agent || true
     pkill -9 cf-serverd || true

ci/upgraded-image.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -ex
+
+platform=$1
+upgraded=$platform-upgraded
+if buildah images --format '{{.Name}}:{{.Tag}}' | grep $upgraded; then
+  echo "container image $upgraded already exists. buildah rm $upgraded if you want to regenerate"
+  continue
+fi
+if ! buildah ps | grep $platform; then
+  buildah --name $platform from $platform
+fi
+buildah copy $platform .. /buildscripts/ci
+buildah run $platform apt update -y
+buildah run $platform apt upgrade -y
+buildah tag $platform $platform-$(date +%F)
+buildah commit $platform $upgraded
+buildah rm $platform