From 7d3fd1c760eccd5ddd191a69eb6265a2afb65401 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 17 Jun 2025 11:58:53 +0100 Subject: [PATCH 01/13] Add proxysql cert generation --- .../ansible/openbao-generate-internal-tls.yml | 35 +++++++++++++++++++ .../ansible/vault-generate-internal-tls.yml | 35 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/etc/kayobe/ansible/openbao-generate-internal-tls.yml b/etc/kayobe/ansible/openbao-generate-internal-tls.yml index 2cc9e841a..d96b9f2bd 100644 --- a/etc/kayobe/ansible/openbao-generate-internal-tls.yml +++ b/etc/kayobe/ansible/openbao-generate-internal-tls.yml @@ -54,3 +54,38 @@ dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt" mode: "0600" delegate_to: localhost + +# NOTE(seunghun1ee): Kolla Ansible reuses internal TLS certificate when +# creating certificate for proxysql +# https://opendev.org/openstack/kolla-ansible/src/branch/stable/2025.1/ansible/roles/certificates/tasks/generate.yml#L169-L183 + - name: Generate ProxySQL certificates + when: kolla_enable_proxysql + block: + - name: Copy ProxySQL certificate + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/proxysql-cert.pem" + content: | + {{ internal_cert.data.certificate }} + {{ internal_cert.data.issuing_ca }} + mode: "0600" + delegate_to: localhost + + - name: Copy ProxySQL certificate key + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/proxysql-key.pem" + content: | + {{ internal_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + +# NOTE(seunghun1ee): ProxySQL only expects root CA to be named ``root.crt`` because of +# https://opendev.org/openstack/kolla-ansible/src/branch/stable/2025.1/ansible/roles/loadbalancer/templates/proxysql/proxysql.json.j2#L36 +# Make a duplicate of ``openbao.crt`` named ``root.crt`` + - name: Copy root CA for ProxySQL + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/root.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/vault-generate-internal-tls.yml b/etc/kayobe/ansible/vault-generate-internal-tls.yml index a585d1bc9..3095b6615 100644 --- a/etc/kayobe/ansible/vault-generate-internal-tls.yml +++ b/etc/kayobe/ansible/vault-generate-internal-tls.yml @@ -54,3 +54,38 @@ dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" mode: "0600" delegate_to: localhost + +# NOTE(seunghun1ee): Kolla Ansible reuses internal TLS certificate when +# creating certificate for proxysql +# https://opendev.org/openstack/kolla-ansible/src/branch/stable/2025.1/ansible/roles/certificates/tasks/generate.yml#L169-L183 + - name: Generate ProxySQL certificates + when: kolla_enable_proxysql + block: + - name: Copy ProxySQL certificate + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/proxysql-cert.pem" + content: | + {{ internal_cert.data.certificate }} + {{ internal_cert.data.issuing_ca }} + mode: "0600" + delegate_to: localhost + + - name: Copy ProxySQL certificate key + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/proxysql-key.pem" + content: | + {{ internal_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + +# NOTE(seunghun1ee): ProxySQL only expects root CA to be named ``root.crt`` because of +# https://opendev.org/openstack/kolla-ansible/src/branch/stable/2025.1/ansible/roles/loadbalancer/templates/proxysql/proxysql.json.j2#L36 +# Make a duplicate of ``vault.crt`` named ``root.crt`` + - name: Copy root CA for ProxySQL + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/root.crt" + mode: "0600" + delegate_to: localhost From 76627f73d76fe3c42de5d14909247f306a8ec43d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 17 Jun 2025 14:55:06 +0000 Subject: [PATCH 02/13] Use testing kolla ansible branch --- etc/kayobe/stackhpc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 88cc16df6..3edd221dc 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -145,7 +145,7 @@ stackhpc_kolla_source_version: stackhpc/20.0.0.14 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" -stackhpc_kolla_ansible_source_version: stackhpc/20.0.0.3 +stackhpc_kolla_ansible_source_version: fix-service-deploy ############################################################################### # Container image registry @@ -169,7 +169,7 @@ stackhpc_enable_kayobe_check: true # Whether or not to check the installed Kolla-Ansible version when running Kayobe commands. # Default is true. -stackhpc_enable_kolla_ansible_check: true +stackhpc_enable_kolla_ansible_check: false ############################################################################### # Octavia Amphora image From 5726affcdbaae8202f8e65177d3b42b702af6e15 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 18 Jun 2025 16:23:03 +0100 Subject: [PATCH 03/13] Temp: bump kayobe to 18.0.0.11 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 63fe6aa65..9fcd7f67b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.0.0.0rc1.2 +kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/18.0.0.11 ansible-modules-hashivault>=5.2.1 jmespath From c76a66b3a3b2885a35f98d0850a0264bb52a6e87 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Fri, 20 Jun 2025 14:12:23 +0000 Subject: [PATCH 04/13] Temporary fix for K-A bug #2115064 https://bugs.launchpad.net/kolla-ansible/+bug/2115064 --- etc/kayobe/environments/ci-multinode/kolla/config/nova.conf | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/nova.conf diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/nova.conf b/etc/kayobe/environments/ci-multinode/kolla/config/nova.conf new file mode 100644 index 000000000..d1df062fa --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/nova.conf @@ -0,0 +1,5 @@ +#NOTE(seunghun1ee): This needs to be set until +# https://bugs.launchpad.net/kolla-ansible/+bug/2115064 +# is fixed +[cinder] +catalog_info = block-storage::internalURL From 9c3e32536ffb6a79d3c931c45b0a046bd622d07b Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 24 Jun 2025 09:55:29 +0100 Subject: [PATCH 05/13] Test: Set rabbit stream max segment to 5KB and age to 60s skip-checks: true --- etc/kayobe/environments/ci-multinode/kolla/globals.yml | 4 ++++ etc/kayobe/kolla-image-tags.yml | 3 +++ etc/kayobe/stackhpc.yml | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index 2fe551d8d..d58a86efb 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -67,3 +67,7 @@ designate_forwarders_addresses: "1.1.1.1; 8.8.8.8" # Open up ports in firewalld for services on the public API network. enable_external_api_firewalld: true external_api_firewalld_zone: "{{ public_net_name | net_zone }}" + +om_enable_rabbitmq_stream_fanout: true +om_rabbitmq_stream_max_segment_size_bytes: 5000 +om_rabbitmq_stream_segment_max_age: 60 diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 77b8a8740..455614bea 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -4,5 +4,8 @@ # where the key is the OS distro and the value is the tag to deploy. kolla_image_tags: openstack: + rocky-9: 2025.1-rocky-9-20250624T160903 + ubuntu-noble: 2025.1-ubuntu-noble-20250624T160903 + bifrost: rocky-9: 2025.1-rocky-9-20250616T133037 ubuntu-noble: 2025.1-ubuntu-noble-20250613T131221 diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 3edd221dc..c51c28f00 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -145,7 +145,7 @@ stackhpc_kolla_source_version: stackhpc/20.0.0.14 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" -stackhpc_kolla_ansible_source_version: fix-service-deploy +stackhpc_kolla_ansible_source_version: rabbitmq-stream-fix ############################################################################### # Container image registry From 176277258c2fc241791326a607f5d66b1cc9106b Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 25 Jun 2025 14:13:06 +0100 Subject: [PATCH 06/13] Temp: bump cephadm collection to 1.19.3 Until https://github.com/stackhpc/stackhpc-kayobe-config/pull/1726 is merged --- etc/kayobe/ansible/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 7c83c00ec..c39978d27 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,7 +1,7 @@ --- collections: - name: stackhpc.cephadm - version: 1.19.1 + version: 1.19.3 # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the # pulp_glue Python library being installed. - name: pulp.squeezer From c9dcde1c81f07dcb7ed5d73370044f9149f3cb4f Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 25 Jun 2025 14:13:36 +0100 Subject: [PATCH 07/13] Bump Ceph to v19.2.2 --- etc/kayobe/cephadm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 64c087081..2e48ad197 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -12,7 +12,7 @@ cephadm_ceph_release: "squid" cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}" # Ceph container image tag. -cephadm_image_tag: "v19.2.1" +cephadm_image_tag: "v19.2.2" # HAProxy container image. cephadm_haproxy_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/haproxy:{{ cephadm_haproxy_image_tag }}" From 291009b46672bebb2565f7387a68bfd5ecb4331e Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 25 Jun 2025 14:16:12 +0100 Subject: [PATCH 08/13] Add test cinder_cluster_name for CI --- etc/kayobe/environments/ci-multinode/kolla/globals.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index d58a86efb..7eeaeeeb8 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -68,6 +68,9 @@ designate_forwarders_addresses: "1.1.1.1; 8.8.8.8" enable_external_api_firewalld: true external_api_firewalld_zone: "{{ public_net_name | net_zone }}" +# Test cinder cluster +cinder_cluster_name: "stackhpc_test_cinder_cluster" + om_enable_rabbitmq_stream_fanout: true om_rabbitmq_stream_max_segment_size_bytes: 5000 om_rabbitmq_stream_segment_max_age: 60 From e485890e61861714d1cec615cf2a226deefb53c5 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Thu, 26 Jun 2025 08:29:24 +0100 Subject: [PATCH 09/13] Temp: run ipv6 tests only by default --- .automation.conf/config.sh | 1 + .automation.conf/tempest/load-lists/ipv6-tests | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 .automation.conf/tempest/load-lists/ipv6-tests diff --git a/.automation.conf/config.sh b/.automation.conf/config.sh index 76f2a59a0..21c037ed3 100644 --- a/.automation.conf/config.sh +++ b/.automation.conf/config.sh @@ -30,6 +30,7 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then export TEMPEST_CONCURRENCY=4 # Uncomment this to perform a full tempest test + export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=ipv6-tests # export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full # export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode-tempest-full fi diff --git a/.automation.conf/tempest/load-lists/ipv6-tests b/.automation.conf/tempest/load-lists/ipv6-tests new file mode 100644 index 000000000..e0429671b --- /dev/null +++ b/.automation.conf/tempest/load-lists/ipv6-tests @@ -0,0 +1,8 @@ +tempest.scenario.test_network_v6.TestGettingAddress.test_dhcp6_stateless_from_os[compute,id-d7e1f858-187c-45a6-89c9-bdafde619a9f,network,slow] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os[compute,id-76f26acd-9688-42b4-bc3e-cd134c4cb09e,network,slow] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_dhcpv6_stateless[compute,id-cf1c4425-766b-45b8-be35-e2959728eb00,network,slow] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_slaac[compute,id-9178ad42-10e4-47e9-8987-e02b170cc5cd,network,slow] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_slaac_from_os[compute,id-b6399d76-4438-4658-bcf5-0d6c8584fde2,network,slow] +tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_dhcpv6_stateless[compute,id-7ab23f41-833b-4a16-a7c9-5b42fe6d4123,network,slow] +tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_slaac[compute,id-dec222b1-180c-4098-b8c5-cc1b8342d611,network,slow] +tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os[compute,id-2c92df61-29f0-4eaa-bee3-7c65bef62a43,network,slow] From 9e2331d7129266c22b63a75a5e2505e76c3c2ab3 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 30 Jun 2025 12:54:58 +0100 Subject: [PATCH 10/13] Test mariadb cert config change --- etc/kayobe/ansible/openbao-generate-backend-tls.yml | 9 +++++++++ etc/kayobe/ansible/vault-generate-backend-tls.yml | 9 +++++++++ etc/kayobe/stackhpc.yml | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/openbao-generate-backend-tls.yml b/etc/kayobe/ansible/openbao-generate-backend-tls.yml index f43513ff1..e19c6e6a5 100644 --- a/etc/kayobe/ansible/openbao-generate-backend-tls.yml +++ b/etc/kayobe/ansible/openbao-generate-backend-tls.yml @@ -81,3 +81,12 @@ {{ backend_cert.data.private_key }} mode: "0600" delegate_to: localhost + + - name: Copy intermediate CA + no_log: true + ansible.builtin.copy: + remote: true + src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-INT.crt" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/int.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/vault-generate-backend-tls.yml b/etc/kayobe/ansible/vault-generate-backend-tls.yml index 32e502fbe..9d3795185 100644 --- a/etc/kayobe/ansible/vault-generate-backend-tls.yml +++ b/etc/kayobe/ansible/vault-generate-backend-tls.yml @@ -80,3 +80,12 @@ {{ backend_cert.data.private_key }} mode: "0600" delegate_to: localhost + + - name: Copy intermediate CA + no_log: true + ansible.builtin.copy: + remote: true + src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-INT.crt" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/int.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index c51c28f00..98ee62b8b 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -145,7 +145,7 @@ stackhpc_kolla_source_version: stackhpc/20.0.0.14 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" -stackhpc_kolla_ansible_source_version: rabbitmq-stream-fix +stackhpc_kolla_ansible_source_version: test-mariadb-cert ############################################################################### # Container image registry From 432898017883d695ca26fc2a0765bde2beef2c2d Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 1 Jul 2025 13:16:23 +0100 Subject: [PATCH 11/13] Test proxysql with no TLS --- etc/kayobe/environments/ci-multinode/kolla/globals.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index 7eeaeeeb8..b399215da 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -15,6 +15,10 @@ kolla_enable_tls_backend: "yes" # Enable RabbitMQ TLS rabbitmq_enable_tls: "yes" +# Disable database TLS +database_enable_tls_internal: false +database_enable_tls_backend: false + ############################################################################ # Most development environments will use nested virtualisation, and we can't # guarantee that nested KVM support is available. Use QEMU as a lowest common From a084e9b890bba7c1191c6b9d8597f17077954b03 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 2 Jul 2025 11:41:59 +0100 Subject: [PATCH 12/13] Use fix-service-deploy branch for K-A --- etc/kayobe/stackhpc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 98ee62b8b..3edd221dc 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -145,7 +145,7 @@ stackhpc_kolla_source_version: stackhpc/20.0.0.14 # Kolla Ansible source repository. stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" -stackhpc_kolla_ansible_source_version: test-mariadb-cert +stackhpc_kolla_ansible_source_version: fix-service-deploy ############################################################################### # Container image registry From 785cccc5cc153e645c59f9c387a65b60fa30600b Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Thu, 3 Jul 2025 07:57:28 +0000 Subject: [PATCH 13/13] Revert "Test mariadb cert config change" This reverts commit 9e2331d7129266c22b63a75a5e2505e76c3c2ab3. --- etc/kayobe/ansible/openbao-generate-backend-tls.yml | 9 --------- etc/kayobe/ansible/vault-generate-backend-tls.yml | 9 --------- 2 files changed, 18 deletions(-) diff --git a/etc/kayobe/ansible/openbao-generate-backend-tls.yml b/etc/kayobe/ansible/openbao-generate-backend-tls.yml index e19c6e6a5..f43513ff1 100644 --- a/etc/kayobe/ansible/openbao-generate-backend-tls.yml +++ b/etc/kayobe/ansible/openbao-generate-backend-tls.yml @@ -81,12 +81,3 @@ {{ backend_cert.data.private_key }} mode: "0600" delegate_to: localhost - - - name: Copy intermediate CA - no_log: true - ansible.builtin.copy: - remote: true - src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-INT.crt" - dest: "{{ kayobe_env_config_path }}/kolla/certificates/int.crt" - mode: "0600" - delegate_to: localhost diff --git a/etc/kayobe/ansible/vault-generate-backend-tls.yml b/etc/kayobe/ansible/vault-generate-backend-tls.yml index 9d3795185..32e502fbe 100644 --- a/etc/kayobe/ansible/vault-generate-backend-tls.yml +++ b/etc/kayobe/ansible/vault-generate-backend-tls.yml @@ -80,12 +80,3 @@ {{ backend_cert.data.private_key }} mode: "0600" delegate_to: localhost - - - name: Copy intermediate CA - no_log: true - ansible.builtin.copy: - remote: true - src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-INT.crt" - dest: "{{ kayobe_env_config_path }}/kolla/certificates/int.crt" - mode: "0600" - delegate_to: localhost