Enable complete log aggregation in all integration tests (#310)

siegfriedweber · siegfriedweber · commit 084bd7304780 · 2023-02-08T09:54:25.000Z
# Description

Enable complete log aggregation in all integration tests
diff --git a/tests/templates/kuttl/orphaned-resources/00-assert.yaml b/tests/templates/kuttl/orphaned-resources/00-assert.yaml
diff --git a/tests/templates/kuttl/orphaned-resources/00-assert.yaml.j2 b/tests/templates/kuttl/orphaned-resources/00-assert.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}
diff --git a/tests/templates/kuttl/orphaned-resources/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/orphaned-resources/00-install-vector-aggregator-discovery-configmap.yaml.j2
@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}
diff --git a/tests/templates/kuttl/orphaned-resources/01-assert.yaml b/tests/templates/kuttl/orphaned-resources/01-assert.yaml
@@ -6,31 +6,7 @@ timeout: 600
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: test-hdfs-namenode-default
-status:
-  readyReplicas: 2
-  replicas: 2
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: test-hdfs-journalnode-default
-status:
-  readyReplicas: 1
-  replicas: 1
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: test-hdfs-datanode-default
-status:
-  readyReplicas: 1
-  replicas: 1
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: test-hdfs-datanode-remove
+  name: test-zk-server-default
 status:
   readyReplicas: 1
   replicas: 1
diff --git a/tests/templates/kuttl/orphaned-resources/01-install-zookeeper.yaml.j2 b/tests/templates/kuttl/orphaned-resources/01-install-zookeeper.yaml.j2
@@ -7,8 +7,15 @@ spec:
   image:
     productVersion: "{{ test_scenario['values']['zookeeper-latest'].split('-stackable')[0] }}"
     stackableVersion: "{{ test_scenario['values']['zookeeper-latest'].split('-stackable')[1] }}"
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  clusterConfig:
+    logging:
+      vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
   servers:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           min: '300m'
diff --git a/tests/templates/kuttl/orphaned-resources/02-assert.yaml b/tests/templates/kuttl/orphaned-resources/02-assert.yaml
@@ -0,0 +1,36 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: test-hdfs-namenode-default
+status:
+  readyReplicas: 2
+  replicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: test-hdfs-journalnode-default
+status:
+  readyReplicas: 1
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: test-hdfs-datanode-default
+status:
+  readyReplicas: 1
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: test-hdfs-datanode-remove
+status:
+  readyReplicas: 1
+  replicas: 1
diff --git a/tests/templates/kuttl/orphaned-resources/02-install-hdfs.yaml.j2 b/tests/templates/kuttl/orphaned-resources/02-install-hdfs.yaml.j2
@@ -8,9 +8,14 @@ spec:
     productVersion: "{{ test_scenario['values']['hadoop-latest'].split('-stackable')[0] }}"
     stackableVersion: "{{ test_scenario['values']['hadoop-latest'].split('-stackable')[1] }}"
   zookeeperConfigMapName: test-znode
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
   dfsReplication: 1
   nameNodes:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           max: '1'
@@ -22,6 +27,8 @@ spec:
         replicas: 2
   dataNodes:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           max: '1'
@@ -35,6 +42,8 @@ spec:
         replicas: 1
   journalNodes:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           max: '1'
diff --git a/tests/templates/kuttl/orphaned-resources/04-change-datanode-rolegroup.yaml.j2 b/tests/templates/kuttl/orphaned-resources/04-change-datanode-rolegroup.yaml.j2
@@ -6,6 +6,8 @@ metadata:
 spec:
   dataNodes:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           max: '1'
diff --git a/tests/templates/kuttl/smoke/00-assert.yaml.j2 b/tests/templates/kuttl/smoke/00-assert.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2
@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/01-assert.yaml b/tests/templates/kuttl/smoke/01-assert.yaml
diff --git a/tests/templates/kuttl/smoke/01-install-zk.yaml.j2 b/tests/templates/kuttl/smoke/01-install-zk.yaml.j2
@@ -7,8 +7,15 @@ spec:
   image:
     productVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[0] }}"
     stackableVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[1] }}"
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  clusterConfig:
+    logging:
+      vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
   servers:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           min: '300m'
diff --git a/tests/templates/kuttl/smoke/02-assert.yaml b/tests/templates/kuttl/smoke/02-assert.yaml
diff --git a/tests/templates/kuttl/smoke/02-assert.yaml.j2 b/tests/templates/kuttl/smoke/02-assert.yaml.j2
diff --git a/tests/templates/kuttl/smoke/02-install-hdfs.yaml.j2 b/tests/templates/kuttl/smoke/02-install-hdfs.yaml.j2
@@ -14,9 +14,14 @@ spec:
     productVersion: "{{ test_scenario['values']['hadoop'].split('-stackable')[0] }}"
     stackableVersion: "{{ test_scenario['values']['hadoop'].split('-stackable')[1] }}"
   zookeeperConfigMapName: hdfs-zk
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
   dfsReplication: 1
   nameNodes:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           max: '1'
@@ -28,6 +33,8 @@ spec:
         replicas: 2
   dataNodes:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           max: '1'
@@ -53,6 +60,8 @@ spec:
         replicas: {{ test_scenario['values']['number-of-datanodes'] }}
   journalNodes:
     config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
           max: '1'
diff --git a/tests/templates/kuttl/smoke/03-assert.yaml b/tests/templates/kuttl/smoke/03-assert.yaml
@@ -1,5 +1,12 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-commands:
-  - script: kubectl exec -n $NAMESPACE webhdfs-0 -- python /tmp/webhdfs.py ls
+timeout: 300
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: webhdfs
+status:
+  readyReplicas: 1
+  replicas: 1
diff --git a/tests/templates/kuttl/smoke/03-webhdfs.yaml b/tests/templates/kuttl/smoke/03-webhdfs.yaml
diff --git a/tests/templates/kuttl/smoke/04-assert.yaml b/tests/templates/kuttl/smoke/04-assert.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: kubectl exec -n $NAMESPACE webhdfs-0 -- python /tmp/webhdfs.py ls
diff --git a/tests/templates/kuttl/smoke/04-create-file.yaml b/tests/templates/kuttl/smoke/04-create-file.yaml
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
@@ -2,18 +2,18 @@
 dimensions:
   - name: hadoop
     values:
-      - 3.2.2-stackable0.7.0
-      - 3.3.3-stackable0.3.0
-      - 3.3.4-stackable0.3.0
+      - 3.2.2-stackable23.4.0-rc1
+      - 3.3.3-stackable23.4.0-rc1
+      - 3.3.4-stackable23.4.0-rc1
   - name: hadoop-latest
     values:
-      - 3.3.4-stackable0.3.0
+      - 3.3.4-stackable23.4.0-rc1
   - name: zookeeper
     values:
-      - 3.8.0-stackable0.9.0
+      - 3.8.0-stackable23.4.0-rc1
   - name: zookeeper-latest
     values:
-      - 3.8.0-stackable0.9.0
+      - 3.8.0-stackable23.4.0-rc1
   - name: number-of-datanodes
     values:
       - "1"
